diff --git a/tests/integration/generated/test_debug_chrysalis.cfg b/tests/integration/generated/test_debug_chrysalis.cfg new file mode 100644 index 00000000..957651d1 --- /dev/null +++ b/tests/integration/generated/test_debug_chrysalis.cfg @@ -0,0 +1,149 @@ +[default] +case = v2.LR.historical_0201 +constraint = "" +dry_run = "False" +environment_commands = "" +input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_debug.py +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201" +partition = "debug" +qos = "regular" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/unique_id" + +[climo] +active = True +walltime = "00:30:00" +years = "1850:1854:2", "1850:1854:4", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "" + + [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] + frequency = "diurnal_8xdaily" + input_files = "eam.h4" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly_climo ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + vars = "" + +[ts] +active = True +e3sm_to_cmip_environment_commands = "" +walltime = "00:30:00" +years = "1850:1854:2", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ atm_daily_180x360_aave ]] + frequency = "daily" + input_files = "eam.h1" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ atm_monthly_glb ]] + # Note global average won't work for 3D variables. + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + mapping_file = "glb" + years = "1850:1860:5", + + [[ land_monthly ]] + e3sm_to_cmip_environment_commands = "" + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + vars = "LAISHA,LAISUN" + ts_fmt = "cmip" + + [[ lnd_monthly_glb ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "glb" + vars = "LAISHA,LAISUN" + years = "1850:1860:5", + + [[ rof_monthly ]] + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + +[e3sm_diags] +active = True +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +grid = '180x360_aave' +ref_final_yr = 2014 +ref_start_yr = 1985 +# TODO: this directory is missing OMI-MLS +sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +walltime = "00:30:00" +years = "1850:1854:2", "1850:1854:4", + + [[ atm_monthly_180x360_aave ]] + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + partition = "compute" + qos = "regular" + sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", + walltime = "5:00:00" + + [[ atm_monthly_180x360_aave_mvm ]] + # Test model-vs-model using the same files as the reference + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + diff_title = "Difference" + partition = "compute" + qos = "regular" + ref_final_yr = 1851 + ref_name = "v2.LR.historical_0201" + ref_start_yr = 1850 + ref_years = "1850-1851", + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim" + run_type = "model_vs_model" + short_ref_name = "v2.LR.historical_0201" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 + ts_subsection = "atm_monthly_180x360_aave" + walltime = "5:00:00" + years = "1852-1853", + + [[ lnd_monthly_mvm_lnd ]] + # Test model-vs-model using the same files as the reference + climo_subsection = "land_monthly_climo" + diff_title = "Difference" + #grid = 'native' + partition = "compute" + qos = "regular" + ref_final_yr = 1851 + ref_name = "v2.LR.historical_0201" + ref_start_yr = 1850 + ref_years = "1850-1851", + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim" + run_type = "model_vs_model" + sets = "lat_lon_land", + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 diff --git a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg index 6c0559c4..05c6db7f 100644 --- a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg @@ -81,7 +81,7 @@ years = "1985:1995:5" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_cdat_migrated_sets_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_cdat_migrated_sets_chrysalis.cfg new file mode 100644 index 00000000..0cc9f437 --- /dev/null +++ b/tests/integration/generated/test_min_case_e3sm_diags_cdat_migrated_sets_chrysalis.cfg @@ -0,0 +1,125 @@ +[default] +case = "v3.LR.historical_0051" +constraint = "" +dry_run = "False" +environment_commands = "" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_cdat_migrated_output/unique_id/v3.LR.historical_0051" +partition = "debug" +qos = "regular" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_e3sm_diags_cdat_migrated_www/unique_id" +years = "1985:1989:2", + +[climo] +active = True +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "" + + [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] + frequency = "diurnal_8xdaily" + input_files = "eam.h3" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly_climo ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + vars = "" + +[ts] +active = True +e3sm_to_cmip_environment_commands = "" +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ rof_monthly ]] + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + ts_fmt = "cmip" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[tc_analysis] +active = True +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_weekly_comprehensive_v3_scratch/unique_id/v3.LR.historical_0051" +walltime = "00:30:00" + +[e3sm_diags] +active = True +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "compute" +qos = "regular" +ref_final_yr = 1986 +ref_start_yr = 1985 +ref_years = "1985-1986", +# Also lat_lon_land for mvm +sets="lat_lon","lat_lon_river","cosp_histogram","zonal_mean_xy","zonal_mean_2d","zonal_mean_2d_stratosphere","aerosol_aeronet","polar","meridional_mean_2d","area_mean_time_series","aerosol_budget","annual_cycle_zonal_mean","diurnal_cycle", +short_name = "v3.LR.historical_0051" +ts_num_years = 2 +walltime = "5:00:00" +years = "1987:1989:2" + + [[ atm_monthly_180x360_aave ]] + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology' + + [[ atm_monthly_180x360_aave_mvm ]] + # Test model-vs-model using the same files as the reference + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + diff_title = "Difference" + partition = "compute" + qos = "regular" + ref_name = "v3.LR.historical_0051" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_cdat_migrated_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim" + run_type = "model_vs_model" + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 + ts_subsection = "atm_monthly_180x360_aave" + + [[ lnd_monthly_mvm_lnd ]] + # Test model-vs-model using the same files as the reference + climo_subsection = "land_monthly_climo" + diff_title = "Difference" + partition = "compute" + qos = "regular" + ref_name = "v3.LR.historical_0051" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_cdat_migrated_output/unique_id/v3.LR.historical_0051/post/lnd/180x360_aave/clim" + run_type = "model_vs_model" + sets = "lat_lon_land", + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg index 416771ae..782f5b91 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg index 036e17a7..ee85a81f 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg index 5a686029..99433c41 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg @@ -25,7 +25,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg index 72219931..8c8892ae 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg @@ -25,7 +25,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg index 481b7fc1..66070972 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg index b0109858..596cccb1 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg index bb585138..d6cc26b4 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg index 797adc44..860efacb 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg @@ -27,7 +27,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg index f7d43fc6..7b0dd8aa 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg @@ -27,7 +27,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg index ee24e496..8b02b1d9 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg @@ -19,7 +19,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg index 3e42bc99..8b782b23 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg @@ -19,7 +19,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg index 7a997083..d17b484c 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg @@ -25,7 +25,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_weekly_bundles_chrysalis.cfg b/tests/integration/generated/test_weekly_bundles_chrysalis.cfg index c4c1b5db..df561ac7 100644 --- a/tests/integration/generated/test_weekly_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_bundles_chrysalis.cfg @@ -103,7 +103,7 @@ years = "1985:1989:2", [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' ref_final_yr = 1989 ref_start_yr = 1985 diff --git a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg index 7b6987e1..d753eae4 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg @@ -84,7 +84,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index 6d37de66..2cbbe392 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -86,7 +86,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/template_debug.cfg b/tests/integration/template_debug.cfg new file mode 100644 index 00000000..2635585c --- /dev/null +++ b/tests/integration/template_debug.cfg @@ -0,0 +1,149 @@ +[default] +case = v2.LR.historical_0201 +constraint = "#expand constraint#" +dry_run = "#expand dry_run#" +environment_commands = "#expand environment_commands#" +input = "#expand user_input#/E3SMv2/v2.LR.historical_0201" +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_debug.py +output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201" +partition = "#expand partition_short#" +qos = "#expand qos_short#" +www = "#expand user_www#zppy_test_debug_www/#expand unique_id#" + +[climo] +active = True +walltime = "00:30:00" +years = "1850:1854:2", "1850:1854:4", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "" + + [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] + frequency = "diurnal_8xdaily" + input_files = "eam.h4" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly_climo ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + vars = "" + +[ts] +active = True +e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" +walltime = "00:30:00" +years = "1850:1854:2", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ atm_daily_180x360_aave ]] + frequency = "daily" + input_files = "eam.h1" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ atm_monthly_glb ]] + # Note global average won't work for 3D variables. + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + mapping_file = "glb" + years = "1850:1860:5", + + [[ land_monthly ]] + e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + vars = "LAISHA,LAISUN" + ts_fmt = "cmip" + + [[ lnd_monthly_glb ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "glb" + vars = "LAISHA,LAISUN" + years = "1850:1860:5", + + [[ rof_monthly ]] + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + +[e3sm_diags] +active = True +environment_commands = "#expand diags_environment_commands#" +grid = '180x360_aave' +ref_final_yr = 2014 +ref_start_yr = 1985 +# TODO: this directory is missing OMI-MLS +sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +walltime = "00:30:00" +years = "1850:1854:2", "1850:1854:4", + + [[ atm_monthly_180x360_aave ]] + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + partition = "#expand partition_long#" + qos = "#expand qos_long#" + sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", + walltime = "#expand diags_walltime#" + + [[ atm_monthly_180x360_aave_mvm ]] + # Test model-vs-model using the same files as the reference + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + diff_title = "Difference" + partition = "#expand partition_long#" + qos = "#expand qos_long#" + ref_final_yr = 1851 + ref_name = "v2.LR.historical_0201" + ref_start_yr = 1850 + ref_years = "1850-1851", + reference_data_path = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201/post/atm/180x360_aave/clim" + run_type = "model_vs_model" + short_ref_name = "v2.LR.historical_0201" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 + ts_subsection = "atm_monthly_180x360_aave" + walltime = "#expand diags_walltime#" + years = "1852-1853", + + [[ lnd_monthly_mvm_lnd ]] + # Test model-vs-model using the same files as the reference + climo_subsection = "land_monthly_climo" + diff_title = "Difference" + #grid = 'native' + partition = "#expand partition_long#" + qos = "#expand qos_long#" + ref_final_yr = 1851 + ref_name = "v2.LR.historical_0201" + ref_start_yr = 1850 + ref_years = "1850-1851", + reference_data_path = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201/post/lnd/180x360_aave/clim" + run_type = "model_vs_model" + sets = "lat_lon_land", + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 diff --git a/tests/integration/template_min_case_e3sm_diags_cdat_migrated_sets.cfg b/tests/integration/template_min_case_e3sm_diags_cdat_migrated_sets.cfg new file mode 100644 index 00000000..ec77ec4b --- /dev/null +++ b/tests/integration/template_min_case_e3sm_diags_cdat_migrated_sets.cfg @@ -0,0 +1,125 @@ +[default] +case = "#expand case_name#" +constraint = "#expand constraint#" +dry_run = "#expand dry_run#" +environment_commands = "#expand environment_commands#" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "#expand user_output#zppy_min_case_e3sm_diags_cdat_migrated_output/#expand unique_id#/#expand case_name#" +partition = "#expand partition_short#" +qos = "#expand qos_short#" +www = "#expand user_www#zppy_min_case_e3sm_diags_cdat_migrated_www/#expand unique_id#" +years = "1985:1989:2", + +[climo] +active = True +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "" + + [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] + frequency = "diurnal_8xdaily" + input_files = "eam.h3" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly_climo ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + vars = "" + +[ts] +active = True +e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ rof_monthly ]] + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + ts_fmt = "cmip" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[tc_analysis] +active = True +scratch = "#expand scratch#zppy_weekly_comprehensive_v3_scratch/#expand unique_id#/#expand case_name#" +walltime = "00:30:00" + +[e3sm_diags] +active = True +environment_commands = "#expand diags_environment_commands#" +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "#expand partition_long#" +qos = "#expand qos_long#" +ref_final_yr = 1986 +ref_start_yr = 1985 +ref_years = "1985-1986", +# Also lat_lon_land for mvm +sets="lat_lon","lat_lon_river","cosp_histogram","zonal_mean_xy","zonal_mean_2d","zonal_mean_2d_stratosphere","aerosol_aeronet","polar","meridional_mean_2d","area_mean_time_series","aerosol_budget","annual_cycle_zonal_mean","diurnal_cycle", +short_name = "#expand case_name#" +ts_num_years = 2 +walltime = "#expand diags_walltime#" +years = "1987:1989:2" + + [[ atm_monthly_180x360_aave ]] + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology' + + [[ atm_monthly_180x360_aave_mvm ]] + # Test model-vs-model using the same files as the reference + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + diff_title = "Difference" + partition = "#expand partition_long#" + qos = "#expand qos_long#" + ref_name = "#expand case_name#" + reference_data_path = "#expand user_output#zppy_min_case_e3sm_diags_cdat_migrated_output/#expand unique_id#/#expand case_name#/post/atm/180x360_aave/clim" + run_type = "model_vs_model" + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 + ts_subsection = "atm_monthly_180x360_aave" + + [[ lnd_monthly_mvm_lnd ]] + # Test model-vs-model using the same files as the reference + climo_subsection = "land_monthly_climo" + diff_title = "Difference" + partition = "#expand partition_long#" + qos = "#expand qos_long#" + ref_name = "#expand case_name#" + reference_data_path = "#expand user_output#zppy_min_case_e3sm_diags_cdat_migrated_output/#expand unique_id#/#expand case_name#/post/lnd/180x360_aave/clim" + run_type = "model_vs_model" + sets = "lat_lon_land", + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 2c6d6bdf..03009ba3 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -139,7 +139,7 @@ def get_chyrsalis_expansions(config): "constraint": "", # To run this test, replace conda environment with your e3sm_diags dev environment # To use default environment_commands, set to "" - "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731", + "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_cdat_branch_20240731", "diags_walltime": "5:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -273,6 +273,7 @@ def generate_cfgs(unified_testing=False, dry_run=False): "min_case_add_dependencies", "min_case_tc_analysis_simultaneous_1", "min_case_tc_analysis_simultaneous_2", + "min_case_e3sm_diags_cdat_migrated_sets", "min_case_e3sm_diags_depend_on_climo_mvm_1", "min_case_e3sm_diags_depend_on_climo_mvm_2", "min_case_e3sm_diags_depend_on_climo", diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index d874d8e3..98ab2921 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -148,6 +148,7 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 depend_on_climo = set( [ "lat_lon", + "lat_lon_land", "zonal_mean_xy", "zonal_mean_2d", "polar", @@ -271,6 +272,8 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file): # noqa: C901 else: print("...adding to bundle '%s'" % (c["bundle"])) + for d in dependencies: + print(f" d: {d}") print(f" environment_commands={c['environment_commands']}") print_url(c, "e3sm_diags") diff --git a/zppy/templates/e3sm_diags.bash b/zppy/templates/e3sm_diags.bash index b32eed54..4e691035 100644 --- a/zppy/templates/e3sm_diags.bash +++ b/zppy/templates/e3sm_diags.bash @@ -101,19 +101,9 @@ create_links_ts() YYYY=`printf "%04d" ${year}` for file in ${ts_dir_source}/${v}_${YYYY}*.nc do - # Add this time series file to the list of files for cdscan to use - echo ${file} >> ${v}_files.txt + cp ${file} ${ts_dir_destination}/${v}_${YYYY}*.nc done done - # xml file will cover the whole period from year1 to year2 - xml_name=${v}_${begin_year}01_${end_year}12.xml - export CDMS_NO_MPI=true - cdscan -x ${xml_name} -f ${v}_files.txt - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo "ERROR (${error_num})" > {{ prefix }}.status - exit ${error_num} - fi done cd .. } @@ -128,13 +118,7 @@ create_links_ts_rof() mkdir -p ${ts_rof_dir_destination} cd ${ts_rof_dir_destination} v="RIVER_DISCHARGE_OVER_LAND_LIQ" - xml_name=${v}_${begin_year}01_${end_year}12.xml - cdscan -x ${xml_name} ${ts_rof_dir_source}/${v}_*.nc - if [ $? != 0 ]; then - cd {{ scriptDir }} - echo "ERROR (${error_num})" > {{ prefix }}.status - exit ${error_num} - fi + cp ${ts_rof_dir_source}/${v}_*.nc ${v}_*.nc cd .. }