diff --git a/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg new file mode 100644 index 00000000..0f04f763 --- /dev/null +++ b/tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg @@ -0,0 +1,183 @@ +# Test carryover dependency handling + +# Things to check for in the output: +# 1. Check that `grep -v "OK" *status returns nothing (i.e., all jobs worked) +# 2. Check that .settings files include `dependencies` listed out. Check that these look correct. + + +# These tasks can have carryover dependencies: +# 1. e3sm_diags.py +# 2. mpas_analysis.py +# 3. tc_analysis.py + +# These tasks have dependencies, but no carryover: +# 1. global_time_series.py +# 2. ilamb.py + +# These tasks don't have dependencies: +# 1. climo.py +# 2. ts.py + +[default] +case = "v3.LR.historical_0051" +constraint = "" +dry_run = "False" +environment_commands = "" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_carryover_dependencies_output/unique_id/v3.LR.historical_0051" +partition = "debug" +qos = "regular" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_carryover_dependencies_www/unique_id" +years = "1985:1989:2", + +[climo] +active = True +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "" + + [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] + frequency = "diurnal_8xdaily" + input_files = "eam.h3" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly_climo ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + vars = "" + +[ts] +active = True +e3sm_to_cmip_environment_commands = "" +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ rof_monthly ]] + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + + [[ atm_monthly_glb ]] + # Note global average won't work for 3D variables. + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + mapping_file = "glb" + years = "1985:1995:5", + + [[ lnd_monthly_glb ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "glb" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + years = "1985:1995:5", + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + ts_fmt = "cmip" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[tc_analysis] +# We're only doing one run of tc_analysis here, +# so we won't be explictally testing carryover dependency here. +active = True +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_carryover_dependencies_scratch/unique_id/v3.LR.historical_0051" +walltime = "00:30:00" + +[e3sm_diags] +active = True +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20240731" +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "compute" +qos = "regular" +ref_final_yr = 1986 +ref_start_yr = 1985 +ref_years = "1985-1986", +short_name = "v3.LR.historical_0051" +ts_num_years = 2 +walltime = "5:00:00" +years = "1987:1989:2" + + [[ atm_monthly_180x360_aave ]] + # This task will depend on the tc_analysis task + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology' + sets = "lat_lon","tc_analysis" + + [[ atm_monthly_180x360_aave_mvm ]] + # This task will not depend on the tc_analysis task + # Test model-vs-model using the same files as the reference + climo_subsection = "atm_monthly_180x360_aave" + diff_title = "Difference" + partition = "compute" + qos = "regular" + ref_name = "v3.LR.historical_0051" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_carryover_dependencies_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim" + run_type = "model_vs_model" + sets = "lat_lon", + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 + ts_subsection = "atm_monthly_180x360_aave" + +[mpas_analysis] +# The second run should depend on the first run. +active = True +anomalyRefYear = 1985 +climo_years = "1985-1989", "1990-1995", +enso_years = "1985-1989", "1990-1995", +mesh = "IcoswISC30E3r5" +parallelTaskCount = 6 +partition = "compute" +qos = "regular" +shortTermArchive = True +ts_years = "1985-1989", "1985-1995", +walltime = "00:30:00" + +[global_time_series] +active = True +climo_years = "1985-1989", "1990-1995", +experiment_name = "v3.LR.historical_0051" +figstr = "v3.LR.historical_0051" +moc_file=mocTimeSeries_1985-1995.nc +plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +ts_num_years = 5 +ts_years = "1985-1989", "1985-1995", +walltime = "00:30:00" +years = "1985-1995", + +[ilamb] +active = True +nodes = 8 +partition = "compute" +short_name = "v3.LR.historical_0051" +ts_num_years = 2 +walltime = "2:00:00" +years = "1985:1989:4" diff --git a/tests/integration/template_min_case_carryover_dependencies.cfg b/tests/integration/template_min_case_carryover_dependencies.cfg new file mode 100644 index 00000000..acd1900c --- /dev/null +++ b/tests/integration/template_min_case_carryover_dependencies.cfg @@ -0,0 +1,183 @@ +# Test carryover dependency handling + +# Things to check for in the output: +# 1. Check that `grep -v "OK" *status returns nothing (i.e., all jobs worked) +# 2. Check that .settings files include `dependencies` listed out. Check that these look correct. + + +# These tasks can have carryover dependencies: +# 1. e3sm_diags.py +# 2. mpas_analysis.py +# 3. tc_analysis.py + +# These tasks have dependencies, but no carryover: +# 1. global_time_series.py +# 2. ilamb.py + +# These tasks don't have dependencies: +# 1. climo.py +# 2. ts.py + +[default] +case = "#expand case_name#" +constraint = "#expand constraint#" +dry_run = "#expand dry_run#" +environment_commands = "#expand environment_commands#" +input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "#expand user_output#zppy_min_case_carryover_dependencies_output/#expand unique_id#/#expand case_name#" +partition = "#expand partition_short#" +qos = "#expand qos_short#" +www = "#expand user_www#zppy_min_case_carryover_dependencies_www/#expand unique_id#" +years = "1985:1989:2", + +[climo] +active = True +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + vars = "" + + [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] + frequency = "diurnal_8xdaily" + input_files = "eam.h3" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ land_monthly_climo ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + vars = "" + +[ts] +active = True +e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#" +walltime = "00:30:00" + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ rof_monthly ]] + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + + [[ atm_monthly_glb ]] + # Note global average won't work for 3D variables. + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + mapping_file = "glb" + years = "1985:1995:5", + + [[ lnd_monthly_glb ]] + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "glb" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + years = "1985:1995:5", + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc" + ts_fmt = "cmip" + vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" + +[tc_analysis] +# We're only doing one run of tc_analysis here, +# so we won't be explictally testing carryover dependency here. +active = True +scratch = "#expand scratch#zppy_min_case_carryover_dependencies_scratch/#expand unique_id#/#expand case_name#" +walltime = "00:30:00" + +[e3sm_diags] +active = True +environment_commands = "#expand diags_environment_commands#" +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "#expand partition_long#" +qos = "#expand qos_long#" +ref_final_yr = 1986 +ref_start_yr = 1985 +ref_years = "1985-1986", +short_name = "#expand case_name#" +ts_num_years = 2 +walltime = "#expand diags_walltime#" +years = "1987:1989:2" + + [[ atm_monthly_180x360_aave ]] + # This task will depend on the tc_analysis task + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology' + sets = "lat_lon","tc_analysis" + + [[ atm_monthly_180x360_aave_mvm ]] + # This task will not depend on the tc_analysis task + # Test model-vs-model using the same files as the reference + climo_subsection = "atm_monthly_180x360_aave" + diff_title = "Difference" + partition = "#expand partition_long#" + qos = "#expand qos_long#" + ref_name = "#expand case_name#" + reference_data_path = "#expand user_output#zppy_min_case_carryover_dependencies_output/#expand unique_id#/#expand case_name#/post/atm/180x360_aave/clim" + run_type = "model_vs_model" + sets = "lat_lon", + short_ref_name = "same simulation" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 + ts_subsection = "atm_monthly_180x360_aave" + +[mpas_analysis] +# The second run should depend on the first run. +active = True +anomalyRefYear = 1985 +climo_years = "1985-1989", "1990-1995", +enso_years = "1985-1989", "1990-1995", +mesh = "IcoswISC30E3r5" +parallelTaskCount = 6 +partition = "#expand partition_long#" +qos = "#expand qos_long#" +shortTermArchive = True +ts_years = "1985-1989", "1985-1995", +walltime = "#expand mpas_analysis_walltime#" + +[global_time_series] +active = True +climo_years = "1985-1989", "1990-1995", +experiment_name = "#expand case_name#" +figstr = "#expand case_name#" +moc_file=mocTimeSeries_1985-1995.nc +plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +ts_num_years = 5 +ts_years = "1985-1989", "1985-1995", +walltime = "00:30:00" +years = "1985-1995", + +[ilamb] +active = True +nodes = 8 +partition = "#expand partition_long#" +short_name = "#expand case_name#" +ts_num_years = 2 +walltime = "2:00:00" +years = "1985:1989:4" diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 2c6d6bdf..801a7f7c 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -271,6 +271,7 @@ def generate_cfgs(unified_testing=False, dry_run=False): cfg_names = [ "min_case_add_dependencies", + "min_case_carryover_dependencies", "min_case_tc_analysis_simultaneous_1", "min_case_tc_analysis_simultaneous_2", "min_case_e3sm_diags_depend_on_climo_mvm_1",