From a70db5ec72ff72881733d2592bb98cd631edadb9 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Tue, 15 Oct 2024 16:12:56 -0500 Subject: [PATCH] Make TC Analysis parallel --- ...st_min_case_add_dependencies_chrysalis.cfg | 2 +- ...e_e3sm_diags_depend_on_climo_chrysalis.cfg | 2 +- ..._diags_depend_on_climo_mvm_2_chrysalis.cfg | 2 +- ...case_e3sm_diags_depend_on_ts_chrysalis.cfg | 2 +- ...3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg | 2 +- ...ase_e3sm_diags_diurnal_cycle_chrysalis.cfg | 2 +- ...sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg | 2 +- ...3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg | 2 +- ...n_case_e3sm_diags_streamflow_chrysalis.cfg | 2 +- ..._e3sm_diags_streamflow_mvm_2_chrysalis.cfg | 2 +- ..._case_e3sm_diags_tc_analysis_chrysalis.cfg | 13 ++++--- ...e3sm_diags_tc_analysis_mvm_1_chrysalis.cfg | 8 ++--- ...e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg | 19 +++++----- ...m_diags_tc_analysis_parallel_chrysalis.cfg | 36 +++++++++++++++++++ ...s_tropical_subseasonal_mvm_2_chrysalis.cfg | 2 +- ...e_tc_analysis_simultaneous_1_chrysalis.cfg | 8 ++--- ...e_tc_analysis_simultaneous_2_chrysalis.cfg | 8 ++--- .../test_weekly_bundles_chrysalis.cfg | 2 +- ...test_weekly_comprehensive_v2_chrysalis.cfg | 2 +- ...test_weekly_comprehensive_v3_chrysalis.cfg | 2 +- ...mplate_min_case_e3sm_diags_tc_analysis.cfg | 11 +++--- ..._min_case_e3sm_diags_tc_analysis_mvm_1.cfg | 8 ++--- ..._min_case_e3sm_diags_tc_analysis_mvm_2.cfg | 17 ++++----- ...n_case_e3sm_diags_tc_analysis_parallel.cfg | 36 +++++++++++++++++++ ...te_min_case_tc_analysis_simultaneous_1.cfg | 8 ++--- ...te_min_case_tc_analysis_simultaneous_2.cfg | 8 ++--- tests/integration/utils.py | 3 +- zppy/e3sm_diags.py | 10 +----- zppy/tc_analysis.py | 12 ++----- 29 files changed, 146 insertions(+), 87 deletions(-) create mode 100644 tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_parallel_chrysalis.cfg create mode 100644 tests/integration/template_min_case_e3sm_diags_tc_analysis_parallel.cfg diff --git a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg index bad745c1..2656fa7a 100644 --- a/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg @@ -81,7 +81,7 @@ years = "1985:1995:5" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg index 1d0cb6f6..c95c1d0c 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg index be9105e8..8b21a431 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_climo_mvm_2_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg index 74540fd2..9e47abd2 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg @@ -25,7 +25,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg index 78063b90..cc085aa1 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg @@ -25,7 +25,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg index 1439fa4e..11e9983f 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg index ed7ffb6d..251d82c3 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_diurnal_cycle_mvm_2_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg index dcb2ac51..13d3015f 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_lat_lon_land_mvm_2_chrysalis.cfg @@ -24,7 +24,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg index e5ad3278..9a82f303 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg @@ -27,7 +27,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg index 81cc605c..147d851f 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg @@ -27,7 +27,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg index d5851184..df05210a 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_chrysalis.cfg @@ -1,12 +1,12 @@ [default] -case = "v3.LR.historical_0051" +case = "v2.LR.historical_0201" constraint = "" dry_run = "False" environment_commands = "" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_www/unique_id" @@ -14,22 +14,21 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_scratch/unique_id/v3.LR.historical_0051" +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_scratch/unique_id/v2.LR.historical_0201" walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 partition = "compute" qos = "regular" -short_name = "v3.LR.historical_0051" +short_name = "v2.LR.historical_0201" ts_num_years = 2 walltime = "5:00:00" [[ atm_monthly_180x360_aave_tc_analysis ]] - # Note: tc_analysis requires e3sm_diags jobs to run sequentially sets = "tc_analysis", # tc_obs is determined automatically diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_1_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_1_chrysalis.cfg index f9611ca1..aa29ad07 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_1_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_1_chrysalis.cfg @@ -1,12 +1,12 @@ [default] -case = "v3.LR.historical_0051" +case = "v2.LR.historical_0201" constraint = "" dry_run = "False" environment_commands = "" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_2_www/unique_id" @@ -14,5 +14,5 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_1_scratch/unique_id/v3.LR.historical_0051" +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_1_scratch/unique_id/v2.LR.historical_0201" walltime = "00:30:00" diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg index eff8a97f..a226f2ef 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_mvm_2_chrysalis.cfg @@ -1,12 +1,13 @@ +# Run this after _1 completes. (We need tc_analysis for the reference years). [default] -case = "v3.LR.historical_0051" +case = "v2.LR.historical_0201" constraint = "" dry_run = "False" environment_commands = "" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_2_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_2_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_2_www/unique_id" @@ -14,18 +15,18 @@ years = "1995:1997:2", [tc_analysis] active = True -scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_2_scratch/unique_id/v3.LR.historical_0051" +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_2_scratch/unique_id/v2.LR.historical_0201" walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 partition = "compute" qos = "regular" -short_name = "v3.LR.historical_0051" +short_name = "v2.LR.historical_0201" walltime = "5:00:00" [[ atm_monthly_180x360_aave_mvm ]] @@ -33,15 +34,15 @@ walltime = "5:00:00" climo_subsection = "atm_monthly_180x360_aave" diff_title = "Difference" ref_final_yr = 1986 - ref_name = "v3.LR.historical_0051" + ref_name = "v2.LR.historical_0201" ref_start_yr = 1985 ref_years = "1985-1986", # Use _1 as reference - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim" + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim" # reference_data_path_tc determined automatically run_type = "model_vs_model" sets = "tc_analysis", - short_ref_name = "v3.LR.historical_0051" + short_ref_name = "v2.LR.historical_0201" swap_test_ref = False tag = "model_vs_model" years = "1995-1996", diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_parallel_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_parallel_chrysalis.cfg new file mode 100644 index 00000000..c3a7e505 --- /dev/null +++ b/tests/integration/generated/test_min_case_e3sm_diags_tc_analysis_parallel_chrysalis.cfg @@ -0,0 +1,36 @@ +[default] +case = "v2.LR.historical_0201" +constraint = "" +dry_run = "False" +environment_commands = "" +input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201 +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_parallel_output/unique_id/v2.LR.historical_0201" +partition = "debug" +qos = "regular" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_parallel_www/unique_id" +years = "1985:1989:2", + +[tc_analysis] +active = True +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_e3sm_diags_tc_analysis_parallel_scratch/unique_id/v2.LR.historical_0201" +walltime = "00:30:00" + +[e3sm_diags] +active = True +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "compute" +qos = "regular" +short_name = "v2.LR.historical_0201" +ts_num_years = 2 +walltime = "5:00:00" + + [[ atm_monthly_180x360_aave_tc_analysis ]] + # Expecting two viewers: 1985-1986, 1987-1988 + # The generation of these two viewers should occur in parallel. + sets = "tc_analysis", + # tc_obs is determined automatically diff --git a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg index 181cd326..233569cc 100644 --- a/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg @@ -25,7 +25,7 @@ walltime = "00:30:00" [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_min_case_tc_analysis_simultaneous_1_chrysalis.cfg b/tests/integration/generated/test_min_case_tc_analysis_simultaneous_1_chrysalis.cfg index 212baec1..41de5e34 100644 --- a/tests/integration/generated/test_min_case_tc_analysis_simultaneous_1_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_tc_analysis_simultaneous_1_chrysalis.cfg @@ -1,12 +1,12 @@ [default] -case = "v3.LR.historical_0051" +case = "v2.LR.historical_0201" constraint = "" dry_run = "False" environment_commands = "" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_1_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_1_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_1_www/unique_id" @@ -14,5 +14,5 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_1_scratch/unique_id/v3.LR.historical_0051" +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_1_scratch/unique_id/v2.LR.historical_0201" walltime = "00:30:00" diff --git a/tests/integration/generated/test_min_case_tc_analysis_simultaneous_2_chrysalis.cfg b/tests/integration/generated/test_min_case_tc_analysis_simultaneous_2_chrysalis.cfg index 3a116479..71880091 100644 --- a/tests/integration/generated/test_min_case_tc_analysis_simultaneous_2_chrysalis.cfg +++ b/tests/integration/generated/test_min_case_tc_analysis_simultaneous_2_chrysalis.cfg @@ -1,14 +1,14 @@ # Run this immediately after running _1 [default] -case = "v3.LR.historical_0051" +case = "v2.LR.historical_0201" constraint = "" dry_run = "False" environment_commands = "" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051 +input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201 input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_2_output/unique_id/v3.LR.historical_0051" +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_2_output/unique_id/v2.LR.historical_0201" partition = "debug" qos = "regular" www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_2_www/unique_id" @@ -16,5 +16,5 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_2_scratch/unique_id/v3.LR.historical_0051" +scratch = "/lcrc/globalscratch/ac.forsyth2/zppy_min_case_tc_analysis_simultaneous_2_scratch/unique_id/v2.LR.historical_0201" walltime = "00:30:00" diff --git a/tests/integration/generated/test_weekly_bundles_chrysalis.cfg b/tests/integration/generated/test_weekly_bundles_chrysalis.cfg index a562e04b..a16129e3 100644 --- a/tests/integration/generated/test_weekly_bundles_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_bundles_chrysalis.cfg @@ -104,7 +104,7 @@ years = "1985:1989:2", [e3sm_diags] active = True -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' ref_final_yr = 1989 ref_start_yr = 1985 diff --git a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg index a3c6d590..7d91a130 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg @@ -93,7 +93,7 @@ walltime = "00:30:00" active = True climo_diurnal_frequency = "diurnal_8xdaily" climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg index 94037545..405af0ad 100644 --- a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +++ b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg @@ -99,7 +99,7 @@ active = True climo_diurnal_frequency = "diurnal_8xdaily" climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" climo_subsection = "atm_monthly_180x360_aave" -environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003" +environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015" grid = '180x360_aave' multiprocessing = True num_workers = 8 diff --git a/tests/integration/template_min_case_e3sm_diags_tc_analysis.cfg b/tests/integration/template_min_case_e3sm_diags_tc_analysis.cfg index 90c158e7..eefb73a8 100644 --- a/tests/integration/template_min_case_e3sm_diags_tc_analysis.cfg +++ b/tests/integration/template_min_case_e3sm_diags_tc_analysis.cfg @@ -1,12 +1,12 @@ [default] -case = "#expand case_name#" +case = "#expand case_name_v2#" constraint = "#expand constraint#" dry_run = "#expand dry_run#" environment_commands = "#expand environment_commands#" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input = #expand user_input#/E3SMv2/#expand case_name_v2# input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_output/#expand unique_id#/#expand case_name#" +output = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_output/#expand unique_id#/#expand case_name_v2#" partition = "#expand partition_short#" qos = "#expand qos_short#" www = "#expand user_www#zppy_min_case_e3sm_diags_tc_analysis_www/#expand unique_id#" @@ -14,7 +14,7 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "#expand scratch#zppy_min_case_e3sm_diags_tc_analysis_scratch/#expand unique_id#/#expand case_name#" +scratch = "#expand scratch#zppy_min_case_e3sm_diags_tc_analysis_scratch/#expand unique_id#/#expand case_name_v2#" walltime = "00:30:00" [e3sm_diags] @@ -25,11 +25,10 @@ multiprocessing = True num_workers = 8 partition = "#expand partition_long#" qos = "#expand qos_long#" -short_name = "#expand case_name#" +short_name = "#expand case_name_v2#" ts_num_years = 2 walltime = "#expand diags_walltime#" [[ atm_monthly_180x360_aave_tc_analysis ]] - # Note: tc_analysis requires e3sm_diags jobs to run sequentially sets = "tc_analysis", # tc_obs is determined automatically diff --git a/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_1.cfg b/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_1.cfg index 51e0426e..1eb6be42 100644 --- a/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_1.cfg +++ b/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_1.cfg @@ -1,12 +1,12 @@ [default] -case = "#expand case_name#" +case = "#expand case_name_v2#" constraint = "#expand constraint#" dry_run = "#expand dry_run#" environment_commands = "#expand environment_commands#" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input = #expand user_input#/E3SMv2/#expand case_name_v2# input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/#expand unique_id#/#expand case_name#" +output = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/#expand unique_id#/#expand case_name_v2#" partition = "#expand partition_short#" qos = "#expand qos_short#" www = "#expand user_www#zppy_min_case_e3sm_diags_tc_analysis_mvm_2_www/#expand unique_id#" @@ -14,5 +14,5 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "#expand scratch#zppy_min_case_e3sm_diags_tc_analysis_mvm_1_scratch/#expand unique_id#/#expand case_name#" +scratch = "#expand scratch#zppy_min_case_e3sm_diags_tc_analysis_mvm_1_scratch/#expand unique_id#/#expand case_name_v2#" walltime = "00:30:00" diff --git a/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_2.cfg b/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_2.cfg index 3f833ba6..80680e4e 100644 --- a/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_2.cfg +++ b/tests/integration/template_min_case_e3sm_diags_tc_analysis_mvm_2.cfg @@ -1,12 +1,13 @@ +# Run this after _1 completes. (We need tc_analysis for the reference years). [default] -case = "#expand case_name#" +case = "#expand case_name_v2#" constraint = "#expand constraint#" dry_run = "#expand dry_run#" environment_commands = "#expand environment_commands#" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input = #expand user_input#/E3SMv2/#expand case_name_v2# input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_mvm_2_output/#expand unique_id#/#expand case_name#" +output = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_mvm_2_output/#expand unique_id#/#expand case_name_v2#" partition = "#expand partition_short#" qos = "#expand qos_short#" www = "#expand user_www#zppy_min_case_e3sm_diags_tc_analysis_mvm_2_www/#expand unique_id#" @@ -14,7 +15,7 @@ years = "1995:1997:2", [tc_analysis] active = True -scratch = "#expand scratch#zppy_min_case_e3sm_diags_tc_analysis_mvm_2_scratch/#expand unique_id#/#expand case_name#" +scratch = "#expand scratch#zppy_min_case_e3sm_diags_tc_analysis_mvm_2_scratch/#expand unique_id#/#expand case_name_v2#" walltime = "00:30:00" [e3sm_diags] @@ -25,7 +26,7 @@ multiprocessing = True num_workers = 8 partition = "#expand partition_long#" qos = "#expand qos_long#" -short_name = "#expand case_name#" +short_name = "#expand case_name_v2#" walltime = "#expand diags_walltime#" [[ atm_monthly_180x360_aave_mvm ]] @@ -33,15 +34,15 @@ walltime = "#expand diags_walltime#" climo_subsection = "atm_monthly_180x360_aave" diff_title = "Difference" ref_final_yr = 1986 - ref_name = "#expand case_name#" + ref_name = "#expand case_name_v2#" ref_start_yr = 1985 ref_years = "1985-1986", # Use _1 as reference - reference_data_path = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/#expand unique_id#/#expand case_name#/post/atm/180x360_aave/clim" + reference_data_path = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_mvm_1_output/#expand unique_id#/#expand case_name_v2#/post/atm/180x360_aave/clim" # reference_data_path_tc determined automatically run_type = "model_vs_model" sets = "tc_analysis", - short_ref_name = "#expand case_name#" + short_ref_name = "#expand case_name_v2#" swap_test_ref = False tag = "model_vs_model" years = "1995-1996", diff --git a/tests/integration/template_min_case_e3sm_diags_tc_analysis_parallel.cfg b/tests/integration/template_min_case_e3sm_diags_tc_analysis_parallel.cfg new file mode 100644 index 00000000..499ee0ac --- /dev/null +++ b/tests/integration/template_min_case_e3sm_diags_tc_analysis_parallel.cfg @@ -0,0 +1,36 @@ +[default] +case = "#expand case_name_v2#" +constraint = "#expand constraint#" +dry_run = "#expand dry_run#" +environment_commands = "#expand environment_commands#" +input = #expand user_input#/E3SMv2/#expand case_name_v2# +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +output = "#expand user_output#zppy_min_case_e3sm_diags_tc_analysis_parallel_output/#expand unique_id#/#expand case_name_v2#" +partition = "#expand partition_short#" +qos = "#expand qos_short#" +www = "#expand user_www#zppy_min_case_e3sm_diags_tc_analysis_parallel_www/#expand unique_id#" +years = "1985:1989:2", + +[tc_analysis] +active = True +scratch = "#expand scratch#zppy_min_case_e3sm_diags_tc_analysis_parallel_scratch/#expand unique_id#/#expand case_name_v2#" +walltime = "00:30:00" + +[e3sm_diags] +active = True +environment_commands = "#expand diags_environment_commands#" +grid = '180x360_aave' +multiprocessing = True +num_workers = 8 +partition = "#expand partition_long#" +qos = "#expand qos_long#" +short_name = "#expand case_name_v2#" +ts_num_years = 2 +walltime = "#expand diags_walltime#" + + [[ atm_monthly_180x360_aave_tc_analysis ]] + # Expecting two viewers: 1985-1986, 1987-1988 + # The generation of these two viewers should occur in parallel. + sets = "tc_analysis", + # tc_obs is determined automatically diff --git a/tests/integration/template_min_case_tc_analysis_simultaneous_1.cfg b/tests/integration/template_min_case_tc_analysis_simultaneous_1.cfg index 7309eca5..3932f753 100644 --- a/tests/integration/template_min_case_tc_analysis_simultaneous_1.cfg +++ b/tests/integration/template_min_case_tc_analysis_simultaneous_1.cfg @@ -1,12 +1,12 @@ [default] -case = "#expand case_name#" +case = "#expand case_name_v2#" constraint = "#expand constraint#" dry_run = "#expand dry_run#" environment_commands = "#expand environment_commands#" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input = #expand user_input#/E3SMv2/#expand case_name_v2# input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "#expand user_output#zppy_min_case_tc_analysis_simultaneous_1_output/#expand unique_id#/#expand case_name#" +output = "#expand user_output#zppy_min_case_tc_analysis_simultaneous_1_output/#expand unique_id#/#expand case_name_v2#" partition = "#expand partition_short#" qos = "#expand qos_short#" www = "#expand user_www#zppy_min_case_tc_analysis_simultaneous_1_www/#expand unique_id#" @@ -14,5 +14,5 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "#expand scratch#zppy_min_case_tc_analysis_simultaneous_1_scratch/#expand unique_id#/#expand case_name#" +scratch = "#expand scratch#zppy_min_case_tc_analysis_simultaneous_1_scratch/#expand unique_id#/#expand case_name_v2#" walltime = "00:30:00" diff --git a/tests/integration/template_min_case_tc_analysis_simultaneous_2.cfg b/tests/integration/template_min_case_tc_analysis_simultaneous_2.cfg index d9318b58..c27a1bbf 100644 --- a/tests/integration/template_min_case_tc_analysis_simultaneous_2.cfg +++ b/tests/integration/template_min_case_tc_analysis_simultaneous_2.cfg @@ -1,14 +1,14 @@ # Run this immediately after running _1 [default] -case = "#expand case_name#" +case = "#expand case_name_v2#" constraint = "#expand constraint#" dry_run = "#expand dry_run#" environment_commands = "#expand environment_commands#" -input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/#expand case_name# +input = #expand user_input#/E3SMv2/#expand case_name_v2# input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "#expand user_output#zppy_min_case_tc_analysis_simultaneous_2_output/#expand unique_id#/#expand case_name#" +output = "#expand user_output#zppy_min_case_tc_analysis_simultaneous_2_output/#expand unique_id#/#expand case_name_v2#" partition = "#expand partition_short#" qos = "#expand qos_short#" www = "#expand user_www#zppy_min_case_tc_analysis_simultaneous_2_www/#expand unique_id#" @@ -16,5 +16,5 @@ years = "1985:1987:2", [tc_analysis] active = True -scratch = "#expand scratch#zppy_min_case_tc_analysis_simultaneous_2_scratch/#expand unique_id#/#expand case_name#" +scratch = "#expand scratch#zppy_min_case_tc_analysis_simultaneous_2_scratch/#expand unique_id#/#expand case_name_v2#" walltime = "00:30:00" diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 2390d60d..913f3b75 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -138,7 +138,7 @@ def get_chyrsalis_expansions(config): "constraint": "", # To run this test, replace conda environment with your e3sm_diags dev environment # To use default environment_commands, set to "" - "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_1003", + "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20241015", "diags_walltime": "5:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -290,6 +290,7 @@ def generate_cfgs(unified_testing=False, dry_run=False): "min_case_e3sm_diags_streamflow", "min_case_e3sm_diags_tc_analysis_mvm_1", "min_case_e3sm_diags_tc_analysis_mvm_2", + "min_case_e3sm_diags_tc_analysis_parallel", "min_case_e3sm_diags_tc_analysis", "min_case_e3sm_diags_tropical_subseasonal_mvm_1", "min_case_e3sm_diags_tropical_subseasonal_mvm_2", diff --git a/zppy/e3sm_diags.py b/zppy/e3sm_diags.py index 1dbb9bbb..091e8e62 100644 --- a/zppy/e3sm_diags.py +++ b/zppy/e3sm_diags.py @@ -34,8 +34,8 @@ def e3sm_diags(config: ConfigObj, script_dir: str, existing_bundles, job_ids_fil return existing_bundles # --- Generate and submit e3sm_diags scripts --- - dependencies: List[str] = [] for c in tasks: + dependencies: List[str] = [] check_parameters_for_bash(c) c["scriptDir"] = script_dir if "ts_num_years" in c.keys(): @@ -92,14 +92,6 @@ def e3sm_diags(config: ConfigObj, script_dir: str, existing_bundles, job_ids_fil dependFiles=dependencies, fail_on_dependency_skip=c["fail_on_dependency_skip"], ) - - # Due to a `socket.gaierror: [Errno -2] Name or service not known` error when running e3sm_diags with tc_analysis - # on multiple year_sets, if tc_analysis is in sets, then e3sm_diags should be run sequentially. - if "tc_analysis" in c["sets"]: - # Note that this line should still be executed even if jobid == -1 - # The later tc_analysis-using e3sm_diags tasks still depend on this task (and thus will also fail). - # Add to the dependency list - dependencies.append(status_file) else: print(f"...adding to bundle {c['bundle']}") diff --git a/zppy/tc_analysis.py b/zppy/tc_analysis.py index b9346c8a..652989d1 100644 --- a/zppy/tc_analysis.py +++ b/zppy/tc_analysis.py @@ -26,11 +26,10 @@ def tc_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_fi return existing_bundles # --- Generate and submit scripts --- - # There is a `GenerateConnectivityFile: error while loading shared libraries: libnetcdf.so.11: cannot open shared object file: No such file or directory` error - # when multiple year_sets are run simultaneously. Therefore, we will wait for the completion of one year_set before moving on to the next. - dependencies: List[str] = [] - for c in tasks: + + dependencies: List[str] = [] + # Loop over year sets year_sets: List[Tuple[int, int]] = get_years(c["years"]) for s in year_sets: @@ -75,11 +74,6 @@ def tc_analysis(config: ConfigObj, script_dir: str, existing_bundles, job_ids_fi dependFiles=dependencies, fail_on_dependency_skip=c["fail_on_dependency_skip"], ) - - # Note that this line should still be executed even if jobid == -1 - # The later tc_analysis tasks still depend on this task (and thus will also fail). - # Add to the dependency list - dependencies.append(status_file) else: print(f"...adding to bundle {c['bundle']}")