Add benchmarks, remove default_perf_config_file

CliMA · May 14, 2024 · 232e7a2 · 232e7a2
1 parent 3fb75cf
commit 232e7a2
Show file tree

Hide file tree

Showing 32 changed files with 340 additions and 58 deletions.
diff --git a/.buildkite/gpu_pipeline/pipeline.yml b/.buildkite/gpu_pipeline/pipeline.yml
@@ -12,6 +12,7 @@ env:
   SLURM_KILL_BAD_EXIT: 1
   SLURM_GRES_FLAGS: "allow-task-sharing"
   GPU_CONFIG_PATH: "config/gpu_configs/"
+  MODEL_CONFIG_PATH: "config/model_configs/"
   CLIMAATMOS_GC_NSTEPS: 10
 
 steps:
@@ -249,7 +250,7 @@ steps:
           - >
             nsys profile --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_diagedmf/output_active/report
             julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
-            --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_diagedmf.yml
+            --config_file ${MODEL_CONFIG_PATH}aquaplanet_diagedmf.yml
             --job_id gpu_aquaplanet_diagedmf
         artifact_paths: "gpu_aquaplanet_diagedmf/output_active/*"
         agents:
@@ -259,6 +260,15 @@ steps:
           slurm_mem: 32G
           slurm_exclusive:
 
+      - label: "gpu_aquaplanet_diagedmf_benchmark"
+        command: >
+          julia --color=yes --project=perf perf/benchmark.jl
+          --config_file ${MODEL_CONFIG_PATH}aquaplanet_diagedmf.yml
+          --job_id gpu_aquaplanet_diagedmf_benchmark
+        artifact_paths: "gpu_aquaplanet_diagedmf_benchmark/output_active/*"
+        agents:
+          slurm_gpus: 1
+
   - group: "Prognostic EDMF GPU"
     steps:
 
@@ -268,7 +278,7 @@ steps:
           - >
             nsys profile --trace=nvtx,mpi,cuda,osrt --output=gpu_aquaplanet_progedmf/output_active/report
             julia --threads=3 --color=yes --project=examples examples/hybrid/driver.jl
-            --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_progedmf.yml
+            --config_file ${MODEL_CONFIG_PATH}aquaplanet_progedmf.yml
             --job_id gpu_aquaplanet_progedmf
         artifact_paths: "gpu_aquaplanet_progedmf/output_active/*"
         agents:
@@ -277,3 +287,12 @@ steps:
           slurm_ntasks: 1
           slurm_mem: 32G
           slurm_exclusive:
+
+      - label: "gpu_aquaplanet_progedmf_benchmark"
+        command: >
+          julia --color=yes --project=perf perf/benchmark.jl
+          --config_file ${MODEL_CONFIG_PATH}aquaplanet_progedmf.yml
+          --job_id gpu_aquaplanet_progedmf_benchmark
+        artifact_paths: "gpu_aquaplanet_progedmf_benchmark/output_active/*"
+        agents:
+          slurm_gpus: 1
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -845,7 +845,7 @@ steps:
           slurm_gpus: 1
           slurm_mem: 20G
 
-  - group: "GPU Performance"
+  - group: "Benchmarks"
     steps:
 
       - label: "Perf: GPU implicit baro wave wrt h_elem"
@@ -887,9 +887,6 @@ steps:
           --job_id cpu_implicit_barowave
         artifact_paths: "cpu_implicit_barowave/output_active/*"
 
-  - group: "Performance"
-    steps:
-
       # Benchmarks
       - label: ":computer: Benchmark: perf target (default)"
         command: >
@@ -917,6 +914,9 @@ steps:
           slurm_mem: 24GB
           slurm_cpus_per_task: 8
 
+  - group: "Flame graphs"
+    steps:
+
       - label: ":fire: Flame graph: perf target (default)"
         command: >
           julia --color=yes --project=perf perf/flame.jl
@@ -1019,6 +1019,9 @@ steps:
           slurm_gpus: 1
           gres: "gpu:p100:1"
 
+  - group: "Inference/Invalidations"
+    steps:
+
       # Inference
       - label: ":rocket: JET n-failures (inference)"
         command: >

diff --git a/config/default_configs/default_perf.yml b/config/default_configs/default_perf.yml
diff --git a/...g/gpu_configs/gpu_aquaplanet_diagedmf.yml → config/model_configs/aquaplanet_diagedmf.yml b/...g/gpu_configs/gpu_aquaplanet_diagedmf.yml → config/model_configs/aquaplanet_diagedmf.yml
diff --git a/...g/gpu_configs/gpu_aquaplanet_progedmf.yml → config/model_configs/aquaplanet_progedmf.yml b/...g/gpu_configs/gpu_aquaplanet_progedmf.yml → config/model_configs/aquaplanet_progedmf.yml
@@ -6,8 +6,8 @@ z_max: 55000.0
 z_elem: 63
 dz_bottom: 30.0
 dz_top: 3000.0
-moist: equil 
-surface_setup: DefaultMoninObukhov 
+moist: equil
+surface_setup: DefaultMoninObukhov
 rad: allskywithclear
 idealized_insolation: false
 dt_rad: 1hours
@@ -18,14 +18,14 @@ implicit_sgs_advection: true
 approximate_linear_solve_iters: 2
 max_newton_iters_ode: 3
 prognostic_tke: true
-edmfx_upwinding: first_order 
-edmfx_entr_model: "Generalized" 
-edmfx_detr_model: "Generalized" 
+edmfx_upwinding: first_order
+edmfx_entr_model: "Generalized"
+edmfx_detr_model: "Generalized"
 edmfx_nh_pressure: true
 edmfx_filter: true
 edmfx_sgs_mass_flux: true
 edmfx_sgs_diffusive_flux: true
 precip_model: 0M
-dt: 10secs 
-t_end: 3hours 
+dt: 10secs
+t_end: 3hours
 toml: [toml/prognostic_edmfx.toml]
diff --git a/config/perf_configs/bm_perf_target.yml b/config/perf_configs/bm_perf_target.yml
@@ -1 +1,17 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 perf_summary: true
diff --git a/config/perf_configs/bm_perf_target_threaded.yml b/config/perf_configs/bm_perf_target_threaded.yml
@@ -1 +1,17 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 perf_summary: true
diff --git a/config/perf_configs/checkbounds.yml b/config/perf_configs/checkbounds.yml
@@ -1 +1,17 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 dt: "1secs"
diff --git a/config/perf_configs/cpu_implicit_barowave.yml b/config/perf_configs/cpu_implicit_barowave.yml
@@ -1,6 +1,17 @@
-h_elem: 12
-initial_condition: "DryBaroclinicWave"
-t_end: "1mins"
+moist: "equil"
+apply_limiter: true
 z_elem: 25
 dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
 dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
+initial_condition: "DryBaroclinicWave"
+t_end: "1mins"
diff --git a/config/perf_configs/flame_gpu_implicit_barowave_moist.yml b/config/perf_configs/flame_gpu_implicit_barowave_moist.yml
@@ -1,10 +1,20 @@
+moist: "equil"
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+warn_allocations_diagnostics: true
+log_progress: false
 apply_limiter: false
 precip_model: "0M"
 h_elem: 12
 initial_condition: "MoistBaroclinicWave"
 t_end: "1mins"
-z_elem: 25
-dt: "1secs"
-dt_save_to_sol: "Inf"
 rad: ~
-moist: "equil"
diff --git a/config/perf_configs/flame_perf_diagnostics.yml b/config/perf_configs/flame_perf_diagnostics.yml
@@ -1,3 +1,19 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 diagnostics:
   - reduction_time: "average"
     short_name: "ua"

diff --git a/config/perf_configs/flame_perf_gw.yml b/config/perf_configs/flame_perf_gw.yml
@@ -1,3 +1,19 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 dz_bottom: 300.0
 rayleigh_sponge: true
 toml:

diff --git a/config/perf_configs/flame_perf_target.yml b/config/perf_configs/flame_perf_target.yml
@@ -1 +1,16 @@
-dt: "1secs"
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
diff --git a/config/perf_configs/flame_perf_target_callbacks.yml b/config/perf_configs/flame_perf_target_callbacks.yml
@@ -1,3 +1,19 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 dt_save_state_to_disk: "1secs"
 dt_rad: "1secs"
 dt_cloud_fraction: "1secs"
diff --git a/config/perf_configs/flame_perf_target_diagnostic_edmfx.yml b/config/perf_configs/flame_perf_target_diagnostic_edmfx.yml
@@ -1,3 +1,19 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 dt_save_state_to_disk: "600secs"
 vert_diff: "false"
 turbconv: "diagnostic_edmfx"

diff --git a/config/perf_configs/flame_perf_target_edmf.yml b/config/perf_configs/flame_perf_target_edmf.yml
@@ -1,3 +1,19 @@
+moist: "equil"
+apply_limiter: true
+z_elem: 25
+dt: "1secs"
+surface_setup: "DefaultExchangeCoefficients"
+t_end: "10secs"
+vert_diff: "true"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+precip_model: "0M"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 rad: ~
 apply_limiter: false
 dt_save_state_to_disk: "10secs"

diff --git a/config/perf_configs/flame_perf_target_frierson.yml b/config/perf_configs/flame_perf_target_frierson.yml
@@ -1,3 +1,13 @@
+apply_limiter: true
+surface_setup: "DefaultExchangeCoefficients"
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+dt_save_to_sol: "Inf"
+rad: "allskywithclear"
+warn_allocations_diagnostics: true
+log_progress: false
 vert_diff: "FriersonDiffusion"
 pproximate_linear_solve_iters: 2
 use_krylov_method: true

diff --git a/config/perf_configs/flame_perf_target_prognostic_edmfx_aquaplanet.yml b/config/perf_configs/flame_perf_target_prognostic_edmfx_aquaplanet.yml
@@ -1,3 +1,11 @@
+apply_limiter: true
+z_elem: 25
+approximate_linear_solve_iters: 2
+h_elem: 12
+forcing: "held_suarez"
+dt_save_to_sol: "Inf"
+warn_allocations_diagnostics: true
+log_progress: false
 surface_setup: DefaultExchangeCoefficients
 rad: gray
 vert_diff: "false"