Merge pull request #2908 from CliMA/zs/netcdf_rawlevel

set netcdf_output_at_levels default to true

config/default_configs/default_config.yml

@@ -267,7 +267,7 @@ output_default_diagnostics:
   value: true
 netcdf_output_at_levels:
   help: "Do not perform any vertical interpolation in the output NetCDF files. Instead, interpolate horizontally and output the level. "
-  value: false
+  value: true
 netcdf_interpolation_num_points:
   help: "Override the number of interpolation point for the NetCDF output. This configuration has to be a list of integers, e.g. [180, 90, 10]."
   value: ~

config/model_configs/diagnostic_edmfx_bomex_box.yml

@@ -28,7 +28,6 @@ dt: "100secs"
 t_end: "6hours" 
 dt_save_state_to_disk: "10mins"
 toml: [toml/diagnostic_edmfx_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 60]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_bomex_stretched_box.yml

@@ -29,7 +29,6 @@ dt: "200secs"
 t_end: "6hours" 
 dt_save_state_to_disk: "10mins"
 toml: [toml/diagnostic_edmfx_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 30]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_dycoms_rf01_box.yml

@@ -28,7 +28,6 @@ dt: 100secs
 t_end: 4hours 
 dt_save_state_to_disk: 10mins
 toml: [toml/diagnostic_edmfx_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 30]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_dycoms_rf01_explicit_box.yml

@@ -26,7 +26,6 @@ dt: 20secs
 t_end: 4hours 
 dt_save_state_to_disk: 10mins
 toml: [toml/diagnostic_edmfx_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 30]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_gabls_box.yml

@@ -26,7 +26,6 @@ dt: 200secs
 t_end: 9hours
 dt_save_state_to_disk: 10mins
 toml: [toml/diagnostic_edmfx_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 8]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_rico_box.yml

@@ -28,7 +28,6 @@ dt: 100secs
 t_end: 8hours
 dt_save_state_to_disk: 10mins
 toml: [toml/diagnostic_edmfx_trmm_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 100]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_test_box.yml

@@ -13,7 +13,6 @@ config: "box"
 job_id: "diagnostic_edmfx_test_box"
 moist: "equil"
 toml: [toml/diagnostic_edmfx_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 45]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_trmm_box.yml

@@ -30,7 +30,6 @@ t_end: 6hours
 dt_save_state_to_disk: 10mins
 FLOAT_TYPE: "Float64"
 toml: [toml/diagnostic_edmfx_trmm_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 82]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/diagnostic_edmfx_trmm_stretched_box.yml

@@ -32,7 +32,6 @@ t_end: 6hours
 dt_save_state_to_disk: 10mins
 FLOAT_TYPE: "Float64"
 toml: [toml/diagnostic_edmfx_trmm_box.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 30]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_adv_test_column.yml

@@ -17,7 +17,6 @@ t_end: "3600secs"
 dt_save_state_to_disk: "100secs"
 FLOAT_TYPE: "Float64"
 toml: [toml/prognostic_edmfx_advection.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 63]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_bomex_box.yml

@@ -32,7 +32,6 @@ dt: "50secs"
 t_end: "6hours"
 dt_save_state_to_disk: "30mins"
 toml: [toml/prognostic_edmfx_bomex.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 60]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_bomex_column.yml

@@ -27,7 +27,6 @@ dt: "50secs"
 t_end: "6hours"
 dt_save_to_disk: "10mins"
 toml: [toml/prognostic_edmfx_bomex.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 60]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_bomex_explicit_column.yml

@@ -28,7 +28,6 @@ dt: "1secs"
 t_end: "6hours"
 dt_save_to_disk: "10mins"
 toml: [toml/prognostic_edmfx_bomex_explicit.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 60]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_bomex_fixtke_column.yml

@@ -25,7 +25,6 @@ dt: "5secs"
 t_end: "6hours"
 dt_save_state_to_disk: "10mins"
 toml: [toml/prognostic_edmfx_bomex.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 60]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_bomex_stretched_column.yml

@@ -31,7 +31,6 @@ dt: "50secs"
 t_end: "6hours"
 dt_save_state_to_disk: "10mins"
 toml: [toml/prognostic_edmfx_bomex.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 30]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_dycoms_rf01_column.yml

@@ -30,7 +30,6 @@ dt: 40secs
 t_end: 4hours
 dt_save_state_to_disk: 10mins
 toml: [toml/prognostic_edmfx_dycoms_rf01.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 30]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_gabls_column.yml

@@ -28,7 +28,6 @@ t_end: "9hours"
 dt_save_state_to_disk: "30mins"
 perturb_initstate: false
 toml: [toml/prognostic_edmfx.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 8]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_rico_column.yml

@@ -25,7 +25,6 @@ dt: "5secs"
 t_end: "6hours"
 dt_save_state_to_disk: "10mins"
 toml: [toml/prognostic_edmfx.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 100]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_simpleplume_column.yml

@@ -25,7 +25,6 @@ dt: 20secs"
 t_end: "12hours"
 dt_save_to_disk: "10mins"
 toml: [toml/prognostic_edmfx_simpleplume.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [2, 2, 80]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_trmm_column.yml

@@ -25,7 +25,6 @@ t_end: 6hours
 dt_save_state_to_disk: 10mins
 FLOAT_TYPE: "Float64"
 toml: [toml/prognostic_edmfx.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 82]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/prognostic_edmfx_trmm_column_0M.yml

@@ -25,7 +25,6 @@ t_end: 6hours
 dt_save_state_to_disk: 10mins
 FLOAT_TYPE: "Float64"
 toml: [toml/prognostic_edmfx.toml]
-netcdf_output_at_levels: true
 netcdf_interpolation_num_points: [8, 8, 82]
 diagnostics:
   - short_name: [ts, ta, thetaa, ha, pfull, rhoa, ua, va, wa, hur, hus, cl, clw, cli, hussfc, evspsbl]

config/model_configs/sphere_baroclinic_wave_rhoe_topography_dcmip_rs.yml

@@ -7,6 +7,7 @@ t_end: "6days"
 use_reference_state: false
 dt: "200secs"
 job_id: "sphere_baroclinic_wave_rhoe_topography_dcmip_rs"
+netcdf_output_at_levels: false
 diagnostics:
   - short_name: [pfull, wa, va, rv]
     period: 1days

perf/benchmark_netcdf_io.jl

@@ -37,6 +37,7 @@ for device in keys(timings)
         [Dict("short_name" => "rhoa", "period" => config.parsed_args["dt"])]
     config.parsed_args["netcdf_interpolation_num_points"] = [NUM, NUM, NUM]
     config.parsed_args["job_id"] = "flame_perf_io"
+    config.parsed_args["netcdf_output_at_levels"] = false
 
     simulation = CA.get_simulation(config)
 

src/diagnostics/netcdf_writer.jl

@@ -301,6 +301,7 @@ function add_space_coordinates_maybe!(
     space::Spaces.ExtrudedFiniteDifferenceSpace,
     num_points;
     interpolated_physical_z = nothing,
+    disable_vertical_interpolation = false,
 )
 
     hdims_names = vdims_names = []
@@ -319,18 +320,40 @@ function add_space_coordinates_maybe!(
         Spaces.staggering(space),
     )
 
-    if Spaces.grid(space).hypsography isa Grids.Flat
-        vdims_names =
-            add_space_coordinates_maybe!(nc, vertical_space, num_points_vertic)
-    else
-        vdims_names = add_space_coordinates_maybe!(
-            nc,
-            vertical_space,
-            num_points_vertic,
-            interpolated_physical_z;
-            names = ("z_reference",),
-            depending_on_dimensions = hdims_names,
+    no_topography = Spaces.grid(space).hypsography isa Grids.Flat
+
+    if disable_vertical_interpolation
+        zpts = Array(
+            parent(space.grid.vertical_grid.center_local_geometry.coordinates),
         )
+        name = no_topography ? "z" : "z_reference"
+        if !dimension_exists(nc, name, (num_points_vertic,))
+            zpts = Array(
+                parent(
+                    space.grid.vertical_grid.center_local_geometry.coordinates,
+                ),
+            )
+            add_dimension!(nc, name, zpts[:, 1], units = "m", axis = "Z")
+        end
+        vdims_names = [name]
+    else
+        if no_topography
+            vdims_names = add_space_coordinates_maybe!(
+                nc,
+                vertical_space,
+                num_points_vertic,
+            )
+        else
+            disable_vertical_interpolation && error("Not implemented")
+            vdims_names = add_space_coordinates_maybe!(
+                nc,
+                vertical_space,
+                num_points_vertic,
+                interpolated_physical_z;
+                names = ("z_reference",),
+                depending_on_dimensions = hdims_names,
+            )
+        end
     end
 
     return (hdims_names..., vdims_names...)
@@ -704,6 +727,7 @@ function save_diagnostic_to_disk!(
         space,
         writer.num_points;
         writer.interpolated_physical_z,
+        writer.disable_vertical_interpolation,
     )
 
     if haskey(nc, "$(var.short_name)")