Skip to content

Commit

Permalink
Updated detailed timers to recent CAM tag.
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnmauff
johnmauff committed Sep 10, 2024
1 parent ddee772 commit 4396829
Show file tree
Hide file tree
Showing 8 changed files with 296 additions and 69 deletions.
8 changes: 6 additions & 2 deletions src/chemistry/modal_aero/aero_model.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1149,7 +1149,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf)
endif

if (convproc_do_aer) then
call t_startf('ma_convproc')
call t_startf('aero_model_wetdep:NAR:ma_convproc')
call ma_convproc_intr( state, ptend, pbuf, dt, &
nsrflx_mzaer2cnvpr, qsrflx_mzaer2cnvpr, aerdepwetis, dcondt_resusp3d)

Expand All @@ -1174,7 +1174,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf)
end do ! loop over number + chem constituents
end do ! m aerosol modes
end if
call t_stopf('ma_convproc')
call t_stopf('aero_model_wetdep:NAR:ma_convproc')
endif

do m = 1, ntot_amode ! main loop over aerosol modes
Expand Down Expand Up @@ -1320,6 +1320,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf)
qqcw_in(:,:) = fldcw(:,:)
endif

call t_startf('aero_model_wetdep:NAR:wetdepa')
call wetdepa_v2( state%pmid, state%q(:,:,1), state%pdel, &
dep_inputs%cldt, dep_inputs%cldcu, dep_inputs%cmfdqr, &
dep_inputs%evapc, dep_inputs%conicw, dep_inputs%prain, dep_inputs%qme, &
Expand All @@ -1334,6 +1335,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf)
convproc_do_aer=convproc_do_aer, rcscavt=rcscavt, rsscavt=rsscavt, &
sol_facti_in=sol_facti, sol_factic_in=sol_factic, &
convproc_do_evaprain_atonce_in=convproc_do_evaprain_atonce )
call t_stopf('aero_model_wetdep:NAR:wetdepa')

do_hygro_sum_del = .false.
if ( lspec > 0 ) do_hygro_sum_del = .true.
Expand Down Expand Up @@ -1549,6 +1551,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf)
fldcw => qqcw_get_field(pbuf, mm,lchnk)
endif

call t_startf('aero_model_wetdep:NAR:wetdepa')
call wetdepa_v2(state%pmid, state%q(:,:,1), state%pdel, &
dep_inputs%cldt, dep_inputs%cldcu, dep_inputs%cmfdqr, &
dep_inputs%evapc, dep_inputs%conicw, dep_inputs%prain, dep_inputs%qme, &
Expand All @@ -1562,6 +1565,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf)
sol_facti_in=sol_facti, sol_factic_in=sol_factic, &
convproc_do_evaprain_atonce_in=convproc_do_evaprain_atonce, &
bergso_in=dep_inputs%bergso )
call t_stopf('aero_model_wetdep:NAR:wetdepa')

if(convproc_do_aer) then
! save resuspension of cloudborne species
Expand Down
36 changes: 31 additions & 5 deletions src/physics/cam/clubb_intr.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2602,10 +2602,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
! MAIN COMPUTATION BEGINS HERE !
!-----------------------------------------------------------------------------------!

call t_startf("clubb_tend_cam")

nlev = pver + 1 - top_lev

call t_startf('clubb_tend_cam:NAR')
rtp2_zt_out = 0._r8
thl2_zt_out = 0._r8
wp2_zt_out = 0._r8
Expand Down Expand Up @@ -2788,6 +2788,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
call grid_size(state1, grid_dx, grid_dy)

end if
call t_stopf('clubb_tend_cam:NAR')

if (clubb_do_icesuper) then

Expand All @@ -2809,11 +2810,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
qitend(:ncol,:)=0._r8
initend(:ncol,:)=0._r8

call t_startf('clubb_tend_cam:ice_macro_tend')
call ice_macro_tend(naai(1:ncol,top_lev:pver), state1%t(1:ncol,top_lev:pver), &
state1%pmid(1:ncol,top_lev:pver), state1%q(1:ncol,top_lev:pver,1), &
state1%q(1:ncol,top_lev:pver,ixcldice), state1%q(1:ncol,top_lev:pver,ixnumice), &
latsub, hdtime, stend(1:ncol,top_lev:pver), qvtend(1:ncol,top_lev:pver), &
qitend(1:ncol,top_lev:pver), initend(1:ncol,top_lev:pver), ncol*(pver-top_lev+1))
call t_stopf('clubb_tend_cam:ice_macro_tend')

! update local copy of state with the tendencies
ptend_loc%q(:ncol,top_lev:pver,1)=qvtend(:ncol,top_lev:pver)
Expand All @@ -2835,6 +2838,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
call outfld( 'NITENDICE', initend, pcols, lchnk )

endif
call t_startf('clubb_tend_cam:NAR')


! Determine CLUBB time step and make it sub-step friendly
Expand Down Expand Up @@ -2996,7 +3000,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
mf_qtflx_output(:,:) = 0._r8
end if

call t_startf("clubb_tend_cam_i_loop")

! Determine Coriolis force at given latitude. This is never used
! when CLUBB is implemented in a host model, therefore just set
Expand Down Expand Up @@ -3239,7 +3242,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
! Perturbed winds are not used in CAM
upwp_sfc_pert = 0.0_r8
vpwp_sfc_pert = 0.0_r8
call t_stopf('clubb_tend_cam:NAR')

call t_startf('clubb_tend_cam:flip-index')
! Need to flip arrays around for CLUBB core
do k=1,nlev+1
do i=1,ncol
Expand Down Expand Up @@ -3453,6 +3458,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
if (macmic_it==1) vpwp_clubb_gw_mc(:ncol,:) = 0._r8
if (macmic_it==1) thlp2_clubb_gw_mc(:ncol,:) = 0._r8
if (macmic_it==1) wpthlp_clubb_gw_mc(:ncol,:) = 0._r8
call t_stopf('clubb_tend_cam:flip-index')

do t=1,nadv ! do needed number of "sub" timesteps for each CAM step

Expand All @@ -3466,6 +3472,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
!###################### CALL MF DIAGNOSTIC PLUMES ######################
!#######################################################################
if (do_clubb_mf) then
call t_startf('clubb_tend_cam:do_clubb_mf')

do k=2,pverp
do i=1, ncol
Expand Down Expand Up @@ -3516,10 +3523,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
((rho_ds_zm(i,k) * mf_thlflx(i,k)) - (rho_ds_zm(i,k-1) * mf_thlflx(i,k-1)))
end do
end do
call t_stopf('clubb_tend_cam:do_clubb_mf')

end if

! Advance CLUBB CORE one timestep in the future
call t_startf('clubb_tend_cam:advance_clubb_core_api')
call advance_clubb_core_api( gr, pverp+1-top_lev, ncol, &
l_implemented, dtime, fcor, sfc_elevation, &
hydromet_dim, &
Expand Down Expand Up @@ -3564,6 +3573,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
wprcp_out, w_up_in_cloud_out, w_down_in_cloud_out, &
cloudy_updraft_frac_out, cloudy_downdraft_frac_out, &
rcm_in_layer_out, cloud_cover_out, invrs_tau_zm_out )
call t_stopf('clubb_tend_cam:advance_clubb_core_api')

! Note that CLUBB does not produce an error code specific to any column, and
! one value only for the entire chunk
Expand All @@ -3587,12 +3597,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
end do
end do

call t_startf('clubb_tend_cam:update_xp2_mc_api')
call update_xp2_mc_api( gr, nlev+1, ncol, dtime, cloud_frac_inout, &
rcm_inout, rvm_in, thlm_in, wm_zt, &
exner, pre_in, pdf_params_chnk(lchnk), &
rtp2_mc_out, thlp2_mc_out, &
wprtp_mc_out, wpthlp_mc_out, &
rtpthlp_mc_out)
call t_stopf('clubb_tend_cam:update_xp2_mc_api')

do k=1,nlev+1
do i=1,ncol
Expand All @@ -3610,6 +3622,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &


if (do_cldcool) then
call t_startf('clubb_tend_cam:do_cldcool')

rcm_out_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, rcm_inout )
qrl_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, qrl_clubb )
Expand All @@ -3624,19 +3637,23 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
thlp2_in(i,:) = thlp2_in(i,:) + thlp2_rad_out(i,:) * dtime
thlp2_in(i,:) = max(thl_tol**2,thlp2_in(i,:))
end do
call t_stopf('clubb_tend_cam:do_cldcool')

end if

! Check to see if stats should be output, here stats are read into
! output arrays to make them conformable to CAM output
if (stats_metadata%l_stats) then
call t_startf('clubb_tend_cam:stats_end_timestep_clubb')
do i=1, ncol
call stats_end_timestep_clubb(i, stats_zt(i), stats_zm(i), stats_rad_zt(i), stats_rad_zm(i), stats_sfc(i), &
out_zt, out_zm, out_radzt, out_radzm, out_sfc)
end do
call t_stopf('clubb_tend_cam:stats_end_timestep_clubb')
end if

enddo ! end time loop
call t_startf('clubb_tend_cam:NAR')

if (clubb_do_adv) then
if (macmic_it == cld_macmic_num_steps) then
Expand All @@ -3662,12 +3679,16 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &

end if
end if
call t_stopf('clubb_tend_cam:NAR')

! Convert RTP2 and THLP2 to thermo grid for output
call t_startf('clubb_tend_cam:NAR')
rtp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, rtp2_in )
thl2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, thlp2_in )
wp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, wp2_in )
call t_stopf('clubb_tend_cam:NAR')

call t_startf('clubb_tend_cam:flip-index')
! Arrays need to be "flipped" to CAM grid
do k=1, nlev+1
do i=1, ncol
Expand Down Expand Up @@ -3727,6 +3748,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &

end do
end do
call t_stopf('clubb_tend_cam:flip-index')
call t_startf('clubb_tend_cam:NAR')

! Accumulate vars through macmic subcycle
upwp_clubb_gw_mc(:ncol,:) = upwp_clubb_gw_mc(:ncol,:) + upwp(:ncol,:)
Expand Down Expand Up @@ -4026,7 +4049,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
end if
end do

call t_stopf("clubb_tend_cam_i_loop")

call outfld('KVH_CLUBB', khzm, pcols, lchnk)

Expand Down Expand Up @@ -4073,9 +4095,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
! then advances it's predictive equations second, this can lead to
! RHliq > 1 directly before microphysics is called. Therefore, we use
! ice_macro_tend to enforce RHliq <= 1 everywhere before microphysics is called.
call t_stopf('clubb_tend_cam:NAR')

if (clubb_do_liqsupersat) then

call t_startf('clubb_cam_tend:do_liqsupersat')
! -------------------------------------- !
! Ice Saturation Adjustment Computation !
! -------------------------------------- !
Expand Down Expand Up @@ -4127,7 +4151,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
end where

call outfld( 'FQTENDICE', fqtend, pcols, lchnk )
call t_stopf('clubb_cam_tend:do_liqsupersat')
end if
call t_startf('clubb_tend_cam:NAR')

! ------------------------------------------------------------ !
! The rest of the code deals with diagnosing variables !
Expand Down Expand Up @@ -4552,8 +4578,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, &
enddo

endif

call t_stopf("clubb_tend_cam")
call t_stopf('clubb_tend_cam:NAR')

return
#endif
Expand Down
30 changes: 30 additions & 0 deletions src/physics/cam/micro_pumas_cam.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1887,6 +1887,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)

nan_array = nan

call t_startf('micro_pumas_cam_tend:NAR')
call phys_getopts(use_subcol_microp_out=use_subcol_microp)

! Set the col_type flag to grid or subcolumn dependent on the value of use_subcol_microp
Expand Down Expand Up @@ -2354,6 +2355,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)
frzimm(:ncol,:top_lev-1)=0._r8
frzcnt(:ncol,:top_lev-1)=0._r8
frzdep(:ncol,:top_lev-1)=0._r8
call t_stopf('micro_pumas_cam_tend:NAR')

do it = 1, num_steps

Expand Down Expand Up @@ -2402,6 +2404,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)

end select
case(2:3)
call t_startf('micro_pumas_cam_tend:micro_pumas_tend')
call micro_pumas_tend( &
ncol, nlev, dtime/num_steps,&
state_loc%t(:ncol,top_lev:), state_loc%q(:ncol,top_lev:,ixq), &
Expand Down Expand Up @@ -2470,8 +2473,10 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)
tnd_qsnow(:ncol,top_lev:),tnd_nsnow(:ncol,top_lev:),re_ice(:ncol,top_lev:),&
prer_evap(:ncol,top_lev:), &
frzimm(:ncol,top_lev:), frzcnt(:ncol,top_lev:), frzdep(:ncol,top_lev:) )
call t_stopf('micro_pumas_cam_tend:micro_pumas_tend')
end select

call t_startf('micro_pumas_cam_tend:NAR')
call handle_errmsg(errstring, subname="micro_pumas_tend")

call physics_ptend_init(ptend_loc, psetcols, "micro_pumas", &
Expand Down Expand Up @@ -2945,17 +2950,25 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)
rel_fn_grid = 10._r8

ncic_grid = 1.e8_r8
call t_stopf('micro_pumas_cam_tend:NAR')

do k = top_lev, pver
call t_startf('micro_pumas_cam_tend:DTO');
!$acc data copyin (mg_liq_props,icwmrst_grid(:ngrdcol,k),rho_grid(:ngrdcol,k)) &
!$acc copy (ncic_grid(:ngrdcol,k)) &
!$acc copyout (mu_grid(:ngrdcol,k),lambdac_grid(:ngrdcol,k))
call t_stopf('micro_pumas_cam_tend:DTO');
call t_startf('micro_pumas_cam_tend:ACCR');
call size_dist_param_liq(mg_liq_props, icwmrst_grid(:ngrdcol,k), &
ncic_grid(:ngrdcol,k), rho_grid(:ngrdcol,k), &
mu_grid(:ngrdcol,k), lambdac_grid(:ngrdcol,k), ngrdcol)
call t_stopf('micro_pumas_cam_tend:ACCR');
call t_startf('micro_pumas_cam_tend:DTO');
!$acc end data
call t_stopf('micro_pumas_cam_tend:DTO');
end do

call t_startf('micro_pumas_cam_tend:NAR')
where (icwmrst_grid(:ngrdcol,top_lev:) > qsmall)
rel_fn_grid(:ngrdcol,top_lev:) = &
(mu_grid(:ngrdcol,top_lev:) + 3._r8)/ &
Expand All @@ -2971,17 +2984,25 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)
! Calculate ncic on the grid
ncic_grid(:ngrdcol,top_lev:) = nc_grid(:ngrdcol,top_lev:) / &
max(mincld,liqcldf_grid(:ngrdcol,top_lev:))
call t_stopf('micro_pumas_cam_tend:NAR')

do k = top_lev, pver
call t_startf('micro_pumas_cam_tend:DTO');
!$acc data copyin (mg_liq_props,icwmrst_grid(:ngrdcol,k), rho_grid(:ngrdcol,k)) &
!$acc copy (ncic_grid(:ngrdcol,k)) &
!$acc copyout (mu_grid(:ngrdcol,k),lambdac_grid(:ngrdcol,k))
call t_stopf('micro_pumas_cam_tend:DTO');
call t_startf('micro_pumas_cam_tend:ACCR');
call size_dist_param_liq(mg_liq_props, icwmrst_grid(:ngrdcol,k), &
ncic_grid(:ngrdcol,k), rho_grid(:ngrdcol,k), &
mu_grid(:ngrdcol,k), lambdac_grid(:ngrdcol,k), ngrdcol)
call t_stopf('micro_pumas_cam_tend:ACCR');
call t_startf('micro_pumas_cam_tend:DTO');
!$acc end data
call t_stopf('micro_pumas_cam_tend:DTO');
end do

call t_startf('micro_pumas_cam_tend:NAR')
where (icwmrst_grid(:ngrdcol,top_lev:) >= qsmall)
rel_grid(:ngrdcol,top_lev:) = &
(mu_grid(:ngrdcol,top_lev:) + 3._r8) / &
Expand Down Expand Up @@ -3075,16 +3096,24 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)

niic_grid(:ngrdcol,top_lev:) = ni_grid(:ngrdcol,top_lev:) / &
max(mincld,icecldf_grid(:ngrdcol,top_lev:))
call t_stopf('micro_pumas_cam_tend:NAR')

do k = top_lev, pver
call t_startf('micro_pumas_cam_tend:DTO');
!$acc data copyin (mg_ice_props, icimrst_grid(:ngrdcol,k)) &
!$acc copy (niic_grid(:ngrdcol,k)) &
!$acc copyout (rei_grid(:ngrdcol,k))
call t_stopf('micro_pumas_cam_tend:DTO');
call t_startf('micro_pumas_cam_tend:ACCR');
call size_dist_param_basic(mg_ice_props,icimrst_grid(:ngrdcol,k), &
niic_grid(:ngrdcol,k),rei_grid(:ngrdcol,k),ngrdcol)
call t_stopf('micro_pumas_cam_tend:ACCR');
call t_startf('micro_pumas_cam_tend:DTO');
!$acc end data
call t_stopf('micro_pumas_cam_tend:DTO');
end do

call t_startf('micro_pumas_cam_tend:NAR')
where (icimrst_grid(:ngrdcol,top_lev:) >= qsmall)
rei_grid(:ngrdcol,top_lev:) = 1.5_r8/rei_grid(:ngrdcol,top_lev:) &
* 1.e6_r8
Expand Down Expand Up @@ -3613,6 +3642,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf)
if (qsatfac_idx <= 0) then
deallocate(qsatfac)
end if
call t_stopf('micro_pumas_cam_tend:NAR')

end subroutine micro_pumas_cam_tend

Expand Down
Loading

0 comments on commit 4396829

Please sign in to comment.