From 43968298437d470603cfa4b09d744614314f8d6d Mon Sep 17 00:00:00 2001 From: johnmauff Date: Tue, 10 Sep 2024 09:22:57 -0600 Subject: [PATCH] Updated detailed timers to recent CAM tag. --- src/chemistry/modal_aero/aero_model.F90 | 8 ++- src/physics/cam/clubb_intr.F90 | 36 ++++++++-- src/physics/cam/micro_pumas_cam.F90 | 30 ++++++++ src/physics/cam/physpkg.F90 | 82 +++++++++++++--------- src/physics/cam/wv_saturation.F90 | 43 ++++++++++++ src/physics/cam7/micro_pumas_cam.F90 | 32 +++++++++ src/physics/cam7/physpkg.F90 | 92 +++++++++++++++++-------- src/physics/rrtmgp/radiation.F90 | 42 +++++++++++ 8 files changed, 296 insertions(+), 69 deletions(-) diff --git a/src/chemistry/modal_aero/aero_model.F90 b/src/chemistry/modal_aero/aero_model.F90 index d5be58e465..ce47244878 100644 --- a/src/chemistry/modal_aero/aero_model.F90 +++ b/src/chemistry/modal_aero/aero_model.F90 @@ -1149,7 +1149,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf) endif if (convproc_do_aer) then - call t_startf('ma_convproc') + call t_startf('aero_model_wetdep:NAR:ma_convproc') call ma_convproc_intr( state, ptend, pbuf, dt, & nsrflx_mzaer2cnvpr, qsrflx_mzaer2cnvpr, aerdepwetis, dcondt_resusp3d) @@ -1174,7 +1174,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf) end do ! loop over number + chem constituents end do ! m aerosol modes end if - call t_stopf('ma_convproc') + call t_stopf('aero_model_wetdep:NAR:ma_convproc') endif do m = 1, ntot_amode ! main loop over aerosol modes @@ -1320,6 +1320,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf) qqcw_in(:,:) = fldcw(:,:) endif + call t_startf('aero_model_wetdep:NAR:wetdepa') call wetdepa_v2( state%pmid, state%q(:,:,1), state%pdel, & dep_inputs%cldt, dep_inputs%cldcu, dep_inputs%cmfdqr, & dep_inputs%evapc, dep_inputs%conicw, dep_inputs%prain, dep_inputs%qme, & @@ -1334,6 +1335,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf) convproc_do_aer=convproc_do_aer, rcscavt=rcscavt, rsscavt=rsscavt, & sol_facti_in=sol_facti, sol_factic_in=sol_factic, & convproc_do_evaprain_atonce_in=convproc_do_evaprain_atonce ) + call t_stopf('aero_model_wetdep:NAR:wetdepa') do_hygro_sum_del = .false. if ( lspec > 0 ) do_hygro_sum_del = .true. @@ -1549,6 +1551,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf) fldcw => qqcw_get_field(pbuf, mm,lchnk) endif + call t_startf('aero_model_wetdep:NAR:wetdepa') call wetdepa_v2(state%pmid, state%q(:,:,1), state%pdel, & dep_inputs%cldt, dep_inputs%cldcu, dep_inputs%cmfdqr, & dep_inputs%evapc, dep_inputs%conicw, dep_inputs%prain, dep_inputs%qme, & @@ -1562,6 +1565,7 @@ subroutine aero_model_wetdep( state, dt, dlf, cam_out, ptend, pbuf) sol_facti_in=sol_facti, sol_factic_in=sol_factic, & convproc_do_evaprain_atonce_in=convproc_do_evaprain_atonce, & bergso_in=dep_inputs%bergso ) + call t_stopf('aero_model_wetdep:NAR:wetdepa') if(convproc_do_aer) then ! save resuspension of cloudborne species diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 06d70a98da..ee89264cdd 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2602,10 +2602,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! MAIN COMPUTATION BEGINS HERE ! !-----------------------------------------------------------------------------------! - call t_startf("clubb_tend_cam") nlev = pver + 1 - top_lev + call t_startf('clubb_tend_cam:NAR') rtp2_zt_out = 0._r8 thl2_zt_out = 0._r8 wp2_zt_out = 0._r8 @@ -2788,6 +2788,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call grid_size(state1, grid_dx, grid_dy) end if + call t_stopf('clubb_tend_cam:NAR') if (clubb_do_icesuper) then @@ -2809,11 +2810,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & qitend(:ncol,:)=0._r8 initend(:ncol,:)=0._r8 + call t_startf('clubb_tend_cam:ice_macro_tend') call ice_macro_tend(naai(1:ncol,top_lev:pver), state1%t(1:ncol,top_lev:pver), & state1%pmid(1:ncol,top_lev:pver), state1%q(1:ncol,top_lev:pver,1), & state1%q(1:ncol,top_lev:pver,ixcldice), state1%q(1:ncol,top_lev:pver,ixnumice), & latsub, hdtime, stend(1:ncol,top_lev:pver), qvtend(1:ncol,top_lev:pver), & qitend(1:ncol,top_lev:pver), initend(1:ncol,top_lev:pver), ncol*(pver-top_lev+1)) + call t_stopf('clubb_tend_cam:ice_macro_tend') ! update local copy of state with the tendencies ptend_loc%q(:ncol,top_lev:pver,1)=qvtend(:ncol,top_lev:pver) @@ -2835,6 +2838,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call outfld( 'NITENDICE', initend, pcols, lchnk ) endif + call t_startf('clubb_tend_cam:NAR') ! Determine CLUBB time step and make it sub-step friendly @@ -2996,7 +3000,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & mf_qtflx_output(:,:) = 0._r8 end if - call t_startf("clubb_tend_cam_i_loop") ! Determine Coriolis force at given latitude. This is never used ! when CLUBB is implemented in a host model, therefore just set @@ -3239,7 +3242,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Perturbed winds are not used in CAM upwp_sfc_pert = 0.0_r8 vpwp_sfc_pert = 0.0_r8 + call t_stopf('clubb_tend_cam:NAR') + call t_startf('clubb_tend_cam:flip-index') ! Need to flip arrays around for CLUBB core do k=1,nlev+1 do i=1,ncol @@ -3453,6 +3458,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (macmic_it==1) vpwp_clubb_gw_mc(:ncol,:) = 0._r8 if (macmic_it==1) thlp2_clubb_gw_mc(:ncol,:) = 0._r8 if (macmic_it==1) wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 + call t_stopf('clubb_tend_cam:flip-index') do t=1,nadv ! do needed number of "sub" timesteps for each CAM step @@ -3466,6 +3472,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !###################### CALL MF DIAGNOSTIC PLUMES ###################### !####################################################################### if (do_clubb_mf) then + call t_startf('clubb_tend_cam:do_clubb_mf') do k=2,pverp do i=1, ncol @@ -3516,10 +3523,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ((rho_ds_zm(i,k) * mf_thlflx(i,k)) - (rho_ds_zm(i,k-1) * mf_thlflx(i,k-1))) end do end do + call t_stopf('clubb_tend_cam:do_clubb_mf') end if ! Advance CLUBB CORE one timestep in the future + call t_startf('clubb_tend_cam:advance_clubb_core_api') call advance_clubb_core_api( gr, pverp+1-top_lev, ncol, & l_implemented, dtime, fcor, sfc_elevation, & hydromet_dim, & @@ -3564,6 +3573,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wprcp_out, w_up_in_cloud_out, w_down_in_cloud_out, & cloudy_updraft_frac_out, cloudy_downdraft_frac_out, & rcm_in_layer_out, cloud_cover_out, invrs_tau_zm_out ) + call t_stopf('clubb_tend_cam:advance_clubb_core_api') ! Note that CLUBB does not produce an error code specific to any column, and ! one value only for the entire chunk @@ -3587,12 +3597,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + call t_startf('clubb_tend_cam:update_xp2_mc_api') call update_xp2_mc_api( gr, nlev+1, ncol, dtime, cloud_frac_inout, & rcm_inout, rvm_in, thlm_in, wm_zt, & exner, pre_in, pdf_params_chnk(lchnk), & rtp2_mc_out, thlp2_mc_out, & wprtp_mc_out, wpthlp_mc_out, & rtpthlp_mc_out) + call t_stopf('clubb_tend_cam:update_xp2_mc_api') do k=1,nlev+1 do i=1,ncol @@ -3610,6 +3622,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (do_cldcool) then + call t_startf('clubb_tend_cam:do_cldcool') rcm_out_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, rcm_inout ) qrl_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, qrl_clubb ) @@ -3624,19 +3637,23 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & thlp2_in(i,:) = thlp2_in(i,:) + thlp2_rad_out(i,:) * dtime thlp2_in(i,:) = max(thl_tol**2,thlp2_in(i,:)) end do + call t_stopf('clubb_tend_cam:do_cldcool') end if ! Check to see if stats should be output, here stats are read into ! output arrays to make them conformable to CAM output if (stats_metadata%l_stats) then + call t_startf('clubb_tend_cam:stats_end_timestep_clubb') do i=1, ncol call stats_end_timestep_clubb(i, stats_zt(i), stats_zm(i), stats_rad_zt(i), stats_rad_zm(i), stats_sfc(i), & out_zt, out_zm, out_radzt, out_radzm, out_sfc) end do + call t_stopf('clubb_tend_cam:stats_end_timestep_clubb') end if enddo ! end time loop + call t_startf('clubb_tend_cam:NAR') if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then @@ -3662,12 +3679,16 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end if + call t_stopf('clubb_tend_cam:NAR') ! Convert RTP2 and THLP2 to thermo grid for output + call t_startf('clubb_tend_cam:NAR') rtp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, rtp2_in ) thl2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, thlp2_in ) wp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, wp2_in ) + call t_stopf('clubb_tend_cam:NAR') + call t_startf('clubb_tend_cam:flip-index') ! Arrays need to be "flipped" to CAM grid do k=1, nlev+1 do i=1, ncol @@ -3727,6 +3748,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + call t_stopf('clubb_tend_cam:flip-index') + call t_startf('clubb_tend_cam:NAR') ! Accumulate vars through macmic subcycle upwp_clubb_gw_mc(:ncol,:) = upwp_clubb_gw_mc(:ncol,:) + upwp(:ncol,:) @@ -4026,7 +4049,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end do - call t_stopf("clubb_tend_cam_i_loop") call outfld('KVH_CLUBB', khzm, pcols, lchnk) @@ -4073,9 +4095,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! then advances it's predictive equations second, this can lead to ! RHliq > 1 directly before microphysics is called. Therefore, we use ! ice_macro_tend to enforce RHliq <= 1 everywhere before microphysics is called. + call t_stopf('clubb_tend_cam:NAR') if (clubb_do_liqsupersat) then + call t_startf('clubb_cam_tend:do_liqsupersat') ! -------------------------------------- ! ! Ice Saturation Adjustment Computation ! ! -------------------------------------- ! @@ -4127,7 +4151,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end where call outfld( 'FQTENDICE', fqtend, pcols, lchnk ) + call t_stopf('clubb_cam_tend:do_liqsupersat') end if + call t_startf('clubb_tend_cam:NAR') ! ------------------------------------------------------------ ! ! The rest of the code deals with diagnosing variables ! @@ -4552,8 +4578,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & enddo endif - - call t_stopf("clubb_tend_cam") + call t_stopf('clubb_tend_cam:NAR') + return #endif diff --git a/src/physics/cam/micro_pumas_cam.F90 b/src/physics/cam/micro_pumas_cam.F90 index dae867f9dc..0bbb3a1c74 100644 --- a/src/physics/cam/micro_pumas_cam.F90 +++ b/src/physics/cam/micro_pumas_cam.F90 @@ -1887,6 +1887,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) nan_array = nan + call t_startf('micro_pumas_cam_tend:NAR') call phys_getopts(use_subcol_microp_out=use_subcol_microp) ! Set the col_type flag to grid or subcolumn dependent on the value of use_subcol_microp @@ -2354,6 +2355,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) frzimm(:ncol,:top_lev-1)=0._r8 frzcnt(:ncol,:top_lev-1)=0._r8 frzdep(:ncol,:top_lev-1)=0._r8 + call t_stopf('micro_pumas_cam_tend:NAR') do it = 1, num_steps @@ -2402,6 +2404,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) end select case(2:3) + call t_startf('micro_pumas_cam_tend:micro_pumas_tend') call micro_pumas_tend( & ncol, nlev, dtime/num_steps,& state_loc%t(:ncol,top_lev:), state_loc%q(:ncol,top_lev:,ixq), & @@ -2470,8 +2473,10 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) tnd_qsnow(:ncol,top_lev:),tnd_nsnow(:ncol,top_lev:),re_ice(:ncol,top_lev:),& prer_evap(:ncol,top_lev:), & frzimm(:ncol,top_lev:), frzcnt(:ncol,top_lev:), frzdep(:ncol,top_lev:) ) + call t_stopf('micro_pumas_cam_tend:micro_pumas_tend') end select + call t_startf('micro_pumas_cam_tend:NAR') call handle_errmsg(errstring, subname="micro_pumas_tend") call physics_ptend_init(ptend_loc, psetcols, "micro_pumas", & @@ -2945,17 +2950,25 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) rel_fn_grid = 10._r8 ncic_grid = 1.e8_r8 + call t_stopf('micro_pumas_cam_tend:NAR') do k = top_lev, pver + call t_startf('micro_pumas_cam_tend:DTO'); !$acc data copyin (mg_liq_props,icwmrst_grid(:ngrdcol,k),rho_grid(:ngrdcol,k)) & !$acc copy (ncic_grid(:ngrdcol,k)) & !$acc copyout (mu_grid(:ngrdcol,k),lambdac_grid(:ngrdcol,k)) + call t_stopf('micro_pumas_cam_tend:DTO'); + call t_startf('micro_pumas_cam_tend:ACCR'); call size_dist_param_liq(mg_liq_props, icwmrst_grid(:ngrdcol,k), & ncic_grid(:ngrdcol,k), rho_grid(:ngrdcol,k), & mu_grid(:ngrdcol,k), lambdac_grid(:ngrdcol,k), ngrdcol) + call t_stopf('micro_pumas_cam_tend:ACCR'); + call t_startf('micro_pumas_cam_tend:DTO'); !$acc end data + call t_stopf('micro_pumas_cam_tend:DTO'); end do + call t_startf('micro_pumas_cam_tend:NAR') where (icwmrst_grid(:ngrdcol,top_lev:) > qsmall) rel_fn_grid(:ngrdcol,top_lev:) = & (mu_grid(:ngrdcol,top_lev:) + 3._r8)/ & @@ -2971,17 +2984,25 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) ! Calculate ncic on the grid ncic_grid(:ngrdcol,top_lev:) = nc_grid(:ngrdcol,top_lev:) / & max(mincld,liqcldf_grid(:ngrdcol,top_lev:)) + call t_stopf('micro_pumas_cam_tend:NAR') do k = top_lev, pver + call t_startf('micro_pumas_cam_tend:DTO'); !$acc data copyin (mg_liq_props,icwmrst_grid(:ngrdcol,k), rho_grid(:ngrdcol,k)) & !$acc copy (ncic_grid(:ngrdcol,k)) & !$acc copyout (mu_grid(:ngrdcol,k),lambdac_grid(:ngrdcol,k)) + call t_stopf('micro_pumas_cam_tend:DTO'); + call t_startf('micro_pumas_cam_tend:ACCR'); call size_dist_param_liq(mg_liq_props, icwmrst_grid(:ngrdcol,k), & ncic_grid(:ngrdcol,k), rho_grid(:ngrdcol,k), & mu_grid(:ngrdcol,k), lambdac_grid(:ngrdcol,k), ngrdcol) + call t_stopf('micro_pumas_cam_tend:ACCR'); + call t_startf('micro_pumas_cam_tend:DTO'); !$acc end data + call t_stopf('micro_pumas_cam_tend:DTO'); end do + call t_startf('micro_pumas_cam_tend:NAR') where (icwmrst_grid(:ngrdcol,top_lev:) >= qsmall) rel_grid(:ngrdcol,top_lev:) = & (mu_grid(:ngrdcol,top_lev:) + 3._r8) / & @@ -3075,16 +3096,24 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) niic_grid(:ngrdcol,top_lev:) = ni_grid(:ngrdcol,top_lev:) / & max(mincld,icecldf_grid(:ngrdcol,top_lev:)) + call t_stopf('micro_pumas_cam_tend:NAR') do k = top_lev, pver + call t_startf('micro_pumas_cam_tend:DTO'); !$acc data copyin (mg_ice_props, icimrst_grid(:ngrdcol,k)) & !$acc copy (niic_grid(:ngrdcol,k)) & !$acc copyout (rei_grid(:ngrdcol,k)) + call t_stopf('micro_pumas_cam_tend:DTO'); + call t_startf('micro_pumas_cam_tend:ACCR'); call size_dist_param_basic(mg_ice_props,icimrst_grid(:ngrdcol,k), & niic_grid(:ngrdcol,k),rei_grid(:ngrdcol,k),ngrdcol) + call t_stopf('micro_pumas_cam_tend:ACCR'); + call t_startf('micro_pumas_cam_tend:DTO'); !$acc end data + call t_stopf('micro_pumas_cam_tend:DTO'); end do + call t_startf('micro_pumas_cam_tend:NAR') where (icimrst_grid(:ngrdcol,top_lev:) >= qsmall) rei_grid(:ngrdcol,top_lev:) = 1.5_r8/rei_grid(:ngrdcol,top_lev:) & * 1.e6_r8 @@ -3613,6 +3642,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) if (qsatfac_idx <= 0) then deallocate(qsatfac) end if + call t_stopf('micro_pumas_cam_tend:NAR') end subroutine micro_pumas_cam_tend diff --git a/src/physics/cam/physpkg.F90 b/src/physics/cam/physpkg.F90 index a2aaad5564..8bf86035ca 100644 --- a/src/physics/cam/physpkg.F90 +++ b/src/physics/cam/physpkg.F90 @@ -1577,7 +1577,9 @@ subroutine tphysac (ztodt, cam_in, & call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& fh2o, surfric, obklen, flx_heat) end if + call t_startf('tphysac:aoa_tracers_timestep_tend') call aoa_tracers_timestep_tend(state, ptend, cam_in%cflx, cam_in%landfrac, ztodt) + call t_stopf('tphysac:aoa_tracers_timestep_tend') if ( (trim(cam_take_snapshot_after) == "aoa_tracers_timestep_tend") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then call cam_snapshot_ptend_outfld(ptend, lchnk) @@ -1594,7 +1596,9 @@ subroutine tphysac (ztodt, cam_in, & call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& fh2o, surfric, obklen, flx_heat) end if + call t_startf('tphysac:co2_cycle_set_ptend') call co2_cycle_set_ptend(state, pbuf, ptend) + call t_stopf('tphysac:co2_cycle_set_ptend') if ( (trim(cam_take_snapshot_after) == "co2_cycle_set_ptend") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then call cam_snapshot_ptend_outfld(ptend, lchnk) @@ -1621,8 +1625,10 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat) end if + call t_startf('tphysac:chem_timestep_tend') call chem_timestep_tend(state, ptend, cam_in, cam_out, ztodt, & pbuf, fh2o=fh2o) + call t_stopf('tphysac:chem_timestep_tend') if ( (trim(cam_take_snapshot_after) == "chem_timestep_tend") .and. & @@ -1646,7 +1652,7 @@ subroutine tphysac (ztodt, cam_in, & ! Call vertical diffusion code (pbl, free atmosphere and molecular) !=================================================== - call t_startf('vertical_diffusion_tend') + call t_startf('tphysac:vertical_diffusion_tend') if (trim(cam_take_snapshot_before) == "vertical_diffusion_section") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& @@ -1680,12 +1686,12 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat) end if - call t_stopf ('vertical_diffusion_tend') + call t_stopf ('tphysac:vertical_diffusion_tend') !=================================================== ! Rayleigh friction calculation !=================================================== - call t_startf('rayleigh_friction') + call t_startf('tphysac:rayleigh_friction_tend') call rayleigh_friction_tend( ztodt, state, ptend) if ( ptend%lu ) then call outfld( 'UTEND_RAYLEIGH', ptend%u, pcols, lchnk) @@ -1694,7 +1700,7 @@ subroutine tphysac (ztodt, cam_in, & call outfld( 'VTEND_RAYLEIGH', ptend%v, pcols, lchnk) end if call physics_update(state, ptend, ztodt, tend) - call t_stopf('rayleigh_friction') + call t_stopf('tphysac:rayleigh_friction_tend') if (do_clubb_sgs) then call check_energy_chng(state, tend, "vdiff", nstep, ztodt, zero, zero, zero, zero) @@ -1706,7 +1712,7 @@ subroutine tphysac (ztodt, cam_in, & call check_tracers_chng(state, tracerint, "vdiff", nstep, ztodt, cam_in%cflx) ! aerosol dry deposition processes - call t_startf('aero_drydep') + call t_startf('tphysac:aero_model_drydep') if (trim(cam_take_snapshot_before) == "aero_model_drydep") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& @@ -1725,7 +1731,7 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat) end if - call t_stopf('aero_drydep') + call t_stopf('tphysac:aero_model_drydep') ! CARMA microphysics ! @@ -1735,12 +1741,12 @@ subroutine tphysac (ztodt, cam_in, & ! that cam_out%xxxdryxxx fields have already been set for CAM aerosols and cam_out ! can be added to for CARMA aerosols. if (carma_do_aerosol) then - call t_startf('carma_timestep_tend') + call t_startf('tphysac:carma_timestep_tend') call carma_timestep_tend(state, cam_in, cam_out, ptend, ztodt, pbuf, obklen=obklen, ustar=surfric) call physics_update(state, ptend, ztodt, tend) call check_energy_chng(state, tend, "carma_tend", nstep, ztodt, zero, zero, zero, zero) - call t_stopf('carma_timestep_tend') + call t_stopf('tphysac:carma_timestep_tend') end if @@ -1752,7 +1758,7 @@ subroutine tphysac (ztodt, cam_in, & !=================================================== ! Gravity wave drag !=================================================== - call t_startf('gw_tend') + call t_startf('tphysac:gw_tend') if (trim(cam_take_snapshot_before) == "gw_tend") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& @@ -1781,7 +1787,7 @@ subroutine tphysac (ztodt, cam_in, & ! Check energy integrals call check_energy_chng(state, tend, "gwdrag", nstep, ztodt, zero, & zero, zero, flx_heat) - call t_stopf('gw_tend') + call t_stopf('tphysac:gw_tend') ! QBO relaxation @@ -2323,14 +2329,15 @@ subroutine tphysbc (ztodt, state, & !=================================================== ! Dry adjustment !=================================================== - call t_startf('dry_adjustment') if (trim(cam_take_snapshot_before) == "dadadj_tend") then call cam_snapshot_all_outfld_tphysbc(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if + call t_startf('tphysbc:dadadj_tend') call dadadj_tend(ztodt, state, ptend) + call t_stopf('tphysbc:dadadj_tend') if ( (trim(cam_take_snapshot_after) == "dadadj_tend") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then @@ -2343,26 +2350,26 @@ subroutine tphysbc (ztodt, state, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if - call t_stopf('dry_adjustment') !=================================================== ! Moist convection !=================================================== call t_startf('moist_convection') - call t_startf ('convect_deep_tend') if (trim(cam_take_snapshot_before) == "convect_deep_tend") then call cam_snapshot_all_outfld_tphysbc(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if + call t_startf ('tphysbc:convect_deep_tend') call convect_deep_tend( & cmfmc, cmfcme, & zdu, & rliq, rice, & ztodt, & state, ptend, cam_in%landfrac, pbuf) + call t_stopf('tphysbc:convect_deep_tend') if ( (trim(cam_take_snapshot_after) == "convect_deep_tend") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then @@ -2382,7 +2389,6 @@ subroutine tphysbc (ztodt, state, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if - call t_stopf('convect_deep_tend') call pbuf_get_field(pbuf, prec_dp_idx, prec_dp ) call pbuf_get_field(pbuf, snow_dp_idx, snow_dp ) @@ -2409,7 +2415,6 @@ subroutine tphysbc (ztodt, state, & ! ! Call Hack (1994) convection scheme to deal with shallow/mid-level convection ! - call t_startf ('convect_shallow_tend') if (dlfzm_idx > 0) then call pbuf_get_field(pbuf, dlfzm_idx, dlfzm) @@ -2423,10 +2428,11 @@ subroutine tphysbc (ztodt, state, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if + call t_startf ('tphysbc:convect_shallow_tend') call convect_shallow_tend (ztodt , cmfmc, & dlf , dlf2 , rliq , rliq2, & state , ptend , pbuf, cam_in) - call t_stopf ('convect_shallow_tend') + call t_stopf ('tphysbc:convect_shallow_tend') if ( (trim(cam_take_snapshot_after) == "convect_shallow_tend") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then @@ -2468,9 +2474,9 @@ subroutine tphysbc (ztodt, state, & ! snow are stored in the physics buffer and will be incorporated by the MG microphysics. ! ! Currently CARMA cloud microphysics is only supported with the MG microphysics. - call t_startf('carma_timestep_tend') if (carma_do_cldice .or. carma_do_cldliq) then + call t_startf('tphysbc:carma_timestep_tend') call carma_timestep_tend(state, cam_in, cam_out, ptend, ztodt, pbuf, dlf=dlf, rliq=rliq, & prec_str=prec_str, snow_str=snow_str, prec_sed=prec_sed_carma, snow_sed=snow_sed_carma) call physics_update(state, ptend, ztodt, tend) @@ -2482,16 +2488,16 @@ subroutine tphysbc (ztodt, state, & else call check_energy_chng(state, tend, "carma_tend", nstep, ztodt, zero, prec_str, snow_str, zero) end if + call t_stopf('tphysbc:carma_timestep_tend') end if - call t_stopf('carma_timestep_tend') if( microp_scheme == 'RK' ) then !=================================================== ! Calculate stratiform tendency (sedimentation, detrain, cloud fraction and microphysics ) !=================================================== - call t_startf('rk_stratiform_tend') + call t_startf('tphysbc:rk_stratiform_tend') call rk_stratiform_tend(state, ptend, pbuf, ztodt, & cam_in%icefrac, cam_in%landfrac, cam_in%ocnfrac, & @@ -2504,7 +2510,7 @@ subroutine tphysbc (ztodt, state, & call physics_update(state, ptend, ztodt, tend) call check_energy_chng(state, tend, "cldwat_tend", nstep, ztodt, zero, prec_str, snow_str, zero) - call t_stopf('rk_stratiform_tend') + call t_stopf('tphysbc:rk_stratiform_tend') elseif( microp_scheme == 'MG' ) then ! Start co-substepping of macrophysics and microphysics @@ -2533,7 +2539,6 @@ subroutine tphysbc (ztodt, state, & ! Calculate macrophysical tendency (sedimentation, detrain, cloud fraction) !=================================================== - call t_startf('macrop_tend') ! don't call Park macrophysics if CLUBB is called if (macrop_scheme .ne. 'CLUBB_SGS') then @@ -2543,6 +2548,7 @@ subroutine tphysbc (ztodt, state, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if + call t_startf('tphysbc:macrop_driver_tend') call macrop_driver_tend( & state, ptend, cld_macmic_ztodt, & cam_in%landfrac, cam_in%ocnfrac, cam_in%snowhland, & ! sediment @@ -2550,6 +2556,7 @@ subroutine tphysbc (ztodt, state, & cmfmc, & cam_in%ts, cam_in%sst, zdu, & pbuf, det_s, det_ice) + call t_stopf('tphysbc:macrop_driver_tend') ! Since we "added" the reserved liquid back in this routine, we need ! to account for it in the energy checker @@ -2590,9 +2597,11 @@ subroutine tphysbc (ztodt, state, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if + call t_startf('tphysbc:clubb_tend_cam') call clubb_tend_cam(state, ptend, pbuf, cld_macmic_ztodt,& cmfmc, cam_in, macmic_it, cld_macmic_num_steps, & dlf, det_s, det_ice) + call t_stopf('tphysbc:clubb_tend_cam') ! Since we "added" the reserved liquid back in this routine, we need ! to account for it in the energy checker @@ -2629,7 +2638,6 @@ subroutine tphysbc (ztodt, state, & endif - call t_stopf('macrop_tend') !=================================================== ! Calculate cloud microphysics @@ -2657,11 +2665,10 @@ subroutine tphysbc (ztodt, state, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if - call t_startf('microp_aero_run') + call t_startf('tphysbc:microp_aero_run') call microp_aero_run(state, ptend_aero, cld_macmic_ztodt, pbuf) - call t_stopf('microp_aero_run') + call t_stopf('tphysbc:microp_aero_run') - call t_startf('microp_tend') if (use_subcol_microp) then @@ -2731,7 +2738,9 @@ subroutine tphysbc (ztodt, state, & call physics_tend_dealloc(tend_sc) call physics_ptend_dealloc(ptend_sc) else + call t_startf('tphysbc:microp_driver_tend') call microp_driver_tend(state, ptend, cld_macmic_ztodt, pbuf) + call t_stopf('tphysbc:microp_driver_tend') end if ! combine aero and micro tendencies for the grid call physics_ptend_sum(ptend_aero, ptend, ncol) @@ -2758,7 +2767,6 @@ subroutine tphysbc (ztodt, state, & zero, prec_str(:ncol)/cld_macmic_num_steps, & snow_str(:ncol)/cld_macmic_num_steps, zero) - call t_stopf('microp_tend') prec_sed_macmic(:ncol) = prec_sed_macmic(:ncol) + prec_sed(:ncol) snow_sed_macmic(:ncol) = snow_sed_macmic(:ncol) + snow_sed(:ncol) prec_pcw_macmic(:ncol) = prec_pcw_macmic(:ncol) + prec_pcw(:ncol) @@ -2805,14 +2813,22 @@ subroutine tphysbc (ztodt, state, & ! Do calculations of mode radius and water uptake if: ! 1) modal aerosols are affecting the climate, or ! 2) prognostic modal aerosols are enabled + call t_startf('tphysbc:modal_aero_calcsize_sub') call modal_aero_calcsize_sub(state, ptend, ztodt, pbuf) + call t_stopf('tphysbc:modal_aero_calcsize_sub') ! for prognostic modal aerosols the transfer of mass between aitken and accumulation ! modes is done in conjunction with the dry radius calculation + call t_startf('tphysbc:modal_aero_wateruptake_dr') call modal_aero_wateruptake_dr(state, pbuf) + call t_stopf('tphysbc:modal_aero_wateruptake_dr') call physics_update(state, ptend, ztodt, tend) else + call t_startf('tphysbc:modal_aero_calcsize_diag') call modal_aero_calcsize_diag(state, pbuf) + call t_stopf('tphysbc:modal_aero_calcsize_diag') + call t_startf('tphysbc:modal_aero_wateruptake_dr') call modal_aero_wateruptake_dr(state, pbuf) + call t_stopf('tphysbc:modal_aero_wateruptake_dr') endif endif @@ -2821,7 +2837,9 @@ subroutine tphysbc (ztodt, state, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if + call t_startf('tphysbc:aero_model_wetdep') call aero_model_wetdep( state, ztodt, dlf, cam_out, ptend, pbuf) + call t_stopf('tphysbc:aero_model_wetdep') if ( (trim(cam_take_snapshot_after) == "aero_model_wetdep") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then call cam_snapshot_ptend_outfld(ptend, lchnk) @@ -2839,16 +2857,16 @@ subroutine tphysbc (ztodt, state, & ! NOTE: It needs to follow aero_model_wetdep, so that cam_out%xxxwetxxx ! fields have already been set for CAM aerosols and cam_out can be added ! to for CARMA aerosols. - call t_startf ('carma_wetdep_tend') + call t_startf ('tphysbc:carma_wetdep_tend') call carma_wetdep_tend(state, ptend, ztodt, pbuf, dlf, cam_out) call physics_update(state, ptend, ztodt, tend) - call t_stopf ('carma_wetdep_tend') + call t_stopf ('tphysbc:carma_wetdep_tend') end if - call t_startf ('convect_deep_tend2') + call t_startf ('tphysbc:convect_deep_tend2') call convect_deep_tend_2( state, ptend, ztodt, pbuf ) call physics_update(state, ptend, ztodt, tend) - call t_stopf ('convect_deep_tend2') + call t_stopf ('tphysbc:convect_deep_tend2') ! check tracer integrals call check_tracers_chng(state, tracerint, "cmfmca", nstep, ztodt, zero_tracers) @@ -2881,15 +2899,16 @@ subroutine tphysbc (ztodt, state, & !=================================================== ! Radiation computations !=================================================== - call t_startf('radiation') if (trim(cam_take_snapshot_before) == "radiation_tend") then call cam_snapshot_all_outfld_tphysbc(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf, & flx_heat, cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, det_s, det_ice, net_flx) end if + call t_startf('tphysbc:radiation_tend') call radiation_tend( & state, ptend, pbuf, cam_out, cam_in, net_flx) + call t_stopf('tphysbc:radiation_tend') ! Set net flux used by spectral dycores do i=1,ncol @@ -2909,7 +2928,6 @@ subroutine tphysbc (ztodt, state, & call check_energy_chng(state, tend, "radheat", nstep, ztodt, zero, zero, zero, net_flx) - call t_stopf('radiation') ! Diagnose the location of the tropopause and its location to the history file(s). call t_startf('tropopause') diff --git a/src/physics/cam/wv_saturation.F90 b/src/physics/cam/wv_saturation.F90 index ac94482e20..350cd12c9d 100644 --- a/src/physics/cam/wv_saturation.F90 +++ b/src/physics/cam/wv_saturation.F90 @@ -35,6 +35,7 @@ module wv_saturation use wv_sat_methods, only: & svp_to_qsat => wv_sat_svp_to_qsat, & svp_to_qsat_vect => wv_sat_svp_to_qsat_vect +use perf_mod, only: t_startf, t_stopf implicit none private @@ -761,6 +762,7 @@ subroutine qsat_line(t, p, es, qs, gam, dqsdt, enthalpy) real(r8) :: hltalt ! Modified latent heat for T derivatives real(r8) :: tterm ! Account for d(es)/dT in transition region + call t_startf('qsat:CPU') es = estblf(t) qs = svp_to_qsat(es, p) @@ -781,6 +783,7 @@ subroutine qsat_line(t, p, es, qs, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat:CPU') end subroutine qsat_line @@ -815,9 +818,12 @@ subroutine qsat_vect(t, p, es, qs, vlen, gam, dqsdt, enthalpy) present_dqsdt = present(dqsdt) present_enthalpy = present(enthalpy) + call t_startf('qsat:DTO') !$acc data copyin (t,p) & !$acc copyout (es,qs,gam,dqsdt,enthalpy) & !$acc create (hltalt,tterm) + call t_stopf('qsat:DTO') + call t_startf('qsat:ACCR') call estblf_vect(t, es, vlen) @@ -846,7 +852,10 @@ subroutine qsat_vect(t, p, es, qs, vlen, gam, dqsdt, enthalpy) end if + call t_stopf('qsat:ACCR') + call t_startf('qsat:DTO') !$acc end data + call t_stopf('qsat:DTO') end subroutine qsat_vect subroutine qsat_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) @@ -881,10 +890,13 @@ subroutine qsat_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) present_dqsdt = present(dqsdt) present_enthalpy = present(enthalpy) + call t_startf('qsat:DTO') !$acc data copyin (t,p) & !$acc copyout (es,qs,gam,dqsdt,enthalpy) & !$acc create (hltalt,tterm) + call t_stopf('qsat:DTO') + call t_startf('qsat:ACCR') call estblf_vect(t, es, vlen) call svp_to_qsat_vect(es, p, qs, vlen) @@ -913,8 +925,11 @@ subroutine qsat_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat:ACCR') + call t_startf('qsat:DTO') !$acc end data + call t_stopf('qsat:DTO') end subroutine qsat_2D subroutine qsat_water_line(t, p, es, qs, gam, dqsdt, enthalpy) @@ -942,6 +957,7 @@ subroutine qsat_water_line(t, p, es, qs, gam, dqsdt, enthalpy) ! Local variables real(r8) :: hltalt ! Modified latent heat for T derivatives + call t_startf('qsat_water:CPU') call wv_sat_qsat_water(t, p, es, qs) if (present(gam) .or. present(dqsdt) .or. present(enthalpy)) then @@ -957,6 +973,7 @@ subroutine qsat_water_line(t, p, es, qs, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat_water:CPU') end subroutine qsat_water_line @@ -993,10 +1010,13 @@ subroutine qsat_water_vect(t, p, es, qs, vlen, gam, dqsdt, enthalpy) present_dqsdt = present(dqsdt) present_enthalpy = present(enthalpy) + call t_startf('qsat_water:DTO') !$acc data copyin (t,p) & !$acc copyout (es,qs,gam,dqsdt,enthalpy) & !$acc create (tterm,hltalt) + call t_stopf('qsat_water:DTO') + call t_startf('qsat_water:ACCR') !$acc parallel vector_length(VLENS) default(present) !$acc loop gang vector do i = 1, vlen @@ -1019,8 +1039,11 @@ subroutine qsat_water_vect(t, p, es, qs, vlen, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat_water:ACCR') + call t_startf('qsat_water:DTO') !$acc end data + call t_stopf('qsat_water:DTO') end subroutine qsat_water_vect subroutine qsat_water_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) @@ -1057,10 +1080,13 @@ subroutine qsat_water_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) present_dqsdt = present(dqsdt) present_enthalpy = present(enthalpy) + call t_startf('qsat_water:DTO') !$acc data copyin (t,p) & !$acc copyout (es,qs,gam,dqsdt,enthalpy) & !$acc create (hltalt,tterm) + call t_stopf('qsat_water:DTO') + call t_startf('qsat_water:ACCR') !$acc parallel vector_length(VLENS) default(present) !$acc loop gang vector collapse(2) do k = 1, dim2 @@ -1085,8 +1111,11 @@ subroutine qsat_water_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat_water:ACCR') + call t_startf('qsat_water:DTO') !$acc end data + call t_stopf('qsat_water:DTO') end subroutine qsat_water_2D subroutine qsat_ice_line(t, p, es, qs, gam, dqsdt, enthalpy) @@ -1114,6 +1143,7 @@ subroutine qsat_ice_line(t, p, es, qs, gam, dqsdt, enthalpy) ! Local variables real(r8) :: hltalt ! Modified latent heat for T derivatives + call t_startf('qsat_ice:CPU') call wv_sat_qsat_ice(t, p, es, qs) if (present(gam) .or. present(dqsdt) .or. present(enthalpy)) then @@ -1128,6 +1158,7 @@ subroutine qsat_ice_line(t, p, es, qs, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat_ice:CPU') end subroutine qsat_ice_line @@ -1164,10 +1195,13 @@ subroutine qsat_ice_vect(t, p, es, qs, vlen, gam, dqsdt, enthalpy) present_dqsdt = present(dqsdt) present_enthalpy = present(enthalpy) + call t_startf('qsat_ice:DTO') !$acc data copyin (t,p) & !$acc copyout (es,qs,gam,dqsdt,enthalpy) & !$acc create (hltalt,tterm) + call t_stopf('qsat_ice:DTO') + call t_startf('qsat_ice:ACCR') !$acc parallel vector_length(VLENS) default(present) !$acc loop gang vector do i = 1, vlen @@ -1194,8 +1228,11 @@ subroutine qsat_ice_vect(t, p, es, qs, vlen, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat_ice:ACCR') + call t_startf('qsat_ice:DTO') !$acc end data + call t_stopf('qsat_ice:DTO') end subroutine qsat_ice_vect subroutine qsat_ice_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) @@ -1232,10 +1269,13 @@ subroutine qsat_ice_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) present_dqsdt = present(dqsdt) present_enthalpy = present(enthalpy) + call t_stopf('qsat_ice:DTO') !$acc data copyin (t,p) & !$acc copyout (es,qs,gam,dqsdt,enthalpy) & !$acc create (hltalt,tterm) + call t_stopf('qsat_ice:DTO') + call t_startf('qsat_ice:ACCR') !$acc parallel vector_length(VLENS) default(present) !$acc loop gang vector collapse(2) do k = 1, dim2 @@ -1266,8 +1306,11 @@ subroutine qsat_ice_2D(t, p, es, qs, dim1, dim2, gam, dqsdt, enthalpy) gam=gam, dqsdt=dqsdt) end if + call t_stopf('qsat_ice:ACCR') + call t_startf('qsat_ice:DTO') !$acc end data + call t_stopf('qsat_ice:DTO') end subroutine qsat_ice_2D !--------------------------------------------------------------------- diff --git a/src/physics/cam7/micro_pumas_cam.F90 b/src/physics/cam7/micro_pumas_cam.F90 index f38eda2ade..36795865f2 100644 --- a/src/physics/cam7/micro_pumas_cam.F90 +++ b/src/physics/cam7/micro_pumas_cam.F90 @@ -1938,6 +1938,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) nan_array = nan + call t_startf('micro_pumas_cam_tend:NAR') ! Allocate the proc_rates DDT ! IMPORTANT NOTE -- elements in proc_rates are dimensioned to the nlev dimension while ! all the other arrays in this routine are dimensioned pver. This is required because @@ -2371,9 +2372,11 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) frzimm(:ncol,:top_lev-1)=0._r8 frzcnt(:ncol,:top_lev-1)=0._r8 frzdep(:ncol,:top_lev-1)=0._r8 + call t_stopf('micro_pumas_cam_tend:NAR') do it = 1, num_steps + call t_startf('micro_pumas_cam_tend:micro_pumas_tend') call micro_pumas_tend( & ncol, nlev, dtime/num_steps,& state_loc%t(:ncol,top_lev:), state_loc%q(:ncol,top_lev:,ixq), & @@ -2424,6 +2427,8 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) tnd_qsnow(:ncol,top_lev:),tnd_nsnow(:ncol,top_lev:),re_ice(:ncol,top_lev:),& prer_evap(:ncol,top_lev:), & frzimm(:ncol,top_lev:), frzcnt(:ncol,top_lev:), frzdep(:ncol,top_lev:) ) + call t_stopf('micro_pumas_cam_tend:micro_pumas_tend') + call t_startf('micro_pumas_cam_tend:NAR') call handle_errmsg(errstring, subname="micro_pumas_cam_tend") @@ -2473,9 +2478,11 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) proc_rates%ank(:ncol,:,:) = proc_rates%ank(:ncol,:,:)/num_steps proc_rates%amk_out(:ncol,:,:) = proc_rates%amk_out(:ncol,:,:)/num_steps end if + call t_stopf('micro_pumas_cam_tend:NAR') end do + call t_startf('micro_pumas_cam_tend:NAR') ! Divide ptend by substeps. call physics_ptend_scale(ptend, 1._r8/num_steps, ncol) @@ -2935,6 +2942,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) else rho_grid = rho end if + call t_stopf('micro_pumas_cam_tend:NAR') ! Effective radius for cloud liquid, fixed number. mu_grid = 0._r8 @@ -2944,15 +2952,22 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) ncic_grid = 1.e8_r8 do k = top_lev, pver + call t_startf('micro_pumas_cam_tend:DTO'); !$acc data copyin (mg_liq_props,icwmrst_grid(:ngrdcol,k),rho_grid(:ngrdcol,k)) & !$acc copy (ncic_grid(:ngrdcol,k)) & !$acc copyout (mu_grid(:ngrdcol,k),lambdac_grid(:ngrdcol,k)) + call t_stopf('micro_pumas_cam_tend:DTO'); + call t_startf('micro_pumas_cam_tend:ACCR'); call size_dist_param_liq(mg_liq_props, icwmrst_grid(:ngrdcol,k), & ncic_grid(:ngrdcol,k), rho_grid(:ngrdcol,k), & mu_grid(:ngrdcol,k), lambdac_grid(:ngrdcol,k), ngrdcol) + call t_stopf('micro_pumas_cam_tend:ACCR'); + call t_startf('micro_pumas_cam_tend:DTO'); !$acc end data + call t_stopf('micro_pumas_cam_tend:DTO'); end do + call t_startf('micro_pumas_cam_tend:NAR') where (icwmrst_grid(:ngrdcol,top_lev:) > qsmall) rel_fn_grid(:ngrdcol,top_lev:) = & (mu_grid(:ngrdcol,top_lev:) + 3._r8)/ & @@ -2968,17 +2983,25 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) ! Calculate ncic on the grid ncic_grid(:ngrdcol,top_lev:) = nc_grid(:ngrdcol,top_lev:) / & max(mincld,liqcldf_grid(:ngrdcol,top_lev:)) + call t_stopf('micro_pumas_cam_tend:NAR') do k = top_lev, pver + call t_startf('micro_pumas_cam_tend:DTO'); !$acc data copyin (mg_liq_props,icwmrst_grid(:ngrdcol,k), rho_grid(:ngrdcol,k)) & !$acc copy (ncic_grid(:ngrdcol,k)) & !$acc copyout (mu_grid(:ngrdcol,k),lambdac_grid(:ngrdcol,k)) + call t_stopf('micro_pumas_cam_tend:DTO'); + call t_startf('micro_pumas_cam_tend:ACCR'); call size_dist_param_liq(mg_liq_props, icwmrst_grid(:ngrdcol,k), & ncic_grid(:ngrdcol,k), rho_grid(:ngrdcol,k), & mu_grid(:ngrdcol,k), lambdac_grid(:ngrdcol,k), ngrdcol) + call t_stopf('micro_pumas_cam_tend:ACCR'); + call t_startf('micro_pumas_cam_tend:DTO'); !$acc end data + call t_stopf('micro_pumas_cam_tend:DTO'); end do + call t_startf('micro_pumas_cam_tend:NAR') where (icwmrst_grid(:ngrdcol,top_lev:) >= qsmall) rel_grid(:ngrdcol,top_lev:) = & (mu_grid(:ngrdcol,top_lev:) + 3._r8) / & @@ -3044,16 +3067,24 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) niic_grid(:ngrdcol,top_lev:) = ni_grid(:ngrdcol,top_lev:) / & max(mincld,icecldf_grid(:ngrdcol,top_lev:)) + call t_stopf('micro_pumas_cam_tend:NAR') do k = top_lev, pver + call t_startf('micro_pumas_cam_tend:DTO'); !$acc data copyin (mg_ice_props, icimrst_grid(:ngrdcol,k)) & !$acc copy (niic_grid(:ngrdcol,k)) & !$acc copyout (rei_grid(:ngrdcol,k)) + call t_stopf('micro_pumas_cam_tend:DTO'); + call t_startf('micro_pumas_cam_tend:ACCR'); call size_dist_param_basic(mg_ice_props,icimrst_grid(:ngrdcol,k), & niic_grid(:ngrdcol,k),rei_grid(:ngrdcol,k),ngrdcol) + call t_stopf('micro_pumas_cam_tend:ACCR'); + call t_startf('micro_pumas_cam_tend:DTO'); !$acc end data + call t_stopf('micro_pumas_cam_tend:DTO'); end do + call t_startf('micro_pumas_cam_tend:NAR') where (icimrst_grid(:ngrdcol,top_lev:) >= qsmall) rei_grid(:ngrdcol,top_lev:) = 1.5_r8/rei_grid(:ngrdcol,top_lev:) & * 1.e6_r8 @@ -3659,6 +3690,7 @@ subroutine micro_pumas_cam_tend(state, ptend, dtime, pbuf) if (qsatfac_idx <= 0) then deallocate(qsatfac) end if + call t_stopf('micro_pumas_cam_tend:NAR') end subroutine micro_pumas_cam_tend diff --git a/src/physics/cam7/physpkg.F90 b/src/physics/cam7/physpkg.F90 index d6b23c0fdf..10cfd8d3d5 100644 --- a/src/physics/cam7/physpkg.F90 +++ b/src/physics/cam7/physpkg.F90 @@ -1635,7 +1635,7 @@ subroutine tphysac (ztodt, cam_in, & !=================================================== ! Apply tracer surface fluxes to lowest model layer !=================================================== - call t_startf('clubb_emissions_tend') + call t_startf('tphysac:clubb_emissions_tend') call clubb_emissions_cam(state, cam_in, ptend) @@ -1643,7 +1643,7 @@ subroutine tphysac (ztodt, cam_in, & call check_energy_chng(state, tend, "clubb_emissions_tend", nstep, ztodt, zero, zero, zero, zero) - call t_stopf('clubb_emissions_tend') + call t_stopf('tphysac:clubb_emissions_tend') !=================================================== ! Calculate tendencies from CARMA bin microphysics. @@ -1658,9 +1658,9 @@ subroutine tphysac (ztodt, cam_in, & ! ! Currently CARMA cloud microphysics is only supported with the MG ! microphysics. - call t_startf('carma_timestep_tend') if (carma_do_cldice .or. carma_do_cldliq) then + call t_startf('tphysac:carma_timestep_tend') call carma_timestep_tend(state, cam_in, cam_out, ptend, ztodt, pbuf, dlf=dlf, rliq=rliq, & prec_str=prec_str, snow_str=snow_str, prec_sed=prec_sed_carma, snow_sed=snow_sed_carma) call physics_update(state, ptend, ztodt, tend) @@ -1673,9 +1673,9 @@ subroutine tphysac (ztodt, cam_in, & else call check_energy_chng(state, tend, "carma_tend", nstep, ztodt, zero, prec_str, snow_str, zero) end if + call t_stopf('tphysac:carma_timestep_tend') end if - call t_stopf('carma_timestep_tend') if( microp_scheme == 'MG' ) then ! Start co-substepping of macrophysics and microphysics @@ -1704,7 +1704,7 @@ subroutine tphysac (ztodt, cam_in, & ! Calculate macrophysical tendency (sedimentation, detrain, cloud fraction) !=================================================== - call t_startf('macrop_tend') + call t_startf('tphysac:clubb_tend_cam') ! ===================================================== ! CLUBB call (PBL, shallow convection, macrophysics) @@ -1718,6 +1718,7 @@ subroutine tphysac (ztodt, cam_in, & call clubb_tend_cam(state, ptend, pbuf, cld_macmic_ztodt,& cmfmc, cam_in, macmic_it, cld_macmic_num_steps, & dlf, det_s, det_ice) + call t_stopf('tphysac:clubb_tend_cam') ! Since we "added" the reserved liquid back in this routine, we need ! to account for it in the energy checker @@ -1752,7 +1753,6 @@ subroutine tphysac (ztodt, cam_in, & det_ice(:ncol)/cld_macmic_num_steps, & flx_heat(:ncol)/cld_macmic_num_steps) - call t_stopf('macrop_tend') !=================================================== ! Calculate cloud microphysics @@ -1780,9 +1780,9 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if - call t_startf('microp_aero_run') + call t_startf('tphysac:microp_aero_run') call microp_aero_run(state, ptend_aero, cld_macmic_ztodt, pbuf) - call t_stopf('microp_aero_run') + call t_stopf('tphysac:microp_aero_run') call t_startf('microp_tend') @@ -1854,7 +1854,9 @@ subroutine tphysac (ztodt, cam_in, & call physics_tend_dealloc(tend_sc) call physics_ptend_dealloc(ptend_sc) else + call t_startf('tphysac:microp_driver_tend') call microp_driver_tend(state, ptend, cld_macmic_ztodt, pbuf) + call t_stopf('tphysac:microp_driver_tend') end if ! combine aero and micro tendencies for the grid call physics_ptend_sum(ptend_aero, ptend, ncol) @@ -1929,14 +1931,22 @@ subroutine tphysac (ztodt, cam_in, & ! Do calculations of mode radius and water uptake if: ! 1) modal aerosols are affecting the climate, or ! 2) prognostic modal aerosols are enabled + call t_startf('tphysac:modal_aero_calcsize_sub') call modal_aero_calcsize_sub(state, ptend, ztodt, pbuf) + call t_stopf('tphysac:modal_aero_calcsize_sub') ! for prognostic modal aerosols the transfer of mass between aitken and accumulation ! modes is done in conjunction with the dry radius calculation + call t_startf('tphysac:modal_aero_wateruptake') call modal_aero_wateruptake_dr(state, pbuf) + call t_stopf('tphysac:modal_aero_wateruptake') call physics_update(state, ptend, ztodt, tend) else + call t_startf('tphysac:modal_aero_calcsize_diag') call modal_aero_calcsize_diag(state, pbuf) + call t_stopf('tphysac:modal_aero_calcsize_diag') + call t_startf('tphysac:modal_aero_wateruptake') call modal_aero_wateruptake_dr(state, pbuf) + call t_stopf('tphysac:modal_aero_wateruptake') endif endif @@ -1945,7 +1955,9 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if + call t_startf('tphysac:aero_model_wetdep') call aero_model_wetdep( state, ztodt, dlf, cam_out, ptend, pbuf) + call t_stopf('tphysac:aero_model_wetdep') if ( (trim(cam_take_snapshot_after) == "aero_model_wetdep") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then call cam_snapshot_ptend_outfld(ptend, lchnk) @@ -1965,16 +1977,16 @@ subroutine tphysac (ztodt, cam_in, & ! fields have already been set for CAM aerosols and cam_out can be ! added ! to for CARMA aerosols. - call t_startf ('carma_wetdep_tend') + call t_startf ('tphysac:carma_wetdep_tend') call carma_wetdep_tend(state, ptend, ztodt, pbuf, dlf, cam_out) call physics_update(state, ptend, ztodt, tend) - call t_stopf ('carma_wetdep_tend') + call t_stopf ('tphysac:carma_wetdep_tend') end if - call t_startf ('convect_deep_tend2') + call t_startf ('tphysac:convect_deep_tend2') call convect_deep_tend_2( state, ptend, ztodt, pbuf ) call physics_update(state, ptend, ztodt, tend) - call t_stopf ('convect_deep_tend2') + call t_stopf ('tphysac:convect_deep_tend2') ! check tracer integrals call check_tracers_chng(state, tracerint, "cmfmca", nstep, ztodt, zero_tracers) @@ -2014,8 +2026,11 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if + + call t_startf('tphysac:radiation_tend') call radiation_tend( & state, ptend, pbuf, cam_out, cam_in, net_flx) + call t_stopf('tphysac:radiation_tend') ! Set net flux used by spectral dycores do i=1,ncol @@ -2045,14 +2060,15 @@ subroutine tphysac (ztodt, cam_in, & !=================================================== ! Source/sink terms for advected tracers. !=================================================== - call t_startf('adv_tracer_src_snk') ! Test tracers if (trim(cam_take_snapshot_before) == "aoa_tracers_timestep_tend") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if + call t_startf('tphysac:aoa_tracers_timestep_tend') call aoa_tracers_timestep_tend(state, ptend, cam_in%cflx, cam_in%landfrac, ztodt) + call t_stopf('tphysac:aoa_tracers_timestep_tend') if ( (trim(cam_take_snapshot_after) == "aoa_tracers_timestep_tend") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then call cam_snapshot_ptend_outfld(ptend, lchnk) @@ -2069,7 +2085,9 @@ subroutine tphysac (ztodt, cam_in, & call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if + call t_startf('tphysac:co2_cycle_set_ptend') call co2_cycle_set_ptend(state, pbuf, ptend) + call t_stopf('tphysac:co2_cycle_set_ptend') if ( (trim(cam_take_snapshot_after) == "co2_cycle_set_ptend") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then call cam_snapshot_ptend_outfld(ptend, lchnk) @@ -2096,8 +2114,10 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if + call t_startf('tphysac:chem_timestep_tend') call chem_timestep_tend(state, ptend, cam_in, cam_out, ztodt, & pbuf, fh2o=fh2o) + call t_stopf('tphysac:chem_timestep_tend') if ( (trim(cam_take_snapshot_after) == "chem_timestep_tend") .and. & @@ -2114,28 +2134,30 @@ subroutine tphysac (ztodt, cam_in, & call check_tracers_chng(state, tracerint, "chem_timestep_tend", nstep, ztodt, & cam_in%cflx) end if - call t_stopf('adv_tracer_src_snk') !=================================================== ! Vertical diffusion/pbl calculation ! Call vertical diffusion (apply tracer emissions, molecular diffusion and pbl form drag) !=================================================== - call t_startf('vertical_diffusion_tend') if (trim(cam_take_snapshot_before) == "vertical_diffusion_section") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if + call t_startf('tphysac:vertical_diffusion_tend') call vertical_diffusion_tend (ztodt ,state , cam_in, & surfric ,obklen ,ptend ,ast ,pbuf ) + call t_stopf ('tphysac:vertical_diffusion_tend') !------------------------------------------ ! Call major diffusion for extended model !------------------------------------------ if ( waccmx_is('ionosphere') .or. waccmx_is('neutral') ) then + call t_startf ('tphysac:waccmx_phys_mspd_tend') call waccmx_phys_mspd_tend (ztodt ,state ,ptend) + call t_stopf ('tphysac:waccmx_phys_mspd_tend') endif if ( (trim(cam_take_snapshot_after) == "vertical_diffusion_section") .and. & @@ -2155,12 +2177,11 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if - call t_stopf ('vertical_diffusion_tend') !=================================================== ! Rayleigh friction calculation !=================================================== - call t_startf('rayleigh_friction') + call t_startf('tphysac:rayleigh_friction') call rayleigh_friction_tend( ztodt, state, ptend) if ( ptend%lu ) then call outfld( 'UTEND_RAYLEIGH', ptend%u, pcols, lchnk) @@ -2169,7 +2190,7 @@ subroutine tphysac (ztodt, cam_in, & call outfld( 'VTEND_RAYLEIGH', ptend%v, pcols, lchnk) end if call physics_update(state, ptend, ztodt, tend) - call t_stopf('rayleigh_friction') + call t_stopf('tphysac:rayleigh_friction') if (do_clubb_sgs) then call check_energy_chng(state, tend, "vdiff", nstep, ztodt, zero, zero, zero, zero) @@ -2181,7 +2202,7 @@ subroutine tphysac (ztodt, cam_in, & call check_tracers_chng(state, tracerint, "vdiff", nstep, ztodt, cam_in%cflx) ! aerosol dry deposition processes - call t_startf('aero_drydep') + call t_startf('tphysac:aero_model_drydep') if (trim(cam_take_snapshot_before) == "aero_model_drydep") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& @@ -2200,7 +2221,7 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if - call t_stopf('aero_drydep') + call t_stopf('tphysac:aero_model_drydep') ! CARMA microphysics ! @@ -2214,12 +2235,12 @@ subroutine tphysac (ztodt, cam_in, & ! cam_out ! can be added to for CARMA aerosols. if (carma_do_aerosol) then - call t_startf('carma_timestep_tend') + call t_startf('tphysac:carma_timestep_tend') call carma_timestep_tend(state, cam_in, cam_out, ptend, ztodt, pbuf, obklen=obklen, ustar=surfric) call physics_update(state, ptend, ztodt, tend) call check_energy_chng(state, tend, "carma_tend", nstep, ztodt, zero, zero, zero, zero) - call t_stopf('carma_timestep_tend') + call t_stopf('tphysac:carma_timestep_tend') end if !--------------------------------------------------------------------------------- @@ -2230,7 +2251,7 @@ subroutine tphysac (ztodt, cam_in, & !=================================================== ! Gravity wave drag !=================================================== - call t_startf('gw_tend') + call t_startf('tphysac:gw_tend') if (trim(cam_take_snapshot_before) == "gw_tend") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& @@ -2259,7 +2280,7 @@ subroutine tphysac (ztodt, cam_in, & ! Check energy integrals call check_energy_chng(state, tend, "gwdrag", nstep, ztodt, zero, & zero, zero, flx_heat) - call t_stopf('gw_tend') + call t_stopf('tphysac:gw_tend') ! QBO relaxation @@ -2268,7 +2289,9 @@ subroutine tphysac (ztodt, cam_in, & fh2o, surfric, obklen, flx_heat, cmfmc, dlf, det_s, det_ice, net_flx) end if + call t_startf('tphysac:qbo_relax') call qbo_relax(state, pbuf, ptend) + call t_stopf('tphysac:qbo_relax') if ( (trim(cam_take_snapshot_after) == "qbo_relax") .and. & (trim(cam_take_snapshot_before) == trim(cam_take_snapshot_after))) then call cam_snapshot_ptend_outfld(ptend, lchnk) @@ -2290,7 +2313,9 @@ subroutine tphysac (ztodt, cam_in, & call check_energy_chng(state, tend, "qborelax", nstep, ztodt, zero, zero, zero, zero) ! Lunar tides + call t_startf('tphysac:lunar_tides_tend') call lunar_tides_tend( state, ptend ) + call t_stopf('tphysac:lunar_tides_tend') if ( ptend%lu ) then call outfld( 'UTEND_LUNART', ptend%u, pcols, lchnk) end if @@ -2302,7 +2327,7 @@ subroutine tphysac (ztodt, cam_in, & call check_energy_chng(state, tend, "lunar_tides", nstep, ztodt, zero, zero, zero, zero) ! Ion drag calculation - call t_startf ( 'iondrag' ) + call t_startf ( 'tphysac:iondrag' ) if (trim(cam_take_snapshot_before) == "iondrag_calc_section") then call cam_snapshot_all_outfld_tphysac(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf,& @@ -2350,7 +2375,7 @@ subroutine tphysac (ztodt, cam_in, & ! Check energy integrals call check_energy_chng(state, tend, "iondrag", nstep, ztodt, zero, zero, zero, zero) - call t_stopf ( 'iondrag' ) + call t_stopf ( 'tphysac:iondrag' ) ! Update Nudging values, if needed !---------------------------------- @@ -2743,7 +2768,7 @@ subroutine tphysbc (ztodt, state, & !=================================================== ! Dry adjustment !=================================================== - call t_startf('dry_adjustment') + call t_startf('tphysbc:dadadj_tend') if (trim(cam_take_snapshot_before) == "dadadj_tend") then call cam_snapshot_all_outfld_tphysbc(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf, & @@ -2763,14 +2788,14 @@ subroutine tphysbc (ztodt, state, & cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, net_flx) end if - call t_stopf('dry_adjustment') + call t_stopf('tphysbc:dadadj_tend') !=================================================== ! Moist convection !=================================================== call t_startf('moist_convection') - call t_startf ('convect_deep_tend') + call t_startf ('tphysbc:convect_deep_tend') if (trim(cam_take_snapshot_before) == "convect_deep_tend") then call cam_snapshot_all_outfld_tphysbc(cam_snapshot_before_num, state, tend, cam_in, cam_out, pbuf, & @@ -2802,7 +2827,7 @@ subroutine tphysbc (ztodt, state, & cmfmc, cmfcme, zdu, rliq, rice, dlf, dlf2, rliq2, net_flx) end if - call t_stopf('convect_deep_tend') + call t_stopf('tphysbc:convect_deep_tend') call pbuf_get_field(pbuf, prec_dp_idx, prec_dp ) call pbuf_get_field(pbuf, snow_dp_idx, snow_dp ) @@ -2875,16 +2900,23 @@ subroutine tphysbc (ztodt, state, & ! Run wet deposition routines to intialize aerosols !=================================================== + call t_startf('tphysbc:modal_aero_calcsize') call modal_aero_calcsize_diag(state, pbuf) + call t_stopf('tphysbc:modal_aero_calcsize') + + call t_startf('tphysbc:modal_aero_wateruptake') call modal_aero_wateruptake_dr(state, pbuf) + call t_stopf('tphysbc:modal_aero_wateruptake') !=================================================== ! Radiation computations ! initialize fluxes only, do not update state !=================================================== + call t_startf('tphysbc:radiation_tend') call radiation_tend( & state, ptend, pbuf, cam_out, cam_in, net_flx) + call t_stopf('tphysbc:radiation_tend') end if diff --git a/src/physics/rrtmgp/radiation.F90 b/src/physics/rrtmgp/radiation.F90 index ce9e86fa93..a0de032eea 100644 --- a/src/physics/rrtmgp/radiation.F90 +++ b/src/physics/rrtmgp/radiation.F90 @@ -58,6 +58,7 @@ module radiation use string_utils, only: to_lower use cam_abortutils, only: endrun, handle_allocate_error use cam_logfile, only: iulog +use perf_mod, only: t_startf, t_stopf implicit none @@ -1119,6 +1120,7 @@ subroutine radiation_tend( & if (dosw) then + call t_startf('radiation_tend:NAR:cloud_sw') ! Set cloud optical properties in cloud_sw object. call rrtmgp_set_cloud_sw( & state, pbuf, nlay, nday, idxday, & @@ -1127,6 +1129,7 @@ subroutine radiation_tend( & rd%tot_cld_vistau, rd%tot_icld_vistau, rd%liq_icld_vistau, & rd%ice_icld_vistau, rd%snow_icld_vistau, rd%grau_icld_vistau, & cld_tau_cloudsim, snow_tau_cloudsim, grau_tau_cloudsim ) + call t_stopf('radiation_tend:NAR:cloud_sw') if (write_output) then call radiation_output_cld(lchnk, rd) @@ -1158,19 +1161,28 @@ subroutine radiation_tend( & if (nday > 0) then ! Set gas volume mixing ratios for this call in gas_concs_sw. + call t_startf('radiation_tend:NAR:gases_sw') call rrtmgp_set_gases_sw( & icall, state, pbuf, nlay, nday, & idxday, gas_concs_sw) + call t_stopf('radiation_tend:NAR:gases_sw') + call t_startf('radiation_tend:DTO') ! Compute the gas optics (stored in atm_optics_sw). ! toa_flux is the reference solar source from RRTMGP data. !$acc data copyin(kdist_sw,pmid_day,pint_day,t_day,gas_concs_sw) & !$acc copy(atm_optics_sw) & !$acc copyout(toa_flux) + call t_stopf('radiation_tend:DTO') + + call t_startf('radiation_tend:ACCR') errmsg = kdist_sw%gas_optics( & pmid_day, pint_day, t_day, gas_concs_sw, atm_optics_sw, & toa_flux) + call t_stopf('radiation_tend:ACCR') + call t_startf('radiation_tend:DTO') !$acc end data + call t_stopf('radiation_tend:DTO') call stop_on_err(errmsg, sub, 'kdist_sw%gas_optics') ! Scale the solar source @@ -1182,11 +1194,14 @@ subroutine radiation_tend( & ! Set SW aerosol optical properties in the aer_sw object. ! This call made even when no daylight columns because it does some ! diagnostic aerosol output. + call t_startf('radiation_tend:NAR:aer_sw') call rrtmgp_set_aer_sw( & icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw) + call t_stopf('radiation_tend:NAR:aer_sw') if (nday > 0) then + call t_startf('radiation_tend:DTO') !! ADDED by SS as part of RRTMGP data optimization !$acc data copyin(atm_optics_sw, toa_flux, & !$acc aer_sw, cloud_sw, & @@ -1197,7 +1212,9 @@ subroutine radiation_tend( & !$acc alb_dir, alb_dif,coszrs_day) & !$acc copy(fswc, fswc%flux_net,fswc%flux_up,fswc%flux_dn, & !$acc fsw, fsw%flux_net, fsw%flux_up, fsw%flux_dn) + call t_stopf('radiation_tend:DTO') + call t_startf('radiation_tend:ACCR') ! Increment the gas optics (in atm_optics_sw) by the aerosol optics in aer_sw. errmsg = aer_sw%increment(atm_optics_sw) call stop_on_err(errmsg, sub, 'aer_sw%increment') @@ -1217,7 +1234,11 @@ subroutine radiation_tend( & atm_optics_sw, top_at_1, coszrs_day, toa_flux, & alb_dir, alb_dif, fsw) call stop_on_err(errmsg, sub, 'all-sky rte_sw') + call t_stopf('radiation_tend:ACCR') + + call t_startf('radiation_tend:DTO') !$acc end data + call t_stopf('radiation_tend:DTO') end if @@ -1248,10 +1269,12 @@ subroutine radiation_tend( & call stop_on_err(errmsg, sub, 'sources_lw%alloc') ! Set cloud optical properties in cloud_lw object. + call t_startf('radiation_tend:NAR:cloud_lw') call rrtmgp_set_cloud_lw( & state, pbuf, ncol, nlay, nlaycam, & cld, cldfsnow, cldfgrau, cldfprime, graupel_in_rad, & kdist_lw, cloud_lw, cld_lw_abs_cloudsim, snow_lw_abs_cloudsim, grau_lw_abs_cloudsim) + call t_stopf('radiation_tend:NAR:cloud_lw') ! Initialize object for gas concentrations errmsg = gas_concs_lw%init(gaslist_lc) @@ -1271,8 +1294,11 @@ subroutine radiation_tend( & if (active_calls(icall)) then ! Set gas volume mixing ratios for this call in gas_concs_lw. + call t_startf('radiation_tend:NAR:gases_lw') call rrtmgp_set_gases_lw(icall, state, pbuf, nlay, gas_concs_lw) + call t_stopf('radiation_tend:NAR:gases_lw') + call t_startf('radiation_tend:DTO') ! Compute the gas optics and Planck sources. !$acc data copyin(kdist_lw,pmid_rad,pint_rad,t_rad,t_sfc, & !$acc gas_concs_lw) & @@ -1281,15 +1307,25 @@ subroutine radiation_tend( & !$acc sources_lw%lay_source, sources_lw%sfc_source, & !$acc sources_lw%lev_source_inc, sources_lw%lev_source_dec, & !$acc sources_lw%sfc_source_jac) + call t_stopf('radiation_tend:DTO') + + call t_startf('radiation_tend:ACCR') errmsg = kdist_lw%gas_optics( & pmid_rad, pint_rad, t_rad, t_sfc, gas_concs_lw, & atm_optics_lw, sources_lw) call stop_on_err(errmsg, sub, 'kdist_lw%gas_optics') + call t_stopf('radiation_tend:ACCR') + + call t_startf('radiation_tend:DTO') !$acc end data + call t_stopf('radiation_tend:DTO') ! Set LW aerosol optical properties in the aer_lw object. + call t_startf('radiation_tend:NAR:aer_lw') call rrtmgp_set_aer_lw(icall, state, pbuf, aer_lw) + call t_stopf('radiation_tend:NAR:aer_lw') + call t_startf('radiation_tend:DTO') !! Added by SS as part of RRTMGP data optimization !$acc data copyin(atm_optics_lw, aer_lw, cloud_lw, & !$acc aer_lw%tau, & @@ -1302,6 +1338,8 @@ subroutine radiation_tend( & !$acc emis_sfc) & !$acc copy(flwc, flwc%flux_net,flwc%flux_up,flwc%flux_dn, & !$acc flw, flw%flux_net, flw%flux_up, flw%flux_dn) + call t_stopf('radiation_tend:DTO') + call t_startf('radiation_tend:ACCR') ! Increment the gas optics by the aerosol optics. @@ -1319,7 +1357,11 @@ subroutine radiation_tend( & ! Compute all-sky LW fluxes errmsg = rte_lw(atm_optics_lw, top_at_1, sources_lw, emis_sfc, flw) call stop_on_err(errmsg, sub, 'all-sky rte_lw') + call t_stopf('radiation_tend:ACCR') + + call t_startf('radiation_tend:DTO') !$acc end data + call t_stopf('radiation_tend:DTO') ! Transform RRTMGP outputs to CAM outputs and compute heating rates. call set_lw_diags()