From d702bad63277af84a6b70cfc85e5273b800664de Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 21:12:36 +0100 Subject: [PATCH 1/6] cloudsc2_tl_loki: Fix dummy args in validate_taylor_test --- src/cloudsc2_tl_loki/error.F90 | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/cloudsc2_tl_loki/error.F90 b/src/cloudsc2_tl_loki/error.F90 index 8557338..ed9be37 100644 --- a/src/cloudsc2_tl_loki/error.F90 +++ b/src/cloudsc2_tl_loki/error.F90 @@ -30,16 +30,16 @@ SUBROUTINE VALIDATE_TAYLOR_TEST(NPROMA, NLEV, NLAM, NGPTOT, & & PCOVPTOT, PERT5_COVPTOT, PERT_COVPTOT & & ) INTEGER(KIND=JPIM), INTENT(IN) :: NPROMA, NLEV, NLAM, NGPTOT - REAL(KIND=JPRB), INTENT(IN) :: FIELD_T(NPROMA,NLEV,NGPBLKS), PERT5_T(NPROMA,NLEV,NLAM,NGPBLKS), PERT_T(NPROMA,NLEV,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: FIELD_Q(NPROMA,NLEV,NGPBLKS), PERT5_Q(NPROMA,NLEV,NLAM,NGPBLKS), PERT_Q(NPROMA,NLEV,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: FIELD_L(NPROMA,NLEV,NGPBLKS), PERT5_L(NPROMA,NLEV,NLAM,NGPBLKS), PERT_L(NPROMA,NLEV,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: FIELD_I(NPROMA,NLEV,NGPBLKS), PERT5_I(NPROMA,NLEV,NLAM,NGPBLKS), PERT_I(NPROMA,NLEV,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: FIELD_A(NPROMA,NLEV,NGPBLKS), PERT5_A(NPROMA,NLEV,NLAM,NGPBLKS), PERT_A(NPROMA,NLEV,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: PFPLSL(NPROMA,NLEV+1,NGPBLKS), PERT5_FPLSL(NPROMA,NLEV+1,NLAM,NGPBLKS), PERT_FPLSL(NPROMA,NLEV+1,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: PFPLSN(NPROMA,NLEV+1,NGPBLKS), PERT5_FPLSN(NPROMA,NLEV+1,NLAM,NGPBLKS), PERT_FPLSN(NPROMA,NLEV+1,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: PFHPSL(NPROMA,NLEV+1,NGPBLKS), PERT5_FHPSL(NPROMA,NLEV+1,NLAM,NGPBLKS), PERT_FHPSL(NPROMA,NLEV+1,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: PFHPSN(NPROMA,NLEV+1,NGPBLKS), PERT5_FHPSN(NPROMA,NLEV+1,NLAM,NGPBLKS), PERT_FHPSN(NPROMA,NLEV+1,NGPBLKS) - REAL(KIND=JPRB), INTENT(IN) :: PCOVPTOT(NPROMA,NLEV,NGPBLKS), PERT5_COVPTOT(NPROMA,NLEV,NLAM,NGPBLKS), PERT_COVPTOT(NPROMA,NLEV,NGPBLKS) + REAL(KIND=JPRB), INTENT(IN) :: FIELD_T(:,:,:), PERT5_T(:,:,:,:), PERT_T(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: FIELD_Q(:,:,:), PERT5_Q(:,:,:,:), PERT_Q(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: FIELD_L(:,:,:), PERT5_L(:,:,:,:), PERT_L(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: FIELD_I(:,:,:), PERT5_I(:,:,:,:), PERT_I(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: FIELD_A(:,:,:), PERT5_A(:,:,:,:), PERT_A(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: PFPLSL(:,:,:), PERT5_FPLSL(:,:,:,:), PERT_FPLSL(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: PFPLSN(:,:,:), PERT5_FPLSN(:,:,:,:), PERT_FPLSN(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: PFHPSL(:,:,:), PERT5_FHPSL(:,:,:,:), PERT_FHPSL(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: PFHPSN(:,:,:), PERT5_FHPSN(:,:,:,:), PERT_FHPSN(:,:,:) + REAL(KIND=JPRB), INTENT(IN) :: PCOVPTOT(:,:,:), PERT5_COVPTOT(:,:,:,:), PERT_COVPTOT(:,:,:) INTEGER(KIND=JPIM) :: JKGLO, IBL, ICEND, ILAM, NGPBLKS INTEGER(KIND=JPIM) :: ISTART,ITEST,INEGAT,ITEMPNEGAT From e7e3886daf527c06fc8af7a611ca58460c9d32cb Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 21:31:40 +0100 Subject: [PATCH 2/6] Replace deprecated loki_transform_convert macros --- src/cloudsc2_ad_loki/CMakeLists.txt | 118 +++++++++++++-------- src/cloudsc2_nl_loki/CMakeLists.txt | 102 +++++++++++------- src/cloudsc2_tl_loki/CMakeLists.txt | 156 ++++++++++++++++------------ 3 files changed, 230 insertions(+), 146 deletions(-) diff --git a/src/cloudsc2_ad_loki/CMakeLists.txt b/src/cloudsc2_ad_loki/CMakeLists.txt index d7a4dde..a95ffd0 100755 --- a/src/cloudsc2_ad_loki/CMakeLists.txt +++ b/src/cloudsc2_ad_loki/CMakeLists.txt @@ -21,20 +21,27 @@ if( HAVE_CLOUDSC2_AD_LOKI ) ## * Internal "do-nothing" mode for Loki debug ## #################################################### - - loki_transform_convert( - MODE idem FRONTEND ${LOKI_FRONTEND} - CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 - INCLUDES ${COMMON_INCLUDE} - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem + loki_transform( + COMMAND convert OUTPUT - loki-idem/satur.idem.F90 - loki-idem/cloudsc2ad.idem.F90 - loki-idem/cloudsc2tl.idem.F90 - loki-idem/cloudsc_driver_ad_loki_mod.idem.F90 - DEPENDS satur.F90 cloudsc2ad.F90 cloudsc2tl.F90 cloudsc_driver_ad_loki_mod.F90 + loki-idem/satur.idem.F90 + loki-idem/cloudsc2ad.idem.F90 + loki-idem/cloudsc2tl.idem.F90 + loki-idem/cloudsc_driver_ad_loki_mod.idem.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem + DEPENDS + satur.F90 + cloudsc2ad.F90 + cloudsc2tl.F90 + cloudsc_driver_ad_loki_mod.F90 + MODE idem + CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} ) ecbuild_add_executable( TARGET dwarf-cloudsc2-ad-loki-idem @@ -52,7 +59,7 @@ if( HAVE_CLOUDSC2_AD_LOKI ) ecbuild_add_test( TARGET dwarf-cloudsc2-ad-loki-idem-serial COMMAND bin/dwarf-cloudsc2-ad-loki-idem - ARGS + ARGS WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 ) @@ -63,24 +70,35 @@ if( HAVE_CLOUDSC2_AD_LOKI ) ## * Invokes compute kernel as `!$acc vector` ## #################################################### + loki_transform( + COMMAND convert + OUTPUT + loki-scc/satur.scc.F90 + loki-scc/cloudsc2ad.scc.F90 + loki-scc/cloudsc2tl.scc.F90 + loki-scc/cloudsc_driver_ad_loki_mod.scc.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc + DEPENDS + satur.F90 + cloudsc2ad.F90 + cloudsc2tl.F90 + cloudsc_driver_ad_loki_mod.F90 + MODE scc + CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config + DATA_OFFLOAD + REMOVE_OPENMP + CPP + DEFINITIONS + ${CLOUDSC_DEFINITIONS} + CLOUDSC_GPU_TIMING + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} + ) - loki_transform_convert( - MODE scc FRONTEND ${LOKI_FRONTEND} CPP - CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 ${COMMON_MODULE}/yoecldp.F90 - INCLUDES ${COMMON_INCLUDE} - DEFINITIONS CLOUDSC_GPU_TIMING - DATA_OFFLOAD REMOVE_OPENMP - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc - OUTPUT - loki-scc/satur.scc.F90 - loki-scc/cloudsc2ad.scc.F90 - loki-scc/cloudsc2tl.scc.F90 - loki-scc/cloudsc_driver_ad_loki_mod.scc.F90 - DEPENDS - satur.F90 cloudsc2ad.F90 cloudsc2tl.F90 cloudsc_driver_ad_loki_mod.F90 - ) ecbuild_add_executable( TARGET dwarf-cloudsc2-ad-loki-scc SOURCES dwarf_cloudsc.F90 @@ -96,7 +114,7 @@ if( HAVE_CLOUDSC2_AD_LOKI ) ecbuild_add_test( TARGET dwarf-cloudsc2-ad--loki-scc-serial COMMAND bin/dwarf-cloudsc2-ad-loki-scc - ARGS + ARGS WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 ENVIRONMENT NV_ACC_CUDA_HEAPSIZE=9G @@ -108,23 +126,35 @@ if( HAVE_CLOUDSC2_AD_LOKI ) ## * Temporary arrays hoisted to driver ## #################################################### - loki_transform_convert( - MODE scc-hoist FRONTEND ${LOKI_FRONTEND} CPP - CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 ${COMMON_MODULE}/yoecldp.F90 - INCLUDES ${COMMON_INCLUDE} - DEFINITIONS CLOUDSC_GPU_TIMING - DATA_OFFLOAD REMOVE_OPENMP - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-hoist + loki_transform( + COMMAND convert OUTPUT loki-scc-hoist/satur.scc_hoist.F90 loki-scc-hoist/cloudsc2ad.scc_hoist.F90 loki-scc-hoist/cloudsc2tl.scc_hoist.F90 loki-scc-hoist/cloudsc_driver_ad_loki_mod.scc_hoist.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-hoist DEPENDS - satur.F90 cloudsc2ad.F90 cloudsc2tl.F90 cloudsc_driver_ad_loki_mod.F90 - ) + satur.F90 + cloudsc2ad.F90 + cloudsc2tl.F90 + cloudsc_driver_ad_loki_mod.F90 + MODE scc-hoist + CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config + DATA_OFFLOAD + REMOVE_OPENMP + CPP + DEFINITIONS + ${CLOUDSC_DEFINITIONS} + CLOUDSC_GPU_TIMING + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} + ) + ecbuild_add_executable( TARGET dwarf-cloudsc2-ad-loki-scc-hoist SOURCES dwarf_cloudsc.F90 @@ -140,7 +170,7 @@ if( HAVE_CLOUDSC2_AD_LOKI ) ecbuild_add_test( TARGET dwarf-cloudsc2-ad--loki-scc-hoist-serial COMMAND bin/dwarf-cloudsc2-ad-loki-scc-hoist - ARGS + ARGS WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 ENVIRONMENT NV_ACC_CUDA_HEAPSIZE=9G diff --git a/src/cloudsc2_nl_loki/CMakeLists.txt b/src/cloudsc2_nl_loki/CMakeLists.txt index 3e8ef97..be12dcb 100755 --- a/src/cloudsc2_nl_loki/CMakeLists.txt +++ b/src/cloudsc2_nl_loki/CMakeLists.txt @@ -20,16 +20,25 @@ if( HAVE_CLOUDSC2_NL_LOKI ) ## * Internal "do-nothing" mode for Loki debug ## #################################################### - - loki_transform_convert( - MODE idem FRONTEND ${LOKI_FRONTEND} + loki_transform( + COMMAND convert + OUTPUT + loki-idem/satur.idem.F90 + loki-idem/cloudsc2.idem.F90 + loki-idem/cloudsc_driver_loki_mod.idem.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem + DEPENDS + satur.F90 + cloudsc2.F90 + cloudsc_driver_loki_mod.F90 + MODE idem CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 - INCLUDES ${COMMON_INCLUDE} - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem - OUTPUT loki-idem/satur.idem.F90 loki-idem/cloudsc2.idem.F90 loki-idem/cloudsc_driver_loki_mod.idem.F90 - DEPENDS satur.F90 cloudsc2.F90 cloudsc_driver_loki_mod.F90 + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} ) ecbuild_add_executable( TARGET dwarf-cloudsc2-nl-loki-idem @@ -58,19 +67,32 @@ if( HAVE_CLOUDSC2_NL_LOKI ) ## * Invokes compute kernel as `!$acc vector` ## #################################################### - - loki_transform_convert( - MODE scc FRONTEND ${LOKI_FRONTEND} CPP - DIRECTIVE openacc + loki_transform( + COMMAND convert + OUTPUT + loki-scc/satur.scc.F90 + loki-scc/cloudsc2.scc.F90 + loki-scc/cloudsc_driver_loki_mod.scc.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc + DEPENDS + satur.F90 + cloudsc2.F90 + cloudsc_driver_loki_mod.F90 + MODE scc CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 ${COMMON_MODULE}/yoecldp.F90 - INCLUDES ${COMMON_INCLUDE} - DEFINITIONS CLOUDSC_GPU_TIMING - DATA_OFFLOAD REMOVE_OPENMP - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc - OUTPUT loki-scc/satur.scc.F90 loki-scc/cloudsc2.scc.F90 loki-scc/cloudsc_driver_loki_mod.scc.F90 - DEPENDS satur.F90 cloudsc2.F90 cloudsc_driver_loki_mod.F90 + DIRECTIVE openacc + DATA_OFFLOAD + REMOVE_OPENMP + CPP + DEFINITIONS + ${CLOUDSC_DEFINE_STMT_FUNC} + CLOUDSC_GPU_TIMING + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} ) ecbuild_add_executable( TARGET dwarf-cloudsc2-nl-loki-scc @@ -100,22 +122,32 @@ if( HAVE_CLOUDSC2_NL_LOKI ) ## * Temporary arrays hoisted to driver ## #################################################### - - loki_transform_convert( - MODE scc-hoist FRONTEND ${LOKI_FRONTEND} CPP - DIRECTIVE openacc - CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 ${COMMON_MODULE}/yoecldp.F90 - INCLUDES ${COMMON_INCLUDE} - DEFINITIONS CLOUDSC_GPU_TIMING - DATA_OFFLOAD REMOVE_OPENMP - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-hoist + loki_transform( + COMMAND convert OUTPUT - loki-scc-hoist/satur.scc_hoist.F90 + loki-scc-hoist/satur.scc_hoist.F90 loki-scc-hoist/cloudsc2.scc_hoist.F90 loki-scc-hoist/cloudsc_driver_loki_mod.scc_hoist.F90 - DEPENDS satur.F90 cloudsc2.F90 cloudsc_driver_loki_mod.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-hoist + DEPENDS + satur.F90 + cloudsc2.F90 + cloudsc_driver_loki_mod.F90 + MODE scc-hoist + CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config + DIRECTIVE openacc + DATA_OFFLOAD + REMOVE_OPENMP + CPP + DEFINITIONS + ${CLOUDSC_DEFINE_STMT_FUNC} + CLOUDSC_GPU_TIMING + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} ) ecbuild_add_executable( TARGET dwarf-cloudsc2-nl-loki-scc-hoist @@ -135,6 +167,6 @@ if( HAVE_CLOUDSC2_NL_LOKI ) ARGS 1 1280 128 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 - ENVIRONMENT NV_ACC_CUDA_HEAPSIZE=19G + ENVIRONMENT NV_ACC_CUDA_HEAPSIZE=19G ) endif() diff --git a/src/cloudsc2_tl_loki/CMakeLists.txt b/src/cloudsc2_tl_loki/CMakeLists.txt index 48abcdd..6f70584 100755 --- a/src/cloudsc2_tl_loki/CMakeLists.txt +++ b/src/cloudsc2_tl_loki/CMakeLists.txt @@ -20,28 +20,34 @@ if( HAVE_CLOUDSC2_TL_LOKI ) ## * Internal "do-nothing" mode for Loki debug ## #################################################### - - loki_transform_convert( - MODE idem FRONTEND ${LOKI_FRONTEND} CPP + loki_transform( + COMMAND convert + OUTPUT + loki-idem/satur.idem.F90 + loki-idem/cloudsc2.idem.F90 + loki-idem/cloudsc2tl.idem.F90 + loki-idem/cloudsc_driver_tl_loki_mod.idem.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-idem + DEPENDS + satur.F90 + cloudsc2.F90 + cloudsc2tl.F90 + cloudsc_driver_tl_loki_mod.F90 + MODE idem CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 - INCLUDES ${COMMON_INCLUDE} - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-idem - OUTPUT - loki-idem/satur.idem.F90 - loki-idem/cloudsc2.idem.F90 - loki-idem/cloudsc2tl.idem.F90 - loki-idem/cloudsc_driver_tl_loki_mod.idem.F90 - DEPENDS - satur.F90 cloudsc2.F90 cloudsc2tl.F90 cloudsc_driver_tl_loki_mod.F90 + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} ) ecbuild_add_executable( TARGET dwarf-cloudsc2-tl-loki-idem SOURCES dwarf_cloudsc.F90 loki-idem/cloudsc_driver_tl_loki_mod.idem.F90 - loki-idem/cloudsc2tl.idem.F90 + loki-idem/cloudsc2tl.idem.F90 loki-idem/cloudsc2.idem.F90 loki-idem/satur.idem.F90 error.F90 @@ -53,7 +59,7 @@ if( HAVE_CLOUDSC2_TL_LOKI ) ecbuild_add_test( TARGET dwarf-cloudsc2-tl-loki-idem-serial COMMAND bin/dwarf-cloudsc2-tl-loki-idem - ARGS + ARGS WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 ) @@ -64,29 +70,36 @@ if( HAVE_CLOUDSC2_TL_LOKI ) ## * Removes horizontal vector loops ## ## * Invokes compute kernel as `!$acc vector` ## #################################################### - - - loki_transform_convert( - MODE scc FRONTEND ${LOKI_FRONTEND} CPP - CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 ${COMMON_MODULE}/yoecldp.F90 - INCLUDES ${COMMON_INCLUDE} - DEFINITIONS CLOUDSC_GPU_TIMING - DATA_OFFLOAD REMOVE_OPENMP - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc - OUTPUT - loki-scc/satur.scc.F90 - loki-scc/cloudsc2.scc.F90 - loki-scc/cloudsc2tl.scc.F90 - loki-scc/cloudsc_driver_tl_loki_mod.scc.F90 - DEPENDS - satur.F90 - cloudsc2.F90 - cloudsc2tl.F90 - cloudsc_driver_tl_loki_mod.F90 - ) - + + loki_transform( + COMMAND convert + OUTPUT + loki-scc/satur.scc.F90 + loki-scc/cloudsc2.scc.F90 + loki-scc/cloudsc2tl.scc.F90 + loki-scc/cloudsc_driver_tl_loki_mod.scc.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc + DEPENDS + satur.F90 + cloudsc2.F90 + cloudsc2tl.F90 + cloudsc_driver_tl_loki_mod.F90 + MODE scc + CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config + DATA_OFFLOAD + REMOVE_OPENMP + CPP + DEFINITIONS + ${CLOUDSC_DEFINITIONS} + CLOUDSC_GPU_TIMING + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} + ) + ecbuild_add_executable( TARGET dwarf-cloudsc2-tl-loki-scc SOURCES dwarf_cloudsc.F90 @@ -99,43 +112,52 @@ if( HAVE_CLOUDSC2_TL_LOKI ) cloudsc2-common-lib DEFINITIONS ${CLOUDSC_DEFINITIONS} ) - + ecbuild_add_test( TARGET dwarf-cloudsc2-tl--loki-scc-serial COMMAND bin/dwarf-cloudsc2-tl-loki-scc - ARGS + ARGS WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 ENVIRONMENT NV_ACC_CUDA_HEAPSIZE=9G - ) - + ) + #################################################### ## SCC-hoist mode ## ## * SCC with vector loop hoisted ## ## * Kernel is "seq, but args are full blocks ## ## * Temporary arrays hoisted to driver ## #################################################### - - - loki_transform_convert( - MODE scc-hoist FRONTEND ${LOKI_FRONTEND} CPP - CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config - PATH ${CMAKE_CURRENT_SOURCE_DIR} - HEADERS ${COMMON_MODULE}/yomphyder.F90 ${COMMON_MODULE}/yoecldp.F90 - INCLUDES ${COMMON_INCLUDE} - DEFINITIONS CLOUDSC_GPU_TIMING - DATA_OFFLOAD REMOVE_OPENMP - OUTPATH ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-hoist - OUTPUT - loki-scc-hoist/satur.scc_hoist.F90 - loki-scc-hoist/cloudsc2.scc_hoist.F90 - loki-scc-hoist/cloudsc2tl.scc_hoist.F90 - loki-scc-hoist/cloudsc_driver_tl_loki_mod.scc_hoist.F90 - DEPENDS - satur.F90 cloudsc2.F90 cloudsc2tl.F90 cloudsc_driver_tl_loki_mod.F90 - ) - - + + loki_transform( + COMMAND convert + OUTPUT + loki-scc-hoist/satur.scc_hoist.F90 + loki-scc-hoist/cloudsc2.scc_hoist.F90 + loki-scc-hoist/cloudsc2tl.scc_hoist.F90 + loki-scc-hoist/cloudsc_driver_tl_loki_mod.scc_hoist.F90 + BUILDDIR ${CMAKE_CURRENT_BINARY_DIR}/loki-scc-hoist + DEPENDS + satur.F90 + cloudsc2.F90 + cloudsc2tl.F90 + cloudsc_driver_tl_loki_mod.F90 + MODE scc-hoist + CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/cloudsc_loki.config + DATA_OFFLOAD + REMOVE_OPENMP + CPP + DEFINITIONS + ${CLOUDSC_DEFINITIONS} + CLOUDSC_GPU_TIMING + FRONTEND ${LOKI_FRONTEND} + SOURCES + ${CMAKE_CURRENT_SOURCE_DIR} + ${COMMON_MODULE} + INCLUDES + ${COMMON_INCLUDE} + ) + ecbuild_add_executable( TARGET dwarf-cloudsc2-tl-loki-scc-hoist SOURCES dwarf_cloudsc.F90 @@ -148,14 +170,14 @@ if( HAVE_CLOUDSC2_TL_LOKI ) cloudsc2-common-lib DEFINITIONS ${CLOUDSC_DEFINITIONS} ) - + ecbuild_add_test( TARGET dwarf-cloudsc2-tl-loki-scc-hoist-serial COMMAND bin/dwarf-cloudsc2-tl-loki-scc-hoist - ARGS + ARGS WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../.. OMP 1 ENVIRONMENT NV_ACC_CUDA_HEAPSIZE=19G - ) + ) endif() From 784a560d9584247abb79c7f5969f596fd3db47c8 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 21:31:53 +0100 Subject: [PATCH 3/6] Remove redundant bundle option --- bundle.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bundle.yml b/bundle.yml index ab14ad1..16afd32 100755 --- a/bundle.yml +++ b/bundle.yml @@ -9,8 +9,8 @@ cmake : > projects : - ecbuild : - git : https://github.com/ecmwf/ecbuild - version : 3.7.0 + git : https://github.com/ecmwf/ecbuild + version : 3.7.0 bundle : false - loki : @@ -52,10 +52,6 @@ options : help : Frontend parser to use for Loki transformations cmake : LOKI_FRONTEND={{value}} - - cloudsc-fortran : - help : Build the new Fortran version of CLOUDSC [ON|OFF] - cmake : ENABLE_CLOUDSC_FORTRAN={{value}} - - cloudsc2-nl-loki : help : Build the optimized Loki-based GPU version CLOUDSC2-NL [ON|OFF] cmake : ENABLE_CLOUDSC2_NL_LOKI={{value}} From 558b7e4b39b2fc1f162be380756cbb9dab07c41b Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 21:32:57 +0100 Subject: [PATCH 4/6] Update Loki config files to new config format --- src/cloudsc2_ad_loki/cloudsc_loki.config | 58 ++++++++--------------- src/cloudsc2_nl_loki/cloudsc_loki.config | 55 +++++++++------------- src/cloudsc2_tl_loki/cloudsc_loki.config | 59 ++++++++---------------- 3 files changed, 60 insertions(+), 112 deletions(-) diff --git a/src/cloudsc2_ad_loki/cloudsc_loki.config b/src/cloudsc2_ad_loki/cloudsc_loki.config index f525adc..efb96e5 100644 --- a/src/cloudsc2_ad_loki/cloudsc_loki.config +++ b/src/cloudsc2_ad_loki/cloudsc_loki.config @@ -8,47 +8,29 @@ demote_locals = true # Ensure that we are never adding these to the tree, and thus # do not attempt to look up the source files for these. -# TODO: Add type-bound procedure support and adjust scheduler to it -disable = ['performance_timer%start', 'performance_timer%end', 'performance_timer%thread_start', - 'performance_timer%thread_end', 'performance_timer%thread_log', - 'performance_timer%thread_log', 'performance_timer%print_performance'] +disable = ['timer_mod', 'file_io_mod', 'foe*', 'fes*', 'falfaad'] # Define entry point for call-tree transformation -[[routine]] -name = 'cloudsc_driver_ad' -expand = true -role = 'driver' +[routines] -[[routine]] -name = 'cloudsc2ad' -role = 'kernel' -expand = true +[routines.cloudsc_driver_ad] + role = 'driver' + expand = true -[[routine]] -name = 'cloudsc2tl' -role = 'kernel' -expand = true +# Define indices and bounds for array dimensions +[dimensions] -[[routine]] -name = 'satur' -role = 'kernel' -expand = true -############################################## - -[[dimension]] -name = 'horizontal' -size = 'KLON' -index = 'JL' -bounds = ['KIDIA', 'KFDIA'] -aliases = ['NPROMA', 'KDIM%KLON'] - -[[dimension]] -name = 'vertical' -size = 'KLEV' -index = 'JK' - -[[dimension]] -name = 'block_dim' -size = 'NGPBLKS' -index = 'IBL' +[dimensions.horizontal] + size = 'KLON' + index = 'JL' + bounds = ['KIDIA', 'KFDIA'] + aliases = ['NPROMA', 'KDIM%KLON'] + +[dimensions.vertical] + size = 'KLEV' + index = 'JK' + +[dimensions.block_dim] + size = 'NGPBLKS' + index = 'IBL' diff --git a/src/cloudsc2_nl_loki/cloudsc_loki.config b/src/cloudsc2_nl_loki/cloudsc_loki.config index f85ab06..551c460 100644 --- a/src/cloudsc2_nl_loki/cloudsc_loki.config +++ b/src/cloudsc2_nl_loki/cloudsc_loki.config @@ -8,42 +8,29 @@ demote_locals = true # Ensure that we are never adding these to the tree, and thus # do not attempt to look up the source files for these. -# TODO: Add type-bound procedure support and adjust scheduler to it -disable = ['performance_timer%start', 'performance_timer%end', 'performance_timer%thread_start', - 'performance_timer%thread_end', 'performance_timer%thread_log', - 'performance_timer%thread_log', 'performance_timer%print_performance'] +disable = ['timer_mod', 'file_io_mod', 'foe*'] # Define entry point for call-tree transformation -[[routine]] -name = 'cloudsc_driver' -expand = true -role = 'driver' +[routines] -[[routine]] -name = 'cloudsc2' -role = 'kernel' -expand = true +[routines.cloudsc_driver] + role = 'driver' + expand = true -[[routine]] -name = 'satur' -role = 'kernel' -expand = true -############################################## - -[[dimension]] -name = 'horizontal' -size = 'KLON' -index = 'JL' -bounds = ['KIDIA', 'KFDIA'] -aliases = ['NPROMA', 'KDIM%KLON'] - -[[dimension]] -name = 'vertical' -size = 'KLEV' -index = 'JK' - -[[dimension]] -name = 'block_dim' -size = 'NGPBLKS' -index = 'IBL' +# Define indices and bounds for array dimensions +[dimensions] + +[dimensions.horizontal] + size = 'KLON' + index = 'JL' + bounds = ['KIDIA', 'KFDIA'] + aliases = ['NPROMA', 'KDIM%KLON'] + +[dimensions.vertical] + size = 'KLEV' + index = 'JK' + +[dimensions.block_dim] + size = 'NGPBLKS' + index = 'IBL' diff --git a/src/cloudsc2_tl_loki/cloudsc_loki.config b/src/cloudsc2_tl_loki/cloudsc_loki.config index 675d59c..4111fc9 100644 --- a/src/cloudsc2_tl_loki/cloudsc_loki.config +++ b/src/cloudsc2_tl_loki/cloudsc_loki.config @@ -8,50 +8,29 @@ demote_locals = true # Ensure that we are never adding these to the tree, and thus # do not attempt to look up the source files for these. -# TODO: Add type-bound procedure support and adjust scheduler to it -disable = ['performance_timer%start', 'performance_timer%end', 'performance_timer%thread_start', - 'performance_timer%thread_end', 'performance_timer%thread_log', - 'performance_timer%thread_log', 'performance_timer%print_performance', - 'validate_taylor_test'] +disable = ['timer_mod', 'file_io_mod', 'error_mod', 'foe*', 'fes*'] # Define entry point for call-tree transformation -[[routine]] -name = 'cloudsc_driver_tl' -expand = true -role = 'driver' +[routines] -[[routine]] -name = 'cloudsc2' -role = 'kernel' -expand = true +[routines.cloudsc_driver_tl] + role = 'driver' + expand = true -[[routine]] -name = 'cloudsc2tl' -role = 'kernel' -expand = true +# Define indices and bounds for array dimensions +[dimensions] +[dimensions.horizontal] + size = 'KLON' + index = 'JL' + bounds = ['KIDIA', 'KFDIA'] + aliases = ['NPROMA', 'KDIM%KLON'] -[[routine]] -name = 'satur' -role = 'kernel' -expand = true - -############################################## - -[[dimension]] -name = 'horizontal' -size = 'KLON' -index = 'JL' -bounds = ['KIDIA', 'KFDIA'] -aliases = ['NPROMA', 'KDIM%KLON'] - -[[dimension]] -name = 'vertical' -size = 'KLEV' -index = 'JK' - -[[dimension]] -name = 'block_dim' -size = 'NGPBLKS' -index = 'IBL' +[dimensions.vertical] + size = 'KLEV' + index = 'JK' + +[dimensions.block_dim] + size = 'NGPBLKS' + index = 'IBL' From ed2b6f5134dfe7e773c3d3ce8e5be619be307ae4 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 21:40:43 +0100 Subject: [PATCH 5/6] Add Github arch --- arch/github/ubuntu/gnu/9.4.0/env.sh | 9 +++++++ arch/github/ubuntu/gnu/9.4.0/toolchain.cmake | 1 + arch/toolchains/github-ubuntu-gnu.cmake | 28 ++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 arch/github/ubuntu/gnu/9.4.0/env.sh create mode 120000 arch/github/ubuntu/gnu/9.4.0/toolchain.cmake create mode 100644 arch/toolchains/github-ubuntu-gnu.cmake diff --git a/arch/github/ubuntu/gnu/9.4.0/env.sh b/arch/github/ubuntu/gnu/9.4.0/env.sh new file mode 100644 index 0000000..c8eb1fc --- /dev/null +++ b/arch/github/ubuntu/gnu/9.4.0/env.sh @@ -0,0 +1,9 @@ +# Source me to get the correct configure/build/run environment + +export CC=gcc-9 +export CXX=g++-9 +export FC=gfortran-9 +export F77=gfortran-9 +export F90=gfortran-9 + +export ECBUILD_TOOLCHAIN="./toolchain.cmake" diff --git a/arch/github/ubuntu/gnu/9.4.0/toolchain.cmake b/arch/github/ubuntu/gnu/9.4.0/toolchain.cmake new file mode 120000 index 0000000..1c35228 --- /dev/null +++ b/arch/github/ubuntu/gnu/9.4.0/toolchain.cmake @@ -0,0 +1 @@ +../../../../toolchains/github-ubuntu-gnu.cmake \ No newline at end of file diff --git a/arch/toolchains/github-ubuntu-gnu.cmake b/arch/toolchains/github-ubuntu-gnu.cmake new file mode 100644 index 0000000..f7e59fa --- /dev/null +++ b/arch/toolchains/github-ubuntu-gnu.cmake @@ -0,0 +1,28 @@ +# (C) Copyright 1988- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +#################################################################### +# COMPILER +#################################################################### + +set( ECBUILD_FIND_MPI ON ) +set( ENABLE_ACC OFF CACHE STRING "" ) + +#################################################################### +# Compiler FLAGS +#################################################################### + +# General Flags (add to default) +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -ffpe-trap=invalid,zero,overflow") +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -fstack-arrays") +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -fconvert=big-endian") +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -fbacktrace") +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -fno-second-underscore") +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -ffast-math") +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -fno-unsafe-math-optimizations") +set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -foffload=disable") From f7d5539a198f3e7203b4133f483168154eeb0998 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Thu, 22 Feb 2024 22:22:34 +0100 Subject: [PATCH 6/6] Switch ecbundle to Github repository --- cloudsc-bundle | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cloudsc-bundle b/cloudsc-bundle index 771ac48..17ccca2 100755 --- a/cloudsc-bundle +++ b/cloudsc-bundle @@ -1,5 +1,14 @@ #!/usr/bin/env bash +# (C) Copyright 1988- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + # BOOTSTRAP ecbundle-build or ecbundle-create, # and pass arguments to it. @@ -9,7 +18,7 @@ if [[ $# -eq 0 ]]; then exit 1 fi -export ecbundle_VERSION=develop +export ecbundle_VERSION=2.0.0 # Which script do we run? # Partial matches are accepted, e.g. "$0 cr" @@ -25,15 +34,11 @@ shift BUNDLE_DIR="$( cd $( dirname "${BASH_SOURCE[0]}" ) && pwd -P )" -if [[ -z "${BITBUCKET}" ]]; then - export BITBUCKET=ssh://git@git.ecmwf.int -fi - # Download ecbundle scripts if not already available command_exists () { type "$1" &> /dev/null ; } if ! command_exists ${BOOTSTRAPPED} ; then if [[ ! -d ${BUNDLE_DIR}/ecbundle ]]; then - git clone ${BITBUCKET}/escape/ecbundle.git ${BUNDLE_DIR}/ecbundle + git clone https://github.com/ecmwf/ecbundle.git ${BUNDLE_DIR}/ecbundle ( cd ${BUNDLE_DIR}/ecbundle && git checkout ${ecbundle_VERSION} ) fi export PATH=${BUNDLE_DIR}/ecbundle/bin:${PATH}