Merge branch 'develop' into HuiLiu-NOAA-new-surface-operator

.gitignore

@@ -146,8 +146,14 @@ build*/
 install*/
 
 # Ignore the following bundle repositories
-!/bundle/CMakeLists.txt
 /bundle/*
+!/bundle/CMakeLists.txt
+!/bundle/rrfs-test-data
+!/bundle/test-data-release
+!/bundle/ufo-data
+!/bundle/ioda-data
+!/bundle/fv3-jedi-data
+!/bundle/mpas-jedi-data
 
 # 
 expr

CMakeLists.txt

@@ -35,9 +35,7 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 # Handle user options.
 option(BUILD_RDASBUNDLE "Build RDAS Bundle" ON)
-option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
-option(CLONE_RRFSDATA "Clone RRFS test data" OFF)
-option(WORKFLOW_TESTS "Include regional-workflow dependent tests" OFF)
+option(ADD_RRFS_TESTS "Add RRFS tests" OFF)
 option(MPAS_DYCORE "Use MPAS" OFF)
 option(FV3_DYCORE "Use FV3" ON)
 

build.sh

@@ -19,13 +19,13 @@ usage() {
   echo "Usage: $0 -p <prefix> | -t <target> -h"
   echo
   echo "  -p  installation prefix <prefix>    DEFAULT: <none>"
-  echo "  -t  target to build for <target>    DEFAULT: $MACHINE_ID"
   echo "  -c  additional CMake options        DEFAULT: <none>"
   echo "  -v  build with verbose output       DEFAULT: NO"
+  echo "  -j  number of build jobs            DEFAULT: 4 on Orion, 6 on other machines"
   echo "  -f  force a clean build             DEFAULT: NO"
   echo "  -s  only build a subset of the bundle  DEFAULT: NO"
   echo "  -m  select dycore                      DEFAULT: FV3andMPAS"
-  echo "  --notestdata  do not include JCSDA and rrfs ctest data  DEFAULT: NO"
+  echo "  -x  build super executables         DEFAULT: NO"
   echo "  -h  display this message and quit"
   echo
   exit 1
@@ -38,27 +38,26 @@ INSTALL_PREFIX=""
 CMAKE_OPTS=""
 BUILD_TARGET="${MACHINE_ID:-'localhost'}"
 BUILD_VERBOSE="NO"
-CLONE_JCSDADATA="YES"
-CLONE_RRFSDATA="YES"
 CLEAN_BUILD="NO"
 BUILD_JCSDA="YES"
+BUILD_SUPER_EXE="NO"
 DYCORE="FV3andMPAS"
 COMPILER="${COMPILER:-intel}"
 
-while getopts "p:t:c:m:hvfs-:" opt; do
+while getopts "p:c:m:j:hvfsx" opt; do
   case $opt in
     p)
       INSTALL_PREFIX=$OPTARG
       ;;
-    t)
-      BUILD_TARGET=$OPTARG
-      ;;
     c)
       CMAKE_OPTS=$OPTARG
       ;;
     m)
       DYCORE=$OPTARG
       ;;
+    j)
+      BUILD_JOBS=$OPTARG
+      ;;
     v)
       BUILD_VERBOSE=YES
       ;;
@@ -68,12 +67,9 @@ while getopts "p:t:c:m:hvfs-:" opt; do
     s)
       BUILD_JCSDA=NO
       ;;
-    -)
-      if [[ "${OPTARG}" == "notestdata" ]]; then
-        CLONE_JCSDADATA=NO
-        CLONE_RRFSDATA=NO
-      fi 
-      ;;
+    x)
+      BUILD_SUPER_EXE=YES
+      ;; 
     h|\?|:)
       usage
       ;;
@@ -96,7 +92,12 @@ case ${BUILD_TARGET} in
     ;;
 esac
 
-CMAKE_OPTS+=" -DCLONE_JCSDADATA=$CLONE_JCSDADATA -DCLONE_RRFSDATA=$CLONE_RRFSDATA"
+# Set default number of build jobs based on machine
+if [[ $BUILD_TARGET == 'orion' ]]; then # lower due to memory limit on login nodes
+  BUILD_JOBS=${BUILD_JOBS:-4}
+else # hera, hercules, jet
+  BUILD_JOBS=${BUILD_JOBS:-6}
+fi
 
 BUILD_DIR=${BUILD_DIR:-$dir_root/build}
 if [[ $CLEAN_BUILD == 'YES' ]]; then
@@ -145,17 +146,7 @@ if [[ $DYCORE == 'MPAS' || $DYCORE == 'FV3andMPAS' ]]; then
   $dir_root/rrfs-test/scripts/link_mpasjedi_expr.sh
 fi
 
-# JCSDA changed test data things, need to make a dummy CRTM directory
-if [[ $BUILD_TARGET == 'hera' ]]; then
-  if [ -d "$dir_root/bundle/fix/test-data-release/" ]; then rm -rf $dir_root/bundle/fix/test-data-release/; fi
-  if [ -d "$dir_root/bundle/test-data-release/" ]; then rm -rf $dir_root/bundle/test-data-release/; fi
-  mkdir -p $dir_root/bundle/fix/test-data-release/
-  mkdir -p $dir_root/bundle/test-data-release/
-  ln -sf $RDASAPP_TESTDATA/crtm $dir_root/bundle/fix/test-data-release/crtm
-  ln -sf $RDASAPP_TESTDATA/crtm $dir_root/bundle/test-data-release/crtm
-fi
-
-  CMAKE_OPTS+=" -DMPIEXEC_MAX_NUMPROCS:STRING=120"
+CMAKE_OPTS+=" -DMPIEXEC_MAX_NUMPROCS:STRING=120 -DBUILD_SUPER_EXE=$BUILD_SUPER_EXE"
 # Configure
 echo "Configuring ..."
 set -x
@@ -168,11 +159,11 @@ set +x
 echo "Building ..."
 set -x
 if [[ $BUILD_JCSDA == 'YES' ]]; then
-  make -j ${BUILD_JOBS:-6} VERBOSE=$BUILD_VERBOSE
+  make -j $BUILD_JOBS VERBOSE=$BUILD_VERBOSE
 else
   for b in $builddirs; do
     cd $b
-    make -j ${BUILD_JOBS:-6} VERBOSE=$BUILD_VERBOSE
+    make -j $BUILD_JOBS VERBOSE=$BUILD_VERBOSE
     cd ../
   done
 fi

bundle/CMakeLists.txt

@@ -33,8 +33,6 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 # Handle user options.
 option(BUILD_RDASBUNDLE "Build RDAS Bundle" ON)
-option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
-option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF)
 
 # Initialize bundle
 # -----------------
@@ -107,41 +105,11 @@ endif()
     ecbuild_bundle( PROJECT iodaconv SOURCE "../sorc/iodaconv" )
   endif()
 
-# rdas mega executable
-  ecbuild_bundle( PROJECT rdas SOURCE "../mains" )
-
-# ioda, ufo, fv3-jedi, and saber test data
-#---------------------------------
-  if(CLONE_JCSDADATA)
-
-   set(JCSDA_DATA_ROOT "$ENV{RDASAPP_TESTDATA}/jcsda")
-   ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" )
-   ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" )
-   if(FV3_DYCORE)
-     ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" )
-   endif()
-   if(MPAS_DYCORE)
-     set(DESTINATION_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
-     file( COPY "${JCSDA_DATA_ROOT}/mpas-jedi-data" DESTINATION ${DESTINATION_DIR} )
-   endif()
-
-  endif(CLONE_JCSDADATA)
-# rrfs-test data (fv3 and/or mpas)
-#---------------------------------
-  if(CLONE_RRFSDATA)
-     set(RDAS_RRFS_DATA_ROOT "$ENV{RDAS_RRFS_DATA_ROOT}/rrfs-test-data")
-     set(DESTINATION_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
-     if(NOT EXISTS "${DESTINATION_DIR}/rrfs-test-data")
-        file(MAKE_DIRECTORY "${DESTINATION_DIR}/rrfs-test-data")
-     endif()
-     if(FV3_DYCORE)
-     file( COPY "${RDAS_RRFS_DATA_ROOT}/rrfs-data_fv3jedi_2022052619" DESTINATION "${DESTINATION_DIR}/rrfs-test-data" )
-     endif()
-     if(MPAS_DYCORE)
-     file( COPY "${CMAKE_CURRENT_SOURCE_DIR}/../expr/mpas_2024052700/" DESTINATION "${DESTINATION_DIR}/rrfs-test-data/rrfs-data_mpasjedi_2024052700" )
-     endif()
+# rdas tests and super executables
   ecbuild_bundle( PROJECT rrfs-test SOURCE "../rrfs-test" )
-  endif(CLONE_RRFSDATA)
+  if(BUILD_SUPER_EXE)
+    ecbuild_bundle( PROJECT rdas SOURCE "../mains" )
+  endif()
 
 endif(BUILD_RDASBUNDLE)
 

bundle/fv3-jedi-data

@@ -0,0 +1 @@
+../fix/jcsda/fv3-jedi-data

bundle/ioda-data

@@ -0,0 +1 @@
+../fix/jcsda/ioda-data

bundle/mpas-jedi-data/testinput_tier_1/384km/init/x1.4002.init.2018-04-15_00.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/384km/init/x1.4002.init.2018-04-15_00.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/ensemble

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/ensemble

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-14_21.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-14_21.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_00.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_00.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_03.00.00.nc

@@ -0,0 +1 @@
+../../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km/bg/restart.2018-04-15_03.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/mpasout.2018-04-15_00.00.00.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/mpasout.2018-04-15_00.00.00.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/static.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/static.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.grid.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.grid.nc

bundle/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.init.2018-04-14_18.00.00.nc

@@ -0,0 +1 @@
+../../../../fix/.agent/jcsda/mpas-jedi-data/testinput_tier_1/480km_2stream/x1.2562.init.2018-04-14_18.00.00.nc

bundle/rrfs-test-data/rrfs-data_fv3jedi_2022052619

@@ -0,0 +1 @@
+../../fix/rrfs-test-data/rrfs-data_fv3jedi_2022052619

bundle/rrfs-test-data/rrfs-data_mpasjedi_2024052700

@@ -0,0 +1 @@
+../../expr/mpas_2024052700

bundle/test-data-release/crtm

@@ -0,0 +1 @@
+../../fix/crtm

bundle/ufo-data

@@ -0,0 +1 @@
+../fix/jcsda/ufo-data

fix/bumploc/conus12km-401km11levels

@@ -1 +1 @@
-../.agent/bumploc/conus12km-401km11levels
+../.agent/bumploc/conus12km-401km11levels_20240912

fix/crtm

@@ -0,0 +1 @@
+.agent/crtm

fix/expr_data/mpas_2024052700/bkg

@@ -0,0 +1 @@
+../../.agent/expr_data/mpas_2024052700/bkg

fix/expr_data/mpas_2024052700/ens

@@ -0,0 +1 @@
+../../.agent/expr_data/mpas_2024052700/ens

fix/expr_data/mpas_2024052700/obs

@@ -0,0 +1 @@
+../../.agent/expr_data/mpas_2024052700/obs

fix/expr_data/mpas_2024052700/ref

@@ -0,0 +1 @@
+../../.agent/expr_data/mpas_2024052700/ref_20240912

fix/jcsda/fv3-jedi-data

@@ -0,0 +1 @@
+../.agent/jcsda/fv3-jedi-data

fix/jcsda/ioda-data

@@ -0,0 +1 @@
+../.agent/jcsda/ioda-data

fix/jcsda/mpas-jedi-data

@@ -0,0 +1 @@
+../.agent/jcsda/mpas-jedi-data

fix/jcsda/ufo-data

@@ -0,0 +1 @@
+../.agent/jcsda/ufo-data_20240909

fix/rrfs-test-data

@@ -0,0 +1 @@
+.agent/rrfs-test-data

modulefiles/RDAS/hera.intel.lua

@@ -84,14 +84,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/scratch1/NCEPDEV/da/Cory.R.Martin/RDASApp/fix/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/scratch1/NCEPDEV/da/Cory.R.Martin/CI/RDASApp/data")
---setenv("RDAS_RRFS_DATA_ROOT","/scratch2/NCEPDEV/fv3-cam/Ting.Lei/dr-emc-rdas/")
---setenv("RDAS_RRFS_DATA_ROOT", "/scratch1/NCEPDEV/da/Xiaoyan.Zhang/RRFS")
-setenv("RDAS_RRFS_DATA_ROOT", "/scratch1/NCEPDEV/da/Samuel.Degelia/RRFS")
-
---prepend_path("PATH","/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/intel-18.0.5.274/prod_util/1.2.2/bin")
-
 whatis("Name: ".. pkgName)
 whatis("Version: ".. pkgVersion)
 whatis("Category: RDASApp")

modulefiles/RDAS/hercules.intel.lua

@@ -80,10 +80,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/work2/noaa/da/cmartin/RDASApp/fix/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/RDASApp/data")
-prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")
-
 execute{cmd="ulimit -s unlimited",modeA={"load"}}
 
 whatis("Name: ".. pkgName)

modulefiles/RDAS/jet.intel.lua

@@ -85,10 +85,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/lfs5/BMC/nrtrr/RDAS_DATA/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/lfs5/BMC/nrtrr/RDAS_DATA")
-setenv("RDAS_RRFS_DATA_ROOT", "/lfs5/BMC/nrtrr/RDAS_DATA")
-
 whatis("Name: ".. pkgName)
 whatis("Version: ".. pkgVersion)
 whatis("Category: RDASApp")

modulefiles/RDAS/orion.intel.lua

@@ -79,12 +79,6 @@ local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)
 setenv('MPIEXEC_NPROC', mpinproc)
 
-setenv("CRTM_FIX","/work2/noaa/da/cmartin/RDASApp/fix/crtm/2.4.0")
-setenv("RDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/RDASApp/data")
---setenv("RDAS_RRFS_DATA_ROOT","/work/noaa/fv3-cam/tlei/dr-emc-rdas/")
-setenv("RDAS_RRFS_DATA_ROOT", "/work2/noaa/fv3-cam/sdegelia/RRFS")
-prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")
-
 execute{cmd="ulimit -s unlimited",modeA={"load"}}
 
 whatis("Name: ".. pkgName)

rrfs-test/CMakeLists.txt

@@ -18,18 +18,16 @@ add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_letkf"    "mpasj
 add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_getkf"    "mpasjedi_enkf.x")
 add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_bumploc"  "mpasjedi_error_covariance_toolbox.x")
 
-# rrfs data - from rrfs-test-data repo if found, from local directory, or from tarball
-if (CLONE_RRFSDATA)
-   message(STATUS "Use test data from rrfs-test-data repository")
-   set (rrfs-test_data_local "${CMAKE_SOURCE_DIR}/rrfs-test-data/")
-   set (src_yaml "${CMAKE_SOURCE_DIR}/rrfs-test/testinput")
+message(STATUS "Use test data from rrfs-test-data repository")
+set (rrfs-test_data_local "${CMAKE_SOURCE_DIR}/rrfs-test-data/")
+set (src_yaml "${CMAKE_SOURCE_DIR}/rrfs-test/testinput")
 # It's unclear if anyone is using the local data functionality, therefore comment it out
 # for now and add in if someone complains / remove if completely if not.
 #elseif (DEFINED ENV{FV3_JEDI_TESTFILES})
 #  message(STATUS "Use test data from local directory $ENV{FV3_JEDI_TESTFILES}")
 #  # A bit of guesswork here, I don't know if folks using this option stored it in the same directory structure
 #  set (fv3-jedi_data_testinput_tier_1_local "$ENV{FV3_JEDI_TESTFILES}/fv3-jedi-data/testinput_tier_1")
-   if(FV3_DYCORE)
+if(FV3_DYCORE)
 
       message(STATUS "MACHINE_ID is " ${MACHINE_ID})
       if("${MACHINE_ID}" STREQUAL "orion")
@@ -107,5 +105,4 @@ if (CLONE_RRFSDATA)
           set(MPI_ARGS ${RESTORE_MPI_ARGS})
       endif()
 
-   endif()
 endif()