WIP: First attempt to add a generic tracer to OBC scheme

.gitlab-ci.yml

@@ -116,7 +116,7 @@ run:
     - time tar zxf $CACHE_DIR/build-intel-repro-$CI_PIPELINE_ID.tgz
     - time tar zxf $CACHE_DIR/build-pgi-repro-$CI_PIPELINE_ID.tgz
     # time tar zxf $CACHE_DIR/build-gnu-debug-$CI_PIPELINE_ID.tgz
-    - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all -B') > job.sh
+    - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all') > job.sh
     - sbatch --clusters=c3,c4 --nodes=29 --time=0:34:00 --account=gfdl_o --qos=debug --job-name=mom6_regressions --output=log.$CI_PIPELINE_ID --wait job.sh
     - cat log.$CI_PIPELINE_ID
     - test -f restart_results_gnu.tar.gz

.gitmodules

@@ -4,9 +4,3 @@
 [submodule "pkg/GSW-Fortran"]
 	path = pkg/GSW-Fortran
 	url = https://github.com/TEOS-10/GSW-Fortran.git
-[submodule "pkg/MOM6_DA_hooks"]
-	path = pkg/MOM6_DA_hooks
-	url = https://github.com/MJHarrison-GFDL/MOM6_DA_hooks.git
-[submodule "pkg/geoKdTree"]
-	path = pkg/geoKdTree
-	url = https://github.com/travissluka/geoKdTree.git

.testing/Makefile

@@ -20,7 +20,7 @@ MKMF := $(abspath $(DEPS)/mkmf/bin/mkmf)
 
 # FMS framework
 FMS_URL ?= https://github.com/NOAA-GFDL/FMS.git
-FMS_COMMIT ?= 2019.01.01
+FMS_COMMIT ?= 2019.01.03
 FMS := $(DEPS)/fms
 
 #---
@@ -92,11 +92,34 @@ TARGET_SOURCE = $(call SOURCE,build/target_codebase/src) \
   $(wildcard build/target_codebase/config_src/ext*/*.F90)
 FMS_SOURCE = $(call SOURCE,$(DEPS)/fms/src)
 
+#---
+# Python preprocessing environment configuration
+
+HAS_NUMPY = $(shell python -c "import numpy" 2> /dev/null && echo "yes")
+HAS_NETCDF4 = $(shell python -c "import netCDF4" 2> /dev/null && echo "yes")
+
+USE_VENV =
+ifneq ($(HAS_NUMPY), yes)
+  USE_VENV = yes
+endif
+ifneq ($(HAS_NETCDF4), yes)
+  USE_VENV = yes
+endif
+
+# When disabled, activation is a null operation (`true`)
+VENV_PATH =
+VENV_ACTIVATE = true
+ifeq ($(USE_VENV), yes)
+  VENV_PATH = work/local-env
+  VENV_ACTIVATE = . $(VENV_PATH)/bin/activate
+endif
+
+
 #---
 # Rules
 
 .PHONY: all build.regressions
-all: $(foreach b,$(BUILDS),build/$(b)/MOM6)
+all: $(foreach b,$(BUILDS),build/$(b)/MOM6) $(VENV_PATH)
 build.regressions: $(foreach b,symmetric target,build/$(b)/MOM6)
 
 # Executable
@@ -184,6 +207,18 @@ $(LIST_PATHS) $(MKMF):
 	cd $(DEPS)/mkmf; git checkout $(MKMF_COMMIT)
 
 
+#---
+# Python preprocessing
+# NOTE: Some less mature environments (e.g. Arm64 Ubuntu) require explicit
+#   installation of numpy before netCDF4, as well as wheel and cython support.
+work/local-env:
+	python3 -m venv $@
+	. $@/bin/activate \
+	  && pip3 install wheel \
+	  && pip3 install cython \
+	  && pip3 install numpy \
+	  && pip3 install netCDF4
+
 #----
 # Testing
 
@@ -264,22 +299,9 @@ $(eval $(call CMP_RULE,regression,symmetric target))
 
 # TODO: chksum_diag parsing of restart files
 
-
 #---
 # Test run output files
 
-# Generalized MPI environment variable support
-# XXX: Using `-env` in the MPICH test can erroneously producing an `nv` file.
-# $(1): Environment variables
-ifeq ($(shell $(MPIRUN) -x tmp=1 true 2> /dev/null ; echo $$?), 0)
-  MPIRUN_CMD=$(MPIRUN) $(if $(1),-x $(1),)
-else ifeq ($(shell $(MPIRUN) -env tmp=1 true 2> /dev/null ; echo $$? ; rm -f nv), 0)
-  MPIRUN_CMD=$(MPIRUN) $(if $(1),-env $(1),)
-else
-  MPIRUN_CMD=$(1) $(MPIRUN)
-endif
-
-
 # Rule to build work/<tc>/{ocean.stats,chksum_diag}.<tag>
 # $(1): Test configuration name <tag>
 # $(2): Executable type
@@ -293,24 +315,30 @@ work/%/$(1)/ocean.stats work/%/$(1)/chksum_diag: build/$(2)/MOM6
 	if [ $(3) ]; then find build/$(2) -name *.gcda -exec rm -f '{}' \; ; fi
 	mkdir -p $$(@D)
 	cp -rL $$*/* $$(@D)
-	cd $$(@D) && if [ -f Makefile ]; then $(MAKE); fi
+	if [ -f $$(@D)/Makefile ]; then \
+	  $$(VENV_ACTIVATE) \
+	    && cd $$(@D) \
+	    && $(MAKE); \
+	else \
+	  cd $$(@D); \
+	fi
 	mkdir -p $$(@D)/RESTART
 	echo -e "$(4)" > $$(@D)/MOM_override
 	cd $$(@D) \
-	  && $$(call MPIRUN_CMD,$(5)) -n $(6) ../../../$$< 2> std.err > std.out \
+	  && $(5) $(MPIRUN) -n $(6) ../../../$$< 2> std.err > std.out \
 	  || !( \
 	    mkdir -p ../../../results/$$*/ ; \
 	    cat std.out | tee ../../../results/$$*/std.$(1).out | tail -20 ; \
-		cat std.err | tee ../../../results/$$*/std.$(1).err | tail -20 ; \
-		rm ocean.stats chksum_diag ; \
-		echo -e "${FAIL}: $$*.$(1) failed at runtime." \
+	    cat std.err | tee ../../../results/$$*/std.$(1).err | tail -20 ; \
+	    rm ocean.stats chksum_diag ; \
+	    echo -e "${FAIL}: $$*.$(1) failed at runtime." \
 	  )
 	@echo -e "${DONE}: $$*.$(1); no runtime errors."
 	if [ $(3) ]; then \
 	  mkdir -p results/$$* ; \
 	  bash <(curl -s https://codecov.io/bash) -n $$@ \
 	    > work/$$*/codecov.$(1).out \
-		2> work/$$*/codecov.$(1).err ; \
+	    2> work/$$*/codecov.$(1).err ; \
 	fi
 endef
 
@@ -322,7 +350,7 @@ $(eval $(call STAT_RULE,symmetric,symmetric,$(REPORT_COVERAGE),,,1))
 $(eval $(call STAT_RULE,asymmetric,asymmetric,,,,1))
 $(eval $(call STAT_RULE,target,target,,,,1))
 $(eval $(call STAT_RULE,repro,repro,,,,1))
-$(eval $(call STAT_RULE,openmp,openmp,,,,1))
+$(eval $(call STAT_RULE,openmp,openmp,,,GOMP_CPU_AFFINITY=0,1))
 $(eval $(call STAT_RULE,layout,symmetric,,LAYOUT=2$(,)1,,2))
 $(eval $(call STAT_RULE,rotate,symmetric,,ROTATE_INDEX=True\nINDEX_TURNS=1,,1))
 $(eval $(call STAT_RULE,nan,symmetric,,,MALLOC_PERTURB_=256,1))
@@ -339,7 +367,13 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	rm -rf $(@D)
 	mkdir -p $(@D)
 	cp -rL $*/* $(@D)
-	cd work/$*/restart && if [ -f Makefile ]; then $(MAKE); fi
+	if [ -f $(@D)/Makefile ]; then \
+	  $(VENV_ACTIVATE) \
+	    && cd work/$*/restart \
+	    && $(MAKE); \
+	else \
+	  cd work/$*/restart; \
+	fi
 	mkdir -p $(@D)/RESTART
 	# Generate the half-period input namelist
 	# TODO: Assumes that runtime set by DAYMAX, will fail if set by input.nml
@@ -354,8 +388,8 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	cd $(@D) && $(MPIRUN) -n 1 ../../../$< 2> std1.err > std1.out \
 	  || !( \
 	    cat std1.out | tee ../../../results/$*/std.restart1.out | tail ; \
-		cat std1.err | tee ../../../results/$*/std.restart1.err | tail ; \
-		echo -e "${FAIL}: $*.restart failed at runtime." \
+	    cat std1.err | tee ../../../results/$*/std.restart1.err | tail ; \
+	    echo -e "${FAIL}: $*.restart failed at runtime." \
 	  )
 	# Setup the next inputs
 	cd $(@D) && rm -rf INPUT && mv RESTART INPUT
@@ -365,8 +399,8 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	cd $(@D) && $(MPIRUN) -n 1 ../../../$< 2> std2.err > std2.out \
 	  || !( \
 	    cat std2.out | tee ../../../results/$*/std.restart2.out | tail ; \
-		cat std2.err | tee ../../../results/$*/std.restart2.err | tail ; \
-		echo -e "${FAIL}: $*.restart failed at runtime." \
+	    cat std2.err | tee ../../../results/$*/std.restart2.err | tail ; \
+	    echo -e "${FAIL}: $*.restart failed at runtime." \
 	  )
 
 # TODO: Restart checksum diagnostics

.testing/tc2/MOM_input

@@ -297,6 +297,10 @@ BOUND_CORIOLIS = True           !   [Boolean] default = False
                                 ! v-points, and similarly at v-points.  This option would
                                 ! have no effect on the SADOURNY Coriolis scheme if it
                                 ! were possible to use centered difference thickness fluxes.
+PGF_STANLEY_T2_DET_COEFF = 0.5  !   [nondim] default = -1.0
+                                ! The coefficient correlating SGS temperature variance with the mean temperature
+                                ! gradient in the deterministic part of the Stanley form of the Brankart
+                                ! correction. Negative values disable the scheme.
 
 ! === module MOM_hor_visc ===
 LAPLACIAN = True
@@ -426,6 +430,10 @@ KHTH = 1.0                      !   [m2 s-1] default = 0.0
                                 ! The background horizontal thickness diffusivity.
 KHTH_MAX = 900.0                !   [m2 s-1] default = 0.0
                                 ! The maximum horizontal thickness diffusivity.
+STANLEY_PRM_DET_COEFF = 0.5     !   [nondim] default = -1.0
+                                ! The coefficient correlating SGS temperature variance with the mean temperature
+                                ! gradient in the deterministic part of the Stanley parameterization. Negative
+                                ! values disable the scheme.
 
 ! === module MOM_mixed_layer_restrat ===
 FOX_KEMPER_ML_RESTRAT_COEF = 5.0 !   [nondim] default = 0.0

.testing/tc4/.gitignore

@@ -0,0 +1,4 @@
+ocean_hgrid.nc
+sponge.nc
+temp_salt_ic.nc
+topog.nc

.testing/tc4/Makefile

@@ -1,3 +1,8 @@
-ocean_hgrid.nc topog.nc temp_salt_ic.nc sponge.nc:
+OUT=ocean_hgrid.nc sponge.nc temp_salt_ic.nc topog.nc
+
+$(OUT):
 	python build_grid.py
 	python build_data.py
+
+clean:
+	rm -rf $(OUT)

.travis.yml

@@ -14,6 +14,7 @@ addons:
     - mpich libmpich-dev
     - doxygen graphviz flex bison cmake
     - python-numpy python-netcdf4
+    - python3 python3-dev python3-venv python3-pip
     - bc
 
 jobs:

config_src/coupled_driver/MOM_surface_forcing_gfdl.F90

@@ -9,10 +9,8 @@ module MOM_surface_forcing_gfdl
 use MOM_constants,        only : hlv, hlf
 use MOM_cpu_clock,        only : cpu_clock_id, cpu_clock_begin, cpu_clock_end
 use MOM_cpu_clock,        only : CLOCK_SUBCOMPONENT
-use MOM_diag_mediator,    only : diag_ctrl
-use MOM_diag_mediator,    only : safe_alloc_ptr, time_type
+use MOM_diag_mediator,    only : diag_ctrl, safe_alloc_ptr, time_type
 use MOM_domains,          only : pass_vector, pass_var, fill_symmetric_edges
-use MOM_domains,          only : global_field_sum, BITWISE_EXACT_SUM
 use MOM_domains,          only : AGRID, BGRID_NE, CGRID_NE, To_All
 use MOM_domains,          only : To_North, To_East, Omit_Corners
 use MOM_error_handler,    only : MOM_error, WARNING, FATAL, is_root_pe, MOM_mesg

config_src/external/GFDL_ocean_BGC/README.md

@@ -3,4 +3,4 @@ GFDL_ocean_BGC
 
 These APIs reflect those for the GFDL ocean_BGC available at https://github.com/NOAA-GFDL/ocean_BGC.
 
-The modules in this directory do not do any computations. They simple reflect the APIs of the above package.
+The modules in this directory do not do any computations. They simply reflect the APIs of the above package.

config_src/external/ODA_hooks/README.md

@@ -0,0 +1,9 @@
+ODA_hooks
+=========
+
+These APIs reflect those for the ocean data assimilation hooks similar to https://github.com/MJHarrison-GFDL/MOM6_DA_hooks
+
+The modules in this directory do not do any computations. They simply reflect the APIs of the above package.
+
+- kdtree.f90 - would come from https://github.com/travissluka/geoKdTree
+- ocean_da_core.F90, ocean_da_types.F90, write_ocean_obs.F90 were copied from https://github.com/MJHarrison-GFDL/MOM6_DA_hooks

config_src/external/ODA_hooks/kdtree.f90

@@ -0,0 +1,12 @@
+!> A null version of K-d tree from geoKdTree
+module kdtree
+  implicit none
+  private
+
+  public :: kd_root
+
+  !> A K-d tree tpe
+  type kd_root
+    integer :: dummy !< To stop a compiler from doing nothing
+  end type kd_root
+end module kdtree

config_src/external/ODA_hooks/ocean_da_core.F90

@@ -0,0 +1,47 @@
+!> A set of dummy interfaces for compiling the MOM6 DA driver code.
+module ocean_da_core_mod
+  ! MOM modules
+  use MOM_domains, only : MOM_domain_type, domain2D
+  use MOM_time_manager, only : time_type, set_time, get_date
+  ! ODA_tools modules
+  use ocean_da_types_mod, only : ocean_profile_type, grid_type
+  use kdtree, only : kd_root
+
+  implicit none
+  private
+  public :: ocean_da_core_init
+  public :: get_profiles
+
+contains
+
+  !> Initializes the MOM6 DA driver code.
+  subroutine ocean_da_core_init(Domain, global_grid, Profiles, model_time)
+    type(domain2D), pointer, intent(in) :: Domain !< A MOM domain type
+    type(grid_type), pointer, intent(in) :: global_grid !< The global ODA horizontal grid type
+    type(ocean_profile_type), pointer :: Profiles  !< This is an unstructured recursive list of profiles
+                                          !! which are either within the localized domain corresponding
+                                          !! to the Domain argument, or the global profile list (type).
+    type(time_type), intent(in) :: model_time !< The current model time type.
+
+
+
+    Profiles=>NULL()
+    return
+  end subroutine ocean_da_core_init
+
+
+  !> Get profiles obs within the current analysis interval
+  subroutine get_profiles(model_time, Profiles, Current_profiles)
+    type(time_type), intent(in) :: model_time !< The current analysis time.
+    type(ocean_profile_type), pointer :: Profiles !< The full recursive list of profiles.
+    type(ocean_profile_type), pointer :: Current_profiles !< A returned list of profiles for the
+                                                          !! current analysis step.
+
+    Profiles=>NULL()
+    Current_Profiles=>NULL()
+
+    return
+  end subroutine get_profiles
+
+
+end module ocean_da_core_mod