Merge branch 'develop' into feature/gw-ci

parm/atm/jcb-prototype_3dvar.yaml.j2

@@ -16,8 +16,8 @@ observations:
   - conventional_ps
   - gnssro
 #  - gpsro
-#  - mtiasi_metop-a
-#  - mtiasi_metop-b
+#  - iasi_metop-a
+#  - iasi_metop-b
 #  - ompsnp_n20
   - ompsnp_npp
 #  - ompstc_n20

parm/atm/jcb-prototype_lgetkf.yaml.j2

@@ -25,8 +25,8 @@ observations:
   - conventional_ps
   - gnssro
 #  - gpsro
-#  - mtiasi_metop-a
-#  - mtiasi_metop-b
+#  - iasi_metop-a
+#  - iasi_metop-b
 #  - ompsnp_n20
   - ompsnp_npp
 #  - ompstc_n20

parm/atm/jcb-prototype_lgetkf_observer.yaml.j2

@@ -25,8 +25,8 @@ observations:
   - conventional_ps
   - gnssro
 #  - gpsro
-#  - mtiasi_metop-a
-#  - mtiasi_metop-b
+#  - iasi_metop-a
+#  - iasi_metop-b
 #  - ompsnp_n20
   - ompsnp_npp
 #  - ompstc_n20

parm/atm/jcb-prototype_lgetkf_solver.yaml.j2

@@ -25,8 +25,8 @@ observations:
   - conventional_ps
   - gnssro
 #  - gpsro
-#  - mtiasi_metop-a
-#  - mtiasi_metop-b
+#  - iasi_metop-a
+#  - iasi_metop-b
 #  - ompsnp_n20
   - ompsnp_npp
 #  - ompstc_n20

parm/ioda/bufr2ioda/bufr2ioda_mtiasi.yaml → parm/ioda/bufr2ioda/bufr2ioda_iasi.yaml

@@ -94,7 +94,7 @@ observations:
 
     ioda:
       backend: netcdf
-      obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.mtiasi_$(splitvar).tm00.nc" 
+      obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.iasi_$(splitvar).tm00.nc" 
 
       dimensions:
         - name: Channel

test/marine/CMakeLists.txt

@@ -28,6 +28,14 @@ install(FILES ${test_input}
 # bufr to ioda tests:
 ###########################################################################
 
+set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine)
+set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine)
+set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i)
+set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput)
+set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref)
+set(PYIODACONV_DIR "${PROJECT_SOURCE_DIR}/build/lib/python3.10/")
+
+
 # prepare a test.yaml file from test.yaml.in by replacing
 # placeholder patterns __BUFRINPUTDIR__ and __IODAOUTPUTDIR__ and __OCEANBASIN__
 # with actual directory paths
@@ -46,14 +54,6 @@ function(CREATE_CONFIG_FILE
 endfunction()
 
 
-set(TEST_WORKING_DIR ${PROJECT_BINARY_DIR}/test/marine)
-
-set(MARINE_BUFR2IODA_DIR ${PROJECT_SOURCE_DIR}/ush/ioda/bufr2ioda/marine)
-set(MARINE_BUFR2IODA_DIR ${MARINE_BUFR2IODA_DIR}/b2i)
-set(CONFIG_DIR ${PROJECT_SOURCE_DIR}/test/marine/testinput)
-set(TESTREF_DIR ${PROJECT_SOURCE_DIR}/test/marine/testref)
-
-
 function(CHECK_AND_SET_PATH PATH1 PATH2 RESULT_VAR)
     # Check if PATH1 exists
     if(EXISTS ${PATH1})
@@ -157,9 +157,15 @@ function(ADD_INSITU_TEST testname testbufr)
 		COMMAND ${MARINE_BUFR2IODA_DIR}/${TEST}.py -c ${CONFIG_FILE} -t ${TESTREF_DIR}/${TESTREF_FILE}
 		WORKING_DIRECTORY ${TEST_WORKING_DIR}
 	)
+	set_property(
+		TEST test_gdasapp_${TEST}
+		APPEND PROPERTY 
+			ENVIRONMENT "PYTHONPATH=${PYIODACONV_DIR}:$ENV{PYTHONPATH}"
+	)
 endfunction()
 
 
+
 if (GENERATE_BUFR2IODA_TESTS)
 	ADD_INSITU_TEST("profile_argo" "subpfl")
 	ADD_INSITU_TEST("profile_bathy" "bathy")

ush/ioda/bufr2ioda/marine/b2i/b2iconverter/bufr2ioda_converter.py

@@ -64,8 +64,22 @@ def run(self):
         # process query results and set ioda variables
         self.ioda_vars.set_from_query_result(r)
 
+        n_obs = self.ioda_vars.number_of_obs()
+        self.logger.debug(f"Query result has {n_obs} obs")
+        if (n_obs == 0):
+            self.logger.warning(f"No obs! Quitting.")
+            sys.exit(0)
+
         self.ioda_vars.filter()
 
+        n_obs = self.ioda_vars.number_of_obs()
+        self.logger.debug(f"Filtered result has {n_obs} obs")
+        if (n_obs == 0):
+            self.logger.warning(f"No obs! Quitting.")
+            sys.exit(0)
+        self.logger.debug(f"Number of temperature obs = {self.ioda_vars.number_of_temp_obs()}")
+        self.logger.debug(f"Number of salinity obs = {self.ioda_vars.number_of_saln_obs()}")
+
         # set seqNum, PreQC, ObsError, OceanBasin
         self.ioda_vars.additional_vars.construct()
 

ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ioda_variables.py

@@ -44,6 +44,29 @@ def set_salinity_range(self, smin, smax):
         self.S_min = smin
         self.S_max = smax
 
+    def number_of_temp_obs(self):
+        try:
+            if isinstance(self.temp, np.ma.MaskedArray):
+                return self.temp.count()
+            else:
+                return 0
+        # except NameError:
+        except AttributeError:
+            return 0
+
+    def number_of_saln_obs(self):
+        try:
+            if isinstance(self.saln, np.ma.MaskedArray):
+                return self.saln.count()
+            else:
+                return 0
+        # except NameError:
+        except AttributeError:
+            return 0
+
+    def number_of_obs(self):
+        return max(self.number_of_temp_obs(), self.number_of_saln_obs())
+
     def build_query(self):
         q = bufr.QuerySet()
         q.add('year', '*/YEAR')

ush/ioda/bufr2ioda/marine/b2i/b2iconverter/ocean.py

@@ -5,9 +5,6 @@
 import numpy as np
 import numpy.ma as ma
 import math
-import matplotlib.pyplot as plt
-import cartopy.crs as ccrs
-import cartopy.feature as cfeature
 import netCDF4 as nc
 import xarray as xr
 
@@ -20,8 +17,6 @@
 
 # the main method is get_station_basin which returns the ocean basin
 # for a list of station coordinates
-# there are methods for plotting and printing the ocean basin data
-# as well as printing and plotting station basin data
 
 
 class OceanBasin:
@@ -54,32 +49,6 @@ def read_nc_file(self):
             print(f"An IOError occurred: {e}")
             sys.exit(1)
 
-    def print_basin(self):
-        for i in range(n1):
-            for j in range(n2):
-                print(i, j, self.__basin_array[i][j])
-
-    def plot_basin(self):
-        # Create a figure and axes with Cartopy projection
-        fig = plt.figure(figsize=(10, 6))
-        ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
-
-        # Plot the ocean basins using a colormap with 6 colors
-        # cmap = plt.cm.get_cmap('rainbow', 6)  # Choose a colormap with 6 colors
-        cmap = plt.get_cmap('viridis', 6)  # Create a colormap with 6 discrete colors
-        im = ax.pcolormesh(self.__longitudes, self.__latitudes, self.__basin_array, cmap='viridis', shading='auto', transform=ccrs.PlateCarree())
-
-        # Add colorbar
-        cbar = fig.colorbar(im, ax=ax, orientation='vertical', pad=0.05, ticks=np.arange(0, 6))
-        cbar.set_label('Ocean Basin', fontsize=12)
-        # Add title and gridlines
-        ax.set_title('Ocean Basin Map', fontsize=16)
-        ax.coastlines()
-        ax.gridlines(draw_labels=True)
-        # Show the plot
-        plt.show()
-        plt.savefig('ocean_basin.png', dpi=300)
-
     # input: 2 vectors of station coordinates
     # output: a vector of station ocean basin values
     def get_station_basin(self, lat, lon):
@@ -99,33 +68,3 @@ def get_station_basin(self, lat, lon):
                 i2 = round((lon[i] - lon0) / dlon)
                 ocean_basin.append(self.__basin_array[i1][i2])
         return ocean_basin
-
-    def print_station_basin(self, lon, lat, file_path):
-        ocean_basin = self.get_station_basin(lat, lon)
-        with open(file_path, 'w') as file:
-            # Iterate over lon, lat, and ocean_basin arrays simultaneously
-            for lat_val, lon_val, basin_val in zip(lat, lon, ocean_basin):
-                file.write(f"{lat_val} {lon_val} {basin_val}\n")
-
-    def plot_stations(self, lon, lat, png_file):
-        ocean_basin = self.get_station_basin(lon, lat)
-
-        # Initialize the plot
-        plt.figure(figsize=(12, 8))
-        # Create a Cartopy map with PlateCarree projection (latitude/longitude)
-        ax = plt.axes(projection=ccrs.PlateCarree())
-        # Add coastlines and borders
-        ax.coastlines()
-        ax.add_feature(cartopy.feature.BORDERS, linestyle=':', linewidth=0.5)
-
-        # Scatter plot with colored dots for each basin type
-        colors = ['blue', 'green', 'red', 'cyan', 'magenta', 'yellow']
-        for basin_type in range(6):
-            indices = np.where(ocean_basin == basin_type)[0]
-            ax.scatter(lon[indices], lat[indices], color=colors[basin_type], label=f'Basin {basin_type}', alpha=0.7)
-
-        # Add a legend
-        plt.legend(loc='lower left')
-        # Add title and show plot
-        plt.title('Ocean Basins Plot using Cartopy')
-        plt.savefig(png_file, dpi=300)