DAS-1177 - Create end-to-end test, finish unit tests, edit test data …

…files.

hoss/dimension_utilities.py

@@ -12,10 +12,10 @@
 from logging import Logger
 from typing import Dict, Set, Tuple
 
+from pathlib import PurePosixPath
 from netCDF4 import Dataset
 from numpy.ma.core import MaskedArray
 import numpy as np
-from pathlib import PurePosixPath
 
 from harmony.message import Message
 from harmony.message_utility import rgetattr
@@ -109,7 +109,7 @@ def add_bounds_variables(dimensions_nc4: str,
                 write_bounds(prefetch_dataset, dimension_variable)
 
                 logger.info('Artificial bounds added for dimension variable: '
-                f'{dimension_name}')
+                            f'{dimension_name}')
 
 
 def needs_bounds(dimension: VariableFromDmr) -> bool:
@@ -208,16 +208,16 @@ def write_bounds(prefetch_dataset: Dataset,
 
     bounds_data_type = str(dimension_variable.data_type)
     bounds = prefetch_dataset.createVariable(bounds_full_path_name,
-                                              bounds_data_type,
-                                              (variable_dimension,
-                                               bounds_dim,))
+                                             bounds_data_type,
+                                             (variable_dimension,
+                                              bounds_dim,))
 
     # Write data to the new variable in the prefetch dataset.
     bounds[:] = bounds_array[:]
 
     # Update varinfo attributes and references.
     prefetch_dataset[dimension_variable.full_name_path].setncatts({'bounds': bounds_name})
-    dimension_variable.references['bounds'] = {bounds_name,}
+    dimension_variable.references['bounds'] = {bounds_name, }
     dimension_variable.attributes['bounds'] = bounds_name
 
 

tests/data/ATL16_prefetch.dmr

@@ -186,6 +186,39 @@
             <Value>Y</Value>
         </Attribute>
     </Float64>
+    <Float32 name="global_asr_obs_grid">
+        <Dim name="/global_grid_lat"/>
+        <Dim name="/global_grid_lon"/>
+        <Attribute name="long_name" type="String">
+        <Value>
+        global apparent surface reflectance observation grid
+        </Value>
+        </Attribute>
+        <Attribute name="units" type="String">
+        <Value>1</Value>
+        </Attribute>
+        <Attribute name="source" type="String">
+        <Value>
+        L3B ATM ATBD, Section 3.9, Table 4., Section 5.0, Table 5.
+        </Value>
+        </Attribute>
+        <Attribute name="contentType" type="String">
+        <Value>modelResult</Value>
+        </Attribute>
+        <Attribute name="description" type="String">
+        <Value>
+        The global apparent surface reflectance (ASR) observation count two-dimensional gridded parameter. The number of observations used to compute the average global apparent surface reflectance (ASR). Only surface signal detected ASR 25 Hz (high-rate profile) observations (apparent_surf_reflec () > 0.0) in each grid cell are used to compute the cell global ASR (global_asr (i,j)). Grid array organization: global_asr_obs_grid (i,j), where i=longitude index, j=latitude index, as per ATBD Section 2.0. Given the weekly global_grid_lon_scale = 3.0 degrees and the global_grid_lat_scale = 3.0 degrees, the weekly global gridded array dimension is: global_asr_obs_grid (120,60), type: floating point . Reference: global_asr_obs_grid (1,1) = (-180.0 degrees longitude, -90.0 degrees latitude), representing the lower left corner of the gridded parameter cell in the global projection array. The global observation counts are applicable to the atmosphere gridded parameter: global_asr (i,j).
+        </Value>
+        </Attribute>
+        <Attribute name="coordinates" type="String">
+        <Value>global_grid_lat global_grid_lon</Value>
+        </Attribute>
+        <Attribute name="grid_mapping" type="String">
+        <Value>
+        crs_latlon: global_grid_lat crs_latlon: global_grid_lon
+        </Value>
+        </Attribute>
+    </Float32>
     <Attribute name="short_name" type="String">
         <Value>ATL16</Value>
     </Attribute>

tests/data/README.md

@@ -99,15 +99,16 @@
     down to only contain the six required dimension variables.<br><br>
 
 * ATL16_prefetch.nc4
-  - A sample output file that contains only the six required dimension variables for
-    the ATL16 collection.<br><br>
+  - A sample output file that contains the six required dimension variables and one
+    requested science variable in the ATL16 collection.<br><br>
 
 * ATL16_prefetch_group.dmr
   - An example `.dmr` file that is identical to the `ATL16_prefetch.dmr` file
-    except for an additional fabricated nested group variable, whereas all the variables in the  ATL16 collection are in the root directory.<br><br>
+    except for an additional fabricated nested group variable, whereas all the variables in the
+    ATL16 collection are in the root directory.<br><br>
 
 * ATL16_prefetch_group.nc4
-  - A sample output file that is identical to the `ATL16_prefetch.nc4` file except
+  - A sample output file that is nearly identical to the `ATL16_prefetch.nc4` file except
     for an additional fabricated nested group variable (the same one in the
     ATL16_prefetch_group.dmr file).<br><br>
 

tests/test_adapter.py

@@ -32,6 +32,7 @@ def setUpClass(cls):
         cls.atl03_variable = '/gt1r/geophys_corr/geoid'
         cls.gpm_variable = '/Grid/precipitationCal'
         cls.rssmif16d_variable = '/wind_speed'
+        cls.atl16_variable = '/global_asr_obs_grid'
         cls.staging_location = 's3://example-bucket/'
 
         with open('tests/data/ATL03_example.dmr', 'r') as file_handler:
@@ -46,6 +47,9 @@ def setUpClass(cls):
         with open('tests/data/GPM_3IMERGHH_example.dmr', 'r') as file_handler:
             cls.gpm_imerghh_dmr = file_handler.read()
 
+        with open('tests/data/ATL16_prefetch.dmr', 'r') as file_handler:
+            cls.atl16_dmr = file_handler.read()
+
     def setUp(self):
         """ Have to mock mkdtemp, to know where to put mock .dmr content. """
         self.tmp_dir = mkdtemp()
@@ -2008,3 +2012,99 @@ def test_exception_handling(self, mock_stage, mock_download_subset,
 
         mock_stage.assert_not_called()
         mock_rmtree.assert_called_once_with(self.tmp_dir)
+
+    @patch('hoss.dimension_utilities.get_fill_slice')
+    @patch('hoss.utilities.uuid4')
+    @patch('hoss.adapter.mkdtemp')
+    @patch('shutil.rmtree')
+    @patch('hoss.bbox_utilities.download')
+    @patch('hoss.utilities.util_download')
+    @patch('hoss.adapter.stage')
+    def test_edge_aligned_no_bounds_end_to_end(self, mock_stage, mock_util_download,
+                         mock_geojson_download, mock_rmtree,
+                         mock_mkdtemp, mock_uuid,
+                         mock_get_fill_slice):
+        """ Ensure a request for a collection that contains dimension variables
+            with edge-aligned grid cells is correctly processed regardless of
+            whether or not a bounds variable associated with that dimension
+            variable exists.
+
+        """
+        expected_output_basename = 'opendap_url_global_asr_obs_grid_subsetted.nc4'
+        expected_staged_url = f'{self.staging_location}{expected_output_basename}'
+
+        mock_uuid.side_effect = [Mock(hex='uuid'), Mock(hex='uuid2')]
+        mock_mkdtemp.return_value = self.tmp_dir
+        mock_stage.return_value = expected_staged_url
+
+        dmr_path = write_dmr(self.tmp_dir, self.atl16_dmr)
+
+        dimensions_path = f'{self.tmp_dir}/dimensions.nc4'
+        copy('tests/data/ATL16_prefetch.nc4', dimensions_path)
+
+        all_variables_path = f'{self.tmp_dir}/variables.nc4'
+        copy('tests/data/ATL16_variables.nc4', all_variables_path)
+
+        mock_util_download.side_effect = [dmr_path, dimensions_path,
+                                          all_variables_path]
+
+        message = Message({
+            'accessToken': 'fake-token',
+                'callback': 'https://example.com/',
+                'sources': [{
+                    'collection': 'C1238589498-EEDTEST',
+                    'shortName': 'ATL16',
+                    'variables': [{'id': '',
+                                'name': self.atl16_variable,
+                                'fullPath': self.atl16_variable}]}],
+                'stagingLocation': self.staging_location,
+                'subset': {'bbox': [77, 71.5, 88, 74.75]},
+                'user': 'jlovell',
+        })
+
+        hoss = HossAdapter(message, config=config(False), catalog=self.input_stac)
+        _, output_catalog = hoss.invoke()
+
+        # Ensure that there is a single item in the output catalog with the
+        # expected asset:
+        self.assert_expected_output_catalog(output_catalog,
+                                            expected_staged_url,
+                                            expected_output_basename)
+
+        # Ensure the expected requests were made against OPeNDAP.
+        # Q JSL - Why are there two of the same calls in a few of the test_adapter.py tests?
+        self.assertEqual(mock_util_download.call_count,3)
+        mock_util_download.assert_has_calls([
+            call(f'{self.granule_url}.dmr.xml', self.tmp_dir, hoss.logger,
+                    access_token=message.accessToken, data=None, cfg=hoss.config),
+            call(f'{self.granule_url}.dap.nc4', self.tmp_dir, hoss.logger,
+                    access_token=message.accessToken, data=ANY, cfg=hoss.config),
+            call(f'{self.granule_url}.dap.nc4', self.tmp_dir, hoss.logger,
+                    access_token=message.accessToken, data=ANY, cfg=hoss.config),
+        ])
+
+        # Ensure the constraint expression for dimensions data included only
+        # dimension variables and their associated bounds variables.
+        dimensions_data = mock_util_download.call_args_list[1][1].get('data', {})
+        self.assert_valid_request_data(
+            dimensions_data,
+            {'%2Fglobal_grid_lat',
+             '%2Fglobal_grid_lon'}
+        )
+
+        # Ensure the constraint expression contains all the required variables.
+        index_range_data = mock_util_download.call_args_list[2][1].get('data', {})
+        self.assert_valid_request_data(
+            index_range_data,
+            {'%2Fglobal_asr_obs_grid%5B53%3A54%5D%5B85%3A89%5D',
+             '%2Fglobal_grid_lat%5B53%3A54%5D',
+             '%2Fglobal_grid_lon%5B85%3A89%5D'}
+        )
+
+        # Ensure the output was staged with the expected file name
+        mock_stage.assert_called_once_with(f'{self.tmp_dir}/uuid2.nc4',
+                                            expected_output_basename,
+                                            'application/x-netcdf4',
+                                            location=self.staging_location,
+                                            logger=hoss.logger)
+        mock_rmtree.assert_called_once_with(self.tmp_dir)

tests/unit/test_dimension_utilities.py

@@ -328,20 +328,17 @@ def test_add_bounds_variables(self, mock_write_bounds, mock_needs_bounds):
         varinfo_prefetch = VarInfoFromDmr(
                 'tests/data/ATL16_prefetch.dmr'
         )
-        required_dimensions = {'/npolar_grid_lat','/npolar_grid_lon'}
+        required_dimensions = {'/npolar_grid_lat','/npolar_grid_lon',
+                               '/spolar_grid_lat','/spolar_grid_lon',
+                               '/global_grid_lat','/global_grid_lon'}
 
         with self.subTest('Bounds need to be written'):
             mock_needs_bounds.return_value = True
             add_bounds_variables(prefetch_dataset_name,
                                  required_dimensions,
                                  varinfo_prefetch,
                                  self.logger)
-
-            with Dataset(prefetch_dataset_name, 'r+') as prefetch_dataset:
-                for dimension_name in required_dimensions:
-                    dimension_variable = varinfo_prefetch.get_variable(dimension_name)
-                    mock_write_bounds.assert_called_with(prefetch_dataset,
-                                                         dimension_variable)
+            self.assertEqual(mock_write_bounds.call_count, 6)
 
             mock_needs_bounds.reset_mock()
             mock_write_bounds.reset_mock()