Skip to content
Regression

Regression #90

Sign in to view logs

Regression

Regression #90

GitHub Actions / Regression test results for ops failed Jul 1, 2024 in 0s

54 fail, 522 pass in 41m 10s

576 tests   522 ✅  41m 10s ⏱️
  1 suites    0 💤
  1 files     54 ❌

Results for commit 429531c.

Annotations

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2832224417-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 42s]
Raw output 
Failed: Unable to find latitude and longitude variables.
collection_concept_id = 'C2832224417-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2832224417-POCLOUD', 'concept-id': 'G3071779083-POCLOUD', 'concept-type': 'granul...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2832224417-POCLOUD'}]}, 'meta': {'association-details': {'collecti...me': 'look', 'Size': 2, 'Type': 'OTHER'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': -9999.0}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C2832224410')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

dataset = <xarray.Dataset> Size: 232B
Dimensions:                 (ydim_grid: 1, xdim_grid: 1, look: 1,
                        ...                                 -0.43
    history_json:                                       [{"date_time": "2024-...
file_to_subset = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C2832224410/69312933_RSS_SMAP_SSS_L2C_r49943_20240607T102958_2024159_NRT_V06.0_001.nc4')
collection_variable_list = [{'associations': {'collections': [{'concept-id': 'C2832224417-POCLOUD'}]}, 'meta': {'association-details': {'collecti...me': 'look', 'Size': 2, 'Type': 'OTHER'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': -9999.0}], ...}}, ...]

    def get_lat_lon_var_names(dataset: xarray.Dataset, file_to_subset: str, collection_variable_list: List[Dict]):
        # Try getting it from UMM-Var first
        lat_var_json, lon_var_json, _ = get_coordinate_vars_from_umm(collection_variable_list)
        lat_var_name = get_variable_name_from_umm_json(lat_var_json)
        lon_var_name = get_variable_name_from_umm_json(lon_var_json)
    
        if lat_var_name and lon_var_name:
            return lat_var_name, lon_var_name
    
        logging.warning("Unable to find lat/lon vars in UMM-Var")
    
        # If that doesn't work, try using cf-xarray to infer lat/lon variable names
        try:
            latitude = [lat for lat in dataset.cf.coordinates['latitude']
                             if lat.lower() in VALID_LATITUDE_VARIABLE_NAMES][0]
            longitude = [lon for lon in dataset.cf.coordinates['longitude']
                             if lon.lower() in VALID_LONGITUDE_VARIABLE_NAMES][0]
            return latitude, longitude
        except:
            logging.warning("Unable to find lat/lon vars using cf_xarray")
    
        # If that still doesn't work, try using l2ss-py directly
        try:
            # file not able to be flattened unless locally downloaded
            shutil.copy(file_to_subset, 'my_copy_file.nc')
            nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
            # flatten the dataset
            nc_dataset_flattened = podaac.subsetter.group_handling.transform_grouped_dataset(nc_dataset, 'my_copy_file.nc')
    
            args = {
                    'decode_coords': False,
                    'mask_and_scale': False,
                    'decode_times': False
                    }
    
            with xarray.open_dataset(
                xarray.backends.NetCDF4DataStore(nc_dataset_flattened),
                **args
                ) as flat_dataset:
                    # use l2ss-py to find lat and lon names
                    lat_var_names, lon_var_names = podaac.subsetter.subset.compute_coordinate_variable_names(flat_dataset)
    
            os.remove('my_copy_file.nc')
            if lat_var_names and lon_var_names:
                lat_var_name = lat_var_names.split('__')[-1] if isinstance(lat_var_names, str) else lat_var_names[0].split('__')[-1]
                lon_var_name = lon_var_names.split('__')[-1] if isinstance(lon_var_names, str) else lon_var_names[0].split('__')[-1]
                return lat_var_name, lon_var_name
    
        except ValueError:
            logging.warning("Unable to find lat/lon vars using l2ss-py")
    
        # Still no dice, try using the 'units' variable attribute
        for coord_name, coord in dataset.coords.items():
            if 'units' not in coord.attrs:
                continue
            if coord.attrs['units'] == 'degrees_north' and lat_var_name is None:
                lat_var_name = coord_name
            if coord.attrs['units'] == 'degrees_east' and lon_var_name is None:
                lon_var_name = coord_name
        if lat_var_name and lon_var_name:
            return lat_var_name, lon_var_name
        else:
            logging.warning("Unable to find lat/lon vars using 'units' attribute")
    
        # Out of options, fail the test because we couldn't determine lat/lon variables
>       pytest.fail(f"Unable to find latitude and longitude variables.")
E       Failed: Unable to find latitude and longitude variables.

verify_collection.py:358: Failed
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3071779083-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2832224417-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-82.28062499999999%3A82.08362499999998%29&subset=lon%284.504874999999998%3A175.500125%29&granuleId=G3071779083-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 4abf94db-f744-4ef2-ad62-ec9ac43a46ba
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C2832224410/69312933_RSS_SMAP_SSS_L2C_r49943_20240607T102958_2024159_NRT_V06.0_001.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
WARNING  root:verify_collection.py:342 Unable to find lat/lon vars using l2ss-py
WARNING  root:verify_collection.py:355 Unable to find lat/lon vars using 'units' attribute

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1627516287-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 32s]
Raw output 
OSError: [Errno group not found: PRODUCT] 'PRODUCT'
ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f0f2c9cda40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f0f351176d0>

    def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
        if group in {None, "", "/"}:
            # use the root group
            return ds
        else:
            # make sure it's a string
            if not isinstance(group, str):
                raise ValueError("group must be a string or None")
            # support path-like syntax
            path = group.strip("/").split("/")
            for key in path:
                try:
>                   ds = ds.groups[key]
E                   KeyError: 'PRODUCT'

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:190: KeyError

During handling of the above exception, another exception occurred:

collection_concept_id = 'C1627516287-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1627516287-GES_DISC', 'concept-id': 'G2084463561-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1627516287-GES_DISC'}]}, 'meta': {'association-details': {'collect...'URL': 'https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/corner', ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1627516280')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
        lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
        lat_var_name = lat_var_name.split('/')[-1]
        lon_var_name = lon_var_name.split('/')[-1]
    
        with netCDF4.Dataset(subsetted_filepath) as f:
            group_list = []
            def group_walk(groups, nc_d, current_group):
                global subsetted_ds_new
                subsetted_ds_new = None
                # check if the top group has lat or lon variable
                if lat_var_name in list(nc_d.variables.keys()):
                    subsetted_ds_new = subsetted_ds
                else:
                    # if not then we'll need to keep track of the group layers
                    group_list.append(current_group)
    
                # loop through the groups in the current layer
                for g in groups:
                    # end the loop if we've already found latitude
                    if subsetted_ds_new:
                        break
                    # check if the groups have latitude, define the dataset and end the loop if found
                    if lat_var_name in list(nc_d.groups[g].variables.keys()):
                        group_list.append(g)
                        g = '/'.join(group_list)
                        subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
                        break
                    # recall the function on a group that has groups in it and didn't find latitude
                    # this is going 'deeper' into the groups
                    if len(list(nc_d.groups[g].groups.keys())) > 0:
                        group_walk(nc_d.groups[g].groups, nc_d.groups[g], g)
                    else:
                        continue
    
>           group_walk(f.groups, f, '')

verify_collection.py:432: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:423: in group_walk
    subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/api.py:571: in open_dataset
    backend_ds = backend.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:646: in open_dataset
    store = NetCDF4DataStore.open(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:409: in open
    return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:356: in __init__
    self.format = self.ds.data_model
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:418: in ds
    return self._acquire()
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:413: in _acquire
    ds = _nc4_require_group(root, self._group, self._mode)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f0f2c9cda40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f0f351176d0>

    def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
        if group in {None, "", "/"}:
            # use the root group
            return ds
        else:
            # make sure it's a string
            if not isinstance(group, str):
                raise ValueError("group must be a string or None")
            # support path-like syntax
            path = group.strip("/").split("/")
            for key in path:
                try:
                    ds = ds.groups[key]
                except KeyError as e:
                    if mode != "r":
                        ds = create_group(ds, key)
                    else:
                        # wrap error to provide slightly more helpful message
>                       raise OSError(f"group not found: {key}", e)
E                       OSError: [Errno group not found: PRODUCT] 'PRODUCT'

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:196: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G2084463561-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1627516287-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-76.99937499999999%3A-59.951625%29&subset=lon%28-76.6214%3A-1.5866000000000042%29&granuleId=G2084463561-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 3508f774-46c2-4b90-b5b1-eb8964945b07
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1627516280/69313032_S5P_OFFL_L2_CO_20210701T170324_20210701T184453_19257_01_010400_20210703T065107_subsetted.nc4

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2087216530-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 39s]
Raw output 
OSError: [Errno group not found: PRODUCT] 'PRODUCT'
ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f8b6a931b40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f8b6f279b40>

    def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
        if group in {None, "", "/"}:
            # use the root group
            return ds
        else:
            # make sure it's a string
            if not isinstance(group, str):
                raise ValueError("group must be a string or None")
            # support path-like syntax
            path = group.strip("/").split("/")
            for key in path:
                try:
>                   ds = ds.groups[key]
E                   KeyError: 'PRODUCT'

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:190: KeyError

During handling of the above exception, another exception occurred:

collection_concept_id = 'C2087216530-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2087216530-GES_DISC', 'concept-id': 'G3137865928-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2087216530-GES_DISC'}]}, 'meta': {'association-details': {'collect...Var', 'URL': 'https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/layer'}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C2087216530')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
        lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
        lat_var_name = lat_var_name.split('/')[-1]
        lon_var_name = lon_var_name.split('/')[-1]
    
        with netCDF4.Dataset(subsetted_filepath) as f:
            group_list = []
            def group_walk(groups, nc_d, current_group):
                global subsetted_ds_new
                subsetted_ds_new = None
                # check if the top group has lat or lon variable
                if lat_var_name in list(nc_d.variables.keys()):
                    subsetted_ds_new = subsetted_ds
                else:
                    # if not then we'll need to keep track of the group layers
                    group_list.append(current_group)
    
                # loop through the groups in the current layer
                for g in groups:
                    # end the loop if we've already found latitude
                    if subsetted_ds_new:
                        break
                    # check if the groups have latitude, define the dataset and end the loop if found
                    if lat_var_name in list(nc_d.groups[g].variables.keys()):
                        group_list.append(g)
                        g = '/'.join(group_list)
                        subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
                        break
                    # recall the function on a group that has groups in it and didn't find latitude
                    # this is going 'deeper' into the groups
                    if len(list(nc_d.groups[g].groups.keys())) > 0:
                        group_walk(nc_d.groups[g].groups, nc_d.groups[g], g)
                    else:
                        continue
    
>           group_walk(f.groups, f, '')

verify_collection.py:432: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:423: in group_walk
    subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/api.py:571: in open_dataset
    backend_ds = backend.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:646: in open_dataset
    store = NetCDF4DataStore.open(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:409: in open
    return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:356: in __init__
    self.format = self.ds.data_model
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:418: in ds
    return self._acquire()
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:413: in _acquire
    ds = _nc4_require_group(root, self._group, self._mode)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f8b6a931b40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f8b6f279b40>

    def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
        if group in {None, "", "/"}:
            # use the root group
            return ds
        else:
            # make sure it's a string
            if not isinstance(group, str):
                raise ValueError("group must be a string or None")
            # support path-like syntax
            path = group.strip("/").split("/")
            for key in path:
                try:
                    ds = ds.groups[key]
                except KeyError as e:
                    if mode != "r":
                        ds = create_group(ds, key)
                    else:
                        # wrap error to provide slightly more helpful message
>                       raise OSError(f"group not found: {key}", e)
E                       OSError: [Errno group not found: PRODUCT] 'PRODUCT'

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:196: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3137865928-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2087216530-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-76.829675%3A-59.831325%29&subset=lon%28-141.291925%3A146.978925%29&granuleId=G3137865928-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 740bc36f-ca0c-4a8d-a0b8-b6fa20db22ce
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C2087216530/69313041_S5P_OFFL_L2_CH4_20240628T015652_20240628T033822_34755_03_020600_20240629T174010_subsetted.nc4

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2832221740-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 49s]
Raw output 
harmony.harmony.ProcessingFailedException: WorkItem failed: podaac/l2ss-py:2.10.0: Service request failed with an unknown error
collection_concept_id = 'C2832221740-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2832221740-POCLOUD', 'concept-id': 'G3059748792-POCLOUD', 'concept-type': 'granul...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2832221740-POCLOUD'}]}, 'meta': {'association-details': {'collecti...rization_2', 'Size': 2, 'Type': 'OTHER'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': -9999.0}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C2832221740')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
>       harmony_client.wait_for_processing(job_id, show_progress=True)

verify_collection.py:386: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <harmony.harmony.Client object at 0x7f77fa72ded0>
job_id = '08340feb-62bb-4a17-909c-e404c033ccb1', show_progress = True

    def wait_for_processing(self, job_id: str, show_progress: bool = False) -> None:
        """Retrieve a submitted job's completion status in percent.
    
        Args:
            job_id: UUID string for the job you wish to interrogate.
    
        Returns:
            The job's processing progress as a percentage.
    
        :raises
            Exception: This can happen if an invalid job_id is provided or Harmony services
            can't be reached.
        """
        # How often to refresh the screen for progress updates and animating spinners.
        ui_update_interval = 0.33  # in seconds
        running_w_errors_logged = False
    
        intervals = round(self.check_interval / ui_update_interval)
        if show_progress:
            with progressbar.ProgressBar(max_value=100, widgets=progressbar_widgets) as bar:
                progress = 0
                while progress < 100:
                    progress, status, message = self.progress(job_id)
                    if status == 'failed':
>                       raise ProcessingFailedException(job_id, message)
E                       harmony.harmony.ProcessingFailedException: WorkItem failed: podaac/l2ss-py:2.10.0: Service request failed with an unknown error

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:986: ProcessingFailedException
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3059748792-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2832221740-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-82.29042474999999%3A82.03317474999999%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3059748792-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 08340feb-62bb-4a17-909c-e404c033ccb1

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2936721448-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 27s]
Raw output 
Failed: Unable to find latitude and longitude variables.
collection_concept_id = 'C2936721448-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2936721448-POCLOUD', 'concept-id': 'G3062447313-POCLOUD', 'concept-type': 'granul...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2936721448-POCLOUD'}]}, 'meta': {'association-details': {'collecti...rization_2', 'Size': 2, 'Type': 'OTHER'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': -9999.0}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C2936721440')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

dataset = <xarray.Dataset> Size: 240B
Dimensions:                 (ydim_grid: 1, xdim_grid: 1, look: 1,
                        ...                                 -0.43
    history_json:                                           [{"date_time": "2...
file_to_subset = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C2936721440/69313137_RSS_SMAP_SSS_L2C_r47700_20240106T014035_2024006_FNL_V05.3.nc4')
collection_variable_list = [{'associations': {'collections': [{'concept-id': 'C2936721448-POCLOUD'}]}, 'meta': {'association-details': {'collecti...rization_2', 'Size': 2, 'Type': 'OTHER'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': -9999.0}], ...}}, ...]

    def get_lat_lon_var_names(dataset: xarray.Dataset, file_to_subset: str, collection_variable_list: List[Dict]):
        # Try getting it from UMM-Var first
        lat_var_json, lon_var_json, _ = get_coordinate_vars_from_umm(collection_variable_list)
        lat_var_name = get_variable_name_from_umm_json(lat_var_json)
        lon_var_name = get_variable_name_from_umm_json(lon_var_json)
    
        if lat_var_name and lon_var_name:
            return lat_var_name, lon_var_name
    
        logging.warning("Unable to find lat/lon vars in UMM-Var")
    
        # If that doesn't work, try using cf-xarray to infer lat/lon variable names
        try:
            latitude = [lat for lat in dataset.cf.coordinates['latitude']
                             if lat.lower() in VALID_LATITUDE_VARIABLE_NAMES][0]
            longitude = [lon for lon in dataset.cf.coordinates['longitude']
                             if lon.lower() in VALID_LONGITUDE_VARIABLE_NAMES][0]
            return latitude, longitude
        except:
            logging.warning("Unable to find lat/lon vars using cf_xarray")
    
        # If that still doesn't work, try using l2ss-py directly
        try:
            # file not able to be flattened unless locally downloaded
            shutil.copy(file_to_subset, 'my_copy_file.nc')
            nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
            # flatten the dataset
            nc_dataset_flattened = podaac.subsetter.group_handling.transform_grouped_dataset(nc_dataset, 'my_copy_file.nc')
    
            args = {
                    'decode_coords': False,
                    'mask_and_scale': False,
                    'decode_times': False
                    }
    
            with xarray.open_dataset(
                xarray.backends.NetCDF4DataStore(nc_dataset_flattened),
                **args
                ) as flat_dataset:
                    # use l2ss-py to find lat and lon names
                    lat_var_names, lon_var_names = podaac.subsetter.subset.compute_coordinate_variable_names(flat_dataset)
    
            os.remove('my_copy_file.nc')
            if lat_var_names and lon_var_names:
                lat_var_name = lat_var_names.split('__')[-1] if isinstance(lat_var_names, str) else lat_var_names[0].split('__')[-1]
                lon_var_name = lon_var_names.split('__')[-1] if isinstance(lon_var_names, str) else lon_var_names[0].split('__')[-1]
                return lat_var_name, lon_var_name
    
        except ValueError:
            logging.warning("Unable to find lat/lon vars using l2ss-py")
    
        # Still no dice, try using the 'units' variable attribute
        for coord_name, coord in dataset.coords.items():
            if 'units' not in coord.attrs:
                continue
            if coord.attrs['units'] == 'degrees_north' and lat_var_name is None:
                lat_var_name = coord_name
            if coord.attrs['units'] == 'degrees_east' and lon_var_name is None:
                lon_var_name = coord_name
        if lat_var_name and lon_var_name:
            return lat_var_name, lon_var_name
        else:
            logging.warning("Unable to find lat/lon vars using 'units' attribute")
    
        # Out of options, fail the test because we couldn't determine lat/lon variables
>       pytest.fail(f"Unable to find latitude and longitude variables.")
E       Failed: Unable to find latitude and longitude variables.

verify_collection.py:358: Failed
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3062447313-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2936721448-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-82.29044999999999%3A82.08044999999998%29&subset=lon%284.51755%3A175.50045%29&granuleId=G3062447313-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 6d71f849-f13b-4198-b38c-09e2dedd2f6c
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C2936721440/69313137_RSS_SMAP_SSS_L2C_r47700_20240106T014035_2024006_FNL_V05.3.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
WARNING  root:verify_collection.py:342 Unable to find lat/lon vars using l2ss-py
WARNING  root:verify_collection.py:355 Unable to find lat/lon vars using 'units' attribute

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1627516288-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 37s]
Raw output 
OSError: [Errno group not found: PRODUCT] 'PRODUCT'
ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f0f2c78da40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f0f351176d0>

    def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
        if group in {None, "", "/"}:
            # use the root group
            return ds
        else:
            # make sure it's a string
            if not isinstance(group, str):
                raise ValueError("group must be a string or None")
            # support path-like syntax
            path = group.strip("/").split("/")
            for key in path:
                try:
>                   ds = ds.groups[key]
E                   KeyError: 'PRODUCT'

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:190: KeyError

During handling of the above exception, another exception occurred:

collection_concept_id = 'C1627516288-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1627516288-GES_DISC', 'concept-id': 'G2085128317-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1627516288-GES_DISC'}]}, 'meta': {'association-details': {'collect... 'https://cdn.earthdata.nasa.gov/umm/variable/v1.9.0', 'Version': '1.9.0'}, 'Name': 'PRODUCT/ground_pixel', ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1627516281')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
        lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
        lat_var_name = lat_var_name.split('/')[-1]
        lon_var_name = lon_var_name.split('/')[-1]
    
        with netCDF4.Dataset(subsetted_filepath) as f:
            group_list = []
            def group_walk(groups, nc_d, current_group):
                global subsetted_ds_new
                subsetted_ds_new = None
                # check if the top group has lat or lon variable
                if lat_var_name in list(nc_d.variables.keys()):
                    subsetted_ds_new = subsetted_ds
                else:
                    # if not then we'll need to keep track of the group layers
                    group_list.append(current_group)
    
                # loop through the groups in the current layer
                for g in groups:
                    # end the loop if we've already found latitude
                    if subsetted_ds_new:
                        break
                    # check if the groups have latitude, define the dataset and end the loop if found
                    if lat_var_name in list(nc_d.groups[g].variables.keys()):
                        group_list.append(g)
                        g = '/'.join(group_list)
                        subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
                        break
                    # recall the function on a group that has groups in it and didn't find latitude
                    # this is going 'deeper' into the groups
                    if len(list(nc_d.groups[g].groups.keys())) > 0:
                        group_walk(nc_d.groups[g].groups, nc_d.groups[g], g)
                    else:
                        continue
    
>           group_walk(f.groups, f, '')

verify_collection.py:432: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:423: in group_walk
    subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/api.py:571: in open_dataset
    backend_ds = backend.open_dataset(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:646: in open_dataset
    store = NetCDF4DataStore.open(
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:409: in open
    return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:356: in __init__
    self.format = self.ds.data_model
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:418: in ds
    return self._acquire()
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:413: in _acquire
    ds = _nc4_require_group(root, self._group, self._mode)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

ds = <[AttributeError('NetCDF: Not a valid ID') raised in repr()] Group object at 0x7f0f2c78da40>
group = '/METADATA/PRODUCT', mode = 'r'
create_group = <function _netcdf4_create_group at 0x7f0f351176d0>

    def _nc4_require_group(ds, group, mode, create_group=_netcdf4_create_group):
        if group in {None, "", "/"}:
            # use the root group
            return ds
        else:
            # make sure it's a string
            if not isinstance(group, str):
                raise ValueError("group must be a string or None")
            # support path-like syntax
            path = group.strip("/").split("/")
            for key in path:
                try:
                    ds = ds.groups[key]
                except KeyError as e:
                    if mode != "r":
                        ds = create_group(ds, key)
                    else:
                        # wrap error to provide slightly more helpful message
>                       raise OSError(f"group not found: {key}", e)
E                       OSError: [Errno group not found: PRODUCT] 'PRODUCT'

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/xarray/backends/netCDF4_.py:196: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G2085128317-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1627516288-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-76.99937499999999%3A-59.951625%29&subset=lon%28-76.6214%3A-1.5866000000000042%29&granuleId=G2085128317-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 9cf999c7-fdf9-449f-a5b4-4b255c3c6378
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1627516281/69313372_S5P_OFFL_L2_CH4_20210701T170324_20210701T184453_19257_01_010400_20210703T102338_subsetted.nc4

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1729926922-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 26s]
Raw output 
assert False
collection_concept_id = 'C1729926922-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1729926922-GES_DISC', 'concept-id': 'G3138391103-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1729926922-GES_DISC'}]}, 'meta': {'association-details': {'collect...9.989990234375}], 'LongName': 'HDFEOS/SWATHS/Temperature-APriori/Data Fields/Temperature-APrioriPrecision', ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1729926920')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
        lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
        lat_var_name = lat_var_name.split('/')[-1]
        lon_var_name = lon_var_name.split('/')[-1]
    
        with netCDF4.Dataset(subsetted_filepath) as f:
            group_list = []
            def group_walk(groups, nc_d, current_group):
                global subsetted_ds_new
                subsetted_ds_new = None
                # check if the top group has lat or lon variable
                if lat_var_name in list(nc_d.variables.keys()):
                    subsetted_ds_new = subsetted_ds
                else:
                    # if not then we'll need to keep track of the group layers
                    group_list.append(current_group)
    
                # loop through the groups in the current layer
                for g in groups:
                    # end the loop if we've already found latitude
                    if subsetted_ds_new:
                        break
                    # check if the groups have latitude, define the dataset and end the loop if found
                    if lat_var_name in list(nc_d.groups[g].variables.keys()):
                        group_list.append(g)
                        g = '/'.join(group_list)
                        subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
                        break
                    # recall the function on a group that has groups in it and didn't find latitude
                    # this is going 'deeper' into the groups
                    if len(list(nc_d.groups[g].groups.keys())) > 0:
                        group_walk(nc_d.groups[g].groups, nc_d.groups[g], g)
                    else:
                        continue
    
            group_walk(f.groups, f, '')
    
        assert lat_var_name and lon_var_name
    
        var_ds = None
        msk = None
    
        if science_vars := get_science_vars(collection_variables):
            for idx, value in enumerate(science_vars):
                science_var_name = science_vars[idx]['umm']['Name']
                try:
                    var_ds = subsetted_ds_new[science_var_name]
                    msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
                    break
                except Exception:
                    try:
                        # if the variable couldn't be found because the name includes a group, e.g.,
                        # `geolocation/relative_azimuth_angle`,
                        # then try to access the variable after removing the group name.
                        var_ds = subsetted_ds_new[science_var_name.rsplit("/", 1)[-1]]
                        msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
                        break
                    except Exception:
                        var_ds = None
                        msk = None
    
            if var_ds is None and msk is None:
                pytest.fail(f"Unable to find variable from umm-v to use as science variable.")
    
        else:
            # Can't find a science var in UMM-V, just pick one
    
            science_var_name = next(iter([v for v in subsetted_ds_new.variables if
                                        str(v) not in lat_var_name and str(v) not in lon_var_name and 'time' not in str(v)]))
    
            var_ds = subsetted_ds_new[science_var_name]
    
        try:
            msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
            llat = subsetted_ds_new[lat_var_name].where(msk)
            llon = subsetted_ds_new[lon_var_name].where(msk)
        except ValueError:
    
            llat = subsetted_ds_new[lat_var_name]
            llon = subsetted_ds_new[lon_var_name]
    
        lat_max = llat.max()
        lat_min = llat.min()
    
        lon_min = llon.min()
        lon_max = llon.max()
    
        lon_min = (lon_min + 180) % 360 - 180
        lon_max = (lon_max + 180) % 360 - 180
    
        lat_var_fill_value = subsetted_ds_new[lat_var_name].encoding.get('_FillValue')
        lon_var_fill_value = subsetted_ds_new[lon_var_name].encoding.get('_FillValue')
    
        if lat_var_fill_value:
            if (lat_max <= north or np.isclose(lat_max, north)) and (lat_min >= south or np.isclose(lat_min, south)):
                logging.info("Successful Latitude subsetting")
            elif np.isnan(lat_max) and np.isnan(lat_min):
                logging.info("Partial Lat Success - no Data")
            else:
                assert False
    
        if lon_var_fill_value:
            if (lon_max <= east or np.isclose(lon_max, east)) and (lon_min >= west or np.isclose(lon_min, west)):
                logging.info("Successful Longitude subsetting")
            elif np.isnan(lon_max) and np.isnan(lon_min):
                logging.info("Partial Lon Success - no Data")
            else:
>               assert False
E               assert False

verify_collection.py:504: AssertionError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3138391103-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1729926922-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3138391103-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job c4f10be4-0a79-4052-8193-bdde7c26ff97
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1729926920/69313398_MLS-Aura_L2GP-Temperature_v05-03-c01_2024d180_subsetted.nc4
INFO     root:verify_collection.py:492 Successful Latitude subsetting

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1968979550-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 37s]
Raw output 
RuntimeError: NetCDF: Problem with HDF5 dimscales.
collection_concept_id = 'C1968979550-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1968979550-POCLOUD', 'concept-id': 'G3139371404-POCLOUD', 'concept-type': 'granul...e': '2024-06-30T22:29:53.000Z'}, 'GranuleUR': 'S6A_P4_2__HR_RED__ST_134_041_20240630T002353_20240630T012006_F09', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1968979550-POCLOUD'}]}, 'meta': {'association-details': {'collecti...ic atmospheric correction [dac])', 'Dimensions': [{'Name': 'time', 'Size': 923, 'Type': 'TIME_DIMENSION'}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1968979550')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:320: in get_lat_lon_var_names
    nc_dataset_flattened = podaac.subsetter.group_handling.transform_grouped_dataset(nc_dataset, 'my_copy_file.nc')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/podaac/subsetter/group_handling.py:88: in transform_grouped_dataset
    walk(nc_dataset.groups, '')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/podaac/subsetter/group_handling.py:71: in walk
    item.renameDimension(dim_name, new_dim_name)
src/netCDF4/_netCDF4.pyx:2736: in netCDF4._netCDF4.Dataset.renameDimension
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   RuntimeError: NetCDF: Problem with HDF5 dimscales.

src/netCDF4/_netCDF4.pyx:2034: RuntimeError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3139371404-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1968979550-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-66.578993075%3A-64.215319925%29&subset=lon%28170.2907472%3A179.7510448%29&granuleId=G3139371404-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 98dba231-f112-41d9-a1e6-ec7701066f0d
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1968979550/69313589_S6A_P4_2_HR_RED_ST_134_041_20240630T002353_20240630T012006_F09_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1251101596-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 24s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1251101596-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1251101596-GES_DISC', 'concept-id': 'G3056241733-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1251101596-GES_DISC'}]}, 'meta': {'association-details': {'collect... 'Extracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -999.989990234375}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C1251101590')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3056241733-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1251101596-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3056241733-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 13820e67-86c6-40bf-9dc4-44ffc49b8706
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C1251101590/69313588_MLS-Aura_L2GP-HOCl_v04-25-c01_2024d152_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_temporal_subset[C2068529568-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 40s]
Raw output 
harmony.harmony.ProcessingFailedException: WorkItem failed: podaac/l2ss-py:2.10.0: Service request failed with an unknown error
collection_concept_id = 'C2068529568-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2068529568-POCLOUD', 'concept-id': 'G2586738585-POCLOUD', 'concept-type': 'granul...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2068529568-POCLOUD'}]}, 'meta': {'association-details': {'collecti...ze': 4193, 'Type': 'ALONG_TRACK_DIMENSION'}, {'Name': 'ni', 'Size': 243, 'Type': 'CROSS_TRACK_DIMENSION'}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw5/test_temporal_subset_C206852950')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_temporal_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
        temporal_subset = get_half_temporal_extent(start_time, end_time)
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection,
                                          granule_id=[granule_json['meta']['concept-id']],
                                          temporal=temporal_subset)
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
    
>       harmony_client.wait_for_processing(job_id, show_progress=True)

verify_collection.py:530: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <harmony.harmony.Client object at 0x7f881feec430>
job_id = '0d6ed7e4-7594-4efa-877d-378584614590', show_progress = True

    def wait_for_processing(self, job_id: str, show_progress: bool = False) -> None:
        """Retrieve a submitted job's completion status in percent.
    
        Args:
            job_id: UUID string for the job you wish to interrogate.
    
        Returns:
            The job's processing progress as a percentage.
    
        :raises
            Exception: This can happen if an invalid job_id is provided or Harmony services
            can't be reached.
        """
        # How often to refresh the screen for progress updates and animating spinners.
        ui_update_interval = 0.33  # in seconds
        running_w_errors_logged = False
    
        intervals = round(self.check_interval / ui_update_interval)
        if show_progress:
            with progressbar.ProgressBar(max_value=100, widgets=progressbar_widgets) as bar:
                progress = 0
                while progress < 100:
                    progress, status, message = self.progress(job_id)
                    if status == 'failed':
>                       raise ProcessingFailedException(job_id, message)
E                       harmony.harmony.ProcessingFailedException: WorkItem failed: podaac/l2ss-py:2.10.0: Service request failed with an unknown error

../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/harmony/harmony.py:986: ProcessingFailedException
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:511 Using granule G2586738585-POCLOUD for test
INFO     root:verify_collection.py:524 Sending harmony request https://harmony.earthdata.nasa.gov/C2068529568-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=time%28%222023-01-11T10%3A04%3A49%22%3A%222023-01-11T10%3A54%3A15%22%29&granuleId=G2586738585-POCLOUD
INFO     root:verify_collection.py:528 Submitted harmony job 0d6ed7e4-7594-4efa-877d-378584614590

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1266136100-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 43s]
Raw output 
RuntimeError: NetCDF: Problem with HDF5 dimscales.
collection_concept_id = 'C1266136100-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1266136100-GES_DISC', 'concept-id': 'G3138409665-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1266136100-GES_DISC'}]}, 'meta': {'association-details': {'collect...escription': 'Extracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': 65535}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1266136100')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:320: in get_lat_lon_var_names
    nc_dataset_flattened = podaac.subsetter.group_handling.transform_grouped_dataset(nc_dataset, 'my_copy_file.nc')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/podaac/subsetter/group_handling.py:88: in transform_grouped_dataset
    walk(nc_dataset.groups, '')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/podaac/subsetter/group_handling.py:71: in walk
    item.renameDimension(dim_name, new_dim_name)
src/netCDF4/_netCDF4.pyx:2736: in netCDF4._netCDF4.Dataset.renameDimension
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   RuntimeError: NetCDF: Problem with HDF5 dimscales.

src/netCDF4/_netCDF4.pyx:2034: RuntimeError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3138409665-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1266136100-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3138409665-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 7705fea6-2ba4-4ee7-a256-d42ab2b9b570
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C1266136100/69313613_OMI-Aura_L2G-OMCLDRRG_2024m0629_v003-2024m0630t081132_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1968980609-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 1m 19s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1968980609-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1968980609-POCLOUD', 'concept-id': 'G3139371385-POCLOUD', 'concept-type': 'granul...e': '2024-06-30T22:11:45.000Z'}, 'GranuleUR': 'S6A_P4_2__LR_STD__ST_134_041_20240630T002353_20240630T012006_F09', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1968980609-POCLOUD'}]}, 'meta': {'association-details': {'collecti...e', 'Size': 923, 'Type': 'TIME_DIMENSION'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': 32767}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1968980600')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3139371385-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1968980609-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-66.57899225%3A-64.21528775%29&subset=lon%28170.29035134999998%3A179.75103465%29&granuleId=G3139371385-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 66868071-e0db-4145-9817-52434e550849
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C1968980600/69313615_S6A_P4_2_LR_STD_ST_134_041_20240630T002353_20240630T012006_F09_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1627516290-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 41s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1627516290-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1627516290-GES_DISC', 'concept-id': 'G2897233694-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1627516290-GES_DISC'}]}, 'meta': {'association-details': {'collect...tracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': 9.969209968386869e+36}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C1627516290')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G2897233694-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1627516290-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-76.855425%3A-59.695575%29&subset=lon%28-76.722325%3A-1.608674999999998%29&granuleId=G2897233694-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 384847fd-4b1b-4189-9bce-08a2389cd6b6
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C1627516290/69313620_S5P_OFFL_L2_AER_LH_20210701T170324_20210701T184453_19257_01_010400_20210703T102339_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2732717000-LARC_CLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 31s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C2732717000-LARC_CLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2732717000-LARC_CLOUD', 'concept-id': 'G2829287584-LARC_CLOUD', 'concept-type': '...'ProductionDateTime': '2023-12-31T03:03:55+00:00'}, 'GranuleUR': 'TEMPO_HCHO_L2_V01_20231230T232423Z_S011G06.nc', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2732717000-LARC_CLOUD'}]}, 'meta': {'association-details': {'colle...acted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -1.0000000150474662e+30}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C2732717000')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G2829287584-LARC_CLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2732717000-LARC_CLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%2818.470170306725002%3A58.175364208275%29&subset=lon%28-148.0427570339%3A-114.1700878141%29&granuleId=G2829287584-LARC_CLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 0d86bca2-2d13-4b79-a896-d8ed69405557
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw8/test_spatial_subset_C2732717000/69313650_TEMPO_HCHO_L2_V01_20231230T232423Z_S011G06_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1251101402-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 25s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1251101402-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1251101402-GES_DISC', 'concept-id': 'G3056242072-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1251101402-GES_DISC'}]}, 'meta': {'association-details': {'collect... 'Extracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -999.989990234375}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1251101400')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3056242072-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1251101402-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3056242072-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 9435f4b7-939e-431a-b275-32c7e72ae166
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1251101400/69313665_MLS-Aura_L2GP-GPH_v04-25-c01_2024d152_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2799465522-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 25s]
Raw output 
RuntimeError: NetCDF: Problem with HDF5 dimscales.
collection_concept_id = 'C2799465522-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2799465522-POCLOUD', 'concept-id': 'G2958929746-POCLOUD', 'concept-type': 'granul...DateTime': '2024-04-26T10:05:05.710Z'}, 'GranuleUR': 'SWOT_GPR_2PfP008_584_20240104_024259_20240104_033426_swot', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2799465522-POCLOUD'}]}, 'meta': {'association-details': {'collecti...', 'Size': 2278, 'Type': 'TIME_DIMENSION'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': 32767}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C2799465520')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:320: in get_lat_lon_var_names
    nc_dataset_flattened = podaac.subsetter.group_handling.transform_grouped_dataset(nc_dataset, 'my_copy_file.nc')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/podaac/subsetter/group_handling.py:88: in transform_grouped_dataset
    walk(nc_dataset.groups, '')
../../../../.cache/pypoetry/virtualenvs/l2ss-py-autotest-iYz8Sff2-py3.10/lib/python3.10/site-packages/podaac/subsetter/group_handling.py:71: in walk
    item.renameDimension(dim_name, new_dim_name)
src/netCDF4/_netCDF4.pyx:2736: in netCDF4._netCDF4.Dataset.renameDimension
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   RuntimeError: NetCDF: Problem with HDF5 dimscales.

src/netCDF4/_netCDF4.pyx:2034: RuntimeError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G2958929746-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2799465522-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%2853.97788%3A77.06592%29&subset=lon%28111.37832999999999%3A178.24047%29&granuleId=G2958929746-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job abe91ec5-8b9f-4fe7-970c-85a26fa88d34
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw6/test_spatial_subset_C2799465520/69313669_SWOT_GPR_2PfP008_584_20240104_024259_20240104_033426_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1239966773-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 28s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1239966773-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1239966773-GES_DISC', 'concept-id': 'G1655407204-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1239966773-GES_DISC'}]}, 'meta': {'association-details': {'collect...acted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -1.2676506002282294e+30}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1239966770')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G1655407204-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1239966773-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-79.4358005%3A-62.2970595%29&subset=lon%28132.95362525000002%3A175.65138475%29&granuleId=G1655407204-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 894f7555-dbb8-4bbc-ac78-79538aafdb9b
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C1239966770/69313671_OMI-Aura_L2-OMCLDO2Z_2019m1112t1303-o81525_v003-2019m1112t203434_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1251101667-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 22s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1251101667-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1251101667-GES_DISC', 'concept-id': 'G3056241845-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1251101667-GES_DISC'}]}, 'meta': {'association-details': {'collect... 'Extracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -999.989990234375}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1251101660')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3056241845-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1251101667-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3056241845-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 53bc858a-44f6-4523-a7cd-5beaa2f9723b
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw3/test_spatial_subset_C1251101660/69313673_MLS-Aura_L2GP-N2O_v04-25-c01_2024d152_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2628598397-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 25s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C2628598397-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2628598397-POCLOUD', 'concept-id': 'G2767634398-POCLOUD', 'concept-type': 'granul...DateTime': '2023-09-01T10:54:31.434Z'}, 'GranuleUR': 'SWOT_IPR_2PfP002_584_20230831_221231_20230831_230358_swot', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2628598397-POCLOUD'}]}, 'meta': {'association-details': {'collecti...T Nadir Altimeter User Handbook', 'Dimensions': [{'Name': 'time', 'Size': 2885, 'Type': 'TIME_DIMENSION'}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C2628598390')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G2767634398-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2628598397-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%2853.98501%3A77.07038999999999%29&subset=lon%28111.3717%3A178.2403%29&granuleId=G2767634398-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job c5be13d8-75fe-4d54-a6c6-0b3706337756
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C2628598390/69313677_SWOT_IPR_2PfP002_584_20230831_221231_20230831_230358_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1266136073-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 57s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C1266136073-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1266136073-GES_DISC', 'concept-id': 'G3137519220-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1266136073-GES_DISC'}]}, 'meta': {'association-details': {'collect...acted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -1.2676506002282294e+30}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C1266136070')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3137519220-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1266136073-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3137519220-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job 59868012-20d0-435f-8d92-01a21046f9fa
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw1/test_spatial_subset_C1266136070/69313679_OMI-Aura_L2G-OMUVBG_2024m0626_v003-2024m0629t083002_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2799465503-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 1m 36s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C2799465503-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2799465503-POCLOUD', 'concept-id': 'G3139458281-POCLOUD', 'concept-type': 'granul...628T235720_PIC0_01.nc', 'SWOT_L2_LR_PreCalSSH_WindWave_017_278_20240628T230639_20240628T235721_PIC0_01.nc', ...], ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2799465503-POCLOUD'}]}, 'meta': {'association-details': {'collecti...ixels', 'Size': 240, 'Type': 'OTHER'}], 'FillValues': [{'Type': 'SCIENCE_FILLVALUE', 'Value': 2147483647}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C2799465501')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3139458281-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2799465503-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%2853.545795752912504%3A77.60315303058749%29&subset=lon%28112.55996904105001%3A178.27076843695%29&granuleId=G3139458281-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 7f0e6ec1-b836-4d43-9232-18b38644fedc
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C2799465501/69313695_SWOT_L2_LR_SSH_Unsmoothed_017_278_20240628T230637_20240628T235720_PIC0_01_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2296989388-POCLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 32s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C2296989388-POCLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2296989388-POCLOUD', 'concept-id': 'G2802624694-POCLOUD', 'concept-type': 'granul...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2296989388-POCLOUD'}]}, 'meta': {'association-details': {'collecti...er cloud liquid water content.', 'Dimensions': [{'Name': 'time', 'Size': 36882, 'Type': 'TIME_DIMENSION'}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C2296989380')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G2802624694-POCLOUD for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2296989388-POCLOUD/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-73.46747500000001%3A74.10647499999999%29&subset=lon%284.500975000000011%3A175.500025%29&granuleId=G2802624694-POCLOUD
INFO     root:verify_collection.py:385 Submitted harmony job 74fe3cc0-33e0-459e-a423-8839530af67b
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw9/test_spatial_subset_C2296989380/69313709_SWOT_IPRAD_2PaP006_376_20231115_225145_20231115_234320_PIB0_01_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_temporal_subset[C2724057189-LARC_CLOUD] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 7s]
Raw output 
ValueError: time data '2023-12-30T23:24:23+00:00' does not match format '%Y-%m-%dT%H:%M:%S.%fZ'
collection_concept_id = 'C2724057189-LARC_CLOUD', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2724057189-LARC_CLOUD', 'concept-id': 'G2829371222-LARC_CLOUD', 'concept-type': '... 'ProductionDateTime': '2023-12-31T03:28:32+00:00'}, 'GranuleUR': 'TEMPO_NO2_L2_V01_20231230T232423Z_S011G06.nc', ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2724057189-LARC_CLOUD'}]}, 'meta': {'association-details': {'colle...acted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -1.0000000150474662e+30}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw6/test_temporal_subset_C272405710')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_temporal_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
>       temporal_subset = get_half_temporal_extent(start_time, end_time)

verify_collection.py:515: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:167: in get_half_temporal_extent
    start_dt = datetime.strptime(start, '%Y-%m-%dT%H:%M:%S.%fZ')
/opt/hostedtoolcache/Python/3.10.14/x64/lib/python3.10/_strptime.py:568: in _strptime_datetime
    tt, fraction, gmtoff_fraction = _strptime(data_string, format)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

data_string = '2023-12-30T23:24:23+00:00', format = '%Y-%m-%dT%H:%M:%S.%fZ'

    def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
        """Return a 2-tuple consisting of a time struct and an int containing
        the number of microseconds based on the input string and the
        format string."""
    
        for index, arg in enumerate([data_string, format]):
            if not isinstance(arg, str):
                msg = "strptime() argument {} must be str, not {}"
                raise TypeError(msg.format(index, type(arg)))
    
        global _TimeRE_cache, _regex_cache
        with _cache_lock:
            locale_time = _TimeRE_cache.locale_time
            if (_getlang() != locale_time.lang or
                time.tzname != locale_time.tzname or
                time.daylight != locale_time.daylight):
                _TimeRE_cache = TimeRE()
                _regex_cache.clear()
                locale_time = _TimeRE_cache.locale_time
            if len(_regex_cache) > _CACHE_MAX_SIZE:
                _regex_cache.clear()
            format_regex = _regex_cache.get(format)
            if not format_regex:
                try:
                    format_regex = _TimeRE_cache.compile(format)
                # KeyError raised when a bad format is found; can be specified as
                # \\, in which case it was a stray % but with a space after it
                except KeyError as err:
                    bad_directive = err.args[0]
                    if bad_directive == "\\":
                        bad_directive = "%"
                    del err
                    raise ValueError("'%s' is a bad directive in format '%s'" %
                                        (bad_directive, format)) from None
                # IndexError only occurs when the format string is "%"
                except IndexError:
                    raise ValueError("stray %% in format '%s'" % format) from None
                _regex_cache[format] = format_regex
        found = format_regex.match(data_string)
        if not found:
>           raise ValueError("time data %r does not match format %r" %
                             (data_string, format))
E           ValueError: time data '2023-12-30T23:24:23+00:00' does not match format '%Y-%m-%dT%H:%M:%S.%fZ'

/opt/hostedtoolcache/Python/3.10.14/x64/lib/python3.10/_strptime.py:349: ValueError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:511 Using granule G2829371222-LARC_CLOUD for test

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C1729925806-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 26s]
Raw output 
assert False
collection_concept_id = 'C1729925806-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C1729925806-GES_DISC', 'concept-id': 'G3138391120-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C1729925806-GES_DISC'}]}, 'meta': {'association-details': {'collect...': [{'Type': 'SCIENCE_FILLVALUE', 'Value': 513}], 'LongName': 'HDFEOS/SWATHS/O3 column/Data Fields/Status', ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C1729925800')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
        lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)
        lat_var_name = lat_var_name.split('/')[-1]
        lon_var_name = lon_var_name.split('/')[-1]
    
        with netCDF4.Dataset(subsetted_filepath) as f:
            group_list = []
            def group_walk(groups, nc_d, current_group):
                global subsetted_ds_new
                subsetted_ds_new = None
                # check if the top group has lat or lon variable
                if lat_var_name in list(nc_d.variables.keys()):
                    subsetted_ds_new = subsetted_ds
                else:
                    # if not then we'll need to keep track of the group layers
                    group_list.append(current_group)
    
                # loop through the groups in the current layer
                for g in groups:
                    # end the loop if we've already found latitude
                    if subsetted_ds_new:
                        break
                    # check if the groups have latitude, define the dataset and end the loop if found
                    if lat_var_name in list(nc_d.groups[g].variables.keys()):
                        group_list.append(g)
                        g = '/'.join(group_list)
                        subsetted_ds_new = xarray.open_dataset(subsetted_filepath, group=g, decode_times=False)
                        break
                    # recall the function on a group that has groups in it and didn't find latitude
                    # this is going 'deeper' into the groups
                    if len(list(nc_d.groups[g].groups.keys())) > 0:
                        group_walk(nc_d.groups[g].groups, nc_d.groups[g], g)
                    else:
                        continue
    
            group_walk(f.groups, f, '')
    
        assert lat_var_name and lon_var_name
    
        var_ds = None
        msk = None
    
        if science_vars := get_science_vars(collection_variables):
            for idx, value in enumerate(science_vars):
                science_var_name = science_vars[idx]['umm']['Name']
                try:
                    var_ds = subsetted_ds_new[science_var_name]
                    msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
                    break
                except Exception:
                    try:
                        # if the variable couldn't be found because the name includes a group, e.g.,
                        # `geolocation/relative_azimuth_angle`,
                        # then try to access the variable after removing the group name.
                        var_ds = subsetted_ds_new[science_var_name.rsplit("/", 1)[-1]]
                        msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
                        break
                    except Exception:
                        var_ds = None
                        msk = None
    
            if var_ds is None and msk is None:
                pytest.fail(f"Unable to find variable from umm-v to use as science variable.")
    
        else:
            # Can't find a science var in UMM-V, just pick one
    
            science_var_name = next(iter([v for v in subsetted_ds_new.variables if
                                        str(v) not in lat_var_name and str(v) not in lon_var_name and 'time' not in str(v)]))
    
            var_ds = subsetted_ds_new[science_var_name]
    
        try:
            msk = np.logical_not(np.isnan(var_ds.data.squeeze()))
            llat = subsetted_ds_new[lat_var_name].where(msk)
            llon = subsetted_ds_new[lon_var_name].where(msk)
        except ValueError:
    
            llat = subsetted_ds_new[lat_var_name]
            llon = subsetted_ds_new[lon_var_name]
    
        lat_max = llat.max()
        lat_min = llat.min()
    
        lon_min = llon.min()
        lon_max = llon.max()
    
        lon_min = (lon_min + 180) % 360 - 180
        lon_max = (lon_max + 180) % 360 - 180
    
        lat_var_fill_value = subsetted_ds_new[lat_var_name].encoding.get('_FillValue')
        lon_var_fill_value = subsetted_ds_new[lon_var_name].encoding.get('_FillValue')
    
        if lat_var_fill_value:
            if (lat_max <= north or np.isclose(lat_max, north)) and (lat_min >= south or np.isclose(lat_min, south)):
                logging.info("Successful Latitude subsetting")
            elif np.isnan(lat_max) and np.isnan(lat_min):
                logging.info("Partial Lat Success - no Data")
            else:
                assert False
    
        if lon_var_fill_value:
            if (lon_max <= east or np.isclose(lon_max, east)) and (lon_min >= west or np.isclose(lon_min, west)):
                logging.info("Successful Longitude subsetting")
            elif np.isnan(lon_max) and np.isnan(lon_min):
                logging.info("Partial Lon Success - no Data")
            else:
>               assert False
E               assert False

verify_collection.py:504: AssertionError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3138391120-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C1729925806-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-85.5%3A85.5%29&subset=lon%28-171.0%3A171.0%29&granuleId=G3138391120-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job c56bac1e-ac5c-4255-aa57-a6667ba680a0
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw2/test_spatial_subset_C1729925800/69313773_MLS-Aura_L2GP-O3_v05-03-c01_2024d180_subsetted.nc4
INFO     root:verify_collection.py:492 Successful Latitude subsetting

Check warning on line 0 in tests.verify_collection

See this annotation in the file changed.

@github-actions github-actions / Regression test results for ops

test_spatial_subset[C2179081553-GES_DISC] (tests.verify_collection) failed 

test-results/ops_test_report.xml [took 1m 51s]
Raw output 
OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'
collection_concept_id = 'C2179081553-GES_DISC', env = 'ops'
granule_json = {'meta': {'collection-concept-id': 'C2179081553-GES_DISC', 'concept-id': 'G3139305336-GES_DISC', 'concept-type': 'gran...pecification': {'Name': 'UMM-G', 'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6', 'Version': '1.6.6'}, ...}}
collection_variables = [{'associations': {'collections': [{'concept-id': 'C2179081553-GES_DISC'}]}, 'meta': {'association-details': {'collect...': 'Extracted from _FillValue metadata attribute', 'Type': 'SCIENCE_FILLVALUE', 'Value': -9999.900390625}], ...}}, ...]
harmony_env = <Environment.PROD: 4>
tmp_path = PosixPath('/tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C2179081550')
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...M1SxaPfRwaV4PnJa7zbVFlZLcvBhRvf4vqzuF5YM7NH6FVZLvYlGKaafF6MKL_It0xU_qyGVtYqXMQZPxtw3-X8U0FJk7UpNs1KArBsF0dKaHHXtrfp8lA'

    @pytest.mark.timeout(600)
    def test_spatial_subset(collection_concept_id, env, granule_json, collection_variables,
                            harmony_env, tmp_path: pathlib.Path, bearer_token):
        test_spatial_subset.__doc__ = f"Verify spatial subset for {collection_concept_id} in {env}"
    
        logging.info("Using granule %s for test", granule_json['meta']['concept-id'])
    
        # Compute a box that is smaller than the granule extent bounding box
        north, south, east, west = get_bounding_box(granule_json)
        east, west, north, south = create_smaller_bounding_box(east, west, north, south, .95)
    
        start_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]
        end_time = granule_json['umm']["TemporalExtent"]["RangeDateTime"]["EndingDateTime"]
    
        # Build harmony request
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        request_bbox = harmony.BBox(w=west, s=south, e=east, n=north)
        request_collection = harmony.Collection(id=collection_concept_id)
        harmony_request = harmony.Request(collection=request_collection, spatial=request_bbox,
                                          granule_id=[granule_json['meta']['concept-id']])
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(harmony_request))
    
        # Submit harmony request and download result
        job_id = harmony_client.submit(harmony_request)
        logging.info("Submitted harmony job %s", job_id)
        harmony_client.wait_for_processing(job_id, show_progress=True)
        subsetted_filepath = None
        for filename in [file_future.result()
                         for file_future
                         in harmony_client.download_all(job_id, directory=f'{tmp_path}', overwrite=True)]:
            logging.info(f'Downloaded: %s', filename)
            subsetted_filepath = pathlib.Path(filename)
    
        # Verify spatial subset worked
        subsetted_ds = xarray.open_dataset(subsetted_filepath, decode_times=False)
        group = None
        # Try to read group in file
>       lat_var_name, lon_var_name = get_lat_lon_var_names(subsetted_ds, subsetted_filepath, collection_variables)

verify_collection.py:398: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:318: in get_lat_lon_var_names
    nc_dataset = netCDF4.Dataset('my_copy_file.nc', mode='r+')
src/netCDF4/_netCDF4.pyx:2469: in netCDF4._netCDF4.Dataset.__init__
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   ???
E   OSError: [Errno -101] NetCDF: HDF error: 'my_copy_file.nc'

src/netCDF4/_netCDF4.pyx:2028: OSError
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:365 Using granule G3139305336-GES_DISC for test
INFO     root:verify_collection.py:381 Sending harmony request https://harmony.earthdata.nasa.gov/C2179081553-GES_DISC/ogc-api-coverages/1.0.0/collections/all/coverage/rangeset?forceAsync=true&subset=lat%28-66.24418899999999%3A-60.465491%29&subset=lon%28-153.6543685%3A-123.3914915%29&granuleId=G3139305336-GES_DISC
INFO     root:verify_collection.py:385 Submitted harmony job f4f4b64a-0124-4ecf-9671-51a92e2179c8
INFO     root:verify_collection.py:391 Downloaded: /tmp/pytest-of-runner/pytest-0/popen-gw7/test_spatial_subset_C2179081550/69313777_2B.GPM.DPRGMI.CORRA2022.20240629-S222513-E235824.058706.V07C_subsetted.nc4
WARNING  root:verify_collection.py:302 Unable to find lat/lon vars in UMM-Var
WARNING  root:verify_collection.py:312 Unable to find lat/lon vars using cf_xarray
View more details on GitHub Actions