Merge pull request #1546 from emlys/task/916

Move GDAL constraint into constraints file and update to GDAL 3.8

.github/workflows/build-and-test.yml

@@ -79,7 +79,6 @@ jobs:
         with:
           fetch-depth: 0 # Fetch complete history for accurate versioning
 
-
       # NOTE: It takes twice as long to save the sample data cache
       # as it does to do a fresh clone (almost 5 minutes vs. 2.5 minutes)
       # Test data is way, way faster by contrast (on the order of a few
@@ -94,7 +93,7 @@ jobs:
         uses: ./.github/actions/setup_env
         with:
           python-version: ${{ matrix.python-version }}
-          requirements-files: requirements.txt requirements-dev.txt
+          requirements-files: requirements.txt requirements-dev.txt constraints_tests.txt
           requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }}
 
       - name: Download previous conda environment.yml

.readthedocs_environment.yml

@@ -11,7 +11,7 @@ channels:
 - nodefaults
 dependencies:
 - python=3.11
-- gdal>=3.4.2,<3.6.0
+- gdal>=3.4.2
 - pip
 - pip:
   - -r requirements.txt

HISTORY.rst

@@ -62,6 +62,9 @@ Unreleased Changes
         versions of InVEST would skip these parameters' type-specific
         validation. Now, these parameters will be validated with their
         type-specific validation checks.
+    * Add support for latest GDAL versions; remove test-specific constraint on
+      GDAL versions from invest requirements.
+      https://github.com/natcap/invest/issues/916
 * Annual Water Yield
     * Added the results_suffix to a few intermediate files where it was
       missing. https://github.com/natcap/invest/issues/1517
@@ -70,6 +73,9 @@ Unreleased Changes
       raster. ``nan`` pixels will now be propertly ignored before calculating
       mean depths along fetch rays.
       https://github.com/natcap/invest/issues/1528
+* HRA
+    * Fixed a bug where habitat and stressor vectors were not being rasterized
+      with the `ALL_TOUCHED=TRUE` setting.
 * SDR
     * Fixed an issue encountered in the sediment deposition function where
       rasters with more than 2^32 pixels would raise a cryptic error relating

constraints_tests.txt

@@ -0,0 +1,7 @@
+# This file contains package constraints needed to run the invest test suite.
+# It follows the pip constraints file format:
+# https://pip.pypa.io/en/stable/user_guide/#constraints-files
+
+# A gdal bug caused our test suite to fail, but this issue is unlikely to
+# occur with regular use of invest. https://github.com/OSGeo/gdal/issues/8497
+GDAL!=3.6.*,!=3.7.*

pyproject.toml

@@ -78,4 +78,4 @@ where = ["src"]
 
 [tool.pytest.ini_options]
 # raise warnings to errors, except for deprecation warnings
-filterwarnings = ["error", "default::DeprecationWarning"]
+filterwarnings = ["error", "default::DeprecationWarning", "default::FutureWarning"]

requirements.txt

@@ -10,7 +10,7 @@
 # scripts/convert-requirements-to-conda-yml.py as though it can only be found
 # on pip.
 
-GDAL>=3.4.2,<3.6.0
+GDAL>=3.4.2
 Pyro4==4.77  # pip-only
 pandas>=1.2.1
 numpy>=1.11.0,!=1.16.0

src/natcap/invest/hra.py

@@ -633,10 +633,6 @@ def execute(args):
         # If the input is a vector, reproject to the AOI SRS and simplify.
         # Rasterization happens in the alignment step.
         elif gis_type == pygeoprocessing.VECTOR_TYPE:
-            # Habitats and stressors are rasterized with ALL_TOUCHED=TRUE
-            if name in habitats_info or name in stressors_info:
-                habitat_stressor_vectors.add(source_filepath)
-
             # Using Shapefile here because its driver appears to not raise a
             # warning if a MultiPolygon geometry is inserted into a Polygon
             # layer, which was happening on a real-world sample dataset while
@@ -680,6 +676,10 @@ def execute(args):
                 dependent_task_list=[reprojected_vector_task]
             ))
 
+            # Habitats and stressors are rasterized with ALL_TOUCHED=TRUE
+            if name in habitats_info or name in stressors_info:
+                habitat_stressor_vectors.add(target_simplified_vector)
+
         # Later operations make use of the habitats rasters or the stressors
         # rasters, so it's useful to collect those here now.
         if name in habitats_info:
@@ -1648,7 +1648,6 @@ def _simplify(source_vector_path, tolerance, target_vector_path,
     for source_feature in source_layer:
         target_feature = ogr.Feature(target_layer_defn)
         source_geom = source_feature.GetGeometryRef()
-
         simplified_geom = source_geom.SimplifyPreserveTopology(tolerance)
         if simplified_geom is not None:
             target_geom = simplified_geom

src/natcap/invest/ndr/ndr_core.pyx

@@ -415,9 +415,9 @@ def ndr_eff_calculation(
 
     # create direction raster in bytes
     def _mfd_to_flow_dir_op(mfd_array):
-        result = numpy.zeros(mfd_array.shape, dtype=numpy.int8)
+        result = numpy.zeros(mfd_array.shape, dtype=numpy.uint8)
         for i in range(8):
-            result[:] |= (((mfd_array >> (i*4)) & 0xF) > 0) << i
+            result[:] |= ((((mfd_array >> (i*4)) & 0xF) > 0) << i).astype(numpy.uint8)
         return result
 
     pygeoprocessing.raster_calculator(

tests/test_validation.py

@@ -2112,6 +2112,7 @@ def test_spatial_overlap_error(self):
         layer = vector.CreateLayer('layer', vector_srs, ogr.wkbPoint)
         new_feature = ogr.Feature(layer.GetLayerDefn())
         new_feature.SetGeometry(ogr.CreateGeometryFromWkt('POINT (1 1)'))
+        layer.CreateFeature(new_feature)
 
         new_feature = None
         layer = None