[Done] Test with newer Python, deprecation fixes, drop Py3.7 support (#…

…747)

* Bump dependencies and update actions files

* Fix warnings

* WIP

* Try with tables 3.7 on py3.10

* Some int32 fixes for windows

.github/workflows/test-conda.yml

@@ -10,13 +10,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: [3.8, 3.9, "3.10"]
+        python: [3.8, 3.9, "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v2
 
       - name: Setup Conda
-        uses: s-weigand/setup-conda@v1.0.5
+        uses: s-weigand/setup-conda@v1
         with:
           activate-conda: false
           conda-channels: conda-forge

.github/workflows/test-pip.yml

@@ -10,32 +10,27 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # 2018
-          - python: 3.7
-            display_name: "2018"
-            deps: "numpy==1.16.* scipy==1.3.* matplotlib==2.2.* pandas==0.23.* tables==3.5.* scikit-learn==0.20.* pyyaml numba==0.45.* llvmlite==0.29.* looseversion>=1.0.1"
-          # 2019
-          - python: 3.8
+          - python: "3.8"
             display_name: "2019"
-            deps: "numpy==1.18.* scipy==1.4.* matplotlib==3.1.* pandas==0.25.* tables==3.6.* scikit-learn==0.22.* pyyaml numba==0.47.* llvmlite==0.31.* looseversion>=1.0.1"
-          # 2020
-          - python: 3.9
+            deps: "numpy==1.18.* scipy==1.4.* matplotlib==3.1.* pandas==1.0.* tables==3.6.* scikit-learn==0.22.* numba==0.47.* llvmlite==0.31.*"
+          - python: "3.9"
             display_name: "2020"
-            deps: "numpy==1.19.* scipy==1.5.* matplotlib==3.3.* pandas==1.1.* tables==3.6.* scikit-learn==0.24.* pyyaml numba==0.53.* llvmlite==0.36.* looseversion>=1.0.1"
-          # 2021 (use Py3.9 & numpy==1.20 due to numba compatibility)
-          - python: 3.9
+            deps: "numpy==1.19.* scipy==1.5.* matplotlib==3.3.* pandas==1.1.* tables==3.6.* scikit-learn==0.24.* numba==0.53.* llvmlite==0.36.*"
+          - python: "3.10"
             display_name: "2021"
-            deps: "numpy==1.20.* scipy==1.7.* matplotlib==3.5.* pandas==1.3.* tables==3.6.* scikit-learn==1.0.* pyyaml numba==0.54.* llvmlite==0.37.* looseversion>=1.0.1"
-          # most recent
-          - python: '3.x'
-            display_name: "latest, no optional deps"
-            deps: "numpy scipy matplotlib pandas pyyaml looseversion"
+            deps: "numpy==1.22.* scipy==1.7.* matplotlib==3.5.* pandas==1.3.* tables==3.7.* scikit-learn==1.0.* numba==0.55.* llvmlite==0.38.*"
+          - python: "3.11"
+            display_name: "2022"
+            deps: "numpy==1.24.* scipy==1.9.* matplotlib==3.6.* pandas==2.0.* tables==3.8.* scikit-learn==1.1.* numba==0.57.* llvmlite==0.40.*"
+          - python: "3.12"
+            display_name: "2023, no numba"  # no numba support yet
+            deps: "numpy==1.26.* scipy==1.11.* matplotlib==3.8.* pandas==2.1.* tables==3.9.* scikit-learn==1.3.*"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -45,9 +40,8 @@ jobs:
       - name: Install python dependencies
         shell: bash
         run: |
-          pip install --disable-pip-version-check --upgrade pip wheel
-          pip install --no-build-isolation pytest ${{ matrix.numpy }};
-          pip install --no-build-isolation ${{ matrix.deps }};
+          pip install --disable-pip-version-check --upgrade pip setuptools wheel
+          pip install -v -e .[test] ${{ matrix.deps }};
           pip list
 
       - name: Run tests

doc/releases/v0.6.txt

@@ -1,3 +1,11 @@
+v0.6.2
+------
+
+trackpy v0.6.2 includes version compatability fixes with newer library versions.
+Also it drops support for Python 3.7 and earlier, NumPy 1.17 and earlier, Pandas 0.x,
+and SciPy 1.2 and earlier.
+
+
 v0.6.1
 ------
 

setup.py

@@ -25,14 +25,11 @@
     author = "Trackpy Contributors",
     author_email = "[email protected]",
     url = "https://github.com/soft-matter/trackpy",
-    install_requires = ['numpy>=1.14', 'scipy>=1.1', 'pandas>=0.22', 'pyyaml', 'matplotlib', "looseversion>=1.0.1"],
-    python_requires=">=3.7",
+    install_requires = ['numpy>=1.18', 'scipy>=1.4', 'pandas>=1', 'pyyaml', 'matplotlib', "looseversion>=1.0.1"],
+    extras_require={"test": "pytest"},
+    python_requires=">=3.8",
     classifiers=[
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
     ],
     packages = ['trackpy', 'trackpy.refine', 'trackpy.linking', 'trackpy.locate_functions'],
     long_description = descr,

trackpy/linking/linking.py

@@ -181,9 +181,9 @@ def link(f, search_range, pos_columns=None, t_column='frame', **kwargs):
 
     # copy the dataframe
     f = f.copy()
-    # coerce t_column to integer type
+    # coerce t_column to integer type (use np.int64 to avoid 32-bit on windows)
     if not np.issubdtype(f[t_column].dtype, np.integer):
-        f[t_column] = f[t_column].astype(int)
+        f[t_column] = f[t_column].astype(np.int64)
     # sort on the t_column
     pandas_sort(f, t_column, inplace=True)
 

trackpy/refine/least_squares.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import warnings
 import numpy as np
@@ -457,6 +458,16 @@ def prepare_subimages(coords, groups, frame_nos, reader, radius):
     return images, meshes, masks
 
 
+def ignore_clip_warnings(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", ".*outside bounds during a minimize step.*", RuntimeWarning, "scipy.optimize.*")
+            return func(*args, **kwargs)
+    return wrapper
+
+
+@ignore_clip_warnings
 def refine_leastsq(f, reader, diameter, separation=None, fit_function='gauss',
                    param_mode=None, param_val=None, constraints=None,
                    bounds=None, compute_error=False, pos_columns=None,

trackpy/tests/test_feature_saving.py

@@ -36,7 +36,7 @@ def _skip_if_no_pytables():
         raise unittest.SkipTest('pytables not installed. Skipping.')
 
     # https://github.com/soft-matter/trackpy/issues/643
-    if tables.get_hdf5_version() == "1.8.5-patch1":
+    if tables.hdf5_version == "1.8.5-patch1":
         raise unittest.SkipTest('this pytables version has an incompatible HDF5 version. Skipping.')
 
 class FeatureSavingTester:

trackpy/tests/test_leastsq.py

@@ -11,7 +11,14 @@
 from trackpy.refine.least_squares import (dimer, trimer, tetramer, dimer_global,
                                           FitFunctions, vect_from_params)
 from trackpy.tests.common import assert_coordinates_close, StrictTestCase
-from scipy.optimize.slsqp import approx_jacobian
+import warnings
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    try:
+        from scipy.optimize.slsqp import approx_jacobian
+    except ImportError:
+        approx_jacobian = None
 
 
 EPSILON = 1E-7
@@ -923,6 +930,8 @@ def get_residual(self, ff, n, params, groups=None):
 
     def compare_jacobian(self, fit_function, ndim, isotropic, n, groups=None,
                          custom_param_mode=None):
+        if approx_jacobian is None:
+            raise SkipTest("This test requires a function that is deprecated in Scipy 2.0")
         ff = FitFunctions(fit_function, ndim, isotropic)
         param_mode = {param: 'var' for param in ff.params}
         param_mode['background'] = 'cluster'

trackpy/tests/test_linking.py

@@ -618,15 +618,16 @@ def f_iter(f, first_frame, last_frame):
         res['particle'] = -1
         for t, ids in link_iter(f_iter(f, 0, int(f['frame'].max())),
                                 search_range, *args, **kwargs):
-            res.loc[res['frame'] == t, 'particle'] = ids
+            if ids:
+                res.loc[res['frame'] == t, 'particle'] = ids
         return pandas_sort(res, ['particle', 'frame']).reset_index(drop=True)
 
     def test_output_dtypes(self):
         N = 5
         f = DataFrame({'x': np.arange(N), 'y': np.ones(N),
                        'frame': np.arange(N)})
         # Integer-typed input
-        f['frame'] = f['frame'].astype(int)
+        f['frame'] = f['frame'].astype(np.int64)
         actual = self.link(f, 5)
 
         # Particle and frame columns should be integer typed

trackpy/tests/test_motion.py

@@ -20,7 +20,7 @@ def random_walk(N):
 def conformity(df):
     """ Organize toy data to look like real data. Be strict about dtypes:
     particle is a float and frame is an integer."""
-    df['frame'] = df['frame'].astype(int)
+    df['frame'] = df['frame'].astype(np.int64)  # pandas maps to int32 on windows!
     df['particle'] = df['particle'].astype(float)
     df['x'] = df['x'].astype(float)
     df['y'] = df['y'].astype(float)

trackpy/tests/test_reproducibility.py

@@ -115,7 +115,7 @@ def test_locate(self):
     def test_link_nomemory(self):
         expected = pd.DataFrame(self.coords_link,
                                 columns=self.pos_columns + ['frame'])
-        expected['frame'] = expected['frame'].astype(int)
+        expected['frame'] = expected['frame'].astype(np.int64)
         actual = tp.link(expected, **self.link_params)
         expected['particle'] = self.expected_link
 
@@ -124,7 +124,7 @@ def test_link_nomemory(self):
     def test_link_memory(self):
         expected = pd.DataFrame(self.coords_link,
                                 columns=self.pos_columns + ['frame'])
-        expected['frame'] = expected['frame'].astype(int)
+        expected['frame'] = expected['frame'].astype(np.int64)
         actual = tp.link(expected, memory=self.memory, **self.link_params)
         expected['particle'] = self.expected_link_memory
 

trackpy/uncertainty.py

@@ -1,5 +1,5 @@
 import numpy as np
-from scipy.ndimage import morphology
+from scipy.ndimage import binary_dilation
 from pandas import DataFrame
 
 from .masks import binary_mask, x_squared_masks
@@ -26,7 +26,7 @@ def measure_noise(image_bp, image_raw, radius):
     background mean, background standard deviation
     """
     structure = binary_mask(radius, image_bp.ndim)
-    background = ~morphology.binary_dilation(image_bp, structure=structure)
+    background = ~binary_dilation(image_bp, structure=structure)
     n_background = background.sum()
     if n_background == 0:  # edge case of no background identified
         return np.nan, np.nan

trackpy/utils.py

@@ -4,7 +4,6 @@
 import sys
 import warnings
 from collections.abc import Hashable
-from contextlib import contextmanager
 from datetime import datetime, timedelta
 from looseversion import LooseVersion
 from multiprocessing.pool import Pool
@@ -198,7 +197,7 @@ def validate_tuple(value, ndim):
 
 
 try:
-    from IPython.core.display import clear_output
+    from IPython.display import clear_output
 except ImportError:
     pass
 
@@ -418,16 +417,14 @@ def get_pool(processes):
     return pool, map_func
 
 
-def stats_mode_scalar(*args, **kwargs):
+def stats_mode_scalar(a):
     """Returns a scalar from scipy.stats.mode().
 
-    Works around new default in scipy 1.11 to eliminate extra axes
-    that were previously removed by trackpy.
-
     See https://docs.scipy.org/doc/scipy-1.9.3/reference/generated/scipy.stats.mode.html
     """
-    result = stats.mode(*args, **kwargs)[0]
     try:
-        return result[0]  # Old behavior
-    except IndexError:
-        return result  # scipy >= 1.11
+        # scipy >= 1.11
+        return stats.mode(a, keepdims=False).mode
+    except TypeError:
+        # Old behavior (keepdims is not accepted)
+        return stats.mode(a).mode[0]