Merge branch 'main' into algorithm-in-estimation

.github/workflows/main.yml

@@ -71,9 +71,10 @@ jobs:
           micromamba activate optimagic
           pytest -m "not slow and not jax"
   run-tests-with-old-pandas:
-    # This job is only for testing if optimagic works with older pandas versions, as
-    # many pandas functions we use will be deprecated in pandas 3. optimagic's behavior
-    # for older verions is handled in src/optimagic/compat.py.
+    # This job is only for testing if optimagic works with pandas<2, as many pandas
+    # functions we use will be deprecated in pandas 3. optimagic's behavior for older
+    # verions is handled in src/optimagic/compat.py. For compatibility with we have to
+    # restrict numpy<2.
     name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with pandas 1
     runs-on: ${{ matrix.os }}
     strategy:
@@ -97,6 +98,32 @@ jobs:
         run: |
           micromamba activate optimagic
           pytest -m "not slow and not jax"
+  run-tests-with-old-numpy:
+    # This job is only for testing if optimagic works with numpy<2. Because we already
+    # test pandas<2 with numpy<2, in this environment we restrict pandas>=2.
+    name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with numpy 1
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+        python-version:
+          - '3.10'
+    steps:
+      - uses: actions/checkout@v4
+      - name: create build environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ./.tools/envs/testenv-numpy.yml
+          cache-environment: true
+          create-args: |
+            python=${{ matrix.python-version }}
+      - name: run pytest
+        shell: bash -l {0}
+        run: |
+          micromamba activate optimagic
+          pytest -m "not slow and not jax"
   code-in-docs:
     name: Run code snippets in documentation
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
         always_run: true
         require_serial: true
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
         args:
@@ -56,7 +56,7 @@ repos:
       - id: yamllint
         exclude: tests/optimagic/optimizers/_pounders/fixtures
   - repo: https://github.com/PyCQA/docformatter
-    rev: v1.7.5
+    rev: eb1df34
     hooks:
       - id: docformatter
         args:
@@ -68,7 +68,7 @@ repos:
           - --blank
         exclude: src/optimagic/optimization/algo_options.py
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.3
+    rev: v0.7.1
     hooks:
       # Run the linter.
       - id: ruff
@@ -85,7 +85,7 @@ repos:
           - pyi
           - jupyter
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.18
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -97,7 +97,7 @@ repos:
           - '88'
         files: (README\.md)
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.18
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -119,12 +119,12 @@ repos:
         args:
           - --drop-empty-cells
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.13.0
     hooks:
       - id: mypy
         files: src|tests
         additional_dependencies:
-          - numpy<2.0
+          - numpy
           - packaging
           - pandas-stubs
           - sqlalchemy-stubs

.tools/envs/testenv-linux.yml

@@ -16,14 +16,14 @@ dependencies:
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum>=0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests

.tools/envs/testenv-numpy.yml

@@ -0,0 +1,37 @@
+---
+name: optimagic
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - pandas>=2
+  - numpy<2
+  - cyipopt>=1.4.0  # dev, tests
+  - pygmo>=2.19.0  # dev, tests
+  - nlopt  # dev, tests
+  - pip  # dev, tests, docs
+  - pytest  # dev, tests
+  - pytest-cov  # tests
+  - pytest-xdist  # dev, tests
+  - statsmodels  # dev, tests
+  - cloudpickle  # run, tests
+  - joblib  # run, tests
+  - plotly  # run, tests
+  - pybaum >= 0.1.2  # run, tests
+  - scipy>=1.2.1  # run, tests
+  - sqlalchemy  # run, tests
+  - seaborn  # dev, tests
+  - mypy=1.13  # dev, tests
+  - pyyaml  # dev, tests
+  - jinja2  # dev, tests
+  - annotated-types  # dev, tests
+  - pip:  # dev, tests, docs
+      - DFO-LS  # dev, tests
+      - Py-BOBYQA  # dev, tests
+      - fides==0.7.4  # dev, tests
+      - kaleido  # dev, tests
+      - types-cffi  # dev, tests
+      - types-openpyxl  # dev, tests
+      - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
+      - -e ../../

.tools/envs/testenv-others.yml

@@ -14,14 +14,14 @@ dependencies:
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum>=0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests

.tools/envs/testenv-pandas.yml

@@ -4,7 +4,8 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - pandas<2.0.0
+  - pandas<2
+  - numpy<2
   - cyipopt>=1.4.0  # dev, tests
   - pygmo>=2.19.0  # dev, tests
   - nlopt  # dev, tests
@@ -15,13 +16,12 @@ dependencies:
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
   - plotly  # run, tests
   - pybaum>=0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests

.tools/update_envs.py

@@ -33,20 +33,27 @@ def main():
     ## test environment others
     test_env_others = deepcopy(test_env)
 
-    ## test environment for pandas version 1
+    ## test environment for pandas version < 2 (requires numpy < 2)
     test_env_pandas = deepcopy(test_env)
-    test_env_pandas = [line for line in test_env_pandas if "pandas" not in line]
-    test_env_pandas.insert(_insert_idx, "  - pandas<2.0.0")
-
-    # create docs testing environment
-
+    for pkg in ["numpy", "pandas"]:
+        test_env_pandas = [line for line in test_env_pandas if pkg not in line]
+        test_env_pandas.insert(_insert_idx, f"  - {pkg}<2")
+
+    ## test environment for numpy version < 2 (with pandas >= 2)
+    test_env_numpy = deepcopy(test_env)
+    for pkg in ["numpy", "pandas"]:
+        test_env_numpy = [line for line in test_env_numpy if pkg not in line]
+    test_env_numpy.insert(_insert_idx, "  - numpy<2")
+    test_env_numpy.insert(_insert_idx, "  - pandas>=2")
+
+    # test environment for documentation
     docs_env = [line for line in lines if _keep_line(line, "docs")]
     docs_env.append("      - -e ../../")  # add local installation
 
     # write environments
     for name, env in zip(
-        ["linux", "others", "pandas"],
-        [test_env_linux, test_env_others, test_env_pandas],
+        ["linux", "others", "pandas", "numpy"],
+        [test_env_linux, test_env_others, test_env_pandas, test_env_numpy],
         strict=False,
     ):
         # Specify newline to avoid wrong line endings on Windows.

docs/source/how_to/how_to_multistart.ipynb

@@ -436,7 +436,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "np.row_stack(res.multistart_info.exploration_sample).shape"
+    "np.vstack(res.multistart_info.exploration_sample).shape"
    ]
   },
   {

environment.yml

@@ -19,7 +19,7 @@ dependencies:
   - toml  # dev
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum>=0.1.2  # run, tests
@@ -32,7 +32,7 @@ dependencies:
   - sphinx-panels  # docs
   - sphinxcontrib-bibtex  # docs
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - furo  # dev, docs
@@ -42,7 +42,7 @@ dependencies:
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests
-      - pre-commit  # dev
+      - pre-commit>=4  # dev
       - -e .  # dev
       # type stubs
       - pandas-stubs  # dev, tests

pyproject.toml

@@ -8,7 +8,7 @@ requires-python = ">=3.10"
 dependencies = [
     "cloudpickle",
     "joblib",
-    "numpy<2.0",
+    "numpy",
     "pandas",
     "plotly",
     "pybaum>=0.1.2",

src/optimagic/algorithms.py

@@ -40,10 +40,12 @@
         name = candidate.__algo_info__.name
         if issubclass(candidate, Algorithm) and candidate is not Algorithm:
             ALL_ALGORITHMS[name] = candidate
-            if candidate.__algo_info__.is_available:
+            if candidate.__algo_info__.is_available:  # type: ignore[attr-defined]
                 AVAILABLE_ALGORITHMS[name] = candidate
 
 
 GLOBAL_ALGORITHMS = [
-    name for name, algo in ALL_ALGORITHMS.items() if algo.__algo_info__.is_global
+    name
+    for name, algo in ALL_ALGORITHMS.items()
+    if algo.__algo_info__.is_global  # type: ignore[attr-defined]
 ]

src/optimagic/differentiation/richardson_extrapolation.py

@@ -272,7 +272,7 @@ def _compute_step_ratio(steps):
 
     """
     ratios = steps[1:, :] / steps[:-1, :]
-    ratios = ratios[np.isfinite(ratios)]
+    finite_ratios = ratios[np.isfinite(ratios)]
 
-    step_ratio = ratios.flat[0]
+    step_ratio = finite_ratios.item(0)
     return step_ratio

src/optimagic/optimization/algorithm.py

@@ -154,7 +154,7 @@ def _solve_internal_problem(
     def __post_init__(self) -> None:
         for field in self.__dataclass_fields__:
             raw_value = getattr(self, field)
-            target_type = self.__dataclass_fields__[field].type
+            target_type = typing.cast(type, self.__dataclass_fields__[field].type)
             if target_type in TYPE_CONVERTERS:
                 try:
                     value = TYPE_CONVERTERS[target_type](raw_value)

src/optimagic/optimization/create_optimization_problem.py

@@ -1,7 +1,7 @@
 import warnings
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Callable, Type, cast
+from typing import Any, Callable, Type
 
 from optimagic import deprecations
 from optimagic.algorithms import ALL_ALGORITHMS
@@ -591,4 +591,4 @@ def pre_process_user_algorithm(
     elif isinstance(algorithm, type) and issubclass(algorithm, Algorithm):
         algorithm = algorithm()
 
-    return cast(Algorithm, algorithm)
+    return algorithm

src/optimagic/optimizers/_pounders/gqtpar.py

@@ -75,7 +75,7 @@ def gqtpar(model, x_candidate, *, k_easy=0.1, k_hard=0.2, maxiter=200):
     zero_threshold = (
         model.square_terms.shape[0]
         * np.finfo(float).eps
-        * np.linalg.norm(model.square_terms, np.Inf)
+        * np.linalg.norm(model.square_terms, np.inf)
     )
     stopping_criteria = {
         "k_easy": k_easy,
@@ -175,7 +175,7 @@ def _get_initial_guess_for_lambdas(
     gradient_norm = np.linalg.norm(main_model.linear_terms)
     model_hessian = main_model.square_terms
 
-    hessian_infinity_norm = np.linalg.norm(model_hessian, np.Inf)
+    hessian_infinity_norm = np.linalg.norm(model_hessian, np.inf)
     hessian_frobenius_norm = np.linalg.norm(model_hessian, "fro")
 
     hessian_gershgorin_lower, hessian_gershgorin_upper = _compute_gershgorin_bounds(

src/optimagic/parameters/kernel_transformations.py

@@ -496,7 +496,7 @@ def _transformation_matrix(dim):
 
     rows = [_unit_vector_or_zeros(i, dim**2) for i in indices]
 
-    transformer = np.row_stack(rows)
+    transformer = np.vstack(rows)
     return transformer
 
 

tests/optimagic/optimization/test_with_nonlinear_constraints.py

@@ -41,7 +41,7 @@ def constraint_func(x):
         return np.array([value - 1, 2 - value])
 
     def constraint_jac(x):
-        return 2 * np.row_stack((x.reshape(1, -1), -x.reshape(1, -1)))
+        return 2 * np.vstack((x.reshape(1, -1), -x.reshape(1, -1)))
 
     constraints_long = om.NonlinearConstraint(
         func=constraint_func,