Add optuna option for searchers

docs/changes/newsfragments/262.doc

@@ -0,0 +1 @@
+Update documentation on Hyperparameter Tuning by `Fede Raimondo_`

docs/changes/newsfragments/262.enh

@@ -0,0 +1 @@
+Refactor how hyperparmeters' distributions are specified by `Fede Raimondo`_

docs/changes/newsfragments/262.feature

@@ -0,0 +1 @@
+Add :class:`~optuna_integration.sklearn.OptunaSearchCV` to the list of available searchers as ``optuna`` by `Fede Raimondo`_

docs/conf.py

@@ -161,6 +161,11 @@
     "joblib": ("https://joblib.readthedocs.io/en/latest/", None),
     "scipy": ("https://docs.scipy.org/doc/scipy/", None),
     "skopt": ("https://scikit-optimize.readthedocs.io/en/latest", None),
+    "optuna": ("https://optuna.readthedocs.io/en/stable", None),
+    "optuna_integration": (
+        "https://optuna-integration.readthedocs.io/en/stable",
+        None,
+    ),
 }
 
 

docs/links.inc

@@ -41,3 +41,4 @@
 
 .. _`DESlib`: https://github.com/scikit-learn-contrib/DESlib
 .. _`scikit-optimize`: https://scikit-optimize.readthedocs.io/en/stable/
+.. _`Optuna`: https://optuna.org

examples/99_docs/run_hyperparameters_docs.py

@@ -253,8 +253,9 @@
 # hyperparameters values.
 #
 # Other searchers that ``julearn`` provides are the
-# :class:`~sklearn.model_selection.RandomizedSearchCV` and
-# :class:`~skopt.BayesSearchCV`.
+# :class:`~sklearn.model_selection.RandomizedSearchCV`, 
+# :class:`~skopt.BayesSearchCV` and
+# :class:`~optuna_integration.sklearn.OptunaSearchCV`.
 #
 # The randomized searcher
 # (:class:`~sklearn.model_selection.RandomizedSearchCV`) is similar to the
@@ -274,6 +275,12 @@
 # :class:`~skopt.BayesSearchCV` documentation, including how to specify
 # the prior distributions of the hyperparameters.
 #
+# The Optuna searcher (:class:`~optuna_integration.sklearn.OptunaSearchCV`)
+# uses the Optuna library to find the best hyperparameter set. Optuna is a
+# hyperparameter optimization framework that has several algorithms to find
+# the best hyperparameter set. For more information, see the
+# `Optuna`_ documentation.
+#
 # We can specify the kind of searcher and its parametrization, by setting the
 # ``search_params`` parameter in the :func:`.run_cross_validation` function.
 # For example, we can use the
@@ -369,6 +376,140 @@
 )
 pprint(model_tuned.best_params_)
 
+###############################################################################
+# An example using optuna searcher is shown below. The searcher is specified
+# as ``"optuna"`` and the hyperparameters are specified as a dictionary with
+# the hyperparameters to tune and their distributions as for the bayesian
+# searcher. However, the optuna searcher behaviour is controlled by a
+# :class:`~optuna.study.Study` object. This object can be passed to the
+# searcher using the ``study`` parameter in the ``search_params`` dictionary.
+# 
+# .. important::
+#    The optuna searcher requires that all the hyperparameters are specified
+#    as distributions, even the categorical ones. 
+#
+# We first modify the pipeline creator so the ``select_k`` parameter is
+# specified as a distribution. We exemplarily use a categorical distribution
+# for the ``class_weight`` hyperparameter, trying the ``"balanced"`` and 
+# ``None`` values.
+
+creator = PipelineCreator(problem_type="classification")
+creator.add("zscore")
+creator.add("select_k", k=(2, 4, "uniform"))
+creator.add(
+    "svm",
+    C=(0.01, 10, "log-uniform"),
+    gamma=(1e-3, 1e-1, "log-uniform"),
+    class_weight=("balanced", None, "categorical")
+)
+print(creator)
+
+###############################################################################
+# We can now use the optuna searcher with 10 trials and 3-fold cross-validation.
+
+import optuna
+
+study = optuna.create_study(
+    direction="maximize",
+    study_name="optuna-concept",
+    load_if_exists=True,
+)
+
+search_params = {
+    "kind": "optuna",
+    "study": study,
+    "cv": 3,
+}
+scores_tuned, model_tuned = run_cross_validation(
+    X=X,
+    y=y,
+    data=df,
+    X_types=X_types,
+    model=creator,
+    return_estimator="all",
+    search_params=search_params,
+)
+
+print(
+    "Scores with best hyperparameter using 10 iterations of "
+    f"optuna and 3-fold CV: {scores_tuned['test_score'].mean()}"
+)
+pprint(model_tuned.best_params_)
+
+###############################################################################
+#
+# Specifying distributions
+# ~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# The hyperparameters can be specified as distributions for the randomized
+# searcher, bayesian searcher and optuna searcher. The distributions are
+# either specified toolbox-specific method or  a tuple convention with the
+# following format: ``(low, high, distribution)`` where the distribution can
+# be either ``"log-uniform"`` or ``"uniform"`` or 
+# ``(a, b, c, d, ..., "categorical")`` where ``a``, ``b``, ``c``, ``d``, etc.
+# are the possible categorical values for the hyperparameter.
+#
+# For example, we can specify the ``C`` and ``gamma`` hyperparameters of the 
+# :class:`~sklearn.svm.SVC` as  log-uniform distributions, while keeping 
+# the ``with_mean`` parameter of the
+# :class:`~sklearn.preprocessing.StandardScaler` as a categorical parameter
+# with two options.
+
+
+creator = PipelineCreator(problem_type="classification")
+creator.add("zscore", with_mean=(True, False, "categorical"))
+creator.add(
+    "svm",
+    C=(0.01, 10, "log-uniform"),
+    gamma=(1e-3, 1e-1, "log-uniform"),
+)
+print(creator)
+
+###############################################################################
+# While this will work for any of the ``random``, ``bayes`` or ``optuna``
+# searcher options, it is important to note that both ``bayes`` and ``optuna``
+# searchers accept further parameters to specify distributions. For example,
+# the ``bayes`` searcher distributions are defined using the 
+# :class:`~skopt.space.space.Categorical`, :class:`~skopt.space.space.Integer`
+# and :class:`~skopt.space.space.Real`.
+#
+# For example, we can define a log-uniform distribution with base 2 for the
+# ``C`` hyperparameter of the :class:`~sklearn.svm.SVC` model:
+from skopt.space import Real
+creator = PipelineCreator(problem_type="classification")
+creator.add("zscore", with_mean=(True, False, "categorical"))
+creator.add(
+    "svm",
+    C=Real(0.01, 10, prior="log-uniform", base=2),
+    gamma=(1e-3, 1e-1, "log-uniform"),
+)
+print(creator)
+
+###############################################################################
+# For the optuna searcher, the distributions are defined using the
+# :class:`~optuna.distributions.CategoricalDistribution`,
+# :class:`~optuna.distributions.FloatDistribution` and
+# :class:`~optuna.distributions.IntDistribution`.
+#
+#
+# For example, we can define a uniform distribution from 0.5 to 0.9 with a 0.05
+# step for the ``n_components`` of a :class:`~sklearn.decomposition.PCA` 
+# transformer, while keeping a log-uniform distribution for the ``C`` and
+# ``gamma`` hyperparameters of the :class:`~sklearn.svm.SVC` model.
+from optuna.distributions import FloatDistribution
+creator = PipelineCreator(problem_type="classification")
+creator.add("zscore")
+creator.add(
+    "pca",
+    n_components=FloatDistribution(0.5, 0.9, step=0.05),
+)
+creator.add(
+    "svm",
+    C=FloatDistribution(0.01, 10, log=True),
+    gamma=(1e-3, 1e-1, "log-uniform"),
+)
+print(creator)
+
 
 ###############################################################################
 #

julearn/api.py

@@ -135,9 +135,18 @@ def run_cross_validation(  # noqa: C901
         Additional parameters in case Hyperparameter Tuning is performed, with
         the following keys:
 
-        * 'kind': The kind of search algorithm to use, e.g.:
-            'grid', 'random' or 'bayes'. Can be any valid julearn searcher name
-            or scikit-learn compatible searcher.
+        * 'kind': The kind of search algorithm to use, Valid options are:
+
+          * ``"grid"`` : :class:`~sklearn.model_selection.GridSearchCV`
+          * ``"random"`` :
+            :class:`~sklearn.model_selection.RandomizedSearchCV`
+          * ``"bayes"`` : :class:`~skopt.BayesSearchCV`
+          * ``"optuna"`` :
+            :class:`~optuna_integration.sklearn.OptunaSearchCV`
+          * user-registered searcher name : see
+            :func:`~julearn.model_selection.register_searcher`
+          * ``scikit-learn``-compatible searcher
+
         * 'cv': If a searcher is going to be used, the cross-validation
             splitting strategy to use. Defaults to same CV as for the model
             evaluation.

julearn/conftest.py

@@ -270,7 +270,7 @@ def search_params(request: FixtureRequest) -> Optional[Dict]:
     scope="function",
 )
 def bayes_search_params(request: FixtureRequest) -> Optional[Dict]:
-    """Return different  search_params argument for BayesSearchCV.
+    """Return different search_params argument for BayesSearchCV.
 
     Parameters
     ----------
@@ -286,6 +286,29 @@ def bayes_search_params(request: FixtureRequest) -> Optional[Dict]:
 
     return request.param
 
+@fixture(
+    params=[
+        {"kind": "optuna", "n_trials": 10, "cv": 3},
+        {"kind": "optuna", "timeout": 20},
+    ],
+    scope="function",
+)
+def optuna_search_params(request: FixtureRequest) -> Optional[Dict]:
+    """Return different search_params argument for OptunaSearchCV.
+
+    Parameters
+    ----------
+    request : pytest.FixtureRequest
+        The request object.
+
+    Returns
+    -------
+    dict or None
+        A dictionary with the search_params argument.
+
+    """
+
+    return request.param
 
 _tuning_params = {
     "zscore": {"with_mean": [True, False]},

julearn/model_selection/__init__.py

@@ -17,5 +17,8 @@
 )
 
 from ._skopt_searcher import register_bayes_searcher
+from ._optuna_searcher import register_optuna_searcher
+
 register_bayes_searcher()
+register_optuna_searcher()
 

julearn/model_selection/_optuna_searcher.py

@@ -0,0 +1,107 @@
+"""Module for registering the BayesSearchCV class from scikit-optimize."""
+
+# Authors: Federico Raimondo <[email protected]>
+# License: AGPL
+
+from typing import Any, Dict
+
+from ..utils import logger
+from .available_searchers import _recreate_reset_copy, register_searcher
+
+
+try:
+    import optuna.distributions as optd
+    from optuna_integration.sklearn import OptunaSearchCV
+except ImportError:
+    from sklearn.model_selection._search import BaseSearchCV
+
+    class OptunaSearchCV(BaseSearchCV):
+        """Dummy class for OptunaSearchCV that raises ImportError.
+
+        This class is used to raise an ImportError when OptunaSearchCV is
+        requested but optuna and optuna-integration ar not installed.
+
+        """
+
+        def __init__(*args, **kwargs):
+            raise ImportError(
+                "OptunaSearchCV requires optuna and optuna-integration to be "
+                "installed."
+            )
+
+
+def register_optuna_searcher():
+    register_searcher("optuna", OptunaSearchCV, "param_distributions")
+
+    # Update the "reset copy" of available searchers
+    _recreate_reset_copy()
+
+
+def _prepare_optuna_hyperparameters_distributions(
+    params_to_tune: Dict[str, Any],
+) -> Dict[str, Any]:
+    """Prepare hyperparameters distributions for OptunaSearchCV.
+
+    This method replaces tuples with distributions for OptunaSearchCV
+    following the skopt convention. That is, if a parameter is a tuple
+    with 3 elements, the first two elements are the bounds of the
+    distribution and the third element is the type of distribution. In case
+    the last element is "categorical", the parameter is considered
+    categorical and all the previous elements are the choices.
+
+    Parameters
+    ----------
+    params_to_tune : dict
+        The parameters to tune.
+
+    Returns
+    -------
+    dict
+        The modified parameters to tune.
+
+    """
+    out = {}
+    for k, v in params_to_tune.items():
+        if isinstance(v, tuple) and len(v) == 3:
+            if v[2] == "uniform":
+                if isinstance(v[0], int) and isinstance(v[1], int):
+                    logger.info(
+                        f"Hyperparameter {k} is uniform integer "
+                        f"[{v[0]}, {v[1]}]"
+                    )
+                    out[k] = optd.IntDistribution(v[0], v[1], log=False)
+                else:
+                    logger.info(
+                        f"Hyperparameter {k} is uniform float [{v[0]}, {v[1]}]"
+                    )
+                    out[k] = optd.FloatDistribution(v[0], v[1], log=False)
+            elif v[2] == "log-uniform":
+                if isinstance(v[0], int) and isinstance(v[1], int):
+                    logger.info(
+                        f"Hyperparameter {k} is log-uniform int "
+                        f"[{v[0]}, {v[1]}]"
+                    )
+                    out[k] = optd.IntDistribution(v[0], v[1], log=True)
+                else:
+                    logger.info(
+                        f"Hyperparameter {k} is log-uniform float "
+                        f"[{v[0]}, {v[1]}]"
+                    )
+                    out[k] = optd.FloatDistribution(v[0], v[1], log=True)
+            elif v[2] == "categorical":
+                logger.info(f"Hyperparameter {k} is categorical with 2 "
+                            f"options: [{v[0]} and {v[1]}]")
+                out[k] = optd.CategoricalDistribution((v[0], v[1]))
+            else:
+                out[k] = v
+        elif (
+            isinstance(v, tuple)
+            and isinstance(v[-1], str)
+            and v[-1] == "categorical"
+        ):
+            logger.info(f"Hyperparameter {k} is categorical [{v[:-1]}]")
+            out[k] = optd.CategoricalDistribution(v[:-1])
+        else:
+            logger.info(f"Hyperparameter {k} as is {v}")
+            out[k] = v
+    return out