Merge pull request #260 from juaml/fraimondo/issue80

[ENH] Bayes optimization for hyperparameters

docs/changes/newsfragments/260.enh

@@ -0,0 +1 @@
+Add :class:`~skopt.BayesSearchCV` to the list of available searchers as 'bayes' by `Fede Raimondo`_

docs/changes/newsfragments/260.misc

@@ -0,0 +1 @@
+Add ``all`` as optional dependencies to install all functional dependencies by `Fede Raimondo`_

docs/conf.py

@@ -160,6 +160,7 @@
     # "sqlalchemy": ("https://docs.sqlalchemy.org/en/20/", None),
     "joblib": ("https://joblib.readthedocs.io/en/latest/", None),
     "scipy": ("https://docs.scipy.org/doc/scipy/", None),
+    "skopt": ("https://scikit-optimize.readthedocs.io/en/latest", None),
 }
 
 

docs/getting_started.rst

@@ -86,4 +86,8 @@ The following optional dependencies are available:
 
 * ``viz``: Visualization tools for ``julearn``. This includes the
   :mod:`.viz` module.
-* ``deslib``: The :mod:`.dynamic` module requires the `deslib`_ package.
+* ``deslib``: The :mod:`.dynamic` module requires the `deslib`_ package. This
+  module is not compatible with newer Python versions and it is unmaintained.
+* ``skopt``: Using the ``"bayes"`` searcher (:class:`~skopt.BayesSearchCV`)
+  requires the `scikit-optimize`_ package.
+* ``all``: Install all optional functional dependencies (except ``deslib``).

docs/links.inc

@@ -40,3 +40,4 @@
 
 
 .. _`DESlib`: https://github.com/scikit-learn-contrib/DESlib
+.. _`scikit-optimize`: https://scikit-optimize.readthedocs.io/en/stable/

examples/03_complex_models/run_hyperparameter_tuning_bayessearch.py

@@ -0,0 +1,95 @@
+"""
+Tuning Hyperparameters using Bayesian Search
+============================================
+
+This example uses the ``fmri`` dataset, performs simple binary classification
+using a Support Vector Machine classifier and analyzes the model.
+
+References
+----------
+
+  Waskom, M.L., Frank, M.C., Wagner, A.D. (2016). Adaptive engagement of
+  cognitive control in context-dependent decision-making. Cerebral Cortex.
+
+.. include:: ../../links.inc
+"""
+
+# Authors: Federico Raimondo <[email protected]>
+# License: AGPL
+
+import numpy as np
+from seaborn import load_dataset
+
+from julearn import run_cross_validation
+from julearn.utils import configure_logging, logger
+from julearn.pipeline import PipelineCreator
+
+
+###############################################################################
+# Set the logging level to info to see extra information.
+configure_logging(level="INFO")
+
+###############################################################################
+# Set the random seed to always have the same example.
+np.random.seed(42)
+
+###############################################################################
+# Load the dataset.
+df_fmri = load_dataset("fmri")
+df_fmri.head()
+
+###############################################################################
+# Set the dataframe in the right format.
+df_fmri = df_fmri.pivot(
+    index=["subject", "timepoint", "event"], columns="region", values="signal"
+)
+
+df_fmri = df_fmri.reset_index()
+df_fmri.head()
+
+###############################################################################
+# Following the hyperparamter tuning example, we will now use a Bayesian
+# search to find the best hyperparameters for the SVM model.
+X = ["frontal", "parietal"]
+y = "event"
+
+creator1 = PipelineCreator(problem_type="classification")
+creator1.add("zscore")
+creator1.add(
+    "svm",
+    kernel=["linear"],
+    C=(1e-6, 1e3, "log-uniform"),
+)
+
+creator2 = PipelineCreator(problem_type="classification")
+creator2.add("zscore")
+creator2.add(
+    "svm",
+    kernel=["rbf"],
+    C=(1e-6, 1e3, "log-uniform"),
+    gamma=(1e-6, 1e1, "log-uniform"),
+)
+
+search_params = {
+    "kind": "bayes",
+    "cv": 2,  # to speed up the example
+    "n_iter": 10,  # 10 iterations of bayesian search to speed up example
+}
+
+
+scores, estimator = run_cross_validation(
+    X=X,
+    y=y,
+    data=df_fmri,
+    model=[creator1, creator2],
+    cv=2,  # to speed up the example
+    search_params=search_params,
+    return_estimator="final",
+)
+
+print(scores["test_score"].mean())
+
+
+###############################################################################
+# It seems that we might have found a better model, but which one is it?
+print(estimator.best_params_)

examples/99_docs/run_hyperparameters_docs.py

@@ -243,22 +243,132 @@
 # tries to find the best combination of values for the hyperparameters using
 # cross-validation.
 #
-# By default, ``julearn`` uses a :class:`~sklearn.model_selection.GridSearchCV`.
-# This searcher is very simple. First, it construct the "grid" of
-# hyperparameters to try. As we see above, we have 3 hyperparameters to tune.
-# So it constructs a 3-dimentional grid with all the possible combinations of
-# the hyperparameters values. The second step is to perform cross-validation
-# on each of the possible combinations of hyperparameters values.
+# By default, ``julearn`` uses a
+# :class:`~sklearn.model_selection.GridSearchCV`.
+# This searcher, specified as ``"grid"`` is very simple. First, it constructs
+# the _grid_ of hyperparameters to try. As we see above, we have 3
+# hyperparameters to tune. So it constructs a 3-dimentional grid with all the
+# possible combinations of the hyperparameters values. The second step is to
+# perform cross-validation on each of the possible combinations of
+# hyperparameters values.
 #
-# Another searcher that ``julearn`` provides is the
-# :class:`~sklearn.model_selection.RandomizedSearchCV`. This searcher is
-# similar to the :class:`~sklearn.model_selection.GridSearchCV`, but instead
-# of trying all the possible combinations of hyperparameters values, it tries
+# Other searchers that ``julearn`` provides are the
+# :class:`~sklearn.model_selection.RandomizedSearchCV` and
+# :class:`~skopt.BayesSearchCV`.
+#
+# The randomized searcher
+# (:class:`~sklearn.model_selection.RandomizedSearchCV`) is similar to the
+# :class:`~sklearn.model_selection.GridSearchCV`, but instead
+# of trying all the possible combinations of hyperparameter values, it tries
 # a random subset of them. This is useful when we have a lot of hyperparameters
-# to tune, since it can be very time consuming to try all the possible, as well
-# as continuous parameters that can be sampled out of a distribution. For
-# more information, see the
+# to tune, since it can be very time consuming to try all the possible
+# combinations, as well as continuous parameters that can be sampled out of a
+# distribution. For more information, see the
 # :class:`~sklearn.model_selection.RandomizedSearchCV` documentation.
+#
+# The Bayesian searcher (:class:`~skopt.BayesSearchCV`) is a bit more
+# complex. It uses Bayesian optimization to find the best hyperparameter set.
+# As with the randomized search, it is useful when we have many
+# hyperparameters to tune, and we don't want to try all the possible
+# combinations due to computational constraints. For more information, see the
+# :class:`~skopt.BayesSearchCV` documentation, including how to specify
+# the prior distributions of the hyperparameters.
+#
+# We can specify the kind of searcher and its parametrization, by setting the
+# ``search_params`` parameter in the :func:`.run_cross_validation` function.
+# For example, we can use the
+# :class:`~sklearn.model_selection.RandomizedSearchCV` searcher with
+# 10 iterations of random search.
+
+search_params = {
+    "kind": "random",
+    "n_iter": 10,
+}
+
+scores_tuned, model_tuned = run_cross_validation(
+    X=X,
+    y=y,
+    data=df,
+    X_types=X_types,
+    model=creator,
+    return_estimator="all",
+    search_params=search_params,
+)
+
+print(
+    "Scores with best hyperparameter using 10 iterations of "
+    f"randomized search: {scores_tuned['test_score'].mean()}"
+)
+pprint(model_tuned.best_params_)
+
+###############################################################################
+# We can now see that the best hyperparameter might be different from the grid
+# search. This is because it tried only 10 combinations and not the whole grid.
+# Furthermore, the  :class:`~sklearn.model_selection.RandomizedSearchCV`
+# searcher can sample hyperparameters from distributions, which can be useful
+# when we have continuous hyperparameters.
+# Let's set both ``C`` and ``gamma`` to be sampled from log-uniform
+# distributions. We can do this by setting the hyperparameter values as a
+# tuple with the following format: ``(low, high, distribution)``. The
+# distribution can be either ``"log-uniform"`` or ``"uniform"``.
+
+creator = PipelineCreator(problem_type="classification")
+creator.add("zscore")
+creator.add("select_k", k=[2, 3, 4])
+creator.add(
+    "svm",
+    C=(0.01, 10, "log-uniform"),
+    gamma=(1e-3, 1e-1, "log-uniform"),
+)
+
+print(creator)
+
+scores_tuned, model_tuned = run_cross_validation(
+    X=X,
+    y=y,
+    data=df,
+    X_types=X_types,
+    model=creator,
+    return_estimator="all",
+    search_params=search_params,
+)
+
+print(
+    "Scores with best hyperparameter using 10 iterations of "
+    f"randomized search: {scores_tuned['test_score'].mean()}"
+)
+pprint(model_tuned.best_params_)
+
+
+###############################################################################
+# We can also control the number of cross-validation folds used by the searcher
+# by setting the ``cv`` parameter in the ``search_params`` dictionary. For
+# example, we can use a bayesian search with 3 folds. Fortunately, the
+# :class:`~skopt.BayesSearchCV` searcher also accepts distributions for the
+# hyperparameters.
+
+search_params = {
+    "kind": "bayes",
+    "n_iter": 10,
+    "cv": 3,
+}
+
+scores_tuned, model_tuned = run_cross_validation(
+    X=X,
+    y=y,
+    data=df,
+    X_types=X_types,
+    model=creator,
+    return_estimator="all",
+    search_params=search_params,
+)
+
+print(
+    "Scores with best hyperparameter using 10 iterations of "
+    f"bayesian search and 3-fold CV: {scores_tuned['test_score'].mean()}"
+)
+pprint(model_tuned.best_params_)
+
 
 ###############################################################################
 #