From 26e32dcec6ecfb7e7bdc209d0905fec543d4dbdb Mon Sep 17 00:00:00 2001 From: Valentin Laurent Date: Wed, 6 Nov 2024 18:57:06 +0100 Subject: [PATCH] reorganized files cleaned classification.py fixed regression.py typing import integration tests : added a dummy test + a script to run tests --- .gitignore | 3 +- Makefile | 6 ++ .../classification.py | 84 ++++++++----------- .../tests/test_regression.py | 28 +++++++ .../migration_guide.md | 0 .../regression.py | 16 +++- 6 files changed, 84 insertions(+), 53 deletions(-) rename public_api_v1_classifier.py => mapie_v1/classification.py (50%) create mode 100644 mapie_v1/integration_tests/tests/test_regression.py rename migration_guide.md => mapie_v1/migration_guide.md (100%) rename public_api_v1_regression.py => mapie_v1/regression.py (98%) diff --git a/.gitignore b/.gitignore index 7c5f1ddae..57070a8cf 100644 --- a/.gitignore +++ b/.gitignore @@ -49,7 +49,7 @@ var/ pip-log.txt pip-delete-this-directory.txt -# Unit test / coverage reports +# Tests / coverage reports htmlcov/ .tox/ .coverage @@ -59,6 +59,7 @@ nosetests.xml coverage.xml *,cover .hypothesis/ +mapie_v1/integration_tests/mapie_v0_package # Translations *.mo diff --git a/Makefile b/Makefile index 2f761ce38..3bfc5e376 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,12 @@ type-check: tests: pytest -vs --doctest-modules mapie +integration-tests-v1: + @pip install mapie --no-dependencies --target=./mapie_v1/integration_tests/mapie_v0_package >/dev/null 2>&1 + @mv ./mapie_v1/integration_tests/mapie_v0_package/mapie ./mapie_v1/integration_tests/mapie_v0_package/mapiev0 + @- export PYTHONPATH="${PYTHONPATH}:./mapie_v1/integration_tests/mapie_v0_package"; pytest -vs mapie_v1/integration_tests/tests + @mv ./mapie_v1/integration_tests/mapie_v0_package/mapiev0 ./mapie_v1/integration_tests/mapie_v0_package/mapie + coverage: pytest -vs \ --doctest-modules \ diff --git a/public_api_v1_classifier.py b/mapie_v1/classification.py similarity index 50% rename from public_api_v1_classifier.py rename to mapie_v1/classification.py index 049db279d..da6cf67d0 100644 --- a/public_api_v1_classifier.py +++ b/mapie_v1/classification.py @@ -1,65 +1,48 @@ from __future__ import annotations -import warnings -from typing import Any, Iterable, Optional, Tuple, Union, cast, List +from typing import Optional, Union, List import numpy as np -from sklearn.base import BaseEstimator, ClassifierMixin -from sklearn.model_selection import BaseCrossValidator, BaseShuffleSplit -from sklearn.preprocessing import LabelEncoder -from sklearn.utils import check_random_state -from sklearn.utils.validation import (_check_y, check_is_fitted, indexable) +from sklearn.base import ClassifierMixin +from sklearn.model_selection import BaseCrossValidator from sklearn.linear_model import LogisticRegression from mapie._typing import ArrayLike, NDArray from mapie.conformity_scores import BaseClassificationScore -from mapie.conformity_scores.sets.raps import RAPSConformityScore -from mapie.conformity_scores.sets.lac import LACConformityScore - -from mapie.conformity_scores.utils import ( - check_depreciated_size_raps, check_classification_conformity_score, - check_target -) -from mapie.estimator.classifier import EnsembleClassifier -from mapie.utils import (check_alpha, check_alpha_and_n_samples, check_cv, - check_estimator_classification, check_n_features_in, - check_n_jobs, check_null_weight, check_predict_params, - check_verbose) class SplitConformalClassifier: - - def __init__( + + def __init__( self, estimator: ClassifierMixin = LogisticRegression(), - conformity_score: Union[str, BaseClassificationScore] = "lac", # Can be a string or a BaseClassificationScore object - alpha: Union[float, List[float]] = 0.1, - split_method: str = "simple", # 'simple' (provide test_size in .fit) or 'prefit'. Future API: 'manual' (provide X_calib, Y_calib in .fit) and BaseCrossValidator (restricted to splitters only) + conformity_score: Union[str, BaseClassificationScore] = "lac", + # Can be a string or a BaseClassificationScore object + confidence_level: Union[float, List[float]] = 0.9, + split_method: str = "simple", + # 'simple' (provide test_size in .fit) or 'prefit'. Future API: 'manual' (provide X_calib, Y_calib in .fit) and BaseCrossValidator (restricted to splitters only) n_jobs: Optional[int] = None, random_state: Optional[Union[int, np.random.RandomState]] = None, verbose: int = 0, ) -> None: - pass - + def fit( self, X: ArrayLike, y: ArrayLike, # sample_weight: Optional[ArrayLike] = None, -> in fit_params # groups: Optional[ArrayLike] = None, # Removed, because it is not used in split conformal classifier - test_size: Union[int, float] = 0.1, # -> In __init__ ? + test_size: Union[int, float] = 0.1, # -> In __init__ ? # Future API: X_calib: Optional[ArrayLike] = None, # Must be None if split_method != 'manual' # Future API: y_calib: Optional[ArrayLike] = None, # Must be None if split_method != 'manual' fit_params: Optional[dict] = None, # For example, LBGMClassifier : {'categorical_feature': 'auto'} - predict_params: Optional[dict] = None, # For example, LBGMClassifier : {'pred_leaf': False} + predict_params: Optional[dict] = None, # For example, LBGMClassifier : {'pred_leaf': False} ) -> SplitConformalClassifier: - return self - def predict(self, + def predict(self, X: ArrayLike) -> NDArray: - """ Return ----- @@ -67,36 +50,36 @@ def predict(self, Shape (n_samples,) """ - def predict_sets(self, + def predict_sets(self, X: ArrayLike, - conformoty_score_params: Optional[dict] = None, # Parameters specific to conformal method, For example: include_last_label + conformity_score_params: Optional[dict] = None, + # Parameters specific to conformal method, For example: include_last_label ) -> NDArray: - """ Return ----- An array containing the prediction sets - Shape (n_samples, n_classes) if alpha is float, - Shape (n_samples, n_classes, alpha) if alpha is a list of floats + Shape (n_samples, n_classes) if confidence_level is float, + Shape (n_samples, n_classes, confidence_level) if confidence_level is a list of floats """ pass + class CrossConformalClassifier: - + def __init__( self, estimator: ClassifierMixin = LogisticRegression(), conformity_score: Union[str, BaseClassificationScore] = 'lac', - cross_val : Union[BaseCrossValidator, str] = 5, - alpha: Union[float, List[float]] = 0.1, + cross_val: Union[BaseCrossValidator, str] = 5, + confidence_level: Union[float, List[float]] = 0.9, n_jobs: Optional[int] = None, random_state: Optional[Union[int, np.random.RandomState]] = None, verbose: int = 0, ) -> None: - - pass + pass def fit( self, @@ -105,13 +88,12 @@ def fit( # sample_weight: Optional[ArrayLike] = None, -> in fit_params # groups: Optional[ArrayLike] = None, fit_params: Optional[dict] = None, # For example, LBGMClassifier : {'categorical_feature': 'auto'} - predict_params: Optional[dict] = None, + predict_params: Optional[dict] = None, ) -> CrossConformalClassifier: - pass - def predict(self, - X: ArrayLike): # Parameters specific to conformal method, For example: include_last_label) -> ArrayLike: + def predict(self, + X: ArrayLike): # Parameters specific to conformal method, For example: include_last_label) -> ArrayLike: """ Return @@ -123,14 +105,14 @@ def predict(self, def predict_sets(self, X: ArrayLike, - agg_scores: Optional[str] = "mean", # how to aggregate the scores by the estimators on test data - conformoty_score_params: Optional[dict] = None,): # Parameters specific to conformal method, For example: include_last_label) -> NDArray - + agg_scores: Optional[str] = "mean", # how to aggregate the scores by the estimators on test data + conformity_score_params: Optional[ + dict] = None, ): # Parameters specific to conformal method, For example: include_last_label) -> NDArray + """ Return ----- An array containing the prediction sets - Shape (n_samples, n_classes) if alpha is float, - Shape (n_samples, n_classes, alpha) if alpha is a list of floats + Shape (n_samples, n_classes) if confidence_level is float, + Shape (n_samples, n_classes, confidence_level) if confidence_level is a list of floats """ - \ No newline at end of file diff --git a/mapie_v1/integration_tests/tests/test_regression.py b/mapie_v1/integration_tests/tests/test_regression.py new file mode 100644 index 000000000..6f5d9b01d --- /dev/null +++ b/mapie_v1/integration_tests/tests/test_regression.py @@ -0,0 +1,28 @@ +import numpy as np +from sklearn.model_selection import train_test_split + +from mapie_v1.regression import SplitConformalRegressor +from mapie.tests.test_regression import X_toy, y_toy +from mapiev0.regression import MapieRegressor as MapieRegressorV0 # noqa + + +def test_dummy(): + test_size = 0.5 + alpha = 0.5 + confidence_level = 1 - alpha + random_state = 42 + + v0 = MapieRegressorV0(cv="split", test_size=test_size, random_state=random_state) + v0.fit(X_toy, y_toy) + v0_preds = v0.predict(X_toy) + v0_pred_intervals = v0.predict(X_toy, alpha=alpha) + + X_train, y_train, X_conf, y_conf = train_test_split( + X_toy, y_toy, test_size=test_size, random_state=random_state + ) + v1 = SplitConformalRegressor(confidence_level=confidence_level, random_state=random_state) + v1.fit_conformalize(X_train, y_train, X_conf, y_conf) + v1_preds = v1.predict(X_toy) + v1_pred_intervals = v1.predict_set(X_toy) + np.testing.assert_array_equal(v1_preds, v0_preds) + np.testing.assert_array_equal(v1_pred_intervals, v0_pred_intervals) diff --git a/migration_guide.md b/mapie_v1/migration_guide.md similarity index 100% rename from migration_guide.md rename to mapie_v1/migration_guide.md diff --git a/public_api_v1_regression.py b/mapie_v1/regression.py similarity index 98% rename from public_api_v1_regression.py rename to mapie_v1/regression.py index e87c956e3..6d4c886a4 100644 --- a/public_api_v1_regression.py +++ b/mapie_v1/regression.py @@ -1,4 +1,5 @@ -from typing import Optional, Union, Self, List +from typing import Optional, Union, List +from typing_extensions import Self import numpy as np from sklearn.linear_model import LinearRegression, QuantileRegressor @@ -206,6 +207,19 @@ def predict( """ pass + def fit_conformalize( + self, + X_train: ArrayLike, + y_train: ArrayLike, + X_conf: ArrayLike, + y_conf: ArrayLike, + fit_params: Optional[dict] = None, + predict_params: Optional[dict] = None, + ) -> Self: + """ + Dummy method to fit and conformalize in one step for testing purposes. + """ + pass class CrossConformalRegressor: """