ENH: calculating d-prime from confusion matrices and samples

.gitignore

@@ -3,3 +3,6 @@
 __pycache__
 .idea
 build
+*.DS_Store
+*~
+.*swp

README.md

@@ -0,0 +1,5 @@
+# bangmetric
+
+# License
+
+New BSD

bangmetric/__init__.py

@@ -5,3 +5,7 @@
 from rmse import *  # pyflakes.ignore
 from kernel_analysis import *  # pyflakes.ignore
 from nk import *  # pyflakes.ignore
+from utils import *  # pyflakes.ignore
+from human_metric import *  # pyflakes.ignore
+
+__version__ = '0.0.1'

bangmetric/accuracy.py

@@ -8,49 +8,100 @@
 __all__ = ['accuracy']
 
 import numpy as np
+from .utils import confusion_matrix_stats
 
+DEFAULT_ACCURACY_MODE = 'binary'
 
-def accuracy(y_true, y_pred, balanced=False):
-    """Computes the Accuracy of the predictions (also known as the
-    zero-one score).
+
+def accuracy(A, B=None, mode=DEFAULT_ACCURACY_MODE, \
+        balanced=False, collation=None):
+    """Computes the accuracy of the predictions (also known as the
+    zero-one score).  Depending on the choice of `mode`, this
+    function can take one of the following data format:
+
+    * Binary classification outputs (`mode='binary'`; default)
+    * Confusion matrix (`mode='confusionmat'`)
 
     Parameters
     ----------
-    y_true: array, shape = [n_samples]
-        True values, interpreted as strictly positive or not
-        (i.e. converted to binary).
+    A, B:
+        If `mode` is 'binary' (default):
+
+            A: array, shape = [n_samples]
+                True values, interpreted as strictly positive or not
+                (i.e. converted to binary).
+
+            B: array, shape = [n_samples]
+                Predicted values, interpreted as strictly positive or not
+                (i.e. converted to binary).
 
-    y_pred: array, shape = [n_samples]
-        Predicted values, interpreted as strictly positive or not
-        (i.e. converted to binary).
+        if `mode` is 'confusionmat':
+
+            A: array-like, shape = [n_classes (true), n_classes (pred)]
+                Confusion matrix, where the element M_{rc} means
+                the number of times when the classifier or subject
+                guesses that a test sample in the r-th class
+                belongs to the c-th class.
+
+            B: ignored
 
     balanced: bool, optional (default=False)
         Returns the balanced accuracy (equal weight for positive and
         negative values).
 
+    collation: None or array-like of shape = [n_groupings,
+        n_classes], optional (default=None)
+        Defines how to group entries in `M` to make sub-confusion matrices
+        when `mode` is 'confusionmat'.  See `confusion_matrix_stats()`
+        for details.
+
     Returns
     -------
-    acc: float
-        Accuracy (zero-one score).
+    acc: float or array of shape = [n_groupings]
+        An accuracy score (zero-one score) or array of accuracies,
+        where each element corresponds to each grouping of
+        positives and negatives (when `mode` is 'confusionmat').
+
+    References
+    ----------
+    http://en.wikipedia.org/wiki/Accuracy
     """
-    assert len(y_true) == len(y_pred)
-    assert np.isfinite(y_true).all()
-    assert np.isfinite(y_pred).all()
 
-    # -- "binarize" the arguments
-    y_true = np.array(y_true) > 0
-    assert y_true.ndim == 1
+    if mode == 'binary':
+        y_true, y_pred = A, B
+        assert len(y_true) == len(y_pred)
+        assert np.isfinite(y_true).all()
+        assert np.isfinite(y_pred).all()
+
+        # -- "binarize" the arguments
+        y_true = np.array(y_true) > 0
+        assert y_true.ndim == 1
+
+        y_pred = np.array(y_pred) > 0
+        assert y_pred.ndim == 1
+
+        i_pos = y_true > 0
+        i_neg = ~i_pos
 
-    y_pred = np.array(y_pred) > 0
-    assert y_pred.ndim == 1
+        P = float(i_pos.sum())
+        N = float(i_neg.sum())
+        TP = float((y_true[i_pos] == y_pred[i_pos]).sum())
+        TN = float((y_true[i_neg] == y_pred[i_neg]).sum())
+
+    elif mode == 'confusionmat':
+        # A: confusion mat
+        # row means true classes, col means predicted classes
+        P, N, TP, TN, _, _ = confusion_matrix_stats(A, \
+                collation=collation, fudge_mode='none')
+
+    else:
+        raise ValueError('Invalid mode')
 
     if balanced:
-        pos = y_true > 0
-        neg = ~pos
-        pos_acc = (y_true[pos] == y_pred[pos]).mean()
-        neg_acc = (y_true[neg] == y_pred[neg]).mean()
-        acc = (pos_acc + neg_acc) / 2.
+        sensitivity = TP / P
+        specificity = TN / N
+        acc = (sensitivity + specificity) / 2.
     else:
-        acc = (y_true == y_pred).mean()
+        acc = (TP + TN) / (P + N)
 
     return acc

bangmetric/dprime.py

@@ -2,61 +2,176 @@
 
 # Authors: Nicolas Pinto <[email protected]>
 #          Nicolas Poilvert <[email protected]>
+#          Ha Hong <[email protected]>
 #
 # License: BSD
 
 __all__ = ['dprime']
 
 import numpy as np
+from scipy.stats import norm
+from .utils import confusion_matrix_stats
 
+DEFAULT_DPRIME_MODE = 'binary'
 
-def dprime(y_pred, y_true):
-    """Computes the d-prime sensitivity index of the predictions.
+
+def dprime(A, B=None, mode=DEFAULT_DPRIME_MODE,\
+        max_value=np.inf, min_value=-np.inf,\
+        max_ppf_value=np.inf, min_ppf_value=-np.inf,\
+        **kwargs):
+    """Computes the d-prime sensitivity index of predictions
+    from various data formats.  Depending on the choice of
+    `mode`, this function can take one of the following format:
+
+    * Binary classification outputs (`mode='binary'`; default)
+    * Positive and negative samples (`mode='sample'`)
+    * True positive and false positive rate (`mode='rate'`)
+    * Confusion matrix (`mode='confusionmat'`)
 
     Parameters
     ----------
-    y_true: array, shape = [n_samples]
-        True values, interpreted as strictly positive or not
-        (i.e. converted to binary).
-        Could be in {-1, +1} or {0, 1} or {False, True}.
+    A, B:
+        If `mode` is 'binary' (default):
+
+            A: array, shape = [n_samples],
+                True values, interpreted as strictly positive or not
+                (i.e. converted to binary).
+                Could be in {-1, +1} or {0, 1} or {False, True}.
+
+            B: array, shape = [n_samples],
+                Predicted values (real).
+
+        If `mode` is 'sample':
+
+            A: array-like,
+                Positive sample values (e.g., raw projection values
+                of the positive classifier).
+
+            B: array-like,
+                Negative sample values.
+
+        If `mode` is 'rate':
+
+            A: array-like, shape = [n_groupings]
+                True positive rates
+
+            B: array-like, shape = [n_groupings]
+                False positive rates
+
+        if `mode` is 'confusionmat':
+
+            A: array-like, shape = [n_classes (true), n_classes (pred)]
+                Confusion matrix, where the element M_{rc} means
+                the number of times when the classifier or subject
+                guesses that a test sample in the r-th class
+                belongs to the c-th class.
+
+            B: ignored
+
+    mode: {'binary', 'sample', 'rate'}, optional, (default='binary')
+        Directs the interpretation of A and B.
+
+    max_value: float, optional (default=np.inf)
+        Maximum possible d-prime value.
+
+    min_value: float, optional (default=-np.inf)
+        Minimum possible d-prime value.
+
+    max_ppf_value: float, optional (default=np.inf)
+        Maximum possible ppf value.
+        Used only when mode is 'rate' or 'confusionmat'.
+
+    min_ppf_value: float, optional (default=-np.inf).
+        Minimum possible ppf value.
+        Used only when mode is 'rate' or 'confusionmat'.
 
-    y_pred: array, shape = [n_samples]
-        Predicted values (real).
+    kwargs: named arguments, optional
+        Passed to ``confusion_matrix_stats()`` and used only when `mode`
+        is 'confusionmat'.  By assigning ``collation``,
+        ``fudge_mode``, ``fudge_factor``, etc. one can
+        change the behavior of d-prime computation
+        (see ``confusion_matrix_stats()`` for details).
 
     Returns
     -------
-    dp: float or None
-        d-prime, None if d-prime is undefined
+    dp: float or array of shape = [n_groupings]
+        A d-prime value or array of d-primes, where each element
+        corresponds to each grouping of positives and negatives
+        (when `mode` is 'rate' or 'confusionmat')
 
     References
     ----------
     http://en.wikipedia.org/wiki/D'
+    http://en.wikipedia.org/wiki/Confusion_matrix
     """
 
     # -- basic checks and conversion
-    assert len(y_true) == len(y_pred)
-    assert np.isfinite(y_true).all()
-    assert np.isfinite(y_pred).all()
-
-    y_true = np.array(y_true)
-    assert y_true.ndim == 1
-
-    y_pred = np.array(y_pred)
-    assert y_pred.ndim == 1
-
-    # -- actual computation
-    pos = y_true > 0
-    neg = ~pos
-    pos_mean = y_pred[pos].mean()
-    neg_mean = y_pred[neg].mean()
-    pos_var = y_pred[pos].var(ddof=1)
-    neg_var = y_pred[neg].var(ddof=1)
-
-    num = pos_mean - neg_mean
-    div = np.sqrt((pos_var + neg_var) / 2.)
-    if div == 0:
-        dp = None
+    if mode == 'sample':
+        pos, neg = np.array(A), np.array(B)
+
+    elif mode == 'binary':
+        y_true, y_pred = A, B
+
+        assert len(y_true) == len(y_pred)
+        assert np.isfinite(y_true).all()
+
+        y_true = np.array(y_true)
+        assert y_true.ndim == 1
+
+        y_pred = np.array(y_pred)
+        assert y_pred.ndim == 1
+
+        i_pos = y_true > 0
+        i_neg = ~i_pos
+
+        pos = y_pred[i_pos]
+        neg = y_pred[i_neg]
+
+    elif mode == 'rate':
+        TPR, FPR = np.array(A), np.array(B)
+        assert TPR.shape == FPR.shape
+
+    elif mode == 'confusionmat':
+        # A: confusion mat
+        # row means true classes, col means predicted classes
+        P, N, TP, _, FP, _ = confusion_matrix_stats(A, **kwargs)
+
+        TPR = TP / P
+        FPR = FP / N
+
     else:
+        raise ValueError('Invalid mode')
+
+    # -- compute d'
+    if mode in ['sample', 'binary']:
+        assert np.isfinite(pos).all()
+        assert np.isfinite(neg).all()
+
+        if pos.size <= 1:
+            raise ValueError('Not enough positive samples'\
+                    'to estimate the variance')
+        if neg.size <= 1:
+            raise ValueError('Not enough negative samples'\
+                    'to estimate the variance')
+
+        pos_mean = pos.mean()
+        neg_mean = neg.mean()
+        pos_var = pos.var(ddof=1)
+        neg_var = neg.var(ddof=1)
+
+        num = pos_mean - neg_mean
+        div = np.sqrt((pos_var + neg_var) / 2.)
+
         dp = num / div
 
+    else:   # mode is rate or confusionmat
+        ppfTPR = norm.ppf(TPR)
+        ppfFPR = norm.ppf(FPR)
+        ppfTPR = np.clip(ppfTPR, min_ppf_value, max_ppf_value)
+        ppfFPR = np.clip(ppfFPR, min_ppf_value, max_ppf_value)
+        dp = ppfTPR - ppfFPR
+
+    # from Dan's suggestion about clipping d' values...
+    dp = np.clip(dp, min_value, max_value)
+
     return dp