Implement scoring functions (and make tests work):

- Make PairsClassifierMixin and QuadrupletsClassifierMixin classes, to implement scoring functions
- Implement a new API for supervised wrappers of weakly supervised learning estimators (through the use of base classes, (ex: BaseMMC), from which inherit child classes (ex: MMC and MMC_Supervised) (which is the same idea as in PR scikit-learn-contrib#85
- Delete tests that use tuples learners as transformers (as we do not want to support this behaviour anymore: it is too complicated to allow such different input types (tuples or points) for the same estimator

Author: William de Vazelhes

metric_learn/base_metric.py

@@ -1,9 +1,11 @@
 from numpy.linalg import inv, cholesky
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.utils.validation import check_array
+from sklearn.metrics import roc_auc_score
+import numpy as np
 
 
-class BaseMetricLearner(BaseEstimator, TransformerMixin):
+class BaseMetricLearner(BaseEstimator):
   def __init__(self):
     raise NotImplementedError('BaseMetricLearner should not be instantiated')
 
@@ -19,6 +21,9 @@ def metric(self):
     L = self.transformer()
     return L.T.dot(L)
 
+
+class MetricTransformer(TransformerMixin):
+
   def transformer(self):
     """Computes the transformation matrix from the Mahalanobis matrix.
 
@@ -49,3 +54,105 @@ def transform(self, X=None):
       X = check_array(X, accept_sparse=True)
     L = self.transformer()
     return X.dot(L.T)
+
+
+class _PairsClassifierMixin:
+
+  def predict(self, pairs):
+    """Predicts the learned similarity between input pairs.
+
+    Returns the learned metric value between samples in every pair. It should
+    ideally be low for similar samples and high for dissimilar samples.
+
+    Parameters
+    ----------
+    pairs : array-like, shape=(n_constraints, 2, n_features)
+      A constrained dataset of paired samples.
+
+    Returns
+    -------
+    y_predicted : `numpy.ndarray` of floats, shape=(n_constraints,)
+      The predicted learned metric value between samples in every pair.
+    """
+    pairwise_diffs = pairs[:, 0, :] - pairs[:, 1, :]
+    return np.sqrt(np.sum(pairwise_diffs.dot(self.metric()) * pairwise_diffs,
+                          axis=1))
+
+  def decision_function(self, pairs):
+    return self.predict(pairs)
+
+  def score(self, pairs, y):
+    """Computes score of pairs similarity prediction.
+
+    Returns the ``roc_auc`` score of the fitted metric learner. It is
+    computed in the following way: for every value of a threshold
+    ``t`` we classify all pairs of samples where the predicted distance is
+    inferior to ``t`` as belonging to the "similar" class, and the other as
+    belonging to the "dissimilar" class, and we count false positive and
+    true positives as in a classical ``roc_auc`` curve.
+
+    Parameters
+    ----------
+    pairs : array-like, shape=(n_constraints, 2, n_features)
+      Input Pairs.
+
+    y : array-like, shape=(n_constraints,)
+      The corresponding labels.
+
+    Returns
+    -------
+    score : float
+      The ``roc_auc`` score.
+    """
+    return roc_auc_score(y, self.decision_function(pairs))
+
+
+class _QuadrupletsClassifierMixin:
+
+  def predict(self, quadruplets):
+    """Predicts differences between sample similarities in input quadruplets.
+
+    For each quadruplet of samples, computes the difference between the learned
+    metric of the first pair minus the learned metric of the second pair.
+
+    Parameters
+    ----------
+    quadruplets : array-like, shape=(n_constraints, 4, n_features)
+      Input quadruplets.
+
+    Returns
+    -------
+    prediction : np.ndarray of floats, shape=(n_constraints,)
+      Metric differences.
+    """
+    similar_diffs = quadruplets[:, 0, :] - quadruplets[:, 1, :]
+    dissimilar_diffs = quadruplets[:, 2, :] - quadruplets[:, 3, :]
+    return (np.sqrt(np.sum(similar_diffs.dot(self.metric()) *
+                           similar_diffs, axis=1)) -
+            np.sqrt(np.sum(dissimilar_diffs.dot(self.metric()) *
+                           dissimilar_diffs, axis=1)))
+
+  def decision_function(self, quadruplets):
+    return self.predict(quadruplets)
+
+  def score(self, quadruplets, y=None):
+    """Computes score on an input constrained dataset
+
+    Returns the accuracy score of the following classification task: a record
+    is correctly classified if the predicted similarity between the first two
+    samples is higher than that of the last two.
+
+    Parameters
+    ----------
+    quadruplets : array-like, shape=(n_constraints, 4, n_features)
+      Input quadruplets.
+
+    y : Ignored, for scikit-learn compatibility.
+
+    Returns
+    -------
+    score : float
+      The quadruplets score.
+    """
+    predicted_sign = self.decision_function(quadruplets) < 0
+    return np.sum(predicted_sign) / predicted_sign.shape[0]

metric_learn/covariance.py

@@ -12,10 +12,10 @@
 import numpy as np
 from sklearn.utils.validation import check_array
 
-from .base_metric import BaseMetricLearner
+from .base_metric import BaseMetricLearner, MetricTransformer
 
 
-class Covariance(BaseMetricLearner):
+class Covariance(BaseMetricLearner, MetricTransformer):
   def __init__(self):
     pass
 

metric_learn/itml.py

@@ -19,12 +19,13 @@
 from sklearn.metrics import pairwise_distances
 from sklearn.utils.validation import check_array, check_X_y
 
-from .base_metric import BaseMetricLearner
+from .base_metric import (BaseMetricLearner, _PairsClassifierMixin,
+                          MetricTransformer)
 from .constraints import Constraints, wrap_pairs
 from ._util import vector_norm
 
 
-class ITML(BaseMetricLearner):
+class _BaseITML(BaseMetricLearner):
   """Information Theoretic Metric Learning (ITML)"""
   def __init__(self, gamma=1., max_iter=1000, convergence_threshold=1e-3,
                A0=None, verbose=False):
@@ -80,8 +81,7 @@ def _process_pairs(self, pairs, y, bounds):
     y = y.astype(bool)
     return pairs, y
 
-
-  def fit(self, pairs, y, bounds=None):
+  def _fit(self, pairs, y, bounds=None):
     """Learn the ITML model.
 
     Parameters
@@ -153,7 +153,13 @@ def metric(self):
     return self.A_
 
 
-class ITML_Supervised(ITML):
+class ITML(_BaseITML, _PairsClassifierMixin):
+
+  def fit(self, pairs, y, bounds=None):
+    return self._fit(pairs, y, bounds=bounds)
+
+
+class ITML_Supervised(_BaseITML, MetricTransformer):
   """Information Theoretic Metric Learning (ITML)"""
   def __init__(self, gamma=1., max_iter=1000, convergence_threshold=1e-3,
                num_labeled=np.inf, num_constraints=None, bounds=None, A0=None,
@@ -177,9 +183,9 @@ def __init__(self, gamma=1., max_iter=1000, convergence_threshold=1e-3,
     verbose : bool, optional
         if True, prints information while learning
     """
-    ITML.__init__(self, gamma=gamma, max_iter=max_iter,
-                  convergence_threshold=convergence_threshold,
-                  A0=A0, verbose=verbose)
+    _BaseITML.__init__(self, gamma=gamma, max_iter=max_iter,
+                       convergence_threshold=convergence_threshold,
+                       A0=A0, verbose=verbose)
     self.num_labeled = num_labeled
     self.num_constraints = num_constraints
     self.bounds = bounds
@@ -209,4 +215,4 @@ def fit(self, X, y, random_state=np.random):
     pos_neg = c.positive_negative_pairs(num_constraints,
                                         random_state=random_state)
     pairs, y = wrap_pairs(X, pos_neg)
-    return ITML.fit(self, pairs, y, bounds=self.bounds)
+    return _BaseITML._fit(self, pairs, y, bounds=self.bounds)

metric_learn/lfda.py

@@ -18,10 +18,10 @@
 from sklearn.metrics import pairwise_distances
 from sklearn.utils.validation import check_X_y
 
-from .base_metric import BaseMetricLearner
+from .base_metric import BaseMetricLearner, MetricTransformer
 
 
-class LFDA(BaseMetricLearner):
+class LFDA(BaseMetricLearner, MetricTransformer):
   '''
   Local Fisher Discriminant Analysis for Supervised Dimensionality Reduction
   Sugiyama, ICML 2006

metric_learn/lmnn.py

@@ -17,11 +17,11 @@
 from sklearn.utils.validation import check_X_y, check_array
 from sklearn.metrics import euclidean_distances
 
-from .base_metric import BaseMetricLearner
+from .base_metric import BaseMetricLearner, MetricTransformer
 
 
 # commonality between LMNN implementations
-class _base_LMNN(BaseMetricLearner):
+class _base_LMNN(BaseMetricLearner, MetricTransformer):
   def __init__(self, k=3, min_iter=50, max_iter=1000, learn_rate=1e-7,
                regularization=0.5, convergence_tol=0.001, use_pca=True,
                verbose=False):

metric_learn/lsml.py

@@ -13,11 +13,12 @@
 from six.moves import xrange
 from sklearn.utils.validation import check_array, check_X_y
 
-from .base_metric import BaseMetricLearner
+from .base_metric import (BaseMetricLearner, _QuadrupletsClassifierMixin,
+                          MetricTransformer)
 from .constraints import Constraints, wrap_pairs
 
 
-class LSML(BaseMetricLearner):
+class _BaseLSML(BaseMetricLearner):
   def __init__(self, tol=1e-3, max_iter=1000, prior=None, verbose=False):
     """Initialize LSML.
 
@@ -60,7 +61,7 @@ def _prepare_quadruplets(self, quadruplets, weights):
   def metric(self):
     return self.M_
 
-  def fit(self, quadruplets, weights=None):
+  def _fit(self, quadruplets, weights=None):
     """Learn the LSML model.
 
     Parameters
@@ -140,7 +141,13 @@ def _gradient(self, metric):
     return dMetric
 
 
-class LSML_Supervised(LSML):
+class LSML(_BaseLSML, _QuadrupletsClassifierMixin):
+
+  def fit(self, quadruplets, weights=None):
+    return self._fit(quadruplets, weights=weights)
+
+
+class LSML_Supervised(_BaseLSML, MetricTransformer):
   def __init__(self, tol=1e-3, max_iter=1000, prior=None, num_labeled=np.inf,
                num_constraints=None, weights=None, verbose=False):
     """Initialize the learner.
@@ -160,8 +167,8 @@ def __init__(self, tol=1e-3, max_iter=1000, prior=None, num_labeled=np.inf,
     verbose : bool, optional
         if True, prints information while learning
     """
-    LSML.__init__(self, tol=tol, max_iter=max_iter, prior=prior,
-                  verbose=verbose)
+    _BaseLSML.__init__(self, tol=tol, max_iter=max_iter, prior=prior,
+                       verbose=verbose)
     self.num_labeled = num_labeled
     self.num_constraints = num_constraints
     self.weights = weights
@@ -189,5 +196,6 @@ def fit(self, X, y, random_state=np.random):
     c = Constraints.random_subset(y, self.num_labeled,
                                   random_state=random_state)
     pos_neg = c.positive_negative_pairs(num_constraints, same_length=True,
-                                      random_state=random_state)
-    return LSML.fit(self, X[np.column_stack(pos_neg)], weights=self.weights)
+                                        random_state=random_state)
+    return _BaseLSML._fit(self, X[np.column_stack(pos_neg)],
+                          weights=self.weights)

metric_learn/mlkr.py

@@ -13,12 +13,12 @@
 from sklearn.decomposition import PCA
 from sklearn.utils.validation import check_X_y
 
-from .base_metric import BaseMetricLearner
+from .base_metric import BaseMetricLearner, MetricTransformer
 
 EPS = np.finfo(float).eps
 
 
-class MLKR(BaseMetricLearner):
+class MLKR(BaseMetricLearner, MetricTransformer):
   """Metric Learning for Kernel Regression (MLKR)"""
   def __init__(self, num_dims=None, A0=None, epsilon=0.01, alpha=0.0001,
                max_iter=1000):

metric_learn/mmc.py

@@ -22,13 +22,13 @@
 from sklearn.metrics import pairwise_distances
 from sklearn.utils.validation import check_array, check_X_y
 
-from .base_metric import BaseMetricLearner
+from .base_metric import (BaseMetricLearner, _PairsClassifierMixin,
+                          MetricTransformer)
 from .constraints import Constraints, wrap_pairs
 from ._util import vector_norm
 
 
-
-class MMC(BaseMetricLearner):
+class _BaseMMC(BaseMetricLearner):
   """Mahalanobis Metric for Clustering (MMC)"""
   def __init__(self, max_iter=100, max_proj=10000, convergence_threshold=1e-3,
                A0=None, diagonal=False, diagonal_c=1.0, verbose=False):
@@ -58,8 +58,7 @@ def __init__(self, max_iter=100, max_proj=10000, convergence_threshold=1e-3,
     self.diagonal_c = diagonal_c
     self.verbose = verbose
 
-
-  def fit(self, pairs, y):
+  def _fit(self, pairs, y):
     """Learn the MMC model.
 
     Parameters
@@ -390,7 +389,13 @@ def transformer(self):
       return V.T * np.sqrt(np.maximum(0, w[:,None]))
 
 
-class MMC_Supervised(MMC):
+class MMC(_BaseMMC, _PairsClassifierMixin):
+
+  def fit(self, pairs, y):
+    return self._fit(pairs, y)
+
+
+class MMC_Supervised(_BaseMMC, MetricTransformer):
   """Mahalanobis Metric for Clustering (MMC)"""
   def __init__(self, max_iter=100, max_proj=10000, convergence_threshold=1e-6,
                num_labeled=np.inf, num_constraints=None,
@@ -418,10 +423,10 @@ def __init__(self, max_iter=100, max_proj=10000, convergence_threshold=1e-6,
     verbose : bool, optional
         if True, prints information while learning
     """
-    MMC.__init__(self, max_iter=max_iter, max_proj=max_proj,
-                 convergence_threshold=convergence_threshold,
-                 A0=A0, diagonal=diagonal, diagonal_c=diagonal_c,
-                 verbose=verbose)
+    _BaseMMC.__init__(self, max_iter=max_iter, max_proj=max_proj,
+                      convergence_threshold=convergence_threshold,
+                      A0=A0, diagonal=diagonal, diagonal_c=diagonal_c,
+                      verbose=verbose)
     self.num_labeled = num_labeled
     self.num_constraints = num_constraints
 
@@ -448,4 +453,4 @@ def fit(self, X, y, random_state=np.random):
     pos_neg = c.positive_negative_pairs(num_constraints,
                                         random_state=random_state)
     pairs, y = wrap_pairs(X, pos_neg)
-    return MMC.fit(self, pairs, y)
+    return _BaseMMC._fit(self, pairs, y)

metric_learn/nca.py

@@ -8,12 +8,12 @@
 from six.moves import xrange
 from sklearn.utils.validation import check_X_y
 
-from .base_metric import BaseMetricLearner
+from .base_metric import BaseMetricLearner, MetricTransformer
 
 EPS = np.finfo(float).eps
 
 
-class NCA(BaseMetricLearner):
+class NCA(BaseMetricLearner, MetricTransformer):
   def __init__(self, num_dims=None, max_iter=100, learning_rate=0.01):
     self.num_dims = num_dims
     self.max_iter = max_iter

metric_learn/rca.py

@@ -18,7 +18,7 @@
 from sklearn import decomposition
 from sklearn.utils.validation import check_array
 
-from .base_metric import BaseMetricLearner
+from .base_metric import BaseMetricLearner, MetricTransformer
 from .constraints import Constraints
 
 
@@ -35,7 +35,7 @@ def _chunk_mean_centering(data, chunks):
   return chunk_mask, chunk_data
 
 
-class RCA(BaseMetricLearner):
+class RCA(BaseMetricLearner, MetricTransformer):
   """Relevant Components Analysis (RCA)"""
   def __init__(self, num_dims=None, pca_comps=None):
     """Initialize the learner.