[MRG+1] Threshold for pairs learners

metric_learn/base_metric.py

@@ -1,8 +1,8 @@
 from numpy.linalg import cholesky
 from scipy.spatial.distance import euclidean
 from sklearn.base import BaseEstimator
-from sklearn.utils.validation import _is_arraylike
-from sklearn.metrics import roc_auc_score
+from sklearn.utils.validation import _is_arraylike, check_is_fitted
+from sklearn.metrics import roc_auc_score, accuracy_score
 import numpy as np
 from abc import ABCMeta, abstractmethod
 import six
@@ -317,7 +317,8 @@ def predict(self, pairs):
     y_predicted : `numpy.ndarray` of floats, shape=(n_constraints,)
       The predicted learned metric value between samples in every pair.
     """
-    return self.decision_function(pairs)
+    check_is_fitted(self, ['threshold_', 'transformer_'])
+    return - 2 * (self.decision_function(pairs) > self.threshold_) + 1
 
   def decision_function(self, pairs):
     """Returns the learned metric between input pairs.
@@ -369,6 +370,13 @@ def score(self, pairs, y):
     """
     return roc_auc_score(y, self.decision_function(pairs))
 
+  def set_default_threshold(self, pairs, y):
+    """Returns a threshold that is the mean between the similar metrics
+    mean, and the dissimilar metrics mean"""
+    similar_threshold = np.mean(self.decision_function(pairs[y==1]))
+    dissimilar_threshold = np.mean(self.decision_function(pairs[y==1]))
+    self.threshold_ = np.mean([similar_threshold, dissimilar_threshold])
+
 
 class _QuadrupletsClassifierMixin(BaseMetricLearner):
 
@@ -393,6 +401,7 @@ def predict(self, quadruplets):
     prediction : `numpy.ndarray` of floats, shape=(n_constraints,)
       Predictions of the ordering of pairs, for each quadruplet.
     """
+    check_is_fitted(self, 'transformer_')
     quadruplets = check_input(quadruplets, type_of_inputs='tuples',
                               preprocessor=self.preprocessor_,
                               estimator=self, tuple_size=self._tuple_size)
@@ -435,11 +444,22 @@ def score(self, quadruplets, y=None):
       points, or 2D array of indices of quadruplets if the metric learner
       uses a preprocessor.
 
-    y : Ignored, for scikit-learn compatibility.
+    y : array-like, shape=(n_constraints,) or `None`
+      Labels of constraints. y[i] should be 1 if
+      d(pairs[i, 0], X[i, 1]) is wanted to be larger than
+      d(X[i, 2], X[i, 3]), and -1 if it is wanted to be smaller. If None,
+      `y` will be set to `np.ones(quadruplets.shape[0])`, i.e. we want all
+      first two points to be closer than the last two points in each
+      quadruplet.
 
     Returns
     -------
     score : float
       The quadruplets score.
     """
-    return -np.mean(self.predict(quadruplets))
+    quadruplets = check_input(quadruplets, y, type_of_inputs='tuples',
+                              preprocessor=self.preprocessor_,
+                              estimator=self, tuple_size=self._tuple_size)
+    if y is None:
+      y = np.ones(quadruplets.shape[0])
+    return accuracy_score(y, self.predict(quadruplets))

metric_learn/itml.py

@@ -148,6 +148,11 @@ class ITML(_BaseITML, _PairsClassifierMixin):
   transformer_ : `numpy.ndarray`, shape=(num_dims, n_features)
       The linear transformation ``L`` deduced from the learned Mahalanobis
       metric (See function `transformer_from_metric`.)
+
+  threshold_ : `float`
+      If the distance metric between two points is lower than this threshold,
+      points will be classified as similar, otherwise they will be
+      classified as dissimilar.
   """
 
   def fit(self, pairs, y, bounds=None):
@@ -176,7 +181,9 @@ def fit(self, pairs, y, bounds=None):
     self : object
         Returns the instance.
     """
-    return self._fit(pairs, y, bounds=bounds)
+    self._fit(pairs, y, bounds=bounds)
+    self.threshold_ = np.mean(self.bounds_)
+    return self
 
 
 class ITML_Supervised(_BaseITML, TransformerMixin):

metric_learn/lsml.py

@@ -46,9 +46,15 @@ def __init__(self, tol=1e-3, max_iter=1000, prior=None, verbose=False,
     super(_BaseLSML, self).__init__(preprocessor)
 
   def _fit(self, quadruplets, y=None, weights=None):
-    quadruplets = self._prepare_inputs(quadruplets,
+    quadruplets = self._prepare_inputs(quadruplets, y,
                                        type_of_inputs='tuples')
-
+    if y is None:
+      y = np.ones(quadruplets.shape[0])
+    # we swap the quadruplets where the label is -1 since they are not in
+    # the right order
+    quadruplets_to_swap = quadruplets[y == -1]
+    quadruplets[y == -1] = np.column_stack([quadruplets_to_swap[:, 2:],
+                                            quadruplets_to_swap[:, :2]])
     # check to make sure that no two constrained vectors are identical
     vab = quadruplets[:, 0, :] - quadruplets[:, 1, :]
     vcd = quadruplets[:, 2, :] - quadruplets[:, 3, :]
@@ -144,18 +150,22 @@ class LSML(_BaseLSML, _QuadrupletsClassifierMixin):
       metric (See function `transformer_from_metric`.)
   """
 
-  def fit(self, quadruplets, weights=None):
+  def fit(self, quadruplets, y=None, weights=None):
     """Learn the LSML model.
 
     Parameters
     ----------
     quadruplets : array-like, shape=(n_constraints, 4, n_features) or
                   (n_constraints, 4)
         3D array-like of quadruplets of points or 2D array of quadruplets of
-        indicators. In order to supervise the algorithm in the right way, we
-        should have the four samples ordered in a way such that:
-        d(pairs[i, 0],X[i, 1]) < d(X[i, 2], X[i, 3]) for all 0 <= i <
-        n_constraints.
+        indicators.
+    y : array-like, shape=(n_constraints,) or `None`
+        Labels of constraints. y[i] should be 1 if
+        d(pairs[i, 0], X[i, 1]) is wanted to be larger than
+        d(X[i, 2], X[i, 3]), and -1 if it is wanted to be smaller. If None,
+        `y` will be set to `np.ones(quadruplets.shape[0])`, i.e. we want to
+        put all first two points closer than the last two points in each
+        quadruplet.
     weights : (n_constraints,) array of floats, optional
         scale factor for each constraint
 

metric_learn/mmc.py

@@ -359,6 +359,11 @@ class MMC(_BaseMMC, _PairsClassifierMixin):
   transformer_ : `numpy.ndarray`, shape=(num_dims, n_features)
       The linear transformation ``L`` deduced from the learned Mahalanobis
       metric (See function `transformer_from_metric`.)
+
+  threshold_ : `float`
+      If the distance metric between two points is lower than this threshold,
+      points will be classified as similar, otherwise they will be
+      classified as dissimilar.
   """
 
   def fit(self, pairs, y):
@@ -379,7 +384,9 @@ def fit(self, pairs, y):
     self : object
         Returns the instance.
     """
-    return self._fit(pairs, y)
+    self._fit(pairs, y)
+    self.set_default_threshold(pairs, y)
+    return self
 
 
 class MMC_Supervised(_BaseMMC, TransformerMixin):

metric_learn/sdml.py

@@ -81,6 +81,11 @@ class SDML(_BaseSDML, _PairsClassifierMixin):
   transformer_ : `numpy.ndarray`, shape=(num_dims, n_features)
       The linear transformation ``L`` deduced from the learned Mahalanobis
       metric (See function `transformer_from_metric`.)
+
+  threshold_ : `float`
+      If the distance metric between two points is lower than this threshold,
+      points will be classified as similar, otherwise they will be
+      classified as dissimilar.
   """
 
   def fit(self, pairs, y):
@@ -101,7 +106,9 @@ def fit(self, pairs, y):
     self : object
         Returns the instance.
     """
-    return self._fit(pairs, y)
+    self._fit(pairs, y)
+    self.set_default_threshold(pairs, y)
+    return self
 
 
 class SDML_Supervised(_BaseSDML, TransformerMixin):

test/test_pairs_classifiers.py

@@ -0,0 +1,65 @@
+import pytest
+from sklearn.exceptions import NotFittedError
+from sklearn.model_selection import train_test_split
+
+from test.test_utils import pairs_learners, ids_pairs_learners
+from sklearn.utils.testing import set_random_state
+from sklearn import clone
+import numpy as np
+
+
+@pytest.mark.parametrize('with_preprocessor', [True, False])
+@pytest.mark.parametrize('estimator, build_dataset', pairs_learners,
+                         ids=ids_pairs_learners)
+def test_predict_only_one_or_minus_one(estimator, build_dataset,
+                                       with_preprocessor):
+  """Test that all predicted values are either +1 or -1"""
+  input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+  pairs_train, pairs_test, y_train, y_test = train_test_split(input_data,
+                                                              labels)
+  estimator.fit(pairs_train, y_train)
+  predictions = estimator.predict(pairs_test)
+  assert np.isin(predictions, [-1, 1]).all()
+
+
+@pytest.mark.parametrize('with_preprocessor', [True, False])
+@pytest.mark.parametrize('estimator, build_dataset', pairs_learners,
+                         ids=ids_pairs_learners)
+def test_predict_monotonous(estimator, build_dataset,
+                            with_preprocessor):
+  """Test that there is a threshold distance separating points labeled as
+  similar and points labeled as dissimilar """
+  input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+  pairs_train, pairs_test, y_train, y_test = train_test_split(input_data,
+                                                              labels)
+  estimator.fit(pairs_train, y_train)
+  distances = estimator.score_pairs(pairs_test)
+  predictions = estimator.predict(pairs_test)
+  min_dissimilar = np.min(distances[predictions == -1])
+  max_similar = np.max(distances[predictions == 1])
+  assert max_similar <= min_dissimilar
+  separator = np.mean([min_dissimilar, max_similar])
+  assert (predictions[distances > separator] == -1).all()
+  assert (predictions[distances < separator] == 1).all()
+
+
+@pytest.mark.parametrize('with_preprocessor', [True, False])
+@pytest.mark.parametrize('estimator, build_dataset', pairs_learners,
+                         ids=ids_pairs_learners)
+def test_raise_not_fitted_error_if_not_fitted(estimator, build_dataset,
+                                            with_preprocessor):
+  """Test that a NotFittedError is raised if someone tries to predict and
+  the metric learner has not been fitted."""
+  input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+  with pytest.raises(NotFittedError):
+    estimator.predict(input_data)
+

test/test_quadruplets_classifiers.py

@@ -0,0 +1,65 @@
+import pytest
+from sklearn.exceptions import NotFittedError
+from sklearn.model_selection import train_test_split
+
+from test.test_utils import quadruplets_learners, ids_quadruplets_learners
+from sklearn.utils.testing import set_random_state
+from sklearn import clone
+import numpy as np
+
+
+@pytest.mark.parametrize('with_preprocessor', [True, False])
+@pytest.mark.parametrize('estimator, build_dataset', quadruplets_learners,
+                         ids=ids_quadruplets_learners)
+def test_predict_only_one_or_minus_one(estimator, build_dataset,
+                                       with_preprocessor):
+  """Test that all predicted values are either +1 or -1"""
+  input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+  (quadruplets_train,
+   quadruplets_test, y_train, y_test) = train_test_split(input_data, labels)
+  estimator.fit(quadruplets_train, y_train)
+  predictions = estimator.predict(quadruplets_test)
+  assert np.isin(predictions, [-1, 1]).all()
+
+
+@pytest.mark.parametrize('with_preprocessor', [True, False])
+@pytest.mark.parametrize('estimator, build_dataset', quadruplets_learners,
+                         ids=ids_quadruplets_learners)
+def test_predict_monotonous(estimator, build_dataset,
+                            with_preprocessor):
+  """Test that there is a threshold distance separating points labeled as
+  similar and points labeled as dissimilar """
+  input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+  (quadruplets_train,
+   quadruplets_test, y_train, y_test) = train_test_split(input_data, labels)
+  estimator.fit(quadruplets_train, y_train)
+  distances = estimator.score_quadruplets(quadruplets_test)
+  predictions = estimator.predict(quadruplets_test)
+  min_dissimilar = np.min(distances[predictions == -1])
+  max_similar = np.max(distances[predictions == 1])
+  assert max_similar <= min_dissimilar
+  separator = np.mean([min_dissimilar, max_similar])
+  assert (predictions[distances > separator] == -1).all()
+  assert (predictions[distances < separator] == 1).all()
+
+
+@pytest.mark.parametrize('with_preprocessor', [True, False])
+@pytest.mark.parametrize('estimator, build_dataset', quadruplets_learners,
+                         ids=ids_quadruplets_learners)
+def test_raise_not_fitted_error_if_not_fitted(estimator, build_dataset,
+                                            with_preprocessor):
+  """Test that a NotFittedError is raised if someone tries to predict and
+  the metric learner has not been fitted."""
+  input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+  with pytest.raises(NotFittedError):
+    estimator.predict(input_data)
+

test/test_sklearn_compat.py

@@ -15,9 +15,11 @@
 import numpy as np
 from sklearn.model_selection import (cross_val_score, cross_val_predict,
                                      train_test_split, KFold)
+from sklearn.metrics.scorer import get_scorer
 from sklearn.utils.testing import _get_args
 from test.test_utils import (metric_learners, ids_metric_learners,
-                             mock_preprocessor)
+                             mock_preprocessor, tuples_learners,
+                             ids_tuples_learners)
 
 
 # Wrap the _Supervised methods with a deterministic wrapper for testing.
@@ -88,22 +90,55 @@ def test_mmc(self):
 
 
 @pytest.mark.parametrize('with_preprocessor', [True, False])
-@pytest.mark.parametrize('estimator, build_dataset', metric_learners,
-                         ids=ids_metric_learners)
-def test_cross_validation_is_finite(estimator, build_dataset,
-                                    with_preprocessor):
-  """Tests that validation on metric-learn estimators returns something finite
+@pytest.mark.parametrize('estimator, build_dataset', tuples_learners,
+                         ids=ids_tuples_learners)
+def test_various_scoring_on_tuples_learners(estimator, build_dataset,
+                                            with_preprocessor):
+  """Tests that metric-learn estimators' scoring returns something finite,
+  for other scoring than default scoring. (List of scikit-learn's scores can be
+  found in sklearn.metrics.scorer). For each type of output (predict,
+  predict_proba, decision_function), we test a bunch of scores.
   """
-  if any(hasattr(estimator, method) for method in ["predict", "score"]):
-    input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  input_data, labels, preprocessor, _ = build_dataset(with_preprocessor)
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+
+  # scores that need a predict function: every tuples learner should have a
+  # predict function (whether the pair is of positive samples or negative
+  # samples)
+  for scoring in ['accuracy', 'f1']:
+    check_score_is_finite(scoring, estimator, input_data, labels)
+  # scores that need a predict_proba:
+  if hasattr(estimator, "predict_proba"):
+    for scoring in ['neg_log_loss', 'brier_score']:
+      check_score_is_finite(scoring, estimator, input_data, labels)
+  # scores that need a decision_function: every tuples learner should have a
+  # decision function (the metric between points)
+  for scoring in ['roc_auc', 'average_precision', 'precision', 'recall']:
+    check_score_is_finite(scoring, estimator, input_data, labels)
+
+
+def check_score_is_finite(scoring, estimator, input_data, labels):
     estimator = clone(estimator)
-    estimator.set_params(preprocessor=preprocessor)
-    set_random_state(estimator)
-    if hasattr(estimator, "score"):
-      assert np.isfinite(cross_val_score(estimator, input_data, labels)).all()
-    if hasattr(estimator, "predict"):
-      assert np.isfinite(cross_val_predict(estimator,
-                                           input_data, labels)).all()
+    assert np.isfinite(cross_val_score(estimator, input_data, labels,
+                                       scoring=scoring)).all()
+    estimator.fit(input_data, labels)
+    assert np.isfinite(get_scorer(scoring)(estimator, input_data, labels))
+
+
+@pytest.mark.parametrize('estimator, build_dataset', tuples_learners,
+                         ids=ids_tuples_learners)
+def test_cross_validation_is_finite(estimator, build_dataset):
+  """Tests that validation on metric-learn estimators returns something finite
+  """
+  input_data, labels, preprocessor, _ = build_dataset()
+  estimator = clone(estimator)
+  estimator.set_params(preprocessor=preprocessor)
+  set_random_state(estimator)
+  assert np.isfinite(cross_val_score(estimator, input_data, labels)).all()
+  assert np.isfinite(cross_val_predict(estimator,
+                                       input_data, labels)).all()
 
 
 @pytest.mark.parametrize('with_preprocessor', [True, False])