Added MLKR algorithm

Author: devashishd12

metric_learn/mlkr.py

@@ -0,0 +1,124 @@
+"""
+Metric Learning for Kernel Regression (MLKR), Weinberger et al.,
+
+MLKR is an algorithm for supervised metric learning, which learns a distance
+function by directly minimising the leave-one-out regression error. This
+algorithm can also be viewed as a supervised variation of PCA and can be used
+for dimensionality reduction and high dimensional data visualization.
+"""
+from __future__ import division
+import numpy as np
+from six.moves import xrange
+
+from .base_metric import BaseMetricLearner
+
+class MLKR(BaseMetricLearner):
+    """Metric Learning for Kernel Regression (MLKR)"""
+    def __init__(self, A=None, epsilon=0.01):
+        """
+        MLKR initialization
+
+        Parameters
+        ----------
+        A: Initialization of matrix A. Defaults to the identity matrix.
+        epsilon: Step size for gradient descent
+        """
+        self.params = {
+            "A": A,
+            "epsilon": epsilon
+        }
+
+    def _process_inputs(self, X, y):
+        if X.ndim == 1:
+            X = X[:, np.newaxis]
+        if y.ndim == 1:
+            y == y[:, np.newaxis]
+        self.X = X
+        n, d = X.shape
+        assert y.shape[0] == n
+        return y, n, d
+
+    def fit(self, X, y):
+        """
+        Fit MLKR model
+
+        Parameters:
+        ----------
+        X : (n x d) array of samples
+        y : (n) data labels
+
+        Returns:
+        -------
+        self: Instance of self
+        """
+        y, n, d = self._process_inputs(X, y)
+        alpha = 0.0001  # Stopping criterion
+        if self.params['A'] is None:
+            A = np.identity(d)  # Initialize A as eye matrix
+        else:
+            A = self.params['A']
+            assert A.shape == (d, d)
+        cost = np.Inf
+        # Gradient descent procedure
+        while cost > alpha:
+            K = self._computeK(X, A, n)
+            yhat = self._computeyhat(y, K)
+            sum_i = 0
+            for i in xrange(n):
+                sum_j = 0
+                for j in xrange(n):
+                    sum_j += (yhat[j] - y[j]) * K[i][j] * \
+                             (X[i, :] - X[j, :])[:, np.newaxis].dot \
+                                 ((X[i, :] - X[j, :])[:, np.newaxis].T)
+                sum_i += (yhat[i] - y[i]) * sum_j
+            gradient = 4 * A.dot(sum_i)
+            A -= self.params['epsilon'] * gradient
+            cost = np.sum(np.square(yhat - y))
+        self._transformer = A
+        return self
+
+    def _computeK(self, X, A, n):
+        """
+        Internal helper function to compute K matrix.
+
+        Parameters:
+        ----------
+        X: (n x d) array of samples
+        A: (d x d) 'A' matrix
+        n: number of rows in X
+
+        Returns:
+        -------
+        K: (n x n) K matrix where Kij = exp(-distance(x_i, x_j)) where
+           distance is defined as squared L2 norm of (x_i - x_j)
+        """
+        dist_mat = np.zeros(shape=(n, n))
+        for i in xrange(n):
+            for j in xrange(n):
+                if i == j:
+                    dist = 0
+                else:
+                    dist = np.sum(np.square(A.dot((X[i, :] - X[j, :]))))
+                dist_mat[i, j] = dist
+        return np.exp(-dist_mat)
+
+    def _computeyhat(self, y, K):
+        """
+        Internal helper function to compute yhat matrix.
+
+        Parameters:
+        ----------
+        y: (n) data labels
+        K: (n x n) K matrix
+
+        Returns:
+        -------
+        yhat: (n x 1) yhat matrix
+        """
+        numerator = K.dot(y)
+        denominator = np.sum(K, 1)[:, np.newaxis]
+        yhat = numerator / denominator
+        return yhat
+
+    def transformer(self):
+        return self._transformer