diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 05b4417f427ab..65b821fe2b8a8 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -205,7 +205,7 @@ Performance improvements - Performance improvement in :meth:`IntervalIndex.isin` (:issue:`38353`) - Performance improvement in :meth:`Series.mean` for nullable data types (:issue:`34814`) - Performance improvement in :meth:`Series.isin` for nullable data types (:issue:`38340`) -- +- Performance improvement in :meth:`DataFrame.corr` for method=kendall (:issue:`28329`) .. --------------------------------------------------------------------------- diff --git a/pandas/_libs/algos.pyx b/pandas/_libs/algos.pyx index 76bfb001cea81..080a84bef1e58 100644 --- a/pandas/_libs/algos.pyx +++ b/pandas/_libs/algos.pyx @@ -393,6 +393,100 @@ def nancorr_spearman(ndarray[float64_t, ndim=2] mat, Py_ssize_t minp=1) -> ndarr return result +# ---------------------------------------------------------------------- +# Kendall correlation +# Wikipedia article: https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient + +@cython.boundscheck(False) +@cython.wraparound(False) +def nancorr_kendall(ndarray[float64_t, ndim=2] mat, Py_ssize_t minp=1) -> ndarray: + """ + Perform kendall correlation on a 2d array + + Parameters + ---------- + mat : np.ndarray[float64_t, ndim=2] + Array to compute kendall correlation on + minp : int, default 1 + Minimum number of observations required per pair of columns + to have a valid result. + + Returns + ------- + numpy.ndarray[float64_t, ndim=2] + Correlation matrix + """ + cdef: + Py_ssize_t i, j, k, xi, yi, N, K + ndarray[float64_t, ndim=2] result + ndarray[float64_t, ndim=2] ranked_mat + ndarray[uint8_t, ndim=2] mask + float64_t currj + ndarray[uint8_t, ndim=1] valid + ndarray[int64_t] sorted_idxs + ndarray[float64_t, ndim=1] col + int64_t n_concordant + int64_t total_concordant = 0 + int64_t total_discordant = 0 + float64_t kendall_tau + int64_t n_obs + const int64_t[:] labels_n + + N, K = (mat).shape + + result = np.empty((K, K), dtype=np.float64) + mask = np.isfinite(mat) + + ranked_mat = np.empty((N, K), dtype=np.float64) + # For compatibility when calling rank_1d + labels_n = np.zeros(N, dtype=np.int64) + + for i in range(K): + ranked_mat[:, i] = rank_1d(mat[:, i], labels_n) + + for xi in range(K): + sorted_idxs = ranked_mat[:, xi].argsort() + ranked_mat = ranked_mat[sorted_idxs] + mask = mask[sorted_idxs] + for yi in range(xi + 1, K): + valid = mask[:, xi] & mask[:, yi] + if valid.sum() < minp: + result[xi, yi] = NaN + result[yi, xi] = NaN + else: + # Get columns and order second column using 1st column ranks + if not valid.all(): + col = ranked_mat[valid.nonzero()][:, yi] + else: + col = ranked_mat[:, yi] + n_obs = col.shape[0] + total_concordant = 0 + total_discordant = 0 + for j in range(n_obs - 1): + currj = col[j] + # Count num concordant and discordant pairs + n_concordant = 0 + for k in range(j, n_obs): + if col[k] > currj: + n_concordant += 1 + total_concordant += n_concordant + total_discordant += (n_obs - 1 - j - n_concordant) + # Note: we do total_concordant+total_discordant here which is + # equivalent to the C(n, 2), the total # of pairs, + # listed on wikipedia + kendall_tau = (total_concordant - total_discordant) / \ + (total_concordant + total_discordant) + result[xi, yi] = kendall_tau + result[yi, xi] = kendall_tau + + if mask[:, xi].sum() > minp: + result[xi, xi] = 1 + else: + result[xi, xi] = NaN + + return result + + # ---------------------------------------------------------------------- ctypedef fused algos_t: diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 0a1ea4041a10b..288292589e940 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -8463,8 +8463,7 @@ def corr(self, method="pearson", min_periods=1) -> DataFrame: min_periods : int, optional Minimum number of observations required per pair of columns - to have a valid result. Currently only available for Pearson - and Spearman correlation. + to have a valid result. Returns ------- @@ -8498,7 +8497,9 @@ def corr(self, method="pearson", min_periods=1) -> DataFrame: correl = libalgos.nancorr(mat, minp=min_periods) elif method == "spearman": correl = libalgos.nancorr_spearman(mat, minp=min_periods) - elif method == "kendall" or callable(method): + elif method == "kendall": + correl = libalgos.nancorr_kendall(mat, minp=min_periods) + elif callable(method): if min_periods is None: min_periods = 1 mat = mat.T