ENH: Added DataFrame.compare and Series.compare (GH30429)

doc/source/whatsnew/v1.1.0.rst

@@ -36,6 +36,31 @@ For example:
    ser["2014"]
    ser.loc["May 2015"]
 
+.. _whatsnew_110.differences:
+
+Comparing two `DataFrame` or two `Series` and summarizing the differences
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We've added :meth:`DataFrame.differences` and :meth:`Series.differences` for comparing two `DataFrame` or two `Series` (:issue:`30429`)
+
+.. ipython:: python
+
+   df = pd.DataFrame(
+       {
+           "col1": ["a", "a", "b", "b", "a"],
+           "col2": [1.0, 2.0, 3.0, np.nan, 5.0],
+           "col3": [1.0, 2.0, 3.0, 4.0, 5.0]
+       },
+       columns=["col1", "col2", "col3"],
+   )
+   df
+   df2 = df.copy()
+   df2.loc[0, 'col1'] = 'c'
+   df2.loc[2, 'col3'] = 4.0
+   df2
+   df.differences(df2)
+
+
 .. _whatsnew_110.enhancements.other:
 
 Other enhancements

pandas/core/frame.py

@@ -5405,6 +5405,94 @@ def _construct_result(self, result) -> "DataFrame":
         out.columns = self.columns
         return out
 
+    @Appender(
+        """
+Returns
+-------
+DataFrame
+    DataFrame that shows the differences stacked side by side.
+
+See Also
+--------
+Series.differences: Show differences.
+
+Examples
+--------
+>>> df = pd.DataFrame(
+...     {
+...         "col1": ["a", "a", "b", "b", "a"],
+...         "col2": [1.0, 2.0, 3.0, np.nan, 5.0],
+...         "col3": [1.0, 2.0, 3.0, 4.0, 5.0]
+...     },
+...     columns=["col1", "col2", "col3"],
+... )
+>>> df
+  col1  col2  col3
+0    a   1.0   1.0
+1    a   2.0   2.0
+2    b   3.0   3.0
+3    b   NaN   4.0
+4    a   5.0   5.0
+
+>>> df2 = df.copy()
+>>> df2.loc[0, 'col1'] = 'c'
+>>> df2.loc[2, 'col3'] = 4.0
+>>> df2
+  col1  col2  col3
+0    c   1.0   1.0
+1    a   2.0   2.0
+2    b   3.0   4.0
+3    b   NaN   4.0
+4    a   5.0   5.0
+
+Stack the differences on columns
+
+>>> df.differences(df2)
+  col1       col3
+  self other self other
+0    a     c  NaN   NaN
+2  NaN   NaN  3.0   4.0
+
+Stack the differences on rows
+
+>>> df.differences(df2, axis=0)
+        col1  col3
+0 self     a   NaN
+  other    c   NaN
+2 self   NaN   3.0
+  other  NaN   4.0
+
+Keep all the original indices (rows and columns)
+
+>>> df.differences(df2, keep_indices=True)
+  col1       col2       col3
+  self other self other self other
+0    a     c  NaN   NaN  NaN   NaN
+1  NaN   NaN  NaN   NaN  NaN   NaN
+2  NaN   NaN  NaN   NaN  3.0   4.0
+3  NaN   NaN  NaN   NaN  NaN   NaN
+4  NaN   NaN  NaN   NaN  NaN   NaN
+
+Keep all original indices and data
+
+>>> df.differences(df2, keep_indices=True, keep_values=True)
+  col1       col2       col3
+  self other self other self other
+0    a     c  1.0   1.0  1.0   1.0
+1    a     a  2.0   2.0  2.0   2.0
+2    b     b  3.0   3.0  3.0   4.0
+3    b     b  NaN   NaN  4.0   4.0
+4    a     a  5.0   5.0  5.0   5.0
+"""
+    )
+    @Appender(_shared_docs["differences"] % _shared_doc_kwargs)
+    def differences(
+        self, other: "DataFrame", axis=1, keep_indices=False, keep_values=False
+    ) -> "DataFrame":
+        return super().differences(
+            other=other, axis=axis, keep_indices=keep_indices, keep_values=keep_values
+        )
+
     def combine(
         self, other: "DataFrame", func, fill_value=None, overwrite=True
     ) -> "DataFrame":

pandas/core/generic.py

@@ -8180,6 +8180,90 @@ def ranker(data):
 
         return ranker(data)
 
+    _shared_docs[
+        "differences"
+    ] = """
+        Compare to another %(klass)s and show the differences.
+
+        The axis on which to stack results and how much information to
+        preserve can be customized.
+
+        Note that NaNs are considered not different from other NaNs.
+
+        Parameters
+        ----------
+        other : %(klass)s
+            Object to compare with.
+
+        axis : {0 or 'index', 1 or 'columns'}, default 1
+            Determine how the differences are stacked.
+            * 0, or 'index' : Stack differences on neighbouring rows.
+            * 1, or 'columns' : Stack differences on neighbouring columns.
+
+        keep_indices : bool, default False
+            Whether to keep the rows and columns that are equal, or drop them.
+
+        keep_values : bool, default False
+            Whether to keep the values that are equal, or show as NaNs.
+        """
+
+    @Appender(_shared_docs["differences"] % _shared_doc_kwargs)
+    def differences(self, other, axis=1, keep_indices=False, keep_values=False):
+        from pandas.core.reshape.concat import concat
+
+        mask = ~((self == other) | (self.isna() & other.isna()))
+        keys = ["self", "other"]
+
+        if not keep_values:
+            self = self.where(mask)
+            other = other.where(mask)
+
+        if not keep_indices:
+            if isinstance(self, ABCDataFrame):
+                cmask = mask.any()
+                rmask = mask.any(axis=1)
+                self = self.loc[rmask, cmask]
+                other = other.loc[rmask, cmask]
+            else:
+                self = self[mask]
+                other = other[mask]
+
+        if axis in (1, "columns"):  # This is needed for Series
+            axis = 1
+        else:
+            axis = self._get_axis_number(axis)
+
+        diff = concat([self, other], axis=axis, keys=keys)
+
+        if axis >= self.ndim:
+            # No need to reorganize data if stacking on new axis
+            # This currently applies for stacking two Series on columns
+            return diff
+
+        ax = diff._get_axis(axis)
+        ax_names = np.array(ax.names)
+
+        # set index names to positions to avoid confusion
+        ax.names = np.arange(len(ax_names))
+
+        # bring self-other to inner level
+        order = list(range(1, ax.nlevels)) + [0]
+        if isinstance(diff, ABCDataFrame):
+            diff = diff.reorder_levels(order, axis=axis)
+        else:
+            diff = diff.reorder_levels(order)
+
+        # restore the index names in order
+        diff._get_axis(axis=axis).names = ax_names[order]
+
+        # reorder axis to keep things organized
+        indices = (
+            np.arange(diff.shape[axis]).reshape([2, diff.shape[axis] // 2]).T.flatten()
+        )
+        diff = diff.take(indices, axis=axis)
+
+        return diff
+
     _shared_docs[
         "align"
     ] = """

pandas/core/series.py

@@ -23,7 +23,7 @@
 from pandas._config import get_option
 
 from pandas._libs import index as libindex, lib, properties, reshape, tslibs
-from pandas._typing import Label
+from pandas._typing import FrameOrSeries, Label
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import Appender, Substitution
 from pandas.util._validators import validate_bool_kwarg, validate_percentile
@@ -2607,7 +2607,70 @@ def _binop(self, other, func, level=None, fill_value=None):
         ret = ops._construct_result(self, result, new_index, name)
         return ret
 
+    @Appender(
+        """
+Returns
+-------
+Series or DataFrame
+    If axis is 0 or 'index' the result will be a Series.
+    If axis is 1 or 'columns' the result will be a DataFrame.
+
+See Also
+--------
+DataFrame.differences: Show differences.
+
+Examples
+--------
+>>> s1 = pd.Series(["a", "b", "c", "d", "e"])
+>>> s2 = pd.Series(["a", "a", "c", "b", "e"])
+
+Stack the differences on columns
+
+>>> s1.differences(s2)
+  self other
+1    b     a
+3    d     b
+
+Stack the differences on indices
+
+>>> s1.differences(s2, axis=0)
+1  self     b
+   other    a
+3  self     d
+   other    b
+dtype: object
+
+Keep all the original indices
+
+>>> s1.differences(s2, keep_indices=True)
+  self other
+0  NaN   NaN
+1    b     a
+2  NaN   NaN
+3    d     b
+4  NaN   NaN
+
+Keep all original indices and data
+
+>>> s1.differences(s2, keep_indices=True, keep_values=True)
+  self other
+0    a     a
+1    b     a
+2    c     c
+3    d     b
+4    e     e
+"""
+    )
+    @Appender(generic._shared_docs["differences"] % _shared_doc_kwargs)
+    def differences(
+        self, other: "Series", axis=1, keep_indices=False, keep_values=False
+    ) -> FrameOrSeries:
+        return super().differences(
+            other=other, axis=axis, keep_indices=keep_indices, keep_values=keep_values
+        )
+
     def combine(self, other, func, fill_value=None) -> "Series":
+
         """
         Combine the Series with a Series or scalar according to `func`.