index multi

Author: TomAugspurger

doc/source/whatsnew/v0.24.1.rst

@@ -53,8 +53,8 @@ This allows ``sort=True`` to now mean "always sort". A ``TypeError`` is raised i
 
    idx.union(idx, sort=True)
 
-The same change applies to :meth:`Index.symmetric_difference`, which would previously not
-sort the result when ``sort=True`` but the values could not be compared.
+The same change applies to :meth:`Index.difference` and :meth:`Index.symmetric_difference`, which
+would previously not sort the result when ``sort=True`` but the values could not be compared.
 
 Changed the behavior of :meth:`Index.intersection` with ``sort=True``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

pandas/core/indexes/base.py

@@ -2447,7 +2447,7 @@ def intersection(self, other, sort=False):
 
         return taken
 
-    def difference(self, other, sort=True):
+    def difference(self, other, sort=None):
         """
         Return a new Index with elements from the index that are not in
         `other`.
@@ -2457,11 +2457,24 @@ def difference(self, other, sort=True):
         Parameters
         ----------
         other : Index or array-like
-        sort : bool, default True
-            Sort the resulting index if possible
+        sort : bool or None, default None
+            Whether to sort the resulting index. By default, the
+            values are attempted to be sorted, but any TypeError from
+            incomparable elements is caught by pandas.
+
+            * None : Attempt to sort the result, but catch any TypeErrors
+              from comparing incomparable elements.
+            * False : Do not sort the result.
+            * True : Sort the result, raising a TypeError if any elements
+              cannot be compared.
 
             .. versionadded:: 0.24.0
 
+            .. versionchanged:: 0.24.1
+
+               Added the `None` option, which matches the behavior of
+               pandas 0.23.4 and earlier.
+
         Returns
         -------
         difference : Index
@@ -2492,11 +2505,13 @@ def difference(self, other, sort=True):
         label_diff = np.setdiff1d(np.arange(this.size), indexer,
                                   assume_unique=True)
         the_diff = this.values.take(label_diff)
-        if sort:
+        if sort is None:
             try:
                 the_diff = sorting.safe_sort(the_diff)
             except TypeError:
                 pass
+        elif sort:
+            the_diff = sorting.safe_sort(the_diff)
 
         return this._shallow_copy(the_diff, name=result_name, freq=None)
 
@@ -2508,7 +2523,7 @@ def symmetric_difference(self, other, result_name=None, sort=None):
         ----------
         other : Index or array-like
         result_name : str
-        sort : bool or None, default None.
+        sort : bool or None, default None
             Whether to sort the resulting index. By default, the
             values are attempted to be sorted, but any TypeError from
             incomparable elements is caught by pandas.

pandas/core/indexes/multi.py

@@ -2973,7 +2973,7 @@ def intersection(self, other, sort=False):
             return MultiIndex.from_arrays(lzip(*uniq_tuples), sortorder=0,
                                           names=result_names)
 
-    def difference(self, other, sort=True):
+    def difference(self, other, sort=None):
         """
         Compute set difference of two MultiIndex objects
 
@@ -2993,6 +2993,8 @@ def difference(self, other, sort=True):
         other, result_names = self._convert_can_do_setop(other)
 
         if len(other) == 0:
+            if sort:
+                return self.sort_values()
             return self
 
         if self.equals(other):

pandas/tests/indexes/multi/test_set_ops.py

@@ -174,7 +174,10 @@ def test_difference(idx, sort):
 
     # name from empty array
     result = first.difference([], sort=sort)
-    assert first.equals(result)
+    if sort:
+        assert first.sort_values().equals(result)
+    else:
+        assert first.equals(result)
     assert first.names == result.names
 
     # name from non-empty array
@@ -189,6 +192,36 @@ def test_difference(idx, sort):
         first.difference([1, 2, 3, 4, 5], sort=sort)
 
 
+def test_difference_sort_special():
+    idx = pd.MultiIndex.from_product([[1, 0], ['a', 'b']])
+    # sort=None, the default
+    result = idx.difference([])
+    tm.assert_index_equal(result, idx)
+
+    result = idx.difference([], sort=True)
+    expected = pd.MultiIndex.from_product([[0, 1], ['a', 'b']])
+    tm.assert_index_equal(result, expected)
+
+
+def test_difference_sort_incomparable():
+    idx = pd.MultiIndex.from_product([[1, pd.Timestamp('2000'), 2],
+                                      ['a', 'b']])
+
+    other = pd.MultiIndex.from_product([[3, pd.Timestamp('2000'), 4],
+                                        ['c', 'd']])
+    # sort=None, the default
+    # result = idx.difference(other)
+    # tm.assert_index_equal(result, idx)
+
+    # sort=False
+    result = idx.difference(other)
+    tm.assert_index_equal(result, idx)
+
+    # sort=True, raises
+    with pytest.raises(TypeError):
+        idx.difference(other, sort=True)
+
+
 @pytest.mark.parametrize("sort", [True, False])
 def test_union(idx, sort):
     piece1 = idx[:5][::-1]

pandas/tests/indexes/test_base.py

@@ -3,6 +3,7 @@
 from collections import defaultdict
 from datetime import datetime, timedelta
 import math
+import operator
 import sys
 
 import numpy as np
@@ -1099,22 +1100,28 @@ def test_symmetric_difference(self, sort):
         assert tm.equalContents(result, expected)
         assert result.name is None
 
-    def test_symmetric_difference_incomparable(self):
+    @pytest.mark.parametrize('opname', ['difference', 'symmetric_difference'])
+    def test_difference_incomparable(self, opname):
         a = pd.Index([3, pd.Timestamp('2000'), 1])
         b = pd.Index([2, pd.Timestamp('1999'), 1])
+        op = operator.methodcaller(opname, b)
 
         # sort=None, the default
-        result = a.symmetric_difference(b)
+        result = op(a)
         expected = pd.Index([3, pd.Timestamp('2000'), 2, pd.Timestamp('1999')])
+        if opname == 'difference':
+            expected = expected[:2]
         tm.assert_index_equal(result, expected)
 
         # sort=False
-        result = a.symmetric_difference(b, sort=False)
+        op = operator.methodcaller(opname, b, sort=False)
+        result = op(a)
         tm.assert_index_equal(result, expected)
 
         # sort=True, raises
+        op = operator.methodcaller(opname, b, sort=True)
         with pytest.raises(TypeError, match='Cannot compare'):
-            a.symmetric_difference(b, sort=True)
+            op(a)
 
     @pytest.mark.parametrize("sort", [True, False])
     def test_symmetric_difference_mi(self, sort):