ENH - Index set operation modifications to address issue #23525

pandas/core/indexes/base.py

@@ -2756,6 +2756,16 @@ def _get_reconciled_name_object(self, other):
             return self._shallow_copy(name=name)
         return self
 
+    def _union_inconsistent_dtypes(self, other):
+        this = self.astype('O')
+        # call Index for when `other` is list-like
+        other = Index(other).astype('O')
+        return Index.union(this, other).astype('O')
+
+    def _is_inconsistent(self, other):
+        return (type(self) is not type(other)
+                or not is_dtype_equal(self.dtype, other.dtype))
+
     def union(self, other):
         """
         Form the union of two Index objects and sorts if possible.
@@ -2778,22 +2788,36 @@ def union(self, other):
 
         """
         self._assert_can_do_setop(other)
+
+        if self._is_inconsistent(other):
+            return self._union_inconsistent_dtypes(other)
+
         other = ensure_index(other)
+        return self._union(other)
 
-        if len(other) == 0 or self.equals(other):
-            return self._get_reconciled_name_object(other)
+    def _union(self, other):
 
+        # if is_dtype_equal(self.dtype, other.dtype):
         if len(self) == 0:
             return other._get_reconciled_name_object(self)
+        elif len(other) == 0:
+            return self._get_reconciled_name_object(other)
+
+        if self.equals(other):
+            return self._get_reconciled_name_object(other)
+
 
         # TODO: is_dtype_union_equal is a hack around
         # 1. buggy set ops with duplicates (GH #13432)
         # 2. CategoricalIndex lacking setops (GH #10186)
         # Once those are fixed, this workaround can be removed
-        if not is_dtype_union_equal(self.dtype, other.dtype):
-            this = self.astype('O')
-            other = other.astype('O')
-            return this.union(other)
+
+        # remove? this might be too lenient
+        # if not is_dtype_union_equal(self.dtype, other.dtype):
+        #     this = self.astype('O')
+        #     other = other.astype('O')
+        #     # force object dtype, for empty index cases
+        #     return this.union(other).astype('O')
 
         # TODO(EA): setops-refactor, clean all this up
         if is_period_dtype(self) or is_datetime64tz_dtype(self):

pandas/core/indexes/category.py

@@ -872,6 +872,14 @@ def _delegate_method(self, name, *args, **kwargs):
             return res
         return CategoricalIndex(res, name=self.name)
 
+    def _is_inconsistent(self, other):
+        if type(self) is not type(other):
+            return True
+        elif type(self.dtype) is not type(other.dtype):
+            return True
+        else:
+            return False
+
 
 CategoricalIndex._add_numeric_methods_add_sub_disabled()
 CategoricalIndex._add_numeric_methods_disabled()

pandas/core/indexes/datetimes.py

@@ -578,6 +578,17 @@ def unique(self, level=None):
         result = super(DatetimeIndex, naive).unique(level=level)
         return self._shallow_copy(result.values)
 
+    def _is_inconsistent(self, other):
+        is_inconsistent = super(DatetimeIndex, self)._is_inconsistent(other)
+        if is_inconsistent:
+            if hasattr(other, 'dtype'):
+                # If same base, consider consistent, let UTC logic takeover
+                return self.dtype.base != other.dtype.base
+            else:
+                return True
+        else:
+            return is_inconsistent
+
     def union(self, other):
         """
         Specialized union for DatetimeIndex objects. If combine
@@ -592,10 +603,11 @@ def union(self, other):
         -------
         y : Index or DatetimeIndex
         """
-        self._assert_can_do_setop(other)
+        return super(DatetimeIndex, self).union(other)
 
+    def _union(self, other):
         if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super(DatetimeIndex, self).union(other)
+            return super(DatetimeIndex, self)._union(other)
 
         if not isinstance(other, DatetimeIndex):
             try:
@@ -608,7 +620,7 @@ def union(self, other):
         if this._can_fast_union(other):
             return this._fast_union(other)
         else:
-            result = Index.union(this, other)
+            result = Index._union(this, other)
             if isinstance(result, DatetimeIndex):
                 result._tz = timezones.tz_standardize(this.tz)
                 if (result.freq is None and

pandas/core/indexes/interval.py

@@ -1038,7 +1038,16 @@ def overlaps(self, other):
 
     def _setop(op_name):
         def func(self, other):
-            other = self._as_like_interval_index(other)
+            try:
+                other = self._as_like_interval_index(other)
+            # allow ValueError from this method to raise to catch mixed closed
+            # except only Non-Interval index mismatches.
+            except TypeError:
+                # Currently this will cause difference operations to return
+                # object dtype as opposed to IntervalIndex, unlike other Index
+                # objects that return the same type when using `difference` on
+                # mismatched types
+                return getattr(self.astype('O'), op_name)(other)
 
             # GH 19016: ensure set op will not return a prohibited dtype
             subtypes = [self.dtype.subtype, other.dtype.subtype]
@@ -1059,6 +1068,7 @@ def func(self, other):
 
             return type(self).from_tuples(result, closed=self.closed,
                                           name=result_name)
+
         return func
 
     union = _setop('union')

pandas/core/indexes/numeric.py

@@ -228,6 +228,17 @@ def _assert_safe_casting(cls, data, subarr):
                 raise TypeError('Unsafe NumPy casting, you must '
                                 'explicitly cast')
 
+    def _is_inconsistent(self, other):
+        from pandas.core.indexes.range import RangeIndex
+        is_inconsistent = super(Int64Index, self)._is_inconsistent(other)
+        if is_inconsistent:
+            if type(self) is Int64Index and isinstance(other, RangeIndex):
+                return False
+            else:
+                return True
+        else:
+            return is_inconsistent
+
 
 Int64Index._add_numeric_methods()
 Int64Index._add_logical_methods()

pandas/core/indexes/period.py

@@ -835,6 +835,11 @@ def join(self, other, how='left', level=None, return_indexers=False,
         """
         self._assert_can_do_setop(other)
 
+        if not isinstance(other, PeriodIndex):
+            return self.astype('O').join(other, how=how, level=level,
+                                         return_indexers=return_indexers,
+                                         sort=sort)
+
         result = Int64Index.join(self, other, how=how, level=level,
                                  return_indexers=return_indexers,
                                  sort=sort)
@@ -847,10 +852,9 @@ def join(self, other, how='left', level=None, return_indexers=False,
     def _assert_can_do_setop(self, other):
         super(PeriodIndex, self)._assert_can_do_setop(other)
 
-        if not isinstance(other, PeriodIndex):
-            raise ValueError('can only call with other PeriodIndex-ed objects')
-
-        if self.freq != other.freq:
+        # *Can't* use PeriodIndexes of different freqs
+        # *Can* use PeriodIndex/DatetimeIndex
+        if isinstance(other, PeriodIndex) and self.freq != other.freq:
             msg = DIFFERENT_FREQ_INDEX.format(self.freqstr, other.freqstr)
             raise IncompatibleFrequency(msg)
 

pandas/core/indexes/range.py

@@ -416,6 +416,13 @@ def _extended_gcd(self, a, b):
             old_t, t = t, old_t - quotient * t
         return old_r, old_s, old_t
 
+    def _is_inconsistent(self, other):
+        is_inconsistent = super(RangeIndex, self)._is_inconsistent(other)
+        if is_inconsistent:
+            return not type(other) is Int64Index
+        else:
+            return is_inconsistent
+
     def union(self, other):
         """
         Form the union of two Index objects and sorts if possible
@@ -428,9 +435,11 @@ def union(self, other):
         -------
         union : Index
         """
-        self._assert_can_do_setop(other)
+        return super(RangeIndex, self).union(other)
+
+    def _union(self, other):
         if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super(RangeIndex, self).union(other)
+            return super(RangeIndex, self)._union(other)
 
         if isinstance(other, RangeIndex):
             start_s, step_s = self._start, self._step
@@ -469,7 +478,7 @@ def union(self, other):
                         (end_s - step_o <= end_o)):
                     return RangeIndex(start_r, end_r + step_o, step_o)
 
-        return self._int64index.union(other)
+        return self._int64index._union(other)
 
     @Appender(_index_shared_docs['join'])
     def join(self, other, how='left', level=None, return_indexers=False,

pandas/core/indexes/timedeltas.py

@@ -281,10 +281,11 @@ def union(self, other):
         -------
         y : Index or TimedeltaIndex
         """
-        self._assert_can_do_setop(other)
+        return super(TimedeltaIndex, self).union(other)
 
+    def _union(self, other):
         if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super(TimedeltaIndex, self).union(other)
+            return super(TimedeltaIndex, self)._union(other)
 
         if not isinstance(other, TimedeltaIndex):
             try:
@@ -296,7 +297,7 @@ def union(self, other):
         if this._can_fast_union(other):
             return this._fast_union(other)
         else:
-            result = Index.union(this, other)
+            result = Index._union(this, other)
             if isinstance(result, TimedeltaIndex):
                 if result.freq is None:
                     result.freq = to_offset(result.inferred_freq)

pandas/tests/indexes/common.py

@@ -626,11 +626,7 @@ def test_intersection_base(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with tm.assert_raises_regex(ValueError, msg):
-                        result = first.intersection(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 else:
                     result = first.intersection(case)
@@ -653,11 +649,7 @@ def test_union_base(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with tm.assert_raises_regex(ValueError, msg):
-                        result = first.union(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 else:
                     result = first.union(case)
@@ -684,11 +676,7 @@ def test_difference_base(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with tm.assert_raises_regex(ValueError, msg):
-                        result = first.difference(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 elif isinstance(idx, (DatetimeIndex, TimedeltaIndex)):
                     assert result.__class__ == answer.__class__
@@ -718,11 +706,7 @@ def test_symmetric_difference(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with tm.assert_raises_regex(ValueError, msg):
-                        result = first.symmetric_difference(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 else:
                     result = first.symmetric_difference(case)

pandas/tests/indexes/datetimes/test_datetime.py

@@ -303,10 +303,9 @@ def test_join_with_period_index(self, join_type):
             c_idx_type='p', r_idx_type='dt')
         s = df.iloc[:5, 0]
 
-        with tm.assert_raises_regex(ValueError,
-                                    'can only call with other '
-                                    'PeriodIndex-ed objects'):
-            df.columns.join(s.index, how=join_type)
+        expected = df.columns.astype('O').join(s.index, how=join_type)
+        result = df.columns.join(s.index, how=join_type)
+        tm.assert_index_equal(expected, result)
 
     def test_factorize(self):
         idx1 = DatetimeIndex(['2014-01', '2014-01', '2014-02', '2014-02',

pandas/tests/indexes/datetimes/test_setops.py

@@ -29,10 +29,13 @@ def test_union2(self):
         assert tm.equalContents(union, everything)
 
         # GH 10149
+        expected = first.astype('O').union(
+            pd.Index(second.values, dtype='O')
+        ).astype('O')
         cases = [klass(second.values) for klass in [np.array, Series, list]]
         for case in cases:
             result = first.union(case)
-            assert tm.equalContents(result, everything)
+            assert tm.equalContents(result, expected)
 
     @pytest.mark.parametrize("tz", tz)
     def test_union(self, tz):
@@ -256,11 +259,11 @@ def test_datetimeindex_union_join_empty(self):
         empty = Index([])
 
         result = dti.union(empty)
-        assert isinstance(result, DatetimeIndex)
-        assert result is result
+        tm.assert_index_equal(result, dti.astype('O'))
 
         result = dti.join(empty)
         assert isinstance(result, DatetimeIndex)
+        tm.assert_index_equal(result, dti)
 
     def test_join_nonunique(self):
         idx1 = to_datetime(['2012-11-06 16:00:11.477563',

pandas/tests/indexes/interval/test_interval.py

@@ -835,15 +835,16 @@ def test_symmetric_difference(self, closed):
 
     @pytest.mark.parametrize('op_name', [
         'union', 'intersection', 'difference', 'symmetric_difference'])
-    def test_set_operation_errors(self, closed, op_name):
+    def test_set_incompatible_types(self, closed, op_name):
         index = self.create_index(closed=closed)
         set_op = getattr(index, op_name)
 
         # non-IntervalIndex
-        msg = ('the other index needs to be an IntervalIndex too, but '
-               'was type Int64Index')
-        with tm.assert_raises_regex(TypeError, msg):
-            set_op(Index([1, 2, 3]))
+        expected = getattr(index.astype('O'), op_name)(Index([1, 2, 3]))
+        result = set_op(Index([1, 2, 3]))
+        tm.assert_index_equal(result, expected)
+
+        # Come back to mixed interval types
 
         # mixed closed
         msg = ('can only do set operations between two IntervalIndex objects '

pandas/tests/indexes/period/test_setops.py

@@ -105,9 +105,9 @@ def test_union_misc(self):
         with pytest.raises(period.IncompatibleFrequency):
             index.union(index2)
 
-        msg = 'can only call with other PeriodIndex-ed objects'
-        with tm.assert_raises_regex(ValueError, msg):
-            index.join(index.to_timestamp())
+        # msg = 'can only call with other PeriodIndex-ed objects'
+        # with tm.assert_raises_regex(ValueError, msg):
+        #     index.join(index.to_timestamp())
 
         index3 = period_range('1/1/2000', '1/20/2000', freq='2D')
         with pytest.raises(period.IncompatibleFrequency):

pandas/tests/indexes/test_base.py

@@ -876,7 +876,12 @@ def test_union_identity(self):
         union = first.union(first)
         assert union is first
 
+        # This should no longer be the same object, since [] is not consistent,
+        # both objects will be recast to dtype('O')
         union = first.union([])
+        assert union.equals(first)
+
+        union = first.union(pd.Index([]))
         assert union is first
 
         union = Index([]).union(first)