diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index e06085c4c5c26..5d92aefdb4eb1 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -233,7 +233,7 @@ Other enhancements - Add keyword ``sort`` to :func:`pivot_table` to allow non-sorting of the result (:issue:`39143`) - Add keyword ``dropna`` to :meth:`DataFrame.value_counts` to allow counting rows that include ``NA`` values (:issue:`41325`) - :meth:`Series.replace` will now cast results to ``PeriodDtype`` where possible instead of ``object`` dtype (:issue:`41526`) -- Improved error message in ``corr` and ``cov`` methods on :class:`.Rolling`, :class:`.Expanding`, and :class:`.ExponentialMovingWindow` when ``other`` is not a :class:`DataFrame` or :class:`Series` (:issue:`41741`) +- Improved error message in ``corr`` and ``cov`` methods on :class:`.Rolling`, :class:`.Expanding`, and :class:`.ExponentialMovingWindow` when ``other`` is not a :class:`DataFrame` or :class:`Series` (:issue:`41741`) .. --------------------------------------------------------------------------- @@ -959,6 +959,7 @@ MultiIndex - Bug in :meth:`MultiIndex.equals` incorrectly returning ``True`` when :class:`MultiIndex` containing ``NaN`` even when they are differently ordered (:issue:`38439`) - Bug in :meth:`MultiIndex.intersection` always returning empty when intersecting with :class:`CategoricalIndex` (:issue:`38653`) - Bug in :meth:`MultiIndex.reindex` raising ``ValueError`` with empty MultiIndex and indexing only a specific level (:issue:`41170`) +- Bug in :meth:`MultiIndex.reindex` raising ``TypeError`` when reindexing against a flat :class:`Index` (:issue:`41707`) I/O ^^^ @@ -1073,6 +1074,7 @@ Reshaping - Bug in :meth:`DataFrame.sort_values` not reshaping index correctly after sorting on columns, when ``ignore_index=True`` (:issue:`39464`) - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``ExtensionDtype`` dtypes (:issue:`39454`) - Bug in :meth:`DataFrame.append` returning incorrect dtypes with combinations of ``datetime64`` and ``timedelta64`` dtypes (:issue:`39574`) +- Bug in :meth:`DataFrame.append` with a :class:`DataFrame` with a :class:`MultiIndex` and appending a :class:`Series` whose :class:`Index` is not a :class:`MultiIndex` (:issue:`41707`) - Bug in :meth:`DataFrame.pivot_table` returning a ``MultiIndex`` for a single value when operating on and empty ``DataFrame`` (:issue:`13483`) - Allow :class:`Index` to be passed to the :func:`numpy.all` function (:issue:`40180`) - Bug in :meth:`DataFrame.stack` not preserving ``CategoricalDtype`` in a ``MultiIndex`` (:issue:`36991`) @@ -1127,6 +1129,7 @@ Other - Bug in :meth:`DataFrame.clip` not interpreting missing values as no threshold (:issue:`40420`) - Bug in :class:`Series` backed by :class:`DatetimeArray` or :class:`TimedeltaArray` sometimes failing to set the array's ``freq`` to ``None`` (:issue:`41425`) - Bug in creating a :class:`Series` from a ``range`` object that does not fit in the bounds of ``int64`` dtype (:issue:`30173`) +- Bug in creating a :class:`Series` from a ``dict`` with all-tuple keys and an :class:`Index` that requires reindexing (:issue:`41707`) .. --------------------------------------------------------------------------- diff --git a/pandas/core/frame.py b/pandas/core/frame.py index bc44b23da25d5..6d4b723cdf921 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -8921,10 +8921,7 @@ def append( index = Index([other.name], name=self.index.name) idx_diff = other.index.difference(self.columns) - try: - combined_columns = self.columns.append(idx_diff) - except TypeError: - combined_columns = self.columns.astype(object).append(idx_diff) + combined_columns = self.columns.append(idx_diff) other = ( other.reindex(combined_columns, copy=False) .to_frame() diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py index 6ae906edd1d81..6dcb2a44e7d3d 100644 --- a/pandas/core/indexes/interval.py +++ b/pandas/core/indexes/interval.py @@ -774,9 +774,11 @@ def _get_indexer_pointwise(self, target: Index) -> tuple[np.ndarray, np.ndarray] except KeyError: missing.append(i) locs = np.array([-1]) - except InvalidIndexError as err: - # i.e. non-scalar key - raise TypeError(key) from err + except InvalidIndexError: + # i.e. non-scalar key e.g. a tuple. + # see test_append_different_columns_types_raises + missing.append(i) + locs = np.array([-1]) indexer.append(locs) diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 59882422f5439..805420a83108a 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -2541,9 +2541,11 @@ def reindex( elif (indexer >= 0).all(): target = self.take(indexer) else: - # hopefully? - target = MultiIndex.from_tuples(target) - + try: + target = MultiIndex.from_tuples(target) + except TypeError: + # not all tuples, see test_constructor_dict_multiindex_reindex_flat + return target, indexer if ( preserve_names and target.nlevels == self.nlevels diff --git a/pandas/tests/indexes/multi/test_reindex.py b/pandas/tests/indexes/multi/test_reindex.py index 3b0fcd72f3123..38ff6efec40c9 100644 --- a/pandas/tests/indexes/multi/test_reindex.py +++ b/pandas/tests/indexes/multi/test_reindex.py @@ -115,3 +115,14 @@ def test_reindex_empty_with_level(values): expected_indexer = np.array([], dtype=result_indexer.dtype) tm.assert_index_equal(result, expected) tm.assert_numpy_array_equal(result_indexer, expected_indexer) + + +def test_reindex_not_all_tuples(): + keys = [("i", "i"), ("i", "j"), ("j", "i"), "j"] + mi = MultiIndex.from_tuples(keys[:-1]) + idx = Index(keys) + res, indexer = mi.reindex(idx) + + tm.assert_index_equal(res, idx) + expected = np.array([0, 1, 2, -1], dtype=np.intp) + tm.assert_numpy_array_equal(indexer, expected) diff --git a/pandas/tests/reshape/concat/test_append.py b/pandas/tests/reshape/concat/test_append.py index 62fe1ed3a7c49..43fe72b0776ed 100644 --- a/pandas/tests/reshape/concat/test_append.py +++ b/pandas/tests/reshape/concat/test_append.py @@ -184,18 +184,12 @@ def test_append_preserve_index_name(self): dt.datetime(2013, 1, 3, 7, 12), ] ), + pd.MultiIndex.from_arrays(["A B C".split(), "D E F".split()]), ] - indexes_cannot_append_with_other = [ - pd.MultiIndex.from_arrays(["A B C".split(), "D E F".split()]) - ] - - # error: Unsupported operand types for + ("List[Index]" and "List[MultiIndex]") - all_indexes = ( - indexes_can_append + indexes_cannot_append_with_other # type: ignore[operator] + @pytest.mark.parametrize( + "index", indexes_can_append, ids=lambda x: type(x).__name__ ) - - @pytest.mark.parametrize("index", all_indexes, ids=lambda x: type(x).__name__) def test_append_same_columns_type(self, index): # GH18359 @@ -249,41 +243,6 @@ def test_append_different_columns_types(self, df_columns, series_index): ) tm.assert_frame_equal(result, expected) - @pytest.mark.parametrize( - "index_can_append", indexes_can_append, ids=lambda x: type(x).__name__ - ) - @pytest.mark.parametrize( - "index_cannot_append_with_other", - indexes_cannot_append_with_other, - ids=lambda x: type(x).__name__, - ) - def test_append_different_columns_types_raises( - self, index_can_append, index_cannot_append_with_other - ): - # GH18359 - # Dataframe.append will raise if MultiIndex appends - # or is appended to a different index type - # - # See also test 'test_append_different_columns_types' above for - # appending without raising. - - df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index_can_append) - ser = Series([7, 8, 9], index=index_cannot_append_with_other, name=2) - msg = ( - r"Expected tuple, got (int|long|float|str|" - r"pandas._libs.interval.Interval)|" - r"object of type '(int|float|Timestamp|" - r"pandas._libs.interval.Interval)' has no len\(\)|" - ) - with pytest.raises(TypeError, match=msg): - df.append(ser) - - df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index_cannot_append_with_other) - ser = Series([7, 8, 9], index=index_can_append, name=2) - - with pytest.raises(TypeError, match=msg): - df.append(ser) - def test_append_dtype_coerce(self, sort): # GH 4993 diff --git a/pandas/tests/series/test_constructors.py b/pandas/tests/series/test_constructors.py index 646d1f0ab1508..a540b692f3aec 100644 --- a/pandas/tests/series/test_constructors.py +++ b/pandas/tests/series/test_constructors.py @@ -1700,6 +1700,14 @@ def test_constructor_dict_multiindex(self): result = result.reindex(index=expected.index) tm.assert_series_equal(result, expected) + def test_constructor_dict_multiindex_reindex_flat(self): + # construction involves reindexing with a MultiIndex corner case + data = {("i", "i"): 0, ("i", "j"): 1, ("j", "i"): 2, "j": np.nan} + expected = Series(data) + + result = Series(expected[:-1].to_dict(), index=expected.index) + tm.assert_series_equal(result, expected) + def test_constructor_dict_timedelta_index(self): # GH #12169 : Resample category data with timedelta index # construct Series from dict as data and TimedeltaIndex as index