From 561d052fb957eb1b9918c3fb765af5f328ec4e20 Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 31 Aug 2022 16:10:14 -0700 Subject: [PATCH 1/3] REF: de-duplicate take_nd --- pandas/core/internals/blocks.py | 34 ++++++------------------------- pandas/core/internals/concat.py | 7 ++++++- pandas/core/internals/managers.py | 3 +++ 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 010358d3a21ec..d0d67a01584c9 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -881,8 +881,13 @@ def take_nd( ) # Called from three places in managers, all of which satisfy - # this assertion + # these assertions + if isinstance(self, ExtensionBlock): + # NB: in this case, the 'axis' kwarg will be ignored in the + # algos.take_nd call above. + assert not (self.ndim == 1 and new_mgr_locs is None) assert not (axis == 0 and new_mgr_locs is None) + if new_mgr_locs is None: new_mgr_locs = self._mgr_locs @@ -1752,33 +1757,6 @@ def is_view(self) -> bool: def is_numeric(self): return self.values.dtype._is_numeric - def take_nd( - self, - indexer: npt.NDArray[np.intp], - axis: int = 0, - new_mgr_locs: BlockPlacement | None = None, - fill_value=lib.no_default, - ) -> Block: - """ - Take values according to indexer and return them as a block. - """ - if fill_value is lib.no_default: - fill_value = None - - # TODO(EA2D): special case not needed with 2D EAs - # axis doesn't matter; we are really a single-dim object - # but are passed the axis depending on the calling routing - # if its REALLY axis 0, then this will be a reindex and not a take - new_values = self.values.take(indexer, fill_value=fill_value, allow_fill=True) - - # Called from three places in managers, all of which satisfy - # this assertion - assert not (self.ndim == 1 and new_mgr_locs is None) - if new_mgr_locs is None: - new_mgr_locs = self._mgr_locs - - return self.make_block_same_class(new_values, new_mgr_locs) - def _slice( self, slicer: slice | npt.NDArray[np.bool_] | npt.NDArray[np.intp] ) -> ExtensionArray: diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index 0df8aa5a055b0..db4bf0f1e0d28 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -19,7 +19,9 @@ ArrayLike, DtypeObj, Manager, + Manager2D, Shape, + npt, ) from pandas.util._decorators import cache_readonly @@ -182,7 +184,10 @@ def concat_arrays(to_concat: list) -> ArrayLike: def concatenate_managers( - mgrs_indexers, axes: list[Index], concat_axis: int, copy: bool + mgrs_indexers: list[tuple[Manager2D, dict[int, npt.NDArray[np.intp] | None]]], + axes: list[Index], + concat_axis: int, + copy: bool, ) -> Manager: """ Concatenate block managers into one. diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 9f4c799941afd..67eab3e86b89c 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -725,6 +725,9 @@ def reindex_indexer( result.axes[axis] = new_axis return result + assert isinstance(indexer, np.ndarray) + assert indexer.dtype == np.intp + # some axes don't allow reindexing with dups if not allow_dups: self.axes[axis]._validate_can_reindex(indexer) From 63527a3203fad99df138a12bc291a5fa7aec0be0 Mon Sep 17 00:00:00 2001 From: Brock Date: Wed, 31 Aug 2022 19:58:44 -0700 Subject: [PATCH 2/3] revert --- pandas/core/internals/concat.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index db4bf0f1e0d28..0df8aa5a055b0 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -19,9 +19,7 @@ ArrayLike, DtypeObj, Manager, - Manager2D, Shape, - npt, ) from pandas.util._decorators import cache_readonly @@ -184,10 +182,7 @@ def concat_arrays(to_concat: list) -> ArrayLike: def concatenate_managers( - mgrs_indexers: list[tuple[Manager2D, dict[int, npt.NDArray[np.intp] | None]]], - axes: list[Index], - concat_axis: int, - copy: bool, + mgrs_indexers, axes: list[Index], concat_axis: int, copy: bool ) -> Manager: """ Concatenate block managers into one. From 32e6650d0040b97a5e71fc6e73503655fcb4db28 Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 1 Sep 2022 07:53:30 -0700 Subject: [PATCH 3/3] fix 32bit --- pandas/core/internals/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 67eab3e86b89c..fe5748b240eab 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -725,8 +725,8 @@ def reindex_indexer( result.axes[axis] = new_axis return result + # Should be intp, but in some cases we get int64 on 32bit builds assert isinstance(indexer, np.ndarray) - assert indexer.dtype == np.intp # some axes don't allow reindexing with dups if not allow_dups: