From b4503d5190e7c045a0fa3456ff2c44a8caa507c8 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 24 Jun 2019 21:03:12 -0700 Subject: [PATCH 1/4] Remove checks for is_sparse that are never truthy --- pandas/core/dtypes/concat.py | 5 +---- pandas/core/internals/blocks.py | 3 --- pandas/core/internals/concat.py | 2 -- pandas/core/internals/managers.py | 15 +++++---------- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/pandas/core/dtypes/concat.py b/pandas/core/dtypes/concat.py index a01ba7fc94f22..242885c7a9679 100644 --- a/pandas/core/dtypes/concat.py +++ b/pandas/core/dtypes/concat.py @@ -73,10 +73,7 @@ def _get_series_result_type(result, objs=None): return DataFrame # otherwise it is a SingleBlockManager (axis = 0) - if result._block.is_sparse: - return SparseSeries - else: - return objs[0]._constructor + return objs[0]._constructor def _get_frame_result_type(result, objs): diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 4cc6c86417b3b..d1660adbe8e15 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -62,7 +62,6 @@ class Block(PandasObject): is_bool = False is_object = False is_categorical = False - is_sparse = False is_extension = False _box_to_block_values = True _can_hold_na = False @@ -1189,8 +1188,6 @@ def take_nd(self, indexer, axis, new_mgr_locs=None, fill_tuple=None): # sparse is treated like an ndarray, but needs .get_values() shaping values = self.values - if self.is_sparse: - values = self.get_values() if fill_tuple is None: fill_value = self.fill_value diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index d92c15e1d6f93..8f699ae24230d 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -187,8 +187,6 @@ def get_reindexed_values(self, empty_dtype, upcasted_na): pass elif getattr(self.block, 'is_categorical', False): pass - elif getattr(self.block, 'is_sparse', False): - pass elif getattr(self.block, 'is_extension', False): pass else: diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 7fe34279c0482..f4b0b1d8ed7df 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -102,16 +102,11 @@ def __init__(self, self.blocks = tuple(blocks) # type: Tuple[Block, ...] for block in blocks: - if block.is_sparse: - if len(block.mgr_locs) != 1: - raise AssertionError("Sparse block refers to multiple " - "items") - else: - if self.ndim != block.ndim: - raise AssertionError( - 'Number of Block dimensions ({block}) must equal ' - 'number of axes ({self})'.format(block=block.ndim, - self=self.ndim)) + if self.ndim != block.ndim: + raise AssertionError( + 'Number of Block dimensions ({block}) must equal ' + 'number of axes ({self})'.format(block=block.ndim, + self=self.ndim)) if do_integrity_check: self._verify_integrity() From 5fb9a6270c8ce75b8020c75982fc6b06de6d6a36 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 24 Jun 2019 21:03:52 -0700 Subject: [PATCH 2/4] Remove never-False box_to_block_values --- pandas/core/internals/blocks.py | 1 - pandas/core/internals/managers.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index d1660adbe8e15..bc7366e13c011 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -63,7 +63,6 @@ class Block(PandasObject): is_object = False is_categorical = False is_extension = False - _box_to_block_values = True _can_hold_na = False _can_consolidate = True _verify_integrity = True diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index f4b0b1d8ed7df..a6523e8b5a249 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -961,7 +961,7 @@ def iget(self, i, fastpath=True): """ block = self.blocks[self._blknos[i]] values = block.iget(self._blklocs[i]) - if not fastpath or not block._box_to_block_values or values.ndim != 1: + if not fastpath or values.ndim != 1: return values # fastpath shortcut for select a single-dim from a 2-dim BM From c3e956f554c9008c2fdeea307163434da3c883e6 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 24 Jun 2019 21:12:02 -0700 Subject: [PATCH 3/4] remove no-longer-needed _na_value --- pandas/core/internals/blocks.py | 30 ++++++++++-------------------- pandas/core/internals/managers.py | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index bc7366e13c011..92ea936944a3c 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -180,10 +180,6 @@ def get_values(self, dtype=None): def to_dense(self): return self.values.view() - @property - def _na_value(self): - return np.nan - @property def fill_value(self): return np.nan @@ -1407,6 +1403,9 @@ def quantile(self, qs, interpolation='linear', axis=0): ------- Block """ + # We should always have ndim == 2 becase Series dispatches to DataFrame + assert self.ndim == 2 + if self.is_datetimetz: # TODO: cleanup this special case. # We need to operate on i8 values for datetimetz @@ -1416,8 +1415,7 @@ def quantile(self, qs, interpolation='linear', axis=0): # TODO: NonConsolidatableMixin shape # Usual shape inconsistencies for ExtensionBlocks - if self.ndim > 1: - values = values[None, :] + values = values[None, :] else: values = self.get_values() values, _ = self._try_coerce_args(values, values) @@ -1429,14 +1427,11 @@ def quantile(self, qs, interpolation='linear', axis=0): qs = [qs] if is_empty: - if self.ndim == 1: - result = self._na_value - else: - # create the array of na_values - # 2d len(values) * len(qs) - result = np.repeat(np.array([self.fill_value] * len(qs)), - len(values)).reshape(len(values), - len(qs)) + # create the array of na_values + # 2d len(values) * len(qs) + result = np.repeat(np.array([self.fill_value] * len(qs)), + len(values)).reshape(len(values), + len(qs)) else: # asarray needed for Sparse, see GH#24600 # TODO: Why self.values and not values? @@ -1447,8 +1442,7 @@ def quantile(self, qs, interpolation='linear', axis=0): interpolation=interpolation) result = np.array(result, copy=False) - if self.ndim > 1: - result = result.T + result = result.T if orig_scalar and not lib.is_scalar(result): # result could be scalar in case with is_empty and self.ndim == 1 @@ -2020,10 +2014,6 @@ class DatetimeLikeBlockMixin: def _holder(self): return DatetimeArray - @property - def _na_value(self): - return tslibs.NaT - @property def fill_value(self): return tslibs.iNaT diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index a6523e8b5a249..79a0328abb836 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -1816,7 +1816,7 @@ def _shape_compat(x): def _interleaved_dtype( blocks: List[Block] -) -> Optional[Union[np.dtype, ExtensionDtype]]: + ) -> Optional[Union[np.dtype, ExtensionDtype]]: """Find the common dtype for `blocks`. Parameters From 2b2a6fd6ac8c06861a5cc72f985cc47be62239ea Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 25 Jun 2019 07:21:15 -0700 Subject: [PATCH 4/4] flake8 fixup --- pandas/core/internals/managers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 8954c4fc8e033..26b6920c119dd 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -1815,8 +1815,7 @@ def _shape_compat(x): def _interleaved_dtype( - blocks: List[Block] - ) -> Optional[Union[np.dtype, ExtensionDtype]]: + blocks: List[Block]) -> Optional[Union[np.dtype, ExtensionDtype]]: """Find the common dtype for `blocks`. Parameters