From 1c8141fcb38375493fef672bf959957540a0ee4b Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 16 Mar 2020 19:02:43 -0700 Subject: [PATCH 1/3] REF: Implement core._algos --- pandas/core/_algos/__init__.py | 9 +++++++++ pandas/core/_algos/needsname.py | 29 +++++++++++++++++++++++++++++ pandas/core/arrays/datetimelike.py | 22 ++-------------------- pandas/core/internals/blocks.py | 22 ++-------------------- 4 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 pandas/core/_algos/__init__.py create mode 100644 pandas/core/_algos/needsname.py diff --git a/pandas/core/_algos/__init__.py b/pandas/core/_algos/__init__.py new file mode 100644 index 0000000000000..b8e05fa16986f --- /dev/null +++ b/pandas/core/_algos/__init__.py @@ -0,0 +1,9 @@ +""" +core._algos is for algorithms that operate on ndarray and ExtensionArray. +These should: + +- Assume that any Index, Series, or DataFrame objects have already been unwrapped. +- Assume that any list arguments have already been cast to ndarray/EA. +- Not depend on Index, Series, or DataFrame, nor import any of these. +- May dispatch to ExtensionArray methods, but should not import from core.arrays. +""" diff --git a/pandas/core/_algos/needsname.py b/pandas/core/_algos/needsname.py new file mode 100644 index 0000000000000..eb2de2c5f9872 --- /dev/null +++ b/pandas/core/_algos/needsname.py @@ -0,0 +1,29 @@ +import numpy as np + +from pandas.core.dtypes.common import ensure_platform_int + + +def shift(values: np.ndarray, periods: int, axis: int, fill_value) -> np.ndarray: + new_values = values + + # make sure array sent to np.roll is c_contiguous + f_ordered = values.flags.f_contiguous + if f_ordered: + new_values = new_values.T + axis = new_values.ndim - axis - 1 + + if np.prod(new_values.shape): + new_values = np.roll(new_values, ensure_platform_int(periods), axis=axis) + + axis_indexer = [slice(None)] * values.ndim + if periods > 0: + axis_indexer[axis] = slice(None, periods) + else: + axis_indexer[axis] = slice(periods, None) + new_values[tuple(axis_indexer)] = fill_value + + # restore original order + if f_ordered: + new_values = new_values.T + + return new_values diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 105d9581b1a25..f417d2bd37acc 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -39,6 +39,7 @@ from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna from pandas.core import missing, nanops, ops +from pandas.core._algos.needsname import shift from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts from pandas.core.arrays.base import ExtensionArray, ExtensionOpsMixin import pandas.core.common as com @@ -773,26 +774,7 @@ def shift(self, periods=1, fill_value=None, axis=0): fill_value = self._unbox_scalar(fill_value) - new_values = self._data - - # make sure array sent to np.roll is c_contiguous - f_ordered = new_values.flags.f_contiguous - if f_ordered: - new_values = new_values.T - axis = new_values.ndim - axis - 1 - - new_values = np.roll(new_values, periods, axis=axis) - - axis_indexer = [slice(None)] * self.ndim - if periods > 0: - axis_indexer[axis] = slice(None, periods) - else: - axis_indexer[axis] = slice(periods, None) - new_values[tuple(axis_indexer)] = fill_value - - # restore original order - if f_ordered: - new_values = new_values.T + new_values = shift(self._data, periods, axis, fill_value) return type(self)._simple_new(new_values, dtype=self.dtype) diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 83980e9028e9a..ffad12976b92c 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -29,7 +29,6 @@ from pandas.core.dtypes.common import ( _NS_DTYPE, _TD_DTYPE, - ensure_platform_int, is_bool_dtype, is_categorical, is_categorical_dtype, @@ -65,6 +64,7 @@ isna, ) +from pandas.core._algos.needsname import shift import pandas.core.algorithms as algos from pandas.core.arrays import ( Categorical, @@ -1317,25 +1317,7 @@ def shift(self, periods, axis: int = 0, fill_value=None): # that, handle boolean etc also new_values, fill_value = maybe_upcast(self.values, fill_value) - # make sure array sent to np.roll is c_contiguous - f_ordered = new_values.flags.f_contiguous - if f_ordered: - new_values = new_values.T - axis = new_values.ndim - axis - 1 - - if np.prod(new_values.shape): - new_values = np.roll(new_values, ensure_platform_int(periods), axis=axis) - - axis_indexer = [slice(None)] * self.ndim - if periods > 0: - axis_indexer[axis] = slice(None, periods) - else: - axis_indexer[axis] = slice(periods, None) - new_values[tuple(axis_indexer)] = fill_value - - # restore original order - if f_ordered: - new_values = new_values.T + new_values = shift(new_values, periods, axis, fill_value) return [self.make_block(new_values)] From d818d9627d5b5a6960af2bc2b77c3aea18467c7e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 17 Mar 2020 09:44:00 -0700 Subject: [PATCH 2/3] needsname->transforms --- pandas/core/_algos/{needsname.py => transforms.py} | 4 ++++ pandas/core/arrays/datetimelike.py | 2 +- pandas/core/internals/blocks.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) rename pandas/core/_algos/{needsname.py => transforms.py} (93%) diff --git a/pandas/core/_algos/needsname.py b/pandas/core/_algos/transforms.py similarity index 93% rename from pandas/core/_algos/needsname.py rename to pandas/core/_algos/transforms.py index eb2de2c5f9872..f775b6d733d9c 100644 --- a/pandas/core/_algos/needsname.py +++ b/pandas/core/_algos/transforms.py @@ -1,3 +1,7 @@ +""" +transforms.py is for shape-preserving functions. +""" + import numpy as np from pandas.core.dtypes.common import ensure_platform_int diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index f417d2bd37acc..255cffdaf7dc6 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -39,7 +39,7 @@ from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna from pandas.core import missing, nanops, ops -from pandas.core._algos.needsname import shift +from pandas.core._algos.transforms import shift from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts from pandas.core.arrays.base import ExtensionArray, ExtensionOpsMixin import pandas.core.common as com diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 1c5792becec11..e3450438d0c66 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -64,7 +64,7 @@ isna, ) -from pandas.core._algos.needsname import shift +from pandas.core._algos.transforms import shift import pandas.core.algorithms as algos from pandas.core.arrays import ( Categorical, From 5ebe2f6667dc88978c07f9e0ad6f5738b39f0247 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 18 Mar 2020 12:04:56 -0700 Subject: [PATCH 3/3] _algos -> array_algos --- pandas/core/{_algos => array_algos}/__init__.py | 2 +- pandas/core/{_algos => array_algos}/transforms.py | 0 pandas/core/arrays/datetimelike.py | 2 +- pandas/core/internals/blocks.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename pandas/core/{_algos => array_algos}/__init__.py (80%) rename pandas/core/{_algos => array_algos}/transforms.py (100%) diff --git a/pandas/core/_algos/__init__.py b/pandas/core/array_algos/__init__.py similarity index 80% rename from pandas/core/_algos/__init__.py rename to pandas/core/array_algos/__init__.py index b8e05fa16986f..a7655a013c6cf 100644 --- a/pandas/core/_algos/__init__.py +++ b/pandas/core/array_algos/__init__.py @@ -1,5 +1,5 @@ """ -core._algos is for algorithms that operate on ndarray and ExtensionArray. +core.array_algos is for algorithms that operate on ndarray and ExtensionArray. These should: - Assume that any Index, Series, or DataFrame objects have already been unwrapped. diff --git a/pandas/core/_algos/transforms.py b/pandas/core/array_algos/transforms.py similarity index 100% rename from pandas/core/_algos/transforms.py rename to pandas/core/array_algos/transforms.py diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 255cffdaf7dc6..7510bfd1f67ad 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -39,8 +39,8 @@ from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna from pandas.core import missing, nanops, ops -from pandas.core._algos.transforms import shift from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts +from pandas.core.array_algos.transforms import shift from pandas.core.arrays.base import ExtensionArray, ExtensionOpsMixin import pandas.core.common as com from pandas.core.construction import array, extract_array diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index e3450438d0c66..adeb1ae04a58d 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -64,8 +64,8 @@ isna, ) -from pandas.core._algos.transforms import shift import pandas.core.algorithms as algos +from pandas.core.array_algos.transforms import shift from pandas.core.arrays import ( Categorical, DatetimeArray,