Skip to content

DEPR: disallow int fill_value in shift with dt64/td64 #49362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 28, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ Removal of prior version deprecations/changes
- Removed :meth:`Series.str.__iter__` (:issue:`28277`)
- Removed ``pandas.SparseArray`` in favor of :class:`arrays.SparseArray` (:issue:`30642`)
- Removed ``pandas.SparseSeries`` and ``pandas.SparseDataFrame``, including pickle support. (:issue:`30642`)
- Enforced disallowing passing an integer ``fill_value`` to :meth:`DataFrame.shift` and :meth:`Series.shift`` with datetime64, timedelta64, or period dtypes (:issue:`32591`)
- Enforced disallowing a string column label into ``times`` in :meth:`DataFrame.ewm` (:issue:`43265`)
- Enforced disallowing a tuple of column labels into :meth:`.DataFrameGroupBy.__getitem__` (:issue:`30546`)
- Removed setting Categorical._codes directly (:issue:`41429`)
Expand Down
7 changes: 1 addition & 6 deletions pandas/core/arrays/_mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,16 +252,11 @@ def _validate_searchsorted_value(
@doc(ExtensionArray.shift)
def shift(self, periods: int = 1, fill_value=None, axis: AxisInt = 0):

fill_value = self._validate_shift_value(fill_value)
fill_value = self._validate_scalar(fill_value)
new_values = shift(self._ndarray, periods, axis, fill_value)

return self._from_backing_data(new_values)

def _validate_shift_value(self, fill_value):
# TODO(2.0): after deprecation in datetimelikearraymixin is enforced,
# we can remove this and use validate_fill_value directly
return self._validate_scalar(fill_value)

def __setitem__(self, key, value) -> None:
key = check_array_indexer(self, key)
value = self._validate_setitem_value(value)
Expand Down
31 changes: 0 additions & 31 deletions pandas/core/arrays/datetimelike.py
Original file line number Diff line number Diff line change
Expand Up @@ -603,37 +603,6 @@ def _validate_comparison_value(self, other):

return other

def _validate_shift_value(self, fill_value):
# TODO(2.0): once this deprecation is enforced, use _validate_scalar
if is_valid_na_for_dtype(fill_value, self.dtype):
fill_value = NaT
elif isinstance(fill_value, self._recognized_scalars):
fill_value = self._scalar_type(fill_value)
else:
new_fill: DatetimeLikeScalar

# only warn if we're not going to raise
if self._scalar_type is Period and lib.is_integer(fill_value):
# kludge for #31971 since Period(integer) tries to cast to str
new_fill = Period._from_ordinal(fill_value, freq=self.freq)
else:
new_fill = self._scalar_type(fill_value)

# stacklevel here is chosen to be correct when called from
# DataFrame.shift or Series.shift
warnings.warn(
f"Passing {type(fill_value)} to shift is deprecated and "
"will raise in a future version, pass "
f"{self._scalar_type.__name__} instead.",
FutureWarning,
# There is no way to hard-code the level since this might be
# reached directly or called from the Index or Block method
stacklevel=find_stack_level(),
)
fill_value = new_fill

return self._unbox(fill_value, setitem=True)

def _validate_scalar(
self,
value,
Expand Down
16 changes: 3 additions & 13 deletions pandas/tests/arrays/test_datetimelike.py
Original file line number Diff line number Diff line change
Expand Up @@ -554,22 +554,12 @@ def test_inplace_arithmetic(self):
tm.assert_equal(arr, expected)

def test_shift_fill_int_deprecated(self):
# GH#31971
# GH#31971, enforced in 2.0
data = np.arange(10, dtype="i8") * 24 * 3600 * 10**9
arr = self.array_cls(data, freq="D")

msg = "Passing <class 'int'> to shift"
with tm.assert_produces_warning(FutureWarning, match=msg):
result = arr.shift(1, fill_value=1)

expected = arr.copy()
if self.array_cls is PeriodArray:
fill_val = arr._scalar_type._from_ordinal(1, freq=arr.freq)
else:
fill_val = arr._scalar_type(1)
expected[0] = fill_val
expected[1:] = arr[:-1]
tm.assert_equal(result, expected)
with pytest.raises(TypeError, match="value should be a"):
arr.shift(1, fill_value=1)

def test_median(self, arr1d):
arr = arr1d
Expand Down
19 changes: 6 additions & 13 deletions pandas/tests/frame/methods/test_shift.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,26 +505,19 @@ def test_shift_dt64values_int_fill_deprecated(self):
# GH#31971
ser = Series([pd.Timestamp("2020-01-01"), pd.Timestamp("2020-01-02")])

with tm.assert_produces_warning(FutureWarning):
result = ser.shift(1, fill_value=0)
expected = Series([pd.Timestamp(0), ser[0]])
tm.assert_series_equal(result, expected)
with pytest.raises(TypeError, match="value should be a"):
ser.shift(1, fill_value=0)

df = ser.to_frame()
with tm.assert_produces_warning(FutureWarning):
result = df.shift(1, fill_value=0)
expected = expected.to_frame()
tm.assert_frame_equal(result, expected)
with pytest.raises(TypeError, match="value should be a"):
df.shift(1, fill_value=0)

# axis = 1
df2 = DataFrame({"A": ser, "B": ser})
df2._consolidate_inplace()

with tm.assert_produces_warning(FutureWarning):
result = df2.shift(1, axis=1, fill_value=0)

expected = DataFrame({"A": [pd.Timestamp(0), pd.Timestamp(0)], "B": df2["A"]})
tm.assert_frame_equal(result, expected)
with pytest.raises(TypeError, match="value should be a"):
df2.shift(1, axis=1, fill_value=0)

# same thing but not consolidated
# This isn't great that we get different behavior, but
Expand Down