REF: implement EA.pad_or_backfill

Author: jbrockmendel

doc/source/whatsnew/v2.1.0.rst

@@ -306,6 +306,7 @@ Deprecations
 - Deprecated parameter ``obj`` in :meth:`GroupBy.get_group` (:issue:`53545`)
 - Deprecated positional indexing on :class:`Series` with :meth:`Series.__getitem__` and :meth:`Series.__setitem__`, in a future version ``ser[item]`` will *always* interpret ``item`` as a label, not a position (:issue:`50617`)
 - Deprecated strings ``T``, ``t``, ``L`` and ``l`` denoting units in :func:`to_timedelta` (:issue:`52536`)
+- Deprecated the "method" and "limit" keywords in :meth:`ExtensionArray.fillna`, implement and use :meth:`ExtensionArray.pad_or_backfill` instead (:issue:`53621`)
 - Deprecated the "method" and "limit" keywords on :meth:`Series.fillna`, :meth:`DataFrame.fillna`, :meth:`SeriesGroupBy.fillna`, :meth:`DataFrameGroupBy.fillna`, and :meth:`Resampler.fillna`, use ``obj.bfill()`` or ``obj.ffill()`` instead (:issue:`53394`)
 - Deprecated the ``method`` and ``limit`` keywords in :meth:`DataFrame.replace` and :meth:`Series.replace` (:issue:`33302`)
 - Deprecated values "pad", "ffill", "bfill", "backfill" for :meth:`Series.interpolate` and :meth:`DataFrame.interpolate`, use ``obj.ffill()`` or ``obj.bfill()`` instead (:issue:`53581`)

pandas/core/arrays/_mixins.py

@@ -19,6 +19,7 @@
     AxisInt,
     Dtype,
     F,
+    FillnaOptions,
     PositionalIndexer2D,
     PositionalIndexerTuple,
     ScalarIndexer,
@@ -294,6 +295,37 @@ def _fill_mask_inplace(
         func = missing.get_fill_func(method, ndim=self.ndim)
         func(self._ndarray.T, limit=limit, mask=mask.T)
 
+    def pad_or_backfill(
+        self,
+        *,
+        method: FillnaOptions,
+        limit: int | None = None,
+        limit_area: Literal["inside", "outside"] | None = None,
+        copy: bool = True,
+    ) -> Self:
+        mask = self.isna()
+        if mask.any():
+            # (for now) when self.ndim == 2, we assume axis=0
+            func = missing.get_fill_func(method, ndim=self.ndim)
+
+            npvalues = self._ndarray.T
+            if copy:
+                npvalues = npvalues.copy()
+            func(npvalues, limit=limit, mask=mask.T)
+            npvalues = npvalues.T
+
+            if copy:
+                new_values = self._from_backing_data(npvalues)
+            else:
+                new_values = self
+
+        else:
+            if copy:
+                new_values = self.copy()
+            else:
+                new_values = self
+        return new_values
+
     @doc(ExtensionArray.fillna)
     def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
         value, method = validate_fillna_kwargs(
@@ -309,7 +341,6 @@ def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
 
         if mask.any():
             if method is not None:
-                # TODO: check value is None
                 # (for now) when self.ndim == 2, we assume axis=0
                 func = missing.get_fill_func(method, ndim=self.ndim)
                 npvalues = self._ndarray.T.copy()

pandas/core/arrays/arrow/array.py

@@ -899,6 +899,24 @@ def dropna(self) -> Self:
         """
         return type(self)(pc.drop_null(self._pa_array))
 
+    def pad_or_backfill(
+        self,
+        *,
+        method: FillnaOptions,
+        limit: int | None = None,
+        limit_area: Literal["inside", "outside"] | None = None,
+        copy: bool = True,
+    ) -> Self:
+        if not self._hasna:
+            # TODO(CoW): Not necessary anymore when CoW is the default
+            return self.copy()
+
+        # TODO(3.0): after EA.fillna 'method' deprecation is enforced, we can remove
+        #  this method entirely.
+        return super().pad_or_backfill(
+            method=method, limit=limit, limit_area=limit_area, copy=copy
+        )
+
     @doc(ExtensionArray.fillna)
     def fillna(
         self,

pandas/core/arrays/base.py

@@ -20,6 +20,7 @@
     cast,
     overload,
 )
+import warnings
 
 import numpy as np
 
@@ -35,6 +36,7 @@
     Substitution,
     cache_readonly,
 )
+from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import (
     validate_bool_kwarg,
     validate_fillna_kwargs,
@@ -127,6 +129,7 @@ class ExtensionArray:
     interpolate
     isin
     isna
+    pad_or_backfill
     ravel
     repeat
     searchsorted
@@ -177,6 +180,7 @@ class ExtensionArray:
     methods:
 
     * fillna
+    * pad_or_backfill
     * dropna
     * unique
     * factorize / _values_for_factorize
@@ -782,6 +786,82 @@ def interpolate(
             f"{type(self).__name__} does not implement interpolate"
         )
 
+    def pad_or_backfill(
+        self,
+        *,
+        method: FillnaOptions,
+        limit: int | None = None,
+        limit_area: Literal["inside", "outside"] | None = None,
+        copy: bool = True,
+    ) -> Self:
+        """
+        pad or backfill values, used by Series/DataFrame ffill and bfill.
+
+        Parameters
+        ----------
+        method : {'backfill', 'bfill', 'pad', 'ffill'}
+            Method to use for filling holes in reindexed Series:
+
+            * pad / ffill: propagate last valid observation forward to next valid.
+            * backfill / bfill: use NEXT valid observation to fill gap.
+
+        limit : int, default None
+            This is the maximum number of consecutive
+            NaN values to forward/backward fill. In other words, if there is
+            a gap with more than this number of consecutive NaNs, it will only
+            be partially filled. If method is not specified, this is the
+            maximum number of entries along the entire axis where NaNs will be
+            filled.
+
+        copy : bool, default True
+            Whether to make a copy of the data before filling. If False, then
+            the original should be modified and no new memory should be allocated.
+            For ExtensionArray subclasses that cannot do this, it is at the
+            author's discretion whether to ignore "copy=False" or to raise.
+            The base class implementation ignores the keyword if any NAs are
+            present.
+
+        """
+
+        # If a 3rd-party EA has implemented this functionality in fillna,
+        #  we warn that they need to implement pad_or_backfill instead.
+        if (
+            type(self).fillna is not ExtensionArray.fillna
+            and type(self).pad_or_backfill is ExtensionArray.pad_or_backfill
+        ):
+            # Check for pad_or_backfill here allows us to call
+            #  super().pad_or_backfill without getting this warning
+            warnings.warn(
+                "ExtensionArray.fillna 'method' keyword is deprecated. "
+                "In a future version. arr.pad_or_backfill will be called "
+                "instead. 3rd-party ExtensionArray authors need to implement "
+                "pad_or_backfill.",
+                FutureWarning,
+                stacklevel=find_stack_level(),
+            )
+            return self.fillna(method=method, limit=limit)
+
+        mask = self.isna()
+
+        if mask.any():
+            # NB: the base class does not respect the "copy" keyword
+            meth = missing.clean_fill_method(method)
+
+            npmask = np.asarray(mask)
+            if meth == "pad":
+                indexer = libalgos.get_fill_indexer(npmask, limit=limit)
+                return self.take(indexer, allow_fill=True)
+            else:
+                # i.e. meth == "backfill"
+                indexer = libalgos.get_fill_indexer(npmask[::-1], limit=limit)[::-1]
+                return self[::-1].take(indexer, allow_fill=True)
+
+        else:
+            if not copy:
+                return self
+            new_values = self.copy()
+        return new_values
+
     def fillna(
         self,
         value: object | ArrayLike | None = None,
@@ -803,6 +883,8 @@ def fillna(
             * pad / ffill: propagate last valid observation forward to next valid.
             * backfill / bfill: use NEXT valid observation to fill gap.
 
+            .. deprecated:: 2.1.0
+
         limit : int, default None
             If method is specified, this is the maximum number of consecutive
             NaN values to forward/backward fill. In other words, if there is
@@ -811,11 +893,22 @@ def fillna(
             maximum number of entries along the entire axis where NaNs will be
             filled.
 
+            .. deprecated:: 2.1.0
+
         Returns
         -------
         ExtensionArray
             With NA/NaN filled.
         """
+        if method is not None:
+            warnings.warn(
+                f"The 'method' keyword in {type(self).__name__}.fillna is "
+                "deprecated and will be removed in a future version. "
+                "Use pad_or_backfill instead.",
+                FutureWarning,
+                stacklevel=find_stack_level(),
+            )
+
         value, method = validate_fillna_kwargs(value, method)
 
         mask = self.isna()

pandas/core/arrays/interval.py

@@ -31,6 +31,7 @@
     ArrayLike,
     AxisInt,
     Dtype,
+    FillnaOptions,
     IntervalClosedType,
     NpDtype,
     PositionalIndexer,
@@ -886,6 +887,16 @@ def max(self, *, axis: AxisInt | None = None, skipna: bool = True) -> IntervalOr
         indexer = obj.argsort()[-1]
         return obj[indexer]
 
+    def pad_or_backfill(
+        self,
+        *,
+        method: FillnaOptions,
+        limit: int | None = None,
+        limit_area: Literal["inside", "outside"] | None = None,
+        copy: bool = True,
+    ) -> Self:
+        raise TypeError("Filling by method is not supported for IntervalArray.")
+
     def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
         """
         Fill NA/NaN values using the specified method.

pandas/core/arrays/masked.py

@@ -25,6 +25,7 @@
     AstypeArg,
     AxisInt,
     DtypeObj,
+    FillnaOptions,
     NpDtype,
     PositionalIndexer,
     Scalar,
@@ -178,6 +179,36 @@ def __getitem__(self, item: PositionalIndexer) -> Self | Any:
 
         return self._simple_new(self._data[item], newmask)
 
+    def pad_or_backfill(
+        self,
+        *,
+        method: FillnaOptions,
+        limit: int | None = None,
+        limit_area: Literal["inside", "outside"] | None = None,
+        copy: bool = True,
+    ) -> Self:
+        mask = self._mask
+
+        if mask.any():
+            func = missing.get_fill_func(method, ndim=self.ndim)
+
+            npvalues = self._data.T
+            new_mask = mask.T
+            if copy:
+                npvalues = npvalues.copy()
+                new_mask = new_mask.copy()
+            func(npvalues, limit=limit, mask=new_mask)
+            if copy:
+                return self._simple_new(npvalues.T, new_mask.T)
+            else:
+                return self
+        else:
+            if copy:
+                new_values = self.copy()
+            else:
+                new_values = self
+        return new_values
+
     @doc(ExtensionArray.fillna)
     def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
         value, method = validate_fillna_kwargs(value, method)

pandas/core/arrays/numpy_.py

@@ -233,7 +233,7 @@ def pad_or_backfill(
         self,
         *,
         method: FillnaOptions,
-        limit: int | None,
+        limit: int | None = None,
         limit_area: Literal["inside", "outside"] | None = None,
         copy: bool = True,
     ) -> Self:

pandas/core/arrays/period.py

@@ -77,6 +77,7 @@
     from pandas._typing import (
         AnyArrayLike,
         Dtype,
+        FillnaOptions,
         NpDtype,
         NumpySorter,
         NumpyValueArrayLike,
@@ -789,6 +790,25 @@ def searchsorted(
         m8arr = self._ndarray.view("M8[ns]")
         return m8arr.searchsorted(npvalue, side=side, sorter=sorter)
 
+    def pad_or_backfill(
+        self,
+        *,
+        method: FillnaOptions,
+        limit: int | None = None,
+        limit_area: Literal["inside", "outside"] | None = None,
+        copy: bool = True,
+    ) -> Self:
+        # view as dt64 so we get treated as timelike in core.missing,
+        #  similar to dtl._period_dispatch
+        dta = self.view("M8[ns]")
+        result = dta.pad_or_backfill(
+            method=method, limit=limit, limit_area=limit_area, copy=copy
+        )
+        if copy:
+            return cast("Self", result.view(self.dtype))
+        else:
+            return self
+
     def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
         if method is not None:
             # view as dt64 so we get treated as timelike in core.missing,

pandas/core/arrays/sparse/array.py

@@ -713,6 +713,30 @@ def isna(self):
         mask[self.sp_index.indices] = isna(self.sp_values)
         return type(self)(mask, fill_value=False, dtype=dtype)
 
+    def pad_or_backfill(
+        self,
+        *,
+        method: FillnaOptions,
+        limit: int | None = None,
+        limit_area: Literal["inside", "outside"] | None = None,
+        copy: bool = True,
+    ) -> Self:
+        msg = "pad_or_backfill with 'method' requires high memory usage."
+        warnings.warn(
+            msg,
+            PerformanceWarning,
+            stacklevel=find_stack_level(),
+        )
+        new_values = np.asarray(self)
+        # pad_or_backfill_inplace modifies new_values inplace
+        # error: Argument "method" to "pad_or_backfill_inplace" has incompatible
+        # type "Literal['backfill', 'bfill', 'ffill', 'pad']"; expected
+        # "Literal['pad', 'backfill']"
+        pad_or_backfill_inplace(
+            new_values, method=method, limit=limit  # type: ignore[arg-type]
+        )
+        return type(self)(new_values, fill_value=self.fill_value)
+
     def fillna(
         self,
         value=None,

pandas/core/internals/blocks.py

@@ -1903,9 +1903,9 @@ def pad_or_backfill(
         values = self.values
         if values.ndim == 2 and axis == 1:
             # NDArrayBackedExtensionArray.fillna assumes axis=0
-            new_values = values.T.fillna(method=method, limit=limit).T
+            new_values = values.T.pad_or_backfill(method=method, limit=limit).T
         else:
-            new_values = values.fillna(method=method, limit=limit)
+            new_values = values.pad_or_backfill(method=method, limit=limit)
         return [self.make_block_same_class(new_values)]
 
 

pandas/tests/arrays/test_datetimelike.py

@@ -251,7 +251,7 @@ def test_fillna_method_doesnt_change_orig(self, method):
 
         fill_value = arr[3] if method == "pad" else arr[5]
 
-        result = arr.fillna(method=method)
+        result = arr.pad_or_backfill(method=method)
         assert result[4] == fill_value
 
         # check that the original was not changed