Merge remote-tracking branch 'upstream/master' into datetime-type-inference

Author: dsaxton

doc/source/whatsnew/v1.1.0.rst

@@ -99,6 +99,7 @@ Other enhancements
   ``df.to_csv(path, compression={'method': 'gzip', 'compresslevel': 1}``
   (:issue:`33196`)
 - :meth:`~pandas.core.groupby.GroupBy.transform` has gained ``engine`` and ``engine_kwargs`` arguments that supports executing functions with ``Numba`` (:issue:`32854`)
+- :meth:`~pandas.core.resample.Resampler.interpolate` now supports SciPy interpolation method :class:`scipy.interpolate.CubicSpline` as method ``cubicspline`` (:issue:`33670`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/core/arrays/datetimelike.py

@@ -60,29 +60,24 @@ def _datetimelike_array_cmp(cls, op):
     opname = f"__{op.__name__}__"
     nat_result = opname == "__ne__"
 
-    @unpack_zerodim_and_defer(opname)
-    def wrapper(self, other):
+    class InvalidComparison(Exception):
+        pass
 
+    def _validate_comparison_value(self, other):
         if isinstance(other, str):
             try:
                 # GH#18435 strings get a pass from tzawareness compat
                 other = self._scalar_from_string(other)
             except ValueError:
                 # failed to parse as Timestamp/Timedelta/Period
-                return invalid_comparison(self, other, op)
+                raise InvalidComparison(other)
 
         if isinstance(other, self._recognized_scalars) or other is NaT:
             other = self._scalar_type(other)
             self._check_compatible_with(other)
 
-            other_i8 = self._unbox_scalar(other)
-
-            result = op(self.view("i8"), other_i8)
-            if isna(other):
-                result.fill(nat_result)
-
         elif not is_list_like(other):
-            return invalid_comparison(self, other, op)
+            raise InvalidComparison(other)
 
         elif len(other) != len(self):
             raise ValueError("Lengths must match")
@@ -93,34 +88,50 @@ def wrapper(self, other):
                 other = np.array(other)
 
             if not isinstance(other, (np.ndarray, type(self))):
-                return invalid_comparison(self, other, op)
-
-            if is_object_dtype(other):
-                # We have to use comp_method_OBJECT_ARRAY instead of numpy
-                #  comparison otherwise it would fail to raise when
-                #  comparing tz-aware and tz-naive
-                with np.errstate(all="ignore"):
-                    result = ops.comp_method_OBJECT_ARRAY(
-                        op, self.astype(object), other
-                    )
-                o_mask = isna(other)
+                raise InvalidComparison(other)
+
+            elif is_object_dtype(other.dtype):
+                pass
 
             elif not type(self)._is_recognized_dtype(other.dtype):
-                return invalid_comparison(self, other, op)
+                raise InvalidComparison(other)
 
             else:
                 # For PeriodDType this casting is unnecessary
+                # TODO: use Index to do inference?
                 other = type(self)._from_sequence(other)
                 self._check_compatible_with(other)
 
-                result = op(self.view("i8"), other.view("i8"))
-                o_mask = other._isnan
+        return other
 
-            if o_mask.any():
-                result[o_mask] = nat_result
+    @unpack_zerodim_and_defer(opname)
+    def wrapper(self, other):
 
-        if self._hasnans:
-            result[self._isnan] = nat_result
+        try:
+            other = _validate_comparison_value(self, other)
+        except InvalidComparison:
+            return invalid_comparison(self, other, op)
+
+        dtype = getattr(other, "dtype", None)
+        if is_object_dtype(dtype):
+            # We have to use comp_method_OBJECT_ARRAY instead of numpy
+            #  comparison otherwise it would fail to raise when
+            #  comparing tz-aware and tz-naive
+            with np.errstate(all="ignore"):
+                result = ops.comp_method_OBJECT_ARRAY(op, self.astype(object), other)
+            return result
+
+        if isinstance(other, self._scalar_type) or other is NaT:
+            other_i8 = self._unbox_scalar(other)
+        else:
+            # Then type(other) == type(self)
+            other_i8 = other.asi8
+
+        result = op(self.asi8, other_i8)
+
+        o_mask = isna(other)
+        if self._hasnans | np.any(o_mask):
+            result[self._isnan | o_mask] = nat_result
 
         return result
 

pandas/core/frame.py

@@ -3649,7 +3649,9 @@ def reindexer(value):
     @property
     def _series(self):
         return {
-            item: Series(self._mgr.iget(idx), index=self.index, name=item)
+            item: Series(
+                self._mgr.iget(idx), index=self.index, name=item, fastpath=True
+            )
             for idx, item in enumerate(self.columns)
         }
 

pandas/core/generic.py

@@ -6671,9 +6671,9 @@ def replace(
               values of the index.  Both 'polynomial' and 'spline' require that
               you also specify an `order` (int), e.g.
               ``df.interpolate(method='polynomial', order=5)``.
-            * 'krogh', 'piecewise_polynomial', 'spline', 'pchip', 'akima':
-              Wrappers around the SciPy interpolation methods of similar
-              names. See `Notes`.
+            * 'krogh', 'piecewise_polynomial', 'spline', 'pchip', 'akima',
+              'cubicspline': Wrappers around the SciPy interpolation methods of
+              similar names. See `Notes`.
             * 'from_derivatives': Refers to
               `scipy.interpolate.BPoly.from_derivatives` which
               replaces 'piecewise_polynomial' interpolation method in
@@ -11208,9 +11208,7 @@ def block_accum_func(blk_values):
 
         result = self._mgr.apply(block_accum_func)
 
-        d = self._construct_axes_dict()
-        d["copy"] = False
-        return self._constructor(result, **d).__finalize__(self, method=name)
+        return self._constructor(result).__finalize__(self, method=name)
 
     return set_function_name(cum_func, name, cls)
 

pandas/core/indexes/datetimelike.py

@@ -2,7 +2,7 @@
 Base and utility classes for tseries type pandas objects.
 """
 from datetime import datetime
-from typing import Any, List, Optional, Union
+from typing import Any, List, Optional, Union, cast
 
 import numpy as np
 
@@ -583,6 +583,22 @@ def delete(self, loc):
         arr = type(self._data)._simple_new(new_i8s, dtype=self.dtype, freq=freq)
         return type(self)._simple_new(arr, name=self.name)
 
+    # --------------------------------------------------------------------
+    # Join/Set Methods
+
+    def _wrap_joined_index(self, joined: np.ndarray, other):
+        assert other.dtype == self.dtype, (other.dtype, self.dtype)
+        name = get_op_result_name(self, other)
+
+        if is_period_dtype(self.dtype):
+            freq = self.freq
+        else:
+            self = cast(DatetimeTimedeltaMixin, self)
+            freq = self.freq if self._can_fast_union(other) else None
+        new_data = type(self._data)._simple_new(joined, dtype=self.dtype, freq=freq)
+
+        return type(self)._simple_new(new_data, name=name)
+
 
 class DatetimeTimedeltaMixin(DatetimeIndexOpsMixin, Int64Index):
     """
@@ -878,15 +894,6 @@ def _is_convertible_to_index_for_join(cls, other: Index) -> bool:
             return True
         return False
 
-    def _wrap_joined_index(self, joined: np.ndarray, other):
-        assert other.dtype == self.dtype, (other.dtype, self.dtype)
-        name = get_op_result_name(self, other)
-
-        freq = self.freq if self._can_fast_union(other) else None
-        new_data = type(self._data)._simple_new(joined, dtype=self.dtype, freq=freq)
-
-        return type(self)._simple_new(new_data, name=name)
-
     # --------------------------------------------------------------------
     # List-Like Methods
 

pandas/core/indexes/period.py

@@ -628,6 +628,7 @@ def join(self, other, how="left", level=None, return_indexers=False, sort=False)
                 other, how=how, level=level, return_indexers=return_indexers, sort=sort
             )
 
+        # _assert_can_do_setop ensures we have matching dtype
         result = Int64Index.join(
             self,
             other,
@@ -636,11 +637,7 @@ def join(self, other, how="left", level=None, return_indexers=False, sort=False)
             return_indexers=return_indexers,
             sort=sort,
         )
-
-        if return_indexers:
-            result, lidx, ridx = result
-            return self._apply_meta(result), lidx, ridx
-        return self._apply_meta(result)
+        return result
 
     # ------------------------------------------------------------------------
     # Set Operation Methods
@@ -719,13 +716,6 @@ def _union(self, other, sort):
 
     # ------------------------------------------------------------------------
 
-    def _apply_meta(self, rawarr) -> "PeriodIndex":
-        if not isinstance(rawarr, PeriodIndex):
-            if not isinstance(rawarr, PeriodArray):
-                rawarr = PeriodArray(rawarr, freq=self.freq)
-            rawarr = PeriodIndex._simple_new(rawarr, name=self.name)
-        return rawarr
-
     def memory_usage(self, deep=False):
         result = super().memory_usage(deep=deep)
         if hasattr(self, "_cache") and "_int64index" in self._cache:

pandas/core/missing.py

@@ -112,6 +112,7 @@ def clean_interp_method(method, **kwargs):
         "akima",
         "spline",
         "from_derivatives",
+        "cubicspline",
     ]
     if method in ("spline", "polynomial") and order is None:
         raise ValueError("You must specify the order of the spline or polynomial.")
@@ -293,6 +294,7 @@ def interpolate_1d(
         "piecewise_polynomial",
         "pchip",
         "akima",
+        "cubicspline",
     ]
 
     if method in sp_methods:
@@ -341,14 +343,11 @@ def _interpolate_scipy_wrapper(
         x, new_x = x._values.astype("i8"), new_x.astype("i8")
 
     if method == "pchip":
-        try:
-            alt_methods["pchip"] = interpolate.pchip_interpolate
-        except AttributeError as err:
-            raise ImportError(
-                "Your version of Scipy does not support PCHIP interpolation."
-            ) from err
+        alt_methods["pchip"] = interpolate.pchip_interpolate
     elif method == "akima":
         alt_methods["akima"] = _akima_interpolate
+    elif method == "cubicspline":
+        alt_methods["cubicspline"] = _cubicspline_interpolate
 
     interp1d_methods = [
         "nearest",
@@ -406,7 +405,7 @@ def _from_derivatives(xi, yi, x, order=None, der=0, extrapolate=False):
     der : int or list
         How many derivatives to extract; None for all potentially nonzero
         derivatives (that is a number equal to the number of points), or a
-        list of derivatives to extract. This numberincludes the function
+        list of derivatives to extract. This number includes the function
         value as 0th derivative.
      extrapolate : bool, optional
         Whether to extrapolate to ouf-of-bounds points based on first and last
@@ -446,8 +445,7 @@ def _akima_interpolate(xi, yi, x, der=0, axis=0):
         A 1-D array of real values.  `yi`'s length along the interpolation
         axis must be equal to the length of `xi`. If N-D array, use axis
         parameter to select correct axis.
-    x : scalar or array_like
-        Of length M.
+    x : scalar or array_like of length M.
     der : int or list, optional
         How many derivatives to extract; None for all potentially
         nonzero derivatives (that is a number equal to the number
@@ -478,6 +476,85 @@ def _akima_interpolate(xi, yi, x, der=0, axis=0):
         return [P(x, nu) for nu in der]
 
 
+def _cubicspline_interpolate(xi, yi, x, axis=0, bc_type="not-a-knot", extrapolate=None):
+    """
+    Convenience function for cubic spline data interpolator.
+
+    See `scipy.interpolate.CubicSpline` for details.
+
+    Parameters
+    ----------
+    xi : array_like, shape (n,)
+        1-d array containing values of the independent variable.
+        Values must be real, finite and in strictly increasing order.
+    yi : array_like
+        Array containing values of the dependent variable. It can have
+        arbitrary number of dimensions, but the length along ``axis``
+        (see below) must match the length of ``x``. Values must be finite.
+    x : scalar or array_like, shape (m,)
+    axis : int, optional
+        Axis along which `y` is assumed to be varying. Meaning that for
+        ``x[i]`` the corresponding values are ``np.take(y, i, axis=axis)``.
+        Default is 0.
+    bc_type : string or 2-tuple, optional
+        Boundary condition type. Two additional equations, given by the
+        boundary conditions, are required to determine all coefficients of
+        polynomials on each segment [2]_.
+        If `bc_type` is a string, then the specified condition will be applied
+        at both ends of a spline. Available conditions are:
+        * 'not-a-knot' (default): The first and second segment at a curve end
+          are the same polynomial. It is a good default when there is no
+          information on boundary conditions.
+        * 'periodic': The interpolated functions is assumed to be periodic
+          of period ``x[-1] - x[0]``. The first and last value of `y` must be
+          identical: ``y[0] == y[-1]``. This boundary condition will result in
+          ``y'[0] == y'[-1]`` and ``y''[0] == y''[-1]``.
+        * 'clamped': The first derivative at curves ends are zero. Assuming
+          a 1D `y`, ``bc_type=((1, 0.0), (1, 0.0))`` is the same condition.
+        * 'natural': The second derivative at curve ends are zero. Assuming
+          a 1D `y`, ``bc_type=((2, 0.0), (2, 0.0))`` is the same condition.
+        If `bc_type` is a 2-tuple, the first and the second value will be
+        applied at the curve start and end respectively. The tuple values can
+        be one of the previously mentioned strings (except 'periodic') or a
+        tuple `(order, deriv_values)` allowing to specify arbitrary
+        derivatives at curve ends:
+        * `order`: the derivative order, 1 or 2.
+        * `deriv_value`: array_like containing derivative values, shape must
+          be the same as `y`, excluding ``axis`` dimension. For example, if
+          `y` is 1D, then `deriv_value` must be a scalar. If `y` is 3D with
+          the shape (n0, n1, n2) and axis=2, then `deriv_value` must be 2D
+          and have the shape (n0, n1).
+    extrapolate : {bool, 'periodic', None}, optional
+        If bool, determines whether to extrapolate to out-of-bounds points
+        based on first and last intervals, or to return NaNs. If 'periodic',
+        periodic extrapolation is used. If None (default), ``extrapolate`` is
+        set to 'periodic' for ``bc_type='periodic'`` and to True otherwise.
+
+    See Also
+    --------
+    scipy.interpolate.CubicHermiteSpline
+
+    Returns
+    -------
+    y : scalar or array_like
+        The result, of shape (m,)
+
+    References
+    ----------
+    .. [1] `Cubic Spline Interpolation
+            <https://en.wikiversity.org/wiki/Cubic_Spline_Interpolation>`_
+            on Wikiversity.
+    .. [2] Carl de Boor, "A Practical Guide to Splines", Springer-Verlag, 1978.
+    """
+    from scipy import interpolate
+
+    P = interpolate.CubicSpline(
+        xi, yi, axis=axis, bc_type=bc_type, extrapolate=extrapolate
+    )
+
+    return P(x)
+
+
 def interpolate_2d(
     values, method="pad", axis=0, limit=None, fill_value=None, dtype=None
 ):

pandas/core/series.py

@@ -259,12 +259,8 @@ def __init__(
                     # astype copies
                     data = data.astype(dtype)
                 else:
-                    # need to copy to avoid aliasing issues
+                    # GH#24096 we need to ensure the index remains immutable
                     data = data._values.copy()
-                    if isinstance(data, ABCDatetimeIndex) and data.tz is not None:
-                        # GH#24096 need copy to be deep for datetime64tz case
-                        # TODO: See if we can avoid these copies
-                        data = data._values.copy(deep=True)
                 copy = False
 
             elif isinstance(data, np.ndarray):
@@ -280,6 +276,7 @@ def __init__(
                     index = data.index
                 else:
                     data = data.reindex(index, copy=copy)
+                    copy = False
                 data = data._mgr
             elif is_dict_like(data):
                 data, index = self._init_dict(data, index, dtype)