Merge remote-tracking branch 'upstream/master' into arrow-string-array-dtype

Author: simonjayhawkins

doc/source/whatsnew/v1.2.5.rst

@@ -18,15 +18,10 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.sum` and :meth:`DataFrame.prod` when ``min_count`` and ``numeric_only`` are both given (:issue:`41074`)
 - Regression in :func:`read_csv` when using ``memory_map=True`` with an non-UTF8 encoding (:issue:`40986`)
 - Regression in :meth:`DataFrame.replace` and :meth:`Series.replace` when the values to replace is a NumPy float array (:issue:`40371`)
+- Regression in :func:`ExcelFile` when a corrupt file is opened but not closed (:issue:`41778`)
 
 .. ---------------------------------------------------------------------------
 
-.. _whatsnew_125.deprecations:
-
-Deprecations
-~~~~~~~~~~~~
-
-- Deprecated passing lists as ``key`` to :meth:`DataFrame.xs` and :meth:`Series.xs` (:issue:`41760`)
 
 .. _whatsnew_125.bug_fixes:
 

doc/source/whatsnew/v1.3.0.rst

@@ -740,6 +740,7 @@ Deprecations
 - Deprecated passing arguments as positional (except for ``"codes"``) in :meth:`MultiIndex.codes` (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`Index.set_names` and :meth:`MultiIndex.set_names` (except for ``names``) (:issue:`41485`)
 - Deprecated passing arguments (apart from ``cond`` and ``other``) as positional in :meth:`DataFrame.mask` and :meth:`Series.mask` (:issue:`41485`)
+- Deprecated passing arguments as positional in :meth:`Resampler.interpolate` (other than ``"method"``) (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.clip` and :meth:`Series.clip` (other than ``"upper"`` and ``"lower"``) (:issue:`41485`)
 - Deprecated special treatment of lists with first element a Categorical in the :class:`DataFrame` constructor; pass as ``pd.DataFrame({col: categorical, ...})`` instead (:issue:`38845`)
 - Deprecated behavior of :class:`DataFrame` constructor when a ``dtype`` is passed and the data cannot be cast to that dtype. In a future version, this will raise instead of being silently ignored (:issue:`24435`)
@@ -761,6 +762,7 @@ Deprecations
 - Deprecated passing arguments as positional in :meth:`DataFrame.set_axis` and :meth:`Series.set_axis` (other than ``"labels"``) (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.where` and :meth:`Series.where` (other than ``"cond"`` and ``"other"``) (:issue:`41485`)
 - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_csv` (:issue:`41485`)
+- Deprecated passing lists as ``key`` to :meth:`DataFrame.xs` and :meth:`Series.xs` (:issue:`41760`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.drop` (other than ``"labels"``) and :meth:`Series.drop` (:issue:`41485`)
 - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_table` (:issue:`41485`)
 
@@ -946,6 +948,7 @@ Conversion
 - Bug in :meth:`qcut` raising error when taking ``Float64DType`` as input (:issue:`40730`)
 - Bug in :class:`DataFrame` and :class:`Series` construction with ``datetime64[ns]`` data and ``dtype=object`` resulting in ``datetime`` objects instead of :class:`Timestamp` objects (:issue:`41599`)
 - Bug in :class:`DataFrame` and :class:`Series` construction with ``timedelta64[ns]`` data and ``dtype=object`` resulting in ``np.timedelta64`` objects instead of :class:`Timedelta` objects (:issue:`41599`)
+- Bug in :class:`DataFrame` construction when given a two-dimensional object-dtype ``np.ndarray`` of :class:`Period` or :class:`Interval` objects failing to cast to :class:`PeriodDtype` or :class:`IntervalDtype`, respectively (:issue:`41812`)
 
 Strings
 ^^^^^^^

pandas/_libs/intervaltree.pxi.in

@@ -31,7 +31,9 @@ cdef class IntervalTree(IntervalMixin):
     we are emulating the IndexEngine interface
     """
     cdef readonly:
-        object left, right, root, dtype
+        ndarray left, right
+        IntervalNode root
+        object dtype
         str closed
         object _is_overlapping, _left_sorter, _right_sorter
 
@@ -203,6 +205,41 @@ cdef sort_values_and_indices(all_values, all_indices, subset):
 # Nodes
 # ----------------------------------------------------------------------
 
+@cython.internal
+cdef class IntervalNode:
+    cdef readonly:
+        int64_t n_elements, n_center, leaf_size
+        bint is_leaf_node
+
+    def __repr__(self) -> str:
+        if self.is_leaf_node:
+            return (
+                f"<{type(self).__name__}: {self.n_elements} elements (terminal)>"
+            )
+        else:
+            n_left = self.left_node.n_elements
+            n_right = self.right_node.n_elements
+            n_center = self.n_elements - n_left - n_right
+            return (
+                f"<{type(self).__name__}: "
+                f"pivot {self.pivot}, {self.n_elements} elements "
+                f"({n_left} left, {n_right} right, {n_center} overlapping)>"
+            )
+
+    def counts(self):
+        """
+        Inspect counts on this node
+        useful for debugging purposes
+        """
+        if self.is_leaf_node:
+            return self.n_elements
+        else:
+            m = len(self.center_left_values)
+            l = self.left_node.counts()
+            r = self.right_node.counts()
+            return (m, (l, r))
+
+
 # we need specialized nodes and leaves to optimize for different dtype and
 # closed values
 
@@ -240,7 +277,7 @@ NODE_CLASSES = {}
 
 
 @cython.internal
-cdef class {{dtype_title}}Closed{{closed_title}}IntervalNode:
+cdef class {{dtype_title}}Closed{{closed_title}}IntervalNode(IntervalNode):
     """Non-terminal node for an IntervalTree
 
     Categorizes intervals by those that fall to the left, those that fall to
@@ -252,8 +289,6 @@ cdef class {{dtype_title}}Closed{{closed_title}}IntervalNode:
         int64_t[:] center_left_indices, center_right_indices, indices
         {{dtype}}_t min_left, max_right
         {{dtype}}_t pivot
-        int64_t n_elements, n_center, leaf_size
-        bint is_leaf_node
 
     def __init__(self,
                  ndarray[{{dtype}}_t, ndim=1] left,
@@ -381,31 +416,6 @@ cdef class {{dtype_title}}Closed{{closed_title}}IntervalNode:
             else:
                 result.extend(self.center_left_indices)
 
-    def __repr__(self) -> str:
-        if self.is_leaf_node:
-            return ('<{{dtype_title}}Closed{{closed_title}}IntervalNode: '
-                    '%s elements (terminal)>' % self.n_elements)
-        else:
-            n_left = self.left_node.n_elements
-            n_right = self.right_node.n_elements
-            n_center = self.n_elements - n_left - n_right
-            return ('<{{dtype_title}}Closed{{closed_title}}IntervalNode: '
-                    'pivot %s, %s elements (%s left, %s right, %s '
-                    'overlapping)>' % (self.pivot, self.n_elements,
-                                       n_left, n_right, n_center))
-
-    def counts(self):
-        """
-        Inspect counts on this node
-        useful for debugging purposes
-        """
-        if self.is_leaf_node:
-            return self.n_elements
-        else:
-            m = len(self.center_left_values)
-            l = self.left_node.counts()
-            r = self.right_node.counts()
-            return (m, (l, r))
 
 NODE_CLASSES['{{dtype}}',
              '{{closed}}'] = {{dtype_title}}Closed{{closed_title}}IntervalNode

pandas/_libs/lib.pyi

@@ -73,6 +73,7 @@ def maybe_convert_objects(
     convert_datetime: Literal[False] = ...,
     convert_timedelta: bool = ...,
     convert_period: Literal[False] = ...,
+    convert_interval: Literal[False] = ...,
     convert_to_nullable_integer: Literal[False] = ...,
     dtype_if_all_nat: DtypeObj | None = ...,
 ) -> np.ndarray: ...
@@ -86,6 +87,7 @@ def maybe_convert_objects(
     convert_datetime: bool = ...,
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
+    convert_interval: bool = ...,
     convert_to_nullable_integer: Literal[True] = ...,
     dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...
@@ -99,6 +101,7 @@ def maybe_convert_objects(
     convert_datetime: Literal[True] = ...,
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
+    convert_interval: bool = ...,
     convert_to_nullable_integer: bool = ...,
     dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...
@@ -112,6 +115,7 @@ def maybe_convert_objects(
     convert_datetime: bool = ...,
     convert_timedelta: bool = ...,
     convert_period: Literal[True] = ...,
+    convert_interval: bool = ...,
     convert_to_nullable_integer: bool = ...,
     dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...
@@ -125,6 +129,7 @@ def maybe_convert_objects(
     convert_datetime: bool = ...,
     convert_timedelta: bool = ...,
     convert_period: bool = ...,
+    convert_interval: bool = ...,
     convert_to_nullable_integer: bool = ...,
     dtype_if_all_nat: DtypeObj | None = ...,
 ) -> ArrayLike: ...

pandas/_libs/lib.pyx

@@ -1573,6 +1573,7 @@ def infer_datetimelike_array(arr: ndarray[object]) -> tuple[str, bool]:
         bint seen_timedelta = False, seen_date = False, seen_datetime = False
         bint seen_tz_aware = False, seen_tz_naive = False
         bint seen_nat = False, seen_str = False
+        bint seen_period = False, seen_interval = False
         list objs = []
         object v
 
@@ -1610,9 +1611,25 @@ def infer_datetimelike_array(arr: ndarray[object]) -> tuple[str, bool]:
         elif is_timedelta(v):
             # timedelta, or timedelta64
             seen_timedelta = True
+        elif is_period_object(v):
+            seen_period = True
+            break
+        elif is_interval(v):
+            seen_interval = True
+            break
         else:
             return "mixed", seen_str
 
+    if seen_period:
+        if is_period_array(arr):
+            return "period", seen_str
+        return "mixed", seen_str
+
+    if seen_interval:
+        if is_interval_array(arr):
+            return "interval", seen_str
+        return "mixed", seen_str
+
     if seen_date and not (seen_datetime or seen_timedelta):
         return "date", seen_str
     elif seen_datetime and not seen_timedelta:

pandas/core/arrays/datetimelike.py

@@ -1677,11 +1677,6 @@ class TimelikeOps(DatetimeLikeArrayMixin):
     Common ops for TimedeltaIndex/DatetimeIndex, but not PeriodIndex.
     """
 
-    def copy(self: TimelikeOpsT) -> TimelikeOpsT:
-        result = super().copy()
-        result._freq = self._freq
-        return result
-
     def _round(self, freq, mode, ambiguous, nonexistent):
         # round the local times
         if is_datetime64tz_dtype(self.dtype):

pandas/core/construction.py

@@ -32,7 +32,6 @@
 )
 from pandas.core.dtypes.cast import (
     construct_1d_arraylike_from_scalar,
-    construct_1d_ndarray_preserving_na,
     construct_1d_object_array_from_listlike,
     maybe_cast_to_datetime,
     maybe_cast_to_integer_array,
@@ -48,7 +47,6 @@
     is_integer_dtype,
     is_list_like,
     is_object_dtype,
-    is_string_dtype,
     is_timedelta64_ns_dtype,
 )
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
@@ -570,26 +568,18 @@ def sanitize_array(
         if dtype is not None or len(data) == 0:
             subarr = _try_cast(data, dtype, copy, raise_cast_failure)
         else:
-            # TODO: copy?
             subarr = maybe_convert_platform(data)
             if subarr.dtype == object:
                 subarr = cast(np.ndarray, subarr)
                 subarr = maybe_infer_to_datetimelike(subarr)
 
     subarr = _sanitize_ndim(subarr, data, dtype, index, allow_2d=allow_2d)
 
-    if not (
-        isinstance(subarr.dtype, ExtensionDtype) or isinstance(dtype, ExtensionDtype)
-    ):
+    if isinstance(subarr, np.ndarray):
+        # at this point we should have dtype be None or subarr.dtype == dtype
+        dtype = cast(np.dtype, dtype)
         subarr = _sanitize_str_dtypes(subarr, data, dtype, copy)
 
-        is_object_or_str_dtype = is_object_dtype(dtype) or is_string_dtype(dtype)
-        if is_object_dtype(subarr.dtype) and not is_object_or_str_dtype:
-            inferred = lib.infer_dtype(subarr, skipna=False)
-            if inferred in {"interval", "period"}:
-                subarr = array(subarr)
-                subarr = extract_array(subarr, extract_numpy=True)
-
     return subarr
 
 
@@ -748,6 +738,10 @@ def _try_cast(
             return subarr
         return ensure_wrapped_if_datetimelike(arr).astype(dtype, copy=copy)
 
+    elif dtype.kind == "U":
+        # TODO: test cases with arr.dtype.kind in ["m", "M"]
+        return lib.ensure_string_array(arr, convert_na_value=False, copy=copy)
+
     elif dtype.kind in ["m", "M"]:
         return maybe_cast_to_datetime(arr, dtype)
 
@@ -757,16 +751,12 @@ def _try_cast(
         if is_integer_dtype(dtype):
             # this will raise if we have e.g. floats
 
-            maybe_cast_to_integer_array(arr, dtype)
-            subarr = arr
+            subarr = maybe_cast_to_integer_array(arr, dtype)
         else:
-            subarr = arr
-
-        if not isinstance(subarr, ABCExtensionArray):
             # 4 tests fail if we move this to a try/except/else; see
             #  test_constructor_compound_dtypes, test_constructor_cast_failure
             #  test_constructor_dict_cast2, test_loc_setitem_dtype
-            subarr = construct_1d_ndarray_preserving_na(subarr, dtype, copy=copy)
+            subarr = np.array(arr, dtype=dtype, copy=copy)
 
     except (ValueError, TypeError):
         if raise_cast_failure: