Merge remote-tracking branch 'upstream/master' into datetime-type-inference

Author: dsaxton

doc/source/whatsnew/v1.1.0.rst

@@ -463,6 +463,7 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.tz_localize` incorrectly retaining ``freq`` in some cases where the original freq is no longer valid (:issue:`30511`)
 - Bug in :meth:`DatetimeIndex.intersection` losing ``freq`` and timezone in some cases (:issue:`33604`)
 - Bug in :meth:`DatetimeIndex.get_indexer` where incorrect output would be returned for mixed datetime-like targets (:issue:`33741`)
+- Bug in :class:`DatetimeIndex` addition and subtraction with some types of :class:`DateOffset` objects incorrectly retaining an invalid ``freq`` attribute (:issue:`33779`)
 
 Timedelta
 ^^^^^^^^^

pandas/compat/numpy/function.py

@@ -157,7 +157,7 @@ def validate_argsort_with_ascending(ascending, args, kwargs):
     return ascending
 
 
-CLIP_DEFAULTS = dict(out=None)  # type Dict[str, Any]
+CLIP_DEFAULTS: Dict[str, Any] = dict(out=None)
 validate_clip = CompatValidator(
     CLIP_DEFAULTS, fname="clip", method="both", max_fname_arg_count=3
 )

pandas/conftest.py

@@ -256,7 +256,9 @@ def nselect_method(request):
 # ----------------------------------------------------------------
 # Missing values & co.
 # ----------------------------------------------------------------
-@pytest.fixture(params=[None, np.nan, pd.NaT, float("nan"), np.float("NaN"), pd.NA])
+@pytest.fixture(
+    params=[None, np.nan, pd.NaT, float("nan"), np.float("NaN"), pd.NA], ids=str
+)
 def nulls_fixture(request):
     """
     Fixture for each null type in pandas.

pandas/core/arrays/categorical.py

@@ -330,7 +330,7 @@ def __init__(
                 values = _convert_to_list_like(values)
 
                 # By convention, empty lists result in object dtype:
-                sanitize_dtype = "object" if len(values) == 0 else None
+                sanitize_dtype = np.dtype("O") if len(values) == 0 else None
                 null_mask = isna(values)
                 if null_mask.any():
                     values = [values[idx] for idx in np.where(~null_mask)[0]]

pandas/core/arrays/datetimelike.py

@@ -538,21 +538,6 @@ def __getitem__(self, key):
         This getitem defers to the underlying array, which by-definition can
         only handle list-likes, slices, and integer scalars
         """
-        is_int = lib.is_integer(key)
-        if lib.is_scalar(key) and not is_int:
-            raise IndexError(
-                "only integers, slices (`:`), ellipsis (`...`), "
-                "numpy.newaxis (`None`) and integer or boolean "
-                "arrays are valid indices"
-            )
-
-        getitem = self._data.__getitem__
-        if is_int:
-            val = getitem(key)
-            if lib.is_scalar(val):
-                # i.e. self.ndim == 1
-                return self._box_func(val)
-            return type(self)(val, dtype=self.dtype)
 
         if com.is_bool_indexer(key):
             # first convert to boolean, because check_array_indexer doesn't
@@ -569,6 +554,16 @@ def __getitem__(self, key):
         else:
             key = check_array_indexer(self, key)
 
+        freq = self._get_getitem_freq(key)
+        result = self._data[key]
+        if lib.is_scalar(result):
+            return self._box_func(result)
+        return self._simple_new(result, dtype=self.dtype, freq=freq)
+
+    def _get_getitem_freq(self, key):
+        """
+        Find the `freq` attribute to assign to the result of a __getitem__ lookup.
+        """
         is_period = is_period_dtype(self.dtype)
         if is_period:
             freq = self.freq
@@ -583,11 +578,7 @@ def __getitem__(self, key):
                 # GH#21282 indexing with Ellipsis is similar to a full slice,
                 #  should preserve `freq` attribute
                 freq = self.freq
-
-        result = getitem(key)
-        if lib.is_scalar(result):
-            return self._box_func(result)
-        return self._simple_new(result, dtype=self.dtype, freq=freq)
+        return freq
 
     def __setitem__(
         self,

pandas/core/construction.py

@@ -13,7 +13,7 @@
 
 from pandas._libs import lib
 from pandas._libs.tslibs import IncompatibleFrequency, OutOfBoundsDatetime
-from pandas._typing import ArrayLike, Dtype
+from pandas._typing import ArrayLike, Dtype, DtypeObj
 
 from pandas.core.dtypes.cast import (
     construct_1d_arraylike_from_scalar,
@@ -27,7 +27,6 @@
     maybe_upcast,
 )
 from pandas.core.dtypes.common import (
-    is_categorical_dtype,
     is_datetime64_ns_dtype,
     is_extension_array_dtype,
     is_float_dtype,
@@ -36,9 +35,8 @@
     is_list_like,
     is_object_dtype,
     is_timedelta64_ns_dtype,
-    pandas_dtype,
 )
-from pandas.core.dtypes.dtypes import CategoricalDtype, ExtensionDtype, registry
+from pandas.core.dtypes.dtypes import ExtensionDtype, registry
 from pandas.core.dtypes.generic import (
     ABCExtensionArray,
     ABCIndexClass,
@@ -52,13 +50,12 @@
 if TYPE_CHECKING:
     from pandas.core.series import Series  # noqa: F401
     from pandas.core.indexes.api import Index  # noqa: F401
+    from pandas.core.arrays import ExtensionArray  # noqa: F401
 
 
 def array(
-    data: Sequence[object],
-    dtype: Optional[Union[str, np.dtype, ExtensionDtype]] = None,
-    copy: bool = True,
-) -> ABCExtensionArray:
+    data: Sequence[object], dtype: Optional[Dtype] = None, copy: bool = True,
+) -> "ExtensionArray":
     """
     Create an array.
 
@@ -388,14 +385,16 @@ def extract_array(obj, extract_numpy: bool = False):
 
 
 def sanitize_array(
-    data, index, dtype=None, copy: bool = False, raise_cast_failure: bool = False
-):
+    data,
+    index: Optional["Index"],
+    dtype: Optional[DtypeObj] = None,
+    copy: bool = False,
+    raise_cast_failure: bool = False,
+) -> ArrayLike:
     """
-    Sanitize input data to an ndarray, copy if specified, coerce to the
-    dtype if specified.
+    Sanitize input data to an ndarray or ExtensionArray, copy if specified,
+    coerce to the dtype if specified.
     """
-    if dtype is not None:
-        dtype = pandas_dtype(dtype)
 
     if isinstance(data, ma.MaskedArray):
         mask = ma.getmaskarray(data)
@@ -508,10 +507,7 @@ def sanitize_array(
 
 
 def _try_cast(
-    arr,
-    dtype: Optional[Union[np.dtype, "ExtensionDtype"]],
-    copy: bool,
-    raise_cast_failure: bool,
+    arr, dtype: Optional[DtypeObj], copy: bool, raise_cast_failure: bool,
 ):
     """
     Convert input to numpy ndarray and optionally cast to a given dtype.
@@ -532,13 +528,23 @@ def _try_cast(
         if maybe_castable(arr) and not copy and dtype is None:
             return arr
 
+    if isinstance(dtype, ExtensionDtype) and dtype.kind != "M":
+        # create an extension array from its dtype
+        # DatetimeTZ case needs to go through maybe_cast_to_datetime
+        array_type = dtype.construct_array_type()._from_sequence
+        subarr = array_type(arr, dtype=dtype, copy=copy)
+        return subarr
+
     try:
         # GH#15832: Check if we are requesting a numeric dype and
         # that we can convert the data to the requested dtype.
         if is_integer_dtype(dtype):
-            subarr = maybe_cast_to_integer_array(arr, dtype)
+            # this will raise if we have e.g. floats
+            maybe_cast_to_integer_array(arr, dtype)
+            subarr = arr
+        else:
+            subarr = maybe_cast_to_datetime(arr, dtype)
 
-        subarr = maybe_cast_to_datetime(arr, dtype)
         # Take care in creating object arrays (but iterators are not
         # supported):
         if is_object_dtype(dtype) and (
@@ -552,19 +558,7 @@ def _try_cast(
         # in case of out of bound datetime64 -> always raise
         raise
     except (ValueError, TypeError):
-        if is_categorical_dtype(dtype):
-            # We *do* allow casting to categorical, since we know
-            # that Categorical is the only array type for 'category'.
-            dtype = cast(CategoricalDtype, dtype)
-            subarr = dtype.construct_array_type()(
-                arr, dtype.categories, ordered=dtype.ordered
-            )
-        elif is_extension_array_dtype(dtype):
-            # create an extension array from its dtype
-            dtype = cast(ExtensionDtype, dtype)
-            array_type = dtype.construct_array_type()._from_sequence
-            subarr = array_type(arr, dtype=dtype, copy=copy)
-        elif dtype is not None and raise_cast_failure:
+        if dtype is not None and raise_cast_failure:
             raise
         else:
             subarr = np.array(arr, dtype=object, copy=copy)

pandas/core/dtypes/cast.py

@@ -3,7 +3,7 @@
 """
 
 from datetime import date, datetime, timedelta
-from typing import TYPE_CHECKING, Type
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type
 
 import numpy as np
 
@@ -17,7 +17,7 @@
     iNaT,
 )
 from pandas._libs.tslibs.timezones import tz_compare
-from pandas._typing import Dtype, DtypeObj
+from pandas._typing import ArrayLike, Dtype, DtypeObj
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core.dtypes.common import (
@@ -613,7 +613,7 @@ def _ensure_dtype_type(value, dtype):
     return dtype.type(value)
 
 
-def infer_dtype_from(val, pandas_dtype: bool = False):
+def infer_dtype_from(val, pandas_dtype: bool = False) -> Tuple[DtypeObj, Any]:
     """
     Interpret the dtype from a scalar or array.
 
@@ -630,7 +630,7 @@ def infer_dtype_from(val, pandas_dtype: bool = False):
     return infer_dtype_from_array(val, pandas_dtype=pandas_dtype)
 
 
-def infer_dtype_from_scalar(val, pandas_dtype: bool = False):
+def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> Tuple[DtypeObj, Any]:
     """
     Interpret the dtype from a scalar.
 
@@ -641,7 +641,7 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False):
         If False, scalar belongs to pandas extension types is inferred as
         object
     """
-    dtype = np.object_
+    dtype = np.dtype(object)
 
     # a 1-element ndarray
     if isinstance(val, np.ndarray):
@@ -660,7 +660,7 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False):
         # instead of np.empty (but then you still don't want things
         # coming out as np.str_!
 
-        dtype = np.object_
+        dtype = np.dtype(object)
 
     elif isinstance(val, (np.datetime64, datetime)):
         val = tslibs.Timestamp(val)
@@ -671,30 +671,30 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False):
                 dtype = DatetimeTZDtype(unit="ns", tz=val.tz)
             else:
                 # return datetimetz as object
-                return np.object_, val
+                return np.dtype(object), val
         val = val.value
 
     elif isinstance(val, (np.timedelta64, timedelta)):
         val = tslibs.Timedelta(val).value
         dtype = np.dtype("m8[ns]")
 
     elif is_bool(val):
-        dtype = np.bool_
+        dtype = np.dtype(np.bool_)
 
     elif is_integer(val):
         if isinstance(val, np.integer):
-            dtype = type(val)
+            dtype = np.dtype(type(val))
         else:
-            dtype = np.int64
+            dtype = np.dtype(np.int64)
 
     elif is_float(val):
         if isinstance(val, np.floating):
-            dtype = type(val)
+            dtype = np.dtype(type(val))
         else:
-            dtype = np.float64
+            dtype = np.dtype(np.float64)
 
     elif is_complex(val):
-        dtype = np.complex_
+        dtype = np.dtype(np.complex_)
 
     elif pandas_dtype:
         if lib.is_period(val):
@@ -707,7 +707,8 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False):
     return dtype, val
 
 
-def infer_dtype_from_array(arr, pandas_dtype: bool = False):
+# TODO: try to make the Any in the return annotation more specific
+def infer_dtype_from_array(arr, pandas_dtype: bool = False) -> Tuple[DtypeObj, Any]:
     """
     Infer the dtype from an array.
 
@@ -738,7 +739,7 @@ def infer_dtype_from_array(arr, pandas_dtype: bool = False):
     array(['1', '1'], dtype='<U21')
 
     >>> infer_dtype_from_array([1, '1'])
-    (<class 'numpy.object_'>, [1, '1'])
+    (dtype('O'), [1, '1'])
     """
     if isinstance(arr, np.ndarray):
         return arr.dtype, arr
@@ -755,7 +756,7 @@ def infer_dtype_from_array(arr, pandas_dtype: bool = False):
     # don't force numpy coerce with nan's
     inferred = lib.infer_dtype(arr, skipna=False)
     if inferred in ["string", "bytes", "mixed", "mixed-integer"]:
-        return (np.object_, arr)
+        return (np.dtype(np.object_), arr)
 
     arr = np.asarray(arr)
     return arr.dtype, arr
@@ -1469,7 +1470,7 @@ def find_common_type(types):
     return np.find_common_type(types, [])
 
 
-def cast_scalar_to_array(shape, value, dtype=None):
+def cast_scalar_to_array(shape, value, dtype: Optional[DtypeObj] = None) -> np.ndarray:
     """
     Create np.ndarray of specified shape and dtype, filled with values.
 
@@ -1496,7 +1497,9 @@ def cast_scalar_to_array(shape, value, dtype=None):
     return values
 
 
-def construct_1d_arraylike_from_scalar(value, length: int, dtype):
+def construct_1d_arraylike_from_scalar(
+    value, length: int, dtype: DtypeObj
+) -> ArrayLike:
     """
     create a np.ndarray / pandas type of specified shape and dtype
     filled with values
@@ -1505,7 +1508,7 @@ def construct_1d_arraylike_from_scalar(value, length: int, dtype):
     ----------
     value : scalar value
     length : int
-    dtype : pandas_dtype / np.dtype
+    dtype : pandas_dtype or np.dtype
 
     Returns
     -------
@@ -1517,8 +1520,6 @@ def construct_1d_arraylike_from_scalar(value, length: int, dtype):
         subarr = cls._from_sequence([value] * length, dtype=dtype)
 
     else:
-        if not isinstance(dtype, (np.dtype, type(np.dtype))):
-            dtype = dtype.dtype
 
         if length and is_integer_dtype(dtype) and isna(value):
             # coerce if we have nan for an integer dtype
@@ -1536,7 +1537,7 @@ def construct_1d_arraylike_from_scalar(value, length: int, dtype):
     return subarr
 
 
-def construct_1d_object_array_from_listlike(values):
+def construct_1d_object_array_from_listlike(values) -> np.ndarray:
     """
     Transform any list-like object in a 1-dimensional numpy array of object
     dtype.
@@ -1561,7 +1562,9 @@ def construct_1d_object_array_from_listlike(values):
     return result
 
 
-def construct_1d_ndarray_preserving_na(values, dtype=None, copy: bool = False):
+def construct_1d_ndarray_preserving_na(
+    values, dtype: Optional[DtypeObj] = None, copy: bool = False
+) -> np.ndarray:
     """
     Construct a new ndarray, coercing `values` to `dtype`, preserving NA.