Merge branch 'main' into remove-json-locale

Author: WillAyd

.pre-commit-config.yaml

@@ -287,5 +287,4 @@ repos:
         language: python
         additional_dependencies:
         - autotyping==22.9.0
-        - black==22.6.0
         - libcst==0.4.7

asv_bench/benchmarks/package.py

@@ -11,7 +11,7 @@ def time_import(self):
         #  measurement of the import time we actually care about,
         #  without the subprocess or interpreter overhead
         cmd = [sys.executable, "-X", "importtime", "-c", "import pandas as pd"]
-        p = subprocess.run(cmd, stderr=subprocess.PIPE)
+        p = subprocess.run(cmd, stderr=subprocess.PIPE, check=True)
 
         line = p.stderr.splitlines()[-1]
         field = line.split(b"|")[-2].strip()

doc/source/whatsnew/v1.5.2.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_152:
 
-What's new in 1.5.2 (November ??, 2022)
+What's new in 1.5.2 (November 21, 2022)
 ---------------------------------------
 
 These are the changes in pandas 1.5.2. See :ref:`release` for a full changelog
@@ -15,27 +15,27 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed regression in :meth:`MultiIndex.join` for extension array dtypes (:issue:`49277`)
 - Fixed regression in :meth:`Series.replace` raising ``RecursionError`` with numeric dtype and when specifying ``value=None`` (:issue:`45725`)
+- Fixed regression in arithmetic operations for :class:`DataFrame` with :class:`MultiIndex` columns with different dtypes (:issue:`49769`)
 - Fixed regression in :meth:`DataFrame.plot` preventing :class:`~matplotlib.colors.Colormap` instance
   from being passed using the ``colormap`` argument if Matplotlib 3.6+ is used (:issue:`49374`)
 - Fixed regression in :func:`date_range` returning an invalid set of periods for ``CustomBusinessDay`` frequency and ``start`` date with timezone (:issue:`49441`)
 - Fixed performance regression in groupby operations (:issue:`49676`)
--
+- Fixed regression in :class:`Timedelta` constructor returning object of wrong type when subclassing ``Timedelta`` (:issue:`49579`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_152.bug_fixes:
 
 Bug fixes
 ~~~~~~~~~
 - Bug in the Copy-on-Write implementation losing track of views in certain chained indexing cases (:issue:`48996`)
--
+- Fixed memory leak in :meth:`.Styler.to_excel` (:issue:`49751`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_152.other:
 
 Other
 ~~~~~
 - Reverted ``color`` as an alias for ``c`` and ``size`` as an alias for ``s`` in function :meth:`DataFrame.plot.scatter` (:issue:`49732`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_152.contributors:

doc/source/whatsnew/v2.0.0.rst

@@ -61,8 +61,10 @@ Other enhancements
 - :class:`.CategoricalConversionWarning`, :class:`.InvalidComparison`, :class:`.InvalidVersion`, :class:`.LossySetitemError`, and :class:`.NoBufferPresent` are now exposed in ``pandas.errors`` (:issue:`27656`)
 - Fix ``test`` optional_extra by adding missing test package ``pytest-asyncio`` (:issue:`48361`)
 - :func:`DataFrame.astype` exception message thrown improved to include column name when type conversion is not possible. (:issue:`47571`)
+- :func:`date_range` now supports a ``unit`` keyword ("s", "ms", "us", or "ns") to specify the desired resolution of the output index (:issue:`49106`)
 - :meth:`DataFrame.to_json` now supports a ``mode`` keyword with supported inputs 'w' and 'a'. Defaulting to 'w', 'a' can be used when lines=True and orient='records' to append record oriented json lines to an existing json file. (:issue:`35849`)
 - Added ``name`` parameter to :meth:`IntervalIndex.from_breaks`, :meth:`IntervalIndex.from_arrays` and :meth:`IntervalIndex.from_tuples` (:issue:`48911`)
+-
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_200.notable_bug_fixes:
@@ -546,6 +548,8 @@ Removal of prior version deprecations/changes
 - Changed behavior of setitem-like operations (``__setitem__``, ``fillna``, ``where``, ``mask``, ``replace``, ``insert``, fill_value for ``shift``) on an object with :class:`DatetimeTZDtype` when using a value with a non-matching timezone, the value will be cast to the object's timezone instead of casting both to object-dtype (:issue:`44243`)
 - Changed behavior of :class:`Index`, :class:`Series`, :class:`DataFrame` constructors with floating-dtype data and a :class:`DatetimeTZDtype`, the data are now interpreted as UTC-times instead of wall-times, consistent with how integer-dtype data are treated (:issue:`45573`)
 - Changed behavior of :class:`Series` and :class:`DataFrame` constructors with integer dtype and floating-point data containing ``NaN``, this now raises ``IntCastingNaNError`` (:issue:`40110`)
+- Changed behavior of :class:`Series` and :class:`DataFrame` constructors with an integer ``dtype`` and values that are too large to losslessly cast to this dtype, this now raises ``ValueError`` (:issue:`41734`)
+- Changed behavior of :class:`Series` and :class:`DataFrame` constructors with an integer ``dtype`` and values having either ``datetime64`` or ``timedelta64`` dtypes, this now raises ``TypeError``, use ``values.view("int64")`` instead (:issue:`41770`)
 - Removed the deprecated ``base`` and ``loffset`` arguments from :meth:`pandas.DataFrame.resample`, :meth:`pandas.Series.resample` and :class:`pandas.Grouper`. Use ``offset`` or ``origin`` instead (:issue:`31809`)
 - Changed behavior of :meth:`Series.fillna` and :meth:`DataFrame.fillna` with ``timedelta64[ns]`` dtype and an incompatible ``fill_value``; this now casts to ``object`` dtype instead of raising, consistent with the behavior with other dtypes (:issue:`45746`)
 - Change the default argument of ``regex`` for :meth:`Series.str.replace` from ``True`` to ``False``. Additionally, a single character ``pat`` with ``regex=True`` is now treated as a regular expression instead of a string literal. (:issue:`36695`, :issue:`24804`)

pandas/_libs/tslibs/dtypes.pxd

@@ -4,7 +4,7 @@ from pandas._libs.tslibs.np_datetime cimport NPY_DATETIMEUNIT
 
 
 cpdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit)
-cdef NPY_DATETIMEUNIT abbrev_to_npy_unit(str abbrev)
+cpdef NPY_DATETIMEUNIT abbrev_to_npy_unit(str abbrev)
 cdef NPY_DATETIMEUNIT freq_group_code_to_npy_unit(int freq) nogil
 cpdef int64_t periods_per_day(NPY_DATETIMEUNIT reso=*) except? -1
 cpdef int64_t periods_per_second(NPY_DATETIMEUNIT reso) except? -1

pandas/_libs/tslibs/dtypes.pyi

@@ -10,6 +10,7 @@ def periods_per_second(reso: int) -> int: ...
 def is_supported_unit(reso: int) -> bool: ...
 def npy_unit_to_abbrev(reso: int) -> str: ...
 def get_supported_reso(reso: int) -> int: ...
+def abbrev_to_npy_unit(abbrev: str) -> int: ...
 
 class PeriodDtypeBase:
     _dtype_code: int  # PeriodDtypeCode

pandas/_libs/tslibs/dtypes.pyx

@@ -336,7 +336,7 @@ cpdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit):
         raise NotImplementedError(unit)
 
 
-cdef NPY_DATETIMEUNIT abbrev_to_npy_unit(str abbrev):
+cpdef NPY_DATETIMEUNIT abbrev_to_npy_unit(str abbrev):
     if abbrev == "Y":
         return NPY_DATETIMEUNIT.NPY_FR_Y
     elif abbrev == "M":

pandas/_libs/tslibs/timedeltas.pxd

@@ -22,6 +22,7 @@ cdef class _Timedelta(timedelta):
 
     cpdef timedelta to_pytimedelta(_Timedelta self)
     cdef bint _has_ns(self)
+    cdef bint _is_in_pytimedelta_bounds(self)
     cdef _ensure_components(_Timedelta self)
     cdef inline bint _compare_mismatched_resos(self, _Timedelta other, op)
     cdef _Timedelta _as_creso(self, NPY_DATETIMEUNIT reso, bint round_ok=*)

pandas/_libs/tslibs/timedeltas.pyx

@@ -189,7 +189,7 @@ def ints_to_pytimedelta(ndarray m8values, box=False):
             res_val = <object>NaT
         else:
             if box:
-                res_val = _timedelta_from_value_and_reso(value, reso=reso)
+                res_val = _timedelta_from_value_and_reso(Timedelta, value, reso=reso)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_ns:
                 res_val = timedelta(microseconds=int(value) / 1000)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_us:
@@ -741,7 +741,7 @@ cdef bint _validate_ops_compat(other):
 def _op_unary_method(func, name):
     def f(self):
         new_value = func(self.value)
-        return _timedelta_from_value_and_reso(new_value, self._creso)
+        return _timedelta_from_value_and_reso(Timedelta, new_value, self._creso)
     f.__name__ = name
     return f
 
@@ -804,7 +804,7 @@ def _binary_op_method_timedeltalike(op, name):
             # TODO: more generally could do an overflowcheck in op?
             return NaT
 
-        return _timedelta_from_value_and_reso(res, reso=self._creso)
+        return _timedelta_from_value_and_reso(Timedelta, res, reso=self._creso)
 
     f.__name__ = name
     return f
@@ -935,10 +935,10 @@ cdef _to_py_int_float(v):
 
 
 def _timedelta_unpickle(value, reso):
-    return _timedelta_from_value_and_reso(value, reso)
+    return _timedelta_from_value_and_reso(Timedelta, value, reso)
 
 
-cdef _timedelta_from_value_and_reso(int64_t value, NPY_DATETIMEUNIT reso):
+cdef _timedelta_from_value_and_reso(cls, int64_t value, NPY_DATETIMEUNIT reso):
     # Could make this a classmethod if/when cython supports cdef classmethods
     cdef:
         _Timedelta td_base
@@ -949,13 +949,13 @@ cdef _timedelta_from_value_and_reso(int64_t value, NPY_DATETIMEUNIT reso):
     #  We pass 0 instead, and override seconds, microseconds, days.
     #  In principle we could pass 0 for ns and us too.
     if reso == NPY_FR_ns:
-        td_base = _Timedelta.__new__(Timedelta, microseconds=int(value) // 1000)
+        td_base = _Timedelta.__new__(cls, microseconds=int(value) // 1000)
     elif reso == NPY_DATETIMEUNIT.NPY_FR_us:
-        td_base = _Timedelta.__new__(Timedelta, microseconds=int(value))
+        td_base = _Timedelta.__new__(cls, microseconds=int(value))
     elif reso == NPY_DATETIMEUNIT.NPY_FR_ms:
-        td_base = _Timedelta.__new__(Timedelta, milliseconds=0)
+        td_base = _Timedelta.__new__(cls, milliseconds=0)
     elif reso == NPY_DATETIMEUNIT.NPY_FR_s:
-        td_base = _Timedelta.__new__(Timedelta, seconds=0)
+        td_base = _Timedelta.__new__(cls, seconds=0)
     # Other resolutions are disabled but could potentially be implemented here:
     # elif reso == NPY_DATETIMEUNIT.NPY_FR_m:
     #    td_base = _Timedelta.__new__(Timedelta, minutes=int(value))
@@ -1093,8 +1093,27 @@ cdef class _Timedelta(timedelta):
             #  non-invariant behavior.
             #  see GH#44504
             return hash(self.value)
-        else:
+        elif self._is_in_pytimedelta_bounds() and (
+            self._creso == NPY_FR_ns or self._creso == NPY_DATETIMEUNIT.NPY_FR_us
+        ):
+            # If we can defer to timedelta.__hash__, do so, as that
+            #  ensures the hash is invariant to our _reso.
+            # We can only defer for ns and us, as for these two resos we
+            #  call _Timedelta.__new__ with the correct input in
+            #  _timedelta_from_value_and_reso; so timedelta.__hash__
+            #  will be correct
             return timedelta.__hash__(self)
+        else:
+            # We want to ensure that two equivalent Timedelta objects
+            #  have the same hash.  So we try downcasting to the next-lowest
+            #  resolution.
+            try:
+                obj = (<_Timedelta>self)._as_creso(<NPY_DATETIMEUNIT>(self._creso + 1))
+            except OverflowError:
+                # Doesn't fit, so we're off the hook
+                return hash(self.value)
+            else:
+                return hash(obj)
 
     def __richcmp__(_Timedelta self, object other, int op):
         cdef:
@@ -1152,6 +1171,13 @@ cdef class _Timedelta(timedelta):
         else:
             raise NotImplementedError(self._creso)
 
+    cdef bint _is_in_pytimedelta_bounds(self):
+        """
+        Check if we are within the bounds of datetime.timedelta.
+        """
+        self._ensure_components()
+        return -999999999 <= self._d and self._d <= 999999999
+
     cdef _ensure_components(_Timedelta self):
         """
         compute the components
@@ -1502,7 +1528,7 @@ cdef class _Timedelta(timedelta):
     @classmethod
     def _from_value_and_reso(cls, int64_t value, NPY_DATETIMEUNIT reso):
         # exposing as classmethod for testing
-        return _timedelta_from_value_and_reso(value, reso)
+        return _timedelta_from_value_and_reso(cls, value, reso)
 
     def as_unit(self, str unit, bint round_ok=True):
         """
@@ -1737,7 +1763,7 @@ class Timedelta(_Timedelta):
         if value == NPY_NAT:
             return NaT
 
-        return _timedelta_from_value_and_reso(value, NPY_FR_ns)
+        return _timedelta_from_value_and_reso(cls, value, NPY_FR_ns)
 
     def __setstate__(self, state):
         if len(state) == 1:
@@ -1829,6 +1855,7 @@ class Timedelta(_Timedelta):
                 return NaT
 
             return _timedelta_from_value_and_reso(
+                Timedelta,
                 <int64_t>(other * self.value),
                 reso=self._creso,
             )

pandas/core/arrays/_ranges.py

@@ -22,6 +22,7 @@ def generate_regular_range(
     end: Timestamp | Timedelta | None,
     periods: int | None,
     freq: BaseOffset,
+    unit: str = "ns",
 ) -> npt.NDArray[np.intp]:
     """
     Generate a range of dates or timestamps with the spans between dates
@@ -37,14 +38,28 @@ def generate_regular_range(
         Number of periods in produced date range.
     freq : Tick
         Describes space between dates in produced date range.
+    unit : str, default "ns"
+        The resolution the output is meant to represent.
 
     Returns
     -------
-    ndarray[np.int64] Representing nanoseconds.
+    ndarray[np.int64]
+        Representing the given resolution.
     """
     istart = start.value if start is not None else None
     iend = end.value if end is not None else None
-    stride = freq.nanos
+    freq.nanos  # raises if non-fixed frequency
+    td = Timedelta(freq)
+    try:
+        td = td.as_unit(  # pyright: ignore[reportGeneralTypeIssues]
+            unit, round_ok=False
+        )
+    except ValueError as err:
+        raise ValueError(
+            f"freq={freq} is incompatible with unit={unit}. "
+            "Use a lower freq or a higher unit instead."
+        ) from err
+    stride = int(td.value)
 
     if periods is None and istart is not None and iend is not None:
         b = istart

pandas/core/arrays/datetimes.py

@@ -42,6 +42,7 @@
     tz_convert_from_utc,
     tzconversion,
 )
+from pandas._libs.tslibs.dtypes import abbrev_to_npy_unit
 from pandas._typing import (
     DateTimeErrorChoices,
     IntervalClosedType,
@@ -380,6 +381,8 @@ def _generate_range(  # type: ignore[override]
         ambiguous: TimeAmbiguous = "raise",
         nonexistent: TimeNonexistent = "raise",
         inclusive: IntervalClosedType = "both",
+        *,
+        unit: str | None = None,
     ) -> DatetimeArray:
 
         periods = dtl.validate_periods(periods)
@@ -402,6 +405,17 @@ def _generate_range(  # type: ignore[override]
         if start is NaT or end is NaT:
             raise ValueError("Neither `start` nor `end` can be NaT")
 
+        if unit is not None:
+            if unit not in ["s", "ms", "us", "ns"]:
+                raise ValueError("'unit' must be one of 's', 'ms', 'us', 'ns'")
+        else:
+            unit = "ns"
+
+        if start is not None and unit is not None:
+            start = start.as_unit(unit, round_ok=False)
+        if end is not None and unit is not None:
+            end = end.as_unit(unit, round_ok=False)
+
         left_inclusive, right_inclusive = validate_inclusive(inclusive)
         start, end = _maybe_normalize_endpoints(start, end, normalize)
         tz = _infer_tz_from_endpoints(start, end, tz)
@@ -416,6 +430,7 @@ def _generate_range(  # type: ignore[override]
             end = _maybe_localize_point(
                 end, end_tz, end, freq, tz, ambiguous, nonexistent
             )
+
         if freq is not None:
             # We break Day arithmetic (fixed 24 hour) here and opt for
             # Day to mean calendar day (23/24/25 hour). Therefore, strip
@@ -427,7 +442,7 @@ def _generate_range(  # type: ignore[override]
                     end = end.tz_localize(None)
 
             if isinstance(freq, Tick):
-                i8values = generate_regular_range(start, end, periods, freq)
+                i8values = generate_regular_range(start, end, periods, freq, unit=unit)
             else:
                 xdr = _generate_range(
                     start=start, end=end, periods=periods, offset=freq
@@ -441,8 +456,13 @@ def _generate_range(  # type: ignore[override]
                 if not timezones.is_utc(tz):
                     # short-circuit tz_localize_to_utc which would make
                     #  an unnecessary copy with UTC but be a no-op.
+                    creso = abbrev_to_npy_unit(unit)
                     i8values = tzconversion.tz_localize_to_utc(
-                        i8values, tz, ambiguous=ambiguous, nonexistent=nonexistent
+                        i8values,
+                        tz,
+                        ambiguous=ambiguous,
+                        nonexistent=nonexistent,
+                        creso=creso,
                     )
 
                 # i8values is localized datetime64 array -> have to convert
@@ -477,8 +497,8 @@ def _generate_range(  # type: ignore[override]
                 if not right_inclusive and len(i8values) and i8values[-1] == end_i8:
                     i8values = i8values[:-1]
 
-        dt64_values = i8values.view("datetime64[ns]")
-        dtype = tz_to_dtype(tz)
+        dt64_values = i8values.view(f"datetime64[{unit}]")
+        dtype = tz_to_dtype(tz, unit=unit)
         return cls._simple_new(dt64_values, freq=freq, dtype=dtype)
 
     # -----------------------------------------------------------------

pandas/core/dtypes/cast.py

@@ -41,7 +41,6 @@
     IntCastingNaNError,
     LossySetitemError,
 )
-from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core.dtypes.common import (
@@ -1680,25 +1679,17 @@ def maybe_cast_to_integer_array(
 
     if casted.dtype < arr.dtype:
         # GH#41734 e.g. [1, 200, 923442] and dtype="int8" -> overflows
-        warnings.warn(
-            f"Values are too large to be losslessly cast to {dtype}. "
-            "In a future version this will raise OverflowError. To retain the "
-            f"old behavior, use pd.Series(values).astype({dtype})",
-            FutureWarning,
-            stacklevel=find_stack_level(),
+        raise ValueError(
+            f"Values are too large to be losslessly converted to {dtype}. "
+            f"To cast anyway, use pd.Series(values).astype({dtype})"
         )
-        return casted
 
     if arr.dtype.kind in ["m", "M"]:
         # test_constructor_maskedarray_nonfloat
-        warnings.warn(
-            f"Constructing Series or DataFrame from {arr.dtype} values and "
-            f"dtype={dtype} is deprecated and will raise in a future version. "
-            "Use values.view(dtype) instead.",
-            FutureWarning,
-            stacklevel=find_stack_level(),
+        raise TypeError(
+            f"Constructing a Series or DataFrame from {arr.dtype} values and "
+            f"dtype={dtype} is not supported. Use values.view({dtype}) instead."
         )
-        return casted
 
     # No known cases that get here, but raising explicitly to cover our bases.
     raise ValueError(f"values cannot be losslessly cast to {dtype}")