Merge branch 'main' into bug4638144642

Author: eshirvana

ci/code_checks.sh

@@ -78,8 +78,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS01, SS02, SS03, SS04, SS05, PR03, PR04, PR05, PR06, PR08, PR09, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS03,SS04,SS05,PR03,PR04,PR05,PR06,PR08,PR09,PR10,EX04,RT01,RT04,RT05,SA02,SA03
+    MSG='Validate docstrings (EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT04, RT05, SA02, SA03, SS01, SS02, SS03, SS04, SS05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT04,RT05,SA02,SA03,SS01,SS02,SS03,SS04,SS05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

doc/source/getting_started/tutorials.rst

@@ -75,6 +75,16 @@ Excel charts with pandas, vincent and xlsxwriter
 
 *  `Using Pandas and XlsxWriter to create Excel charts <https://pandas-xlsxwriter-charts.readthedocs.io/>`_
 
+Joyful pandas
+-------------
+
+A tutorial written in Chinese by Yuanhao Geng. It covers the basic operations
+for NumPy and pandas, 4 main data manipulation methods (including indexing, groupby, reshaping
+and concatenation) and 4 main data types (including missing data, string data, categorical
+data and time series data). At the end of each chapter, corresponding exercises are posted.
+All the datasets and related materials can be found in the GitHub repository
+`datawhalechina/joyful-pandas <https://github.com/datawhalechina/joyful-pandas>`_.
+
 Video tutorials
 ---------------
 

doc/source/user_guide/duplicates.rst

@@ -172,7 +172,7 @@ going forward, to ensure that your data pipeline doesn't introduce duplicates.
    >>> deduplicated = raw.groupby(level=0).first()  # remove duplicates
    >>> deduplicated.flags.allows_duplicate_labels = False  # disallow going forward
 
-Setting ``allows_duplicate_labels=True`` on a ``Series`` or ``DataFrame`` with duplicate
+Setting ``allows_duplicate_labels=False`` on a ``Series`` or ``DataFrame`` with duplicate
 labels or performing an operation that introduces duplicate labels on a ``Series`` or
 ``DataFrame`` that disallows duplicates will raise an
 :class:`errors.DuplicateLabelError`.

doc/source/whatsnew/v1.4.2.rst

@@ -31,7 +31,7 @@ Bug fixes
 ~~~~~~~~~
 - Fix some cases for subclasses that define their ``_constructor`` properties as general callables (:issue:`46018`)
 - Fixed "longtable" formatting in :meth:`.Styler.to_latex` when ``column_format`` is given in extended format (:issue:`46037`)
--
+- Fixed incorrect rendering in :meth:`.Styler.format` with ``hyperlinks="html"`` when the url contains a colon or other special characters (:issue:`46389`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.5.0.rst

@@ -527,6 +527,7 @@ Groupby/resample/rolling
 - Bug in :meth:`GroupBy.cummin` and :meth:`GroupBy.cummax` with nullable dtypes incorrectly altering the original data in place (:issue:`46220`)
 - Bug in :meth:`GroupBy.cummax` with ``int64`` dtype with leading value being the smallest possible int64 (:issue:`46382`)
 - Bug in :meth:`GroupBy.max` with empty groups and ``uint64`` dtype incorrectly raising ``RuntimeError`` (:issue:`46408`)
+- Bug in :meth:`.GroupBy.apply` would fail when ``func`` was a string and args or kwargs were supplied (:issue:`46479`)
 -
 
 Reshaping

environment.yml

@@ -18,7 +18,7 @@ dependencies:
   - cython>=0.29.24
 
   # code checks
-  - black=21.5b2
+  - black=22.1.0
   - cpplint
   - flake8=4.0.1
   - flake8-bugbear=21.3.2  # used by flake8, find likely bugs
@@ -86,7 +86,7 @@ dependencies:
   - bottleneck>=1.3.1
   - ipykernel
   - ipython>=7.11.1
-  - jinja2  # pandas.Styler
+  - jinja2<=3.0.3  # pandas.Styler
   - matplotlib>=3.3.2  # pandas.plotting, Series.plot, DataFrame.plot
   - numexpr>=2.7.1
   - scipy>=1.4.1

pandas/_libs/tslibs/conversion.pxd

@@ -31,5 +31,3 @@ cdef int64_t get_datetime64_nanos(object val) except? -1
 cpdef datetime localize_pydatetime(datetime dt, tzinfo tz)
 cdef int64_t cast_from_unit(object ts, str unit) except? -1
 cpdef (int64_t, int) precision_from_unit(str unit)
-
-cdef int64_t normalize_i8_stamp(int64_t local_val) nogil

pandas/_libs/tslibs/conversion.pyx

@@ -31,6 +31,7 @@ from pandas._libs.tslibs.np_datetime cimport (
     NPY_DATETIMEUNIT,
     NPY_FR_ns,
     _string_to_dts,
+    astype_overflowsafe,
     check_dts_bounds,
     dt64_to_dtstruct,
     dtstruct_to_dt64,
@@ -71,6 +72,7 @@ from pandas._libs.tslibs.nattype cimport (
 )
 from pandas._libs.tslibs.tzconversion cimport (
     bisect_right_i8,
+    infer_datetuil_fold,
     localize_tzinfo_api,
     tz_localize_to_utc_single,
 )
@@ -215,54 +217,20 @@ def ensure_datetime64ns(arr: ndarray, copy: bool = True):
     -------
     ndarray with dtype datetime64[ns]
     """
-    cdef:
-        Py_ssize_t i, n = arr.size
-        const int64_t[:] ivalues
-        int64_t[:] iresult
-        NPY_DATETIMEUNIT unit
-        npy_datetimestruct dts
-
-    shape = (<object>arr).shape
-
     if (<object>arr).dtype.byteorder == ">":
         # GH#29684 we incorrectly get OutOfBoundsDatetime if we dont swap
         dtype = arr.dtype
         arr = arr.astype(dtype.newbyteorder("<"))
 
     if arr.size == 0:
+        # Fastpath; doesn't matter but we have old tests for result.base
+        #  being arr.
         result = arr.view(DT64NS_DTYPE)
         if copy:
             result = result.copy()
         return result
 
-    if arr.dtype.kind != "M":
-        raise TypeError("ensure_datetime64ns arr must have datetime64 dtype")
-    unit = get_unit_from_dtype(arr.dtype)
-    if unit == NPY_DATETIMEUNIT.NPY_FR_GENERIC:
-        # without raising explicitly here, we end up with a SystemError
-        # built-in function ensure_datetime64ns returned a result with an error
-        raise ValueError("datetime64/timedelta64 must have a unit specified")
-
-    if unit == NPY_FR_ns:
-        # Check this before allocating result for perf, might save some memory
-        if copy:
-            return arr.copy()
-        return arr
-
-    ivalues = arr.view(np.int64).ravel("K")
-
-    result = np.empty_like(arr, dtype=DT64NS_DTYPE)
-    iresult = result.ravel("K").view(np.int64)
-
-    for i in range(n):
-        if ivalues[i] != NPY_NAT:
-            pandas_datetime_to_datetimestruct(ivalues[i], unit, &dts)
-            iresult[i] = dtstruct_to_dt64(&dts)
-            check_dts_bounds(&dts)
-        else:
-            iresult[i] = NPY_NAT
-
-    return result
+    return astype_overflowsafe(arr, DT64NS_DTYPE, copy=copy)
 
 
 def ensure_timedelta64ns(arr: ndarray, copy: bool = True):
@@ -563,7 +531,7 @@ cdef _TSObject _create_tsobject_tz_using_offset(npy_datetimestruct dts,
         if typ == 'dateutil':
             tdata = <int64_t*>cnp.PyArray_DATA(trans)
             pos = bisect_right_i8(tdata, obj.value, trans.shape[0]) - 1
-            obj.fold = _infer_tsobject_fold(obj, trans, deltas, pos)
+            obj.fold = infer_datetuil_fold(obj.value, trans, deltas, pos)
 
     # Keep the converter same as PyDateTime's
     dt = datetime(obj.dts.year, obj.dts.month, obj.dts.day,
@@ -747,7 +715,7 @@ cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
             local_val = obj.value + deltas[pos]
 
             # dateutil supports fold, so we infer fold from value
-            obj.fold = _infer_tsobject_fold(obj, trans, deltas, pos)
+            obj.fold = infer_datetuil_fold(obj.value, trans, deltas, pos)
         else:
             # All other cases have len(deltas) == 1. As of 2018-07-17
             #  (and 2022-03-07), all test cases that get here have
@@ -759,49 +727,6 @@ cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
     obj.tzinfo = tz
 
 
-cdef inline bint _infer_tsobject_fold(
-    _TSObject obj,
-    const int64_t[:] trans,
-    const int64_t[:] deltas,
-    intp_t pos,
-):
-    """
-    Infer _TSObject fold property from value by assuming 0 and then setting
-    to 1 if necessary.
-
-    Parameters
-    ----------
-    obj : _TSObject
-    trans : ndarray[int64_t]
-        ndarray of offset transition points in nanoseconds since epoch.
-    deltas : int64_t[:]
-        array of offsets corresponding to transition points in trans.
-    pos : intp_t
-        Position of the last transition point before taking fold into account.
-
-    Returns
-    -------
-    bint
-        Due to daylight saving time, one wall clock time can occur twice
-        when shifting from summer to winter time; fold describes whether the
-        datetime-like corresponds  to the first (0) or the second time (1)
-        the wall clock hits the ambiguous time
-
-    References
-    ----------
-    .. [1] "PEP 495 - Local Time Disambiguation"
-           https://www.python.org/dev/peps/pep-0495/#the-fold-attribute
-    """
-    cdef:
-        bint fold = 0
-
-    if pos > 0:
-        fold_delta = deltas[pos - 1] - deltas[pos]
-        if obj.value - fold_delta < trans[pos]:
-            fold = 1
-
-    return fold
-
 cdef inline datetime _localize_pydatetime(datetime dt, tzinfo tz):
     """
     Take a datetime/Timestamp in UTC and localizes to timezone tz.
@@ -835,24 +760,3 @@ cpdef inline datetime localize_pydatetime(datetime dt, tzinfo tz):
     elif isinstance(dt, ABCTimestamp):
         return dt.tz_localize(tz)
     return _localize_pydatetime(dt, tz)
-
-
-# ----------------------------------------------------------------------
-# Normalization
-
-@cython.cdivision(False)
-cdef inline int64_t normalize_i8_stamp(int64_t local_val) nogil:
-    """
-    Round the localized nanosecond timestamp down to the previous midnight.
-
-    Parameters
-    ----------
-    local_val : int64_t
-
-    Returns
-    -------
-    int64_t
-    """
-    cdef:
-        int64_t day_nanos = 24 * 3600 * 1_000_000_000
-    return local_val - (local_val % day_nanos)

pandas/_libs/tslibs/np_datetime.pxd

@@ -52,6 +52,8 @@ cdef extern from "numpy/ndarraytypes.h":
         NPY_FR_as
         NPY_FR_GENERIC
 
+    int64_t NPY_DATETIME_NAT  # elswhere we call this NPY_NAT
+
 cdef extern from "src/datetime/np_datetime.h":
     ctypedef struct pandas_timedeltastruct:
         int64_t days
@@ -67,7 +69,7 @@ cdef extern from "src/datetime/np_datetime.h":
 
 cdef bint cmp_scalar(int64_t lhs, int64_t rhs, int op) except -1
 
-cdef check_dts_bounds(npy_datetimestruct *dts)
+cdef check_dts_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT unit=?)
 
 cdef int64_t dtstruct_to_dt64(npy_datetimestruct* dts) nogil
 cdef void dt64_to_dtstruct(int64_t dt64, npy_datetimestruct* out) nogil
@@ -86,3 +88,9 @@ cdef int _string_to_dts(str val, npy_datetimestruct* dts,
                         bint want_exc) except? -1
 
 cdef NPY_DATETIMEUNIT get_unit_from_dtype(cnp.dtype dtype)
+
+cpdef cnp.ndarray astype_overflowsafe(
+    cnp.ndarray values,  # ndarray[datetime64[anyunit]]
+    cnp.dtype dtype,  # ndarray[datetime64[anyunit]]
+    bint copy=*,
+)

pandas/_libs/tslibs/np_datetime.pyi

@@ -4,3 +4,6 @@ class OutOfBoundsDatetime(ValueError): ...
 
 # only exposed for testing
 def py_get_unit_from_dtype(dtype: np.dtype): ...
+def astype_overflowsafe(
+    arr: np.ndarray, dtype: np.dtype, copy: bool = ...
+) -> np.ndarray: ...