Merge branch 'factorize_na_v2' of https://github.com/rhshadrach/pandas into add_numeric_only_gb

Author: rhshadrach

doc/source/whatsnew/v1.5.0.rst

@@ -481,7 +481,7 @@ Timedelta
 
 Time Zones
 ^^^^^^^^^^
--
+- Bug in :class:`Timestamp` constructor raising when passed a ``ZoneInfo`` tzinfo object (:issue:`46425`)
 -
 
 Numeric
@@ -569,6 +569,8 @@ I/O
 - Bug in Parquet roundtrip for Interval dtype with ``datetime64[ns]`` subtype (:issue:`45881`)
 - Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements (:issue:`45598`)
 - Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)
+- :meth:`to_html` now excludes the ``border`` attribute from ``<table>`` elements when ``border`` keyword is set to ``False``.
+-
 
 Period
 ^^^^^^
@@ -599,7 +601,7 @@ Groupby/resample/rolling
 - Bug in :meth:`GroupBy.cummax` with ``int64`` dtype with leading value being the smallest possible int64 (:issue:`46382`)
 - Bug in :meth:`GroupBy.max` with empty groups and ``uint64`` dtype incorrectly raising ``RuntimeError`` (:issue:`46408`)
 - Bug in :meth:`.GroupBy.apply` would fail when ``func`` was a string and args or kwargs were supplied (:issue:`46479`)
--
+- Bug in :meth:`.Rolling.var` would segfault calculating weighted variance when window size was larger than data size (:issue:`46760`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/algos_common_helper.pxi.in

@@ -65,7 +65,8 @@ def ensure_{{name}}(object arr, copy=True):
         if (<ndarray>arr).descr.type_num == NPY_{{c_type}}:
             return arr
         else:
-            return arr.astype(np.{{dtype}}, copy=copy)
+            # equiv: arr.astype(np.{{dtype}}, copy=copy)
+            return cnp.PyArray_Cast(<ndarray>arr, cnp.NPY_{{c_type}})
     else:
         return np.array(arr, dtype=np.{{dtype}})
 

pandas/_libs/hashtable_class_helper.pxi.in

@@ -658,7 +658,7 @@ cdef class {{name}}HashTable(HashTable):
                             return_inverse=return_inverse)
 
     def factorize(self, const {{dtype}}_t[:] values, Py_ssize_t na_sentinel=-1,
-                  object na_value=None, object mask=None):
+                  object na_value=None, object mask=None, ignore_na=True):
         """
         Calculate unique values and labels (no sorting!)
 
@@ -690,7 +690,7 @@ cdef class {{name}}HashTable(HashTable):
         """
         uniques_vector = {{name}}Vector()
         return self._unique(values, uniques_vector, na_sentinel=na_sentinel,
-                            na_value=na_value, ignore_na=True, mask=mask,
+                            na_value=na_value, ignore_na=ignore_na, mask=mask,
                             return_inverse=True)
 
     def get_labels(self, const {{dtype}}_t[:] values, {{name}}Vector uniques,
@@ -1037,7 +1037,7 @@ cdef class StringHashTable(HashTable):
                             return_inverse=return_inverse)
 
     def factorize(self, ndarray[object] values, Py_ssize_t na_sentinel=-1,
-                  object na_value=None, object mask=None):
+                  object na_value=None, object mask=None, ignore_na=True):
         """
         Calculate unique values and labels (no sorting!)
 
@@ -1067,7 +1067,7 @@ cdef class StringHashTable(HashTable):
         """
         uniques_vector = ObjectVector()
         return self._unique(values, uniques_vector, na_sentinel=na_sentinel,
-                            na_value=na_value, ignore_na=True,
+                            na_value=na_value, ignore_na=ignore_na,
                             return_inverse=True)
 
     def get_labels(self, ndarray[object] values, ObjectVector uniques,
@@ -1290,7 +1290,7 @@ cdef class PyObjectHashTable(HashTable):
                             return_inverse=return_inverse)
 
     def factorize(self, ndarray[object] values, Py_ssize_t na_sentinel=-1,
-                  object na_value=None, object mask=None):
+                  object na_value=None, object mask=None, ignore_na=True):
         """
         Calculate unique values and labels (no sorting!)
 
@@ -1320,7 +1320,7 @@ cdef class PyObjectHashTable(HashTable):
         """
         uniques_vector = ObjectVector()
         return self._unique(values, uniques_vector, na_sentinel=na_sentinel,
-                            na_value=na_value, ignore_na=True,
+                            na_value=na_value, ignore_na=ignore_na,
                             return_inverse=True)
 
     def get_labels(self, ndarray[object] values, ObjectVector uniques,

pandas/_libs/tslib.pyx

@@ -424,7 +424,7 @@ cpdef array_to_datetime(
     """
     cdef:
         Py_ssize_t i, n = len(values)
-        object val, py_dt, tz, tz_out = None
+        object val, tz
         ndarray[int64_t] iresult
         ndarray[object] oresult
         npy_datetimestruct dts
@@ -443,6 +443,8 @@ cpdef array_to_datetime(
         float offset_seconds, tz_offset
         set out_tzoffset_vals = set()
         bint string_to_dts_failed
+        datetime py_dt
+        tzinfo tz_out = None
 
     # specify error conditions
     assert is_raise or is_ignore or is_coerce
@@ -647,6 +649,8 @@ cpdef array_to_datetime(
     return result, tz_out
 
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
 cdef ndarray[object] ignore_errors_out_of_bounds_fallback(ndarray[object] values):
     """
     Fallback for array_to_datetime if an OutOfBoundsDatetime is raised

pandas/_libs/tslibs/conversion.pyx

@@ -56,6 +56,7 @@ from pandas._libs.tslibs.timezones cimport (
     is_fixed_offset,
     is_tzlocal,
     is_utc,
+    is_zoneinfo,
     maybe_get_tz,
     tz_compare,
     utc_pytz as UTC,
@@ -532,7 +533,7 @@ cdef _TSObject _create_tsobject_tz_using_offset(npy_datetimestruct dts,
     # see PEP 495 https://www.python.org/dev/peps/pep-0495/#the-fold-attribute
     if is_utc(tz):
         pass
-    elif is_tzlocal(tz):
+    elif is_tzlocal(tz) or is_zoneinfo(tz):
         localize_tzinfo_api(obj.value, tz, &obj.fold)
     else:
         trans, deltas, typ = get_dst_info(tz)
@@ -581,55 +582,62 @@ cdef _TSObject _convert_str_to_tsobject(object ts, tzinfo tz, str unit,
     """
     cdef:
         npy_datetimestruct dts
-        int out_local = 0, out_tzoffset = 0
-        bint do_parse_datetime_string = False
+        int out_local = 0, out_tzoffset = 0, string_to_dts_failed
+        datetime dt
+        int64_t ival
 
     if len(ts) == 0 or ts in nat_strings:
         ts = NaT
+        obj = _TSObject()
+        obj.value = NPY_NAT
+        obj.tzinfo = tz
+        return obj
     elif ts == 'now':
         # Issue 9000, we short-circuit rather than going
         # into np_datetime_strings which returns utc
-        ts = datetime.now(tz)
+        dt = datetime.now(tz)
     elif ts == 'today':
         # Issue 9000, we short-circuit rather than going
         # into np_datetime_strings which returns a normalized datetime
-        ts = datetime.now(tz)
+        dt = datetime.now(tz)
         # equiv: datetime.today().replace(tzinfo=tz)
     else:
         string_to_dts_failed = _string_to_dts(
             ts, &dts, &out_local,
             &out_tzoffset, False
         )
-        try:
-            if not string_to_dts_failed:
+        if not string_to_dts_failed:
+            try:
                 check_dts_bounds(&dts)
                 if out_local == 1:
                     return _create_tsobject_tz_using_offset(dts,
                                                             out_tzoffset, tz)
                 else:
-                    ts = dtstruct_to_dt64(&dts)
+                    ival = dtstruct_to_dt64(&dts)
                     if tz is not None:
                         # shift for _localize_tso
-                        ts = tz_localize_to_utc_single(ts, tz,
-                                                       ambiguous="raise")
+                        ival = tz_localize_to_utc_single(ival, tz,
+                                                         ambiguous="raise")
 
-        except OutOfBoundsDatetime:
-            # GH#19382 for just-barely-OutOfBounds falling back to dateutil
-            # parser will return incorrect result because it will ignore
-            # nanoseconds
-            raise
+                    return convert_to_tsobject(ival, tz, None, False, False)
 
-        except ValueError:
-            do_parse_datetime_string = True
+            except OutOfBoundsDatetime:
+                # GH#19382 for just-barely-OutOfBounds falling back to dateutil
+                # parser will return incorrect result because it will ignore
+                # nanoseconds
+                raise
 
-        if string_to_dts_failed or do_parse_datetime_string:
-            try:
-                ts = parse_datetime_string(ts, dayfirst=dayfirst,
-                                           yearfirst=yearfirst)
-            except (ValueError, OverflowError):
-                raise ValueError("could not convert string to Timestamp")
+            except ValueError:
+                # Fall through to parse_datetime_string
+                pass
+
+        try:
+            dt = parse_datetime_string(ts, dayfirst=dayfirst,
+                                       yearfirst=yearfirst)
+        except (ValueError, OverflowError):
+            raise ValueError("could not convert string to Timestamp")
 
-    return convert_to_tsobject(ts, tz, unit, dayfirst, yearfirst)
+    return convert_datetime_to_tsobject(dt, tz)
 
 
 cdef inline check_overflows(_TSObject obj):
@@ -688,12 +696,8 @@ cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
     Sets obj.tzinfo inplace, alters obj.dts inplace.
     """
     cdef:
-        ndarray[int64_t] trans
-        int64_t[::1] deltas
         int64_t local_val
-        int64_t* tdata
-        Py_ssize_t pos, ntrans, outpos = -1
-        str typ
+        Py_ssize_t outpos = -1
 
     assert obj.tzinfo is None
 

pandas/_libs/tslibs/timestamps.pyx

@@ -397,6 +397,7 @@ cdef class _Timestamp(ABCTimestamp):
         elif is_datetime64_object(other):
             return type(self)(other) - self
         return NotImplemented
+
     # -----------------------------------------------------------------
 
     cdef int64_t _maybe_convert_value_to_local(self):

pandas/_libs/tslibs/timezones.pxd

@@ -9,6 +9,7 @@ cdef tzinfo utc_pytz
 
 cpdef bint is_utc(tzinfo tz)
 cdef bint is_tzlocal(tzinfo tz)
+cdef bint is_zoneinfo(tzinfo tz)
 
 cdef bint treat_tz_as_pytz(tzinfo tz)
 

pandas/_libs/tslibs/timezones.pyx

@@ -3,6 +3,14 @@ from datetime import (
     timezone,
 )
 
+try:
+    # py39+
+    import zoneinfo
+    from zoneinfo import ZoneInfo
+except ImportError:
+    zoneinfo = None
+    ZoneInfo = None
+
 from cpython.datetime cimport (
     datetime,
     timedelta,
@@ -42,18 +50,43 @@ cdef tzinfo utc_stdlib = timezone.utc
 cdef tzinfo utc_pytz = UTC
 cdef tzinfo utc_dateutil_str = dateutil_gettz("UTC")  # NB: *not* the same as tzutc()
 
+cdef tzinfo utc_zoneinfo = None
+
 
 # ----------------------------------------------------------------------
 
+cdef inline bint is_utc_zoneinfo(tzinfo tz):
+    # Workaround for cases with missing tzdata
+    #  https://github.com/pandas-dev/pandas/pull/46425#discussion_r830633025
+    if tz is None or zoneinfo is None:
+        return False
+
+    global utc_zoneinfo
+    if utc_zoneinfo is None:
+        try:
+            utc_zoneinfo = ZoneInfo("UTC")
+        except zoneinfo.ZoneInfoNotFoundError:
+            return False
+
+    return tz is utc_zoneinfo
+
+
 cpdef inline bint is_utc(tzinfo tz):
     return (
         tz is utc_pytz
         or tz is utc_stdlib
         or isinstance(tz, _dateutil_tzutc)
         or tz is utc_dateutil_str
+        or is_utc_zoneinfo(tz)
     )
 
 
+cdef inline bint is_zoneinfo(tzinfo tz):
+    if ZoneInfo is None:
+        return False
+    return isinstance(tz, ZoneInfo)
+
+
 cdef inline bint is_tzlocal(tzinfo tz):
     return isinstance(tz, _dateutil_tzlocal)
 
@@ -210,6 +243,8 @@ cdef inline bint is_fixed_offset(tzinfo tz):
             return 1
         else:
             return 0
+    elif is_zoneinfo(tz):
+        return 0
     # This also implicitly accepts datetime.timezone objects which are
     # considered fixed
     return 1
@@ -264,6 +299,8 @@ cdef object get_dst_info(tzinfo tz):
         # e.g. pytz.FixedOffset, matplotlib.dates._UTC,
         # psycopg2.tz.FixedOffsetTimezone
         num = int(get_utcoffset(tz, None).total_seconds()) * 1_000_000_000
+        # If we have e.g. ZoneInfo here, the get_utcoffset call will return None,
+        #  so the total_seconds() call will raise AttributeError.
         return (np.array([NPY_NAT + 1], dtype=np.int64),
                 np.array([num], dtype=np.int64),
                 "unknown")
@@ -291,13 +328,13 @@ cdef object get_dst_info(tzinfo tz):
                 # deltas
                 deltas = np.array([v.offset for v in (
                     tz._ttinfo_before,) + tz._trans_idx], dtype='i8')
-                deltas *= 1000000000
+                deltas *= 1_000_000_000
                 typ = 'dateutil'
 
             elif is_fixed_offset(tz):
                 trans = np.array([NPY_NAT + 1], dtype=np.int64)
                 deltas = np.array([tz._ttinfo_std.offset],
-                                  dtype='i8') * 1000000000
+                                  dtype='i8') * 1_000_000_000
                 typ = 'fixed'
             else:
                 # 2018-07-12 this is not reached in the tests, and this case

pandas/_libs/tslibs/tzconversion.pyx

@@ -42,6 +42,7 @@ from pandas._libs.tslibs.timezones cimport (
     is_fixed_offset,
     is_tzlocal,
     is_utc,
+    is_zoneinfo,
     utc_pytz,
 )
 
@@ -60,7 +61,7 @@ cdef int64_t tz_localize_to_utc_single(
     elif is_utc(tz) or tz is None:
         return val
 
-    elif is_tzlocal(tz):
+    elif is_tzlocal(tz) or is_zoneinfo(tz):
         return val - _tz_localize_using_tzinfo_api(val, tz, to_utc=True)
 
     elif is_fixed_offset(tz):
@@ -135,7 +136,7 @@ timedelta-like}
 
     result = np.empty(n, dtype=np.int64)
 
-    if is_tzlocal(tz):
+    if is_tzlocal(tz) or is_zoneinfo(tz):
         for i in range(n):
             v = vals[i]
             if v == NPY_NAT:
@@ -484,8 +485,8 @@ cdef int64_t tz_convert_from_utc_single(
 
     if is_utc(tz):
         return utc_val
-    elif is_tzlocal(tz):
-        return utc_val + _tz_localize_using_tzinfo_api(utc_val, tz, to_utc=False)
+    elif is_tzlocal(tz) or is_zoneinfo(tz):
+        return utc_val + _tz_localize_using_tzinfo_api(utc_val, tz, to_utc=False, fold=fold)
     else:
         trans, deltas, typ = get_dst_info(tz)
         tdata = <int64_t*>cnp.PyArray_DATA(trans)
@@ -569,7 +570,7 @@ cdef const int64_t[:] _tz_convert_from_utc(const int64_t[:] stamps, tzinfo tz):
 
     if is_utc(tz) or tz is None:
         use_utc = True
-    elif is_tzlocal(tz):
+    elif is_tzlocal(tz) or is_zoneinfo(tz):
         use_tzlocal = True
     else:
         trans, deltas, typ = get_dst_info(tz)