ENH: infer resolution in to_datetime, DatetimeIndex

Author: jbrockmendel

pandas/_libs/lib.pyx

@@ -123,16 +123,12 @@ from pandas._libs.missing cimport (
     is_null_datetime64,
     is_null_timedelta64,
 )
-from pandas._libs.tslibs.conversion cimport (
-    _TSObject,
-    convert_to_tsobject,
-)
+from pandas._libs.tslibs.conversion cimport convert_to_tsobject
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
     c_NaT as NaT,
     checknull_with_nat,
 )
-from pandas._libs.tslibs.np_datetime cimport NPY_FR_ns
 from pandas._libs.tslibs.offsets cimport is_offset_object
 from pandas._libs.tslibs.period cimport is_period_object
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
@@ -2523,7 +2519,6 @@ def maybe_convert_objects(ndarray[object] objects,
         ndarray[uint8_t] mask
         Seen seen = Seen()
         object val
-        _TSObject tsobj
         float64_t fnan = NaN
 
     if dtype_if_all_nat is not None:
@@ -2630,9 +2625,9 @@ def maybe_convert_objects(ndarray[object] objects,
                 else:
                     seen.datetime_ = True
                     try:
-                        tsobj = convert_to_tsobject(val, None, None, 0, 0)
-                        tsobj.ensure_reso(NPY_FR_ns)
+                        convert_to_tsobject(val, None, None, 0, 0)
                     except OutOfBoundsDatetime:
+                        # e.g. test_out_of_s_bounds_datetime64
                         seen.object_ = True
                         break
             else:

pandas/_libs/tslib.pyx

@@ -70,7 +70,10 @@ from pandas._libs.tslibs.conversion cimport (
     get_datetime64_nanos,
     parse_pydatetime,
 )
-from pandas._libs.tslibs.dtypes cimport npy_unit_to_abbrev
+from pandas._libs.tslibs.dtypes cimport (
+    get_supported_reso,
+    npy_unit_to_abbrev,
+)
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
     c_NaT as NaT,
@@ -417,7 +420,7 @@ cpdef array_to_datetime(
     bint dayfirst=False,
     bint yearfirst=False,
     bint utc=False,
-    NPY_DATETIMEUNIT creso=NPY_FR_ns,
+    NPY_DATETIMEUNIT creso=NPY_DATETIMEUNIT.NPY_FR_GENERIC,
 ):
     """
     Converts a 1D array of date-like values to a numpy array of either:
@@ -444,18 +447,20 @@ cpdef array_to_datetime(
          yearfirst parsing behavior when encountering datetime strings
     utc : bool, default False
          indicator whether the dates should be UTC
-    creso : NPY_DATETIMEUNIT, default NPY_FR_ns
-        Set to NPY_FR_GENERIC to infer a resolution.
+    creso : NPY_DATETIMEUNIT, default NPY_FR_GENERIC
+        If NPY_FR_GENERIC, conduct inference.
 
     Returns
     -------
     np.ndarray
         May be datetime64[ns] or object dtype
     tzinfo or None
+    str
+        Inferred resolution
     """
     cdef:
         Py_ssize_t i, n = values.size
-        object val, tz
+        object val
         ndarray[int64_t] iresult
         npy_datetimestruct dts
         bint utc_convert = bool(utc)
@@ -467,7 +472,7 @@ cpdef array_to_datetime(
         _TSObject _ts
         float tz_offset
         set out_tzoffset_vals = set()
-        tzinfo tz_out = None
+        tzinfo tz, tz_out = None
         cnp.flatiter it = cnp.PyArray_IterNew(values)
         NPY_DATETIMEUNIT item_reso
         bint infer_reso = creso == NPY_DATETIMEUNIT.NPY_FR_GENERIC
@@ -522,15 +527,14 @@ cpdef array_to_datetime(
 
             elif is_integer_object(val) or is_float_object(val):
                 # these must be ns unit by-definition
-                item_reso = NPY_FR_ns
-                state.update_creso(item_reso)
-                if infer_reso:
-                    creso = state.creso
 
                 if val != val or val == NPY_NAT:
                     iresult[i] = NPY_NAT
                 else:
-                    # we now need to parse this as if unit='ns'
+                    item_reso = NPY_FR_ns
+                    state.update_creso(item_reso)
+                    if infer_reso:
+                        creso = state.creso
                     iresult[i] = cast_from_unit(val, "ns", out_reso=creso)
                     state.found_other = True
 
@@ -552,6 +556,14 @@ cpdef array_to_datetime(
                 _ts = convert_str_to_tsobject(
                     val, None, unit="ns", dayfirst=dayfirst, yearfirst=yearfirst
                 )
+
+                if _ts.value == NPY_NAT:
+                    # e.g. "NaT" string or empty string, we do not consider
+                    #  this as either tzaware or tznaive. See
+                    #  test_to_datetime_with_empty_str_utc_false_format_mixed
+                    iresult[i] = _ts.value
+                    continue
+
                 item_reso = _ts.creso
                 state.update_creso(item_reso)
                 if infer_reso:
@@ -561,12 +573,7 @@ cpdef array_to_datetime(
                 iresult[i] = _ts.value
 
                 tz = _ts.tzinfo
-                if _ts.value == NPY_NAT:
-                    # e.g. "NaT" string or empty string, we do not consider
-                    #  this as either tzaware or tznaive. See
-                    #  test_to_datetime_with_empty_str_utc_false_format_mixed
-                    pass
-                elif tz is not None:
+                if tz is not None:
                     # dateutil timezone objects cannot be hashed, so
                     # store the UTC offsets in seconds instead
                     nsecs = tz.utcoffset(None).total_seconds()
@@ -639,7 +646,7 @@ cpdef array_to_datetime(
         # Otherwise we can use the single reso that we encountered and avoid
         #  a second pass.
         abbrev = npy_unit_to_abbrev(state.creso)
-        result = iresult.view(f"M8[{abbrev}]")
+        result = iresult.view(f"M8[{abbrev}]").reshape(result.shape)
     return result, tz_out
 
 
@@ -803,7 +810,9 @@ def array_to_datetime_with_tz(
         if state.creso_ever_changed:
             # We encountered mismatched resolutions, need to re-parse with
             #  the correct one.
-            return array_to_datetime_with_tz(values, tz=tz, creso=creso)
+            return array_to_datetime_with_tz(
+                values, tz=tz, dayfirst=dayfirst, yearfirst=yearfirst, creso=creso
+            )
 
         # Otherwise we can use the single reso that we encountered and avoid
         #  a second pass.

pandas/_libs/tslibs/conversion.pyx

@@ -710,7 +710,9 @@ cdef int64_t parse_pydatetime(
         result = _ts.value
     else:
         if isinstance(val, _Timestamp):
-            result = (<_Timestamp>val)._as_creso(creso, round_ok=False)._value
+            # TODO: make round_ok an arg here that is True if a user passed a
+            #  dtype to DatetimeIndex/astype and False if we are doing inference?
+            result = (<_Timestamp>val)._as_creso(creso, round_ok=True)._value
         else:
             result = pydatetime_to_dt64(val, dts, reso=creso)
             check_dts_bounds(dts, creso)

pandas/_libs/tslibs/strptime.pyx

@@ -241,7 +241,7 @@ cdef _get_format_regex(str fmt):
 
 
 cdef class DatetimeParseState:
-    def __cinit__(self, NPY_DATETIMEUNIT creso=NPY_DATETIMEUNIT.NPY_FR_ns):
+    def __cinit__(self, NPY_DATETIMEUNIT creso):
         # found_tz and found_naive are specifically about datetime/Timestamp
         #  objects with and without tzinfos attached.
         self.found_tz = False
@@ -295,7 +295,7 @@ def array_strptime(
     bint exact=True,
     errors="raise",
     bint utc=False,
-    NPY_DATETIMEUNIT creso=NPY_FR_ns,
+    NPY_DATETIMEUNIT creso=NPY_DATETIMEUNIT.NPY_FR_GENERIC,
 ):
     """
     Calculates the datetime structs represented by the passed array of strings
@@ -621,7 +621,7 @@ cdef tzinfo _parse_with_format(
             elif len(s) <= 6:
                 item_reso[0] = NPY_DATETIMEUNIT.NPY_FR_us
             else:
-                item_reso[0] = NPY_DATETIMEUNIT.NPY_FR_ns
+                item_reso[0] = NPY_FR_ns
             # Pad to always return nanoseconds
             s += "0" * (9 - len(s))
             us = long(s)

pandas/_testing/asserters.py

@@ -970,6 +970,9 @@ def assert_series_equal(
 
         # datetimelike may have different objects (e.g. datetime.datetime
         # vs Timestamp) but will compare equal
+        # TODO: this works for object-vs-dt64 but not e.g. dt64[ns] vs dt64[us],
+        #  which AFAICT would have been intended at the time
+        #  check_datetimelike_compat was implemented, xref GH#55638
         if not Index(left._values).equals(Index(right._values)):
             msg = (
                 f"[datetimelike_compat=True] {left._values} "

pandas/core/arrays/datetimes.py

@@ -2229,9 +2229,11 @@ def _sequence_to_dt64(
     data, copy = maybe_convert_dtype(data, copy, tz=tz)
     data_dtype = getattr(data, "dtype", None)
 
-    if out_unit is None:
-        out_unit = "ns"
-    out_dtype = np.dtype(f"M8[{out_unit}]")
+    out_dtype = DT64NS_DTYPE
+    out_reso = abbrev_to_npy_unit(None)  # NPY_FR_GENERIC
+    if out_unit is not None:
+        out_dtype = np.dtype(f"M8[{out_unit}]")
+        out_reso = abbrev_to_npy_unit(out_unit)
 
     if data_dtype == object or is_string_dtype(data_dtype):
         # TODO: We do not have tests specific to string-dtypes,
@@ -2255,19 +2257,17 @@ def _sequence_to_dt64(
                 dayfirst=dayfirst,
                 yearfirst=yearfirst,
                 allow_object=False,
-                out_unit=out_unit or "ns",
+                out_reso=out_reso,
             )
             copy = False
             if tz and inferred_tz:
                 #  two timezones: convert to intended from base UTC repr
                 # GH#42505 by convention, these are _already_ UTC
-                assert converted.dtype == out_dtype, converted.dtype
-                result = converted.view(out_dtype)
+                result = converted
 
             elif inferred_tz:
                 tz = inferred_tz
-                assert converted.dtype == out_dtype, converted.dtype
-                result = converted.view(out_dtype)
+                result = converted
 
             else:
                 result, _ = _construct_from_dt64_naive(
@@ -2365,7 +2365,7 @@ def objects_to_datetime64(
     utc: bool = False,
     errors: DateTimeErrorChoices = "raise",
     allow_object: bool = False,
-    out_unit: str = "ns",
+    out_reso: int = 14,
 ):
     """
     Convert data to array of timestamps.
@@ -2381,7 +2381,9 @@ def objects_to_datetime64(
     allow_object : bool
         Whether to return an object-dtype ndarray instead of raising if the
         data contains more than one timezone.
-    out_unit : str, default "ns"
+    out_reso : int, default 14
+        14 corresponds to NPY_FR_GENERIC, which indicates to infer
+        a resolution.
 
     Returns
     -------
@@ -2407,7 +2409,7 @@ def objects_to_datetime64(
         utc=utc,
         dayfirst=dayfirst,
         yearfirst=yearfirst,
-        creso=abbrev_to_npy_unit(out_unit),
+        creso=out_reso,
     )
 
     if tz_parsed is not None:

pandas/core/series.py

@@ -3483,6 +3483,7 @@ def combine_first(self, other) -> Series:
         other = other.reindex(keep_other, copy=False)
 
         if this.dtype.kind == "M" and other.dtype.kind != "M":
+            # TODO: try to match resos?
             other = to_datetime(other)
         combined = concat([this, other])
         combined = combined.reindex(new_index, copy=False)

pandas/core/tools/datetimes.py

@@ -497,7 +497,8 @@ def _convert_listlike_datetimes(
     if tz_parsed is not None:
         # We can take a shortcut since the datetime64 numpy array
         # is in UTC
-        dtype = cast(DatetimeTZDtype, tz_to_dtype(tz_parsed))
+        out_unit = np.datetime_data(result.dtype)[0]
+        dtype = cast(DatetimeTZDtype, tz_to_dtype(tz_parsed, out_unit))
         dt64_values = result.view(f"M8[{dtype.unit}]")
         dta = DatetimeArray._simple_new(dt64_values, dtype=dtype)
         return DatetimeIndex._simple_new(dta, name=name)

pandas/core/window/ewm.py

@@ -7,6 +7,7 @@
 
 import numpy as np
 
+from pandas._libs import lib
 from pandas._libs.tslibs import Timedelta
 import pandas._libs.window.aggregations as window_aggregations
 from pandas.util._decorators import doc
@@ -364,7 +365,15 @@ def __init__(
             if not self.adjust:
                 raise NotImplementedError("times is not supported with adjust=False.")
             if not is_datetime64_ns_dtype(self.times):
-                raise ValueError("times must be datetime64[ns] dtype.")
+                if hasattr(self.times, "dtype") and lib.is_np_dtype(
+                    self.times.dtype, "M"
+                ):
+                    # TODO: is this what we really want?
+                    from pandas import Series
+
+                    self.times = Series(self.times).dt.as_unit("ns")._values
+                else:
+                    raise ValueError("times must be datetime64[ns] dtype.")
             if len(self.times) != len(obj):
                 raise ValueError("times must be the same length as the object.")
             if not isinstance(self.halflife, (str, datetime.timedelta, np.timedelta64)):

pandas/tests/apply/test_frame_apply.py

@@ -1272,7 +1272,7 @@ def test_nuiscance_columns():
 
     result = df.agg(["min"])
     expected = DataFrame(
-        [[1, 1.0, "bar", Timestamp("20130101")]],
+        [[1, 1.0, "bar", Timestamp("20130101").as_unit("ns")]],
         index=["min"],
         columns=df.columns,
     )

pandas/tests/apply/test_series_apply.py

@@ -153,7 +153,7 @@ def func(x):
 def test_apply_box():
     # ufunc will not be boxed. Same test cases as the test_map_box
     vals = [pd.Timestamp("2011-01-01"), pd.Timestamp("2011-01-02")]
-    s = Series(vals)
+    s = Series(vals, dtype="M8[ns]")
     assert s.dtype == "datetime64[ns]"
     # boxed value must be Timestamp instance
     res = s.apply(lambda x: f"{type(x).__name__}_{x.day}_{x.tz}", by_row="compat")
@@ -164,7 +164,7 @@ def test_apply_box():
         pd.Timestamp("2011-01-01", tz="US/Eastern"),
         pd.Timestamp("2011-01-02", tz="US/Eastern"),
     ]
-    s = Series(vals)
+    s = Series(vals, dtype="M8[ns, US/Eastern]")
     assert s.dtype == "datetime64[ns, US/Eastern]"
     res = s.apply(lambda x: f"{type(x).__name__}_{x.day}_{x.tz}", by_row="compat")
     exp = Series(["Timestamp_1_US/Eastern", "Timestamp_2_US/Eastern"])

pandas/tests/arithmetic/test_datetime64.py

@@ -1286,7 +1286,7 @@ def test_dti_add_tick_tzaware(self, tz_aware_fixture, box_with_array):
             ["2010-11-01 05:00", "2010-11-01 06:00", "2010-11-01 07:00"],
             freq="h",
             tz=tz,
-        )
+        ).as_unit("ns")
 
         dates = tm.box_expected(dates, box_with_array)
         expected = tm.box_expected(expected, box_with_array)
@@ -1580,7 +1580,7 @@ def test_dti_add_sub_nonzero_mth_offset(
         mth = getattr(date, op)
         result = mth(offset)
 
-        expected = DatetimeIndex(exp, tz=tz)
+        expected = DatetimeIndex(exp, tz=tz).as_unit("ns")
         expected = tm.box_expected(expected, box_with_array, False)
         tm.assert_equal(result, expected)
 
@@ -2286,7 +2286,7 @@ def test_dti_add_series(self, tz_naive_fixture, names):
         tz = tz_naive_fixture
         index = DatetimeIndex(
             ["2016-06-28 05:30", "2016-06-28 05:31"], tz=tz, name=names[0]
-        )
+        ).as_unit("ns")
         ser = Series([Timedelta(seconds=5)] * 2, index=index, name=names[1])
         expected = Series(index + Timedelta(seconds=5), index=index, name=names[2])