BUG: Series/DataFrame construction from scalars (#38405)

Author: jbrockmendel

doc/source/whatsnew/v1.3.0.rst

@@ -86,7 +86,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
-
+- Bug in :class:`DataFrame` and :class:`Series` constructors sometimes dropping nanoseconds from :class:`Timestamp` (resp. :class:`Timedelta`) ``data``, with ``dtype=datetime64[ns]`` (resp. ``timedelta64[ns]``) (:issue:`38032`)
 -
 -
 

pandas/core/dtypes/cast.py

@@ -20,7 +20,7 @@
 
 import numpy as np
 
-from pandas._libs import lib, tslib, tslibs
+from pandas._libs import lib, tslib
 from pandas._libs.tslibs import (
     NaT,
     OutOfBoundsDatetime,
@@ -145,13 +145,35 @@ def maybe_box_datetimelike(value: Scalar, dtype: Optional[Dtype] = None) -> Scal
     if dtype == object:
         pass
     elif isinstance(value, (np.datetime64, datetime)):
-        value = tslibs.Timestamp(value)
+        value = Timestamp(value)
     elif isinstance(value, (np.timedelta64, timedelta)):
-        value = tslibs.Timedelta(value)
+        value = Timedelta(value)
 
     return value
 
 
+def maybe_unbox_datetimelike(value: Scalar, dtype: DtypeObj) -> Scalar:
+    """
+    Convert a Timedelta or Timestamp to timedelta64 or datetime64 for setting
+    into a numpy array.  Failing to unbox would risk dropping nanoseconds.
+
+    Notes
+    -----
+    Caller is responsible for checking dtype.kind in ["m", "M"]
+    """
+    if is_valid_nat_for_dtype(value, dtype):
+        # GH#36541: can't fill array directly with pd.NaT
+        # > np.empty(10, dtype="datetime64[64]").fill(pd.NaT)
+        # ValueError: cannot convert float NaN to integer
+        value = dtype.type("NaT", "ns")
+    elif isinstance(value, Timestamp):
+        if value.tz is None:
+            value = value.to_datetime64()
+    elif isinstance(value, Timedelta):
+        value = value.to_timedelta64()
+    return value
+
+
 def maybe_downcast_to_dtype(result, dtype: Union[str, np.dtype]):
     """
     try to cast to the specified dtype (e.g. convert back to bool/int
@@ -1361,8 +1383,7 @@ def maybe_cast_to_datetime(value, dtype: Optional[DtypeObj]):
                     raise TypeError(f"cannot convert timedeltalike to dtype [{dtype}]")
 
             if is_scalar(value):
-                if value == iNaT or isna(value):
-                    value = iNaT
+                value = maybe_unbox_datetimelike(value, dtype)
             elif not is_sparse(value):
                 value = np.array(value, copy=False)
 
@@ -1535,11 +1556,8 @@ def construct_1d_arraylike_from_scalar(
             dtype = np.dtype("object")
             if not isna(value):
                 value = ensure_str(value)
-        elif dtype.kind in ["M", "m"] and is_valid_nat_for_dtype(value, dtype):
-            # GH36541: can't fill array directly with pd.NaT
-            # > np.empty(10, dtype="datetime64[64]").fill(pd.NaT)
-            # ValueError: cannot convert float NaN to integer
-            value = dtype.type("NaT", "ns")
+        elif dtype.kind in ["M", "m"]:
+            value = maybe_unbox_datetimelike(value, dtype)
 
         subarr = np.empty(length, dtype=dtype)
         subarr.fill(value)

pandas/core/frame.py

@@ -86,6 +86,7 @@
     maybe_convert_platform,
     maybe_downcast_to_dtype,
     maybe_infer_to_datetimelike,
+    maybe_unbox_datetimelike,
     validate_numeric_casting,
 )
 from pandas.core.dtypes.common import (
@@ -592,6 +593,9 @@ def __init__(
                 ]
                 mgr = arrays_to_mgr(values, columns, index, columns, dtype=None)
             else:
+                if dtype.kind in ["m", "M"]:
+                    data = maybe_unbox_datetimelike(data, dtype)
+
                 # Attempt to coerce to a numpy array
                 try:
                     arr = np.array(data, dtype=dtype, copy=copy)

pandas/tests/dtypes/cast/test_construct_from_scalar.py

@@ -1,7 +1,9 @@
+import numpy as np
+
 from pandas.core.dtypes.cast import construct_1d_arraylike_from_scalar
 from pandas.core.dtypes.dtypes import CategoricalDtype
 
-from pandas import Categorical
+from pandas import Categorical, Timedelta, Timestamp
 import pandas._testing as tm
 
 
@@ -16,3 +18,17 @@ def test_cast_1d_array_like_from_scalar_categorical():
 
     result = construct_1d_arraylike_from_scalar("a", len(expected), cat_type)
     tm.assert_categorical_equal(result, expected)
+
+
+def test_cast_1d_array_like_from_timestamp():
+    # check we dont lose nanoseconds
+    ts = Timestamp.now() + Timedelta(1)
+    res = construct_1d_arraylike_from_scalar(ts, 2, np.dtype("M8[ns]"))
+    assert res[0] == ts
+
+
+def test_cast_1d_array_like_from_timedelta():
+    # check we dont lose nanoseconds
+    td = Timedelta(1)
+    res = construct_1d_arraylike_from_scalar(td, 2, np.dtype("m8[ns]"))
+    assert res[0] == td

pandas/tests/frame/test_constructors.py

@@ -11,7 +11,7 @@
 import pytz
 
 from pandas.compat import is_platform_little_endian
-from pandas.compat.numpy import _np_version_under1p19
+from pandas.compat.numpy import _np_version_under1p19, _np_version_under1p20
 
 from pandas.core.dtypes.common import is_integer_dtype
 from pandas.core.dtypes.dtypes import DatetimeTZDtype, IntervalDtype, PeriodDtype
@@ -2915,3 +2915,50 @@ def test_construction_from_set_raises(self):
         msg = "Set type is unordered"
         with pytest.raises(TypeError, match=msg):
             DataFrame({"a": {1, 2, 3}})
+
+
+def get1(obj):
+    if isinstance(obj, Series):
+        return obj.iloc[0]
+    else:
+        return obj.iloc[0, 0]
+
+
+class TestFromScalar:
+    @pytest.fixture
+    def constructor(self, frame_or_series):
+        if frame_or_series is Series:
+            return functools.partial(Series, index=range(2))
+        else:
+            return functools.partial(DataFrame, index=range(2), columns=range(2))
+
+    @pytest.mark.parametrize("dtype", ["M8[ns]", "m8[ns]"])
+    def test_from_nat_scalar(self, dtype, constructor):
+        obj = constructor(pd.NaT, dtype=dtype)
+        assert np.all(obj.dtypes == dtype)
+        assert np.all(obj.isna())
+
+    def test_from_timedelta_scalar_preserves_nanos(self, constructor):
+        td = Timedelta(1)
+
+        obj = constructor(td, dtype="m8[ns]")
+        assert get1(obj) == td
+
+    def test_from_timestamp_scalar_preserves_nanos(self, constructor):
+        ts = Timestamp.now() + Timedelta(1)
+
+        obj = Series(ts, index=range(1), dtype="M8[ns]")
+        assert get1(obj) == ts
+
+    def test_from_timedelta64_scalar_object(self, constructor, request):
+        if constructor.func is DataFrame and _np_version_under1p20:
+            mark = pytest.mark.xfail(
+                reason="np.array(td64, dtype=object) converts to int"
+            )
+            request.node.add_marker(mark)
+
+        td = Timedelta(1)
+        td64 = td.to_timedelta64()
+
+        obj = constructor(td64, dtype=object)
+        assert isinstance(get1(obj), np.timedelta64)