From 704e7c6c9086dede94c98ad071239966e0a82173 Mon Sep 17 00:00:00 2001 From: MarcusJak Date: Sat, 2 Mar 2024 17:42:26 +0100 Subject: [PATCH 1/2] Nanosecond fixed precision of DatetimeIndex with time zone information, Fix #53473 The expected behaviour is now correct from issue #53473 The out->picoseond value wasn't set which ended with a Json error regarding the ns calculations. --- pandas/_libs/src/datetime/pd_datetime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/src/datetime/pd_datetime.c b/pandas/_libs/src/datetime/pd_datetime.c index 19de51be6e1b2..1de74ee19ce9d 100644 --- a/pandas/_libs/src/datetime/pd_datetime.c +++ b/pandas/_libs/src/datetime/pd_datetime.c @@ -71,7 +71,7 @@ static int convert_pydatetime_to_datetimestruct(PyObject *dtobj, out->min = PyLong_AsLong(PyObject_GetAttrString(obj, "minute")); out->sec = PyLong_AsLong(PyObject_GetAttrString(obj, "second")); out->us = PyLong_AsLong(PyObject_GetAttrString(obj, "microsecond")); - + out->ps = 1000 * PyLong_AsLong(PyObject_GetAttrString(obj, "nanosecond")); if (PyObject_HasAttrString(obj, "tzinfo")) { PyObject *offset = extract_utc_offset(obj); /* Apply the time zone offset if datetime obj is tz-aware */ From 682ba46f923f3273c6d4d66840fc1f29395e9dab Mon Sep 17 00:00:00 2001 From: rgrsst Date: Mon, 4 Mar 2024 12:48:31 +0100 Subject: [PATCH 2/2] fix: Handling datetimes without nanoseconds, adds a test --- pandas/_libs/src/datetime/pd_datetime.c | 4 +++- pandas/tests/io/json/test_pandas.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/src/datetime/pd_datetime.c b/pandas/_libs/src/datetime/pd_datetime.c index 1de74ee19ce9d..1a45c6883fa9d 100644 --- a/pandas/_libs/src/datetime/pd_datetime.c +++ b/pandas/_libs/src/datetime/pd_datetime.c @@ -71,7 +71,9 @@ static int convert_pydatetime_to_datetimestruct(PyObject *dtobj, out->min = PyLong_AsLong(PyObject_GetAttrString(obj, "minute")); out->sec = PyLong_AsLong(PyObject_GetAttrString(obj, "second")); out->us = PyLong_AsLong(PyObject_GetAttrString(obj, "microsecond")); - out->ps = 1000 * PyLong_AsLong(PyObject_GetAttrString(obj, "nanosecond")); + if (PyObject_HasAttrString(obj, "nanosecond")) { + out->ps = 1000 * PyLong_AsLong(PyObject_GetAttrString(obj, "nanosecond")); + } if (PyObject_HasAttrString(obj, "tzinfo")) { PyObject *offset = extract_utc_offset(obj); /* Apply the time zone offset if datetime obj is tz-aware */ diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index db120588b234c..73e52b35dc32e 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1216,6 +1216,26 @@ def test_datetime_tz(self): s_naive = Series(tz_naive) assert stz.to_json() == s_naive.to_json() + def test_tz_nano_datetimes(self): + df = DataFrame( + { + "date": Series( + [ + datetime.datetime( + 2024, 1, 1, 0, 0, 0, 000000, tzinfo=datetime.timezone.utc + ) + ] + ) + } + ) + df.date = df.date + np.timedelta64(1, "ns") + buf = StringIO() + df.to_json(buf, date_unit="ns", orient="columns", date_format="iso") + buf.seek(0) + tm.assert_frame_equal( + read_json(buf), df, check_index_type=False, check_dtype=False + ) + def test_sparse(self): # GH4377 df.to_json segfaults with non-ndarray blocks df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))