From 4d677423d77150bd4489723f5e435a2c9304f7ad Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 22 Jan 2022 11:57:19 -0800 Subject: [PATCH 1/3] BUG: Segfault in to_json with tzaware datetime --- doc/source/whatsnew/v1.4.1.rst | 2 +- pandas/_libs/tslibs/src/datetime/np_datetime.c | 2 +- pandas/tests/io/json/test_pandas.py | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v1.4.1.rst b/doc/source/whatsnew/v1.4.1.rst index f321aabc0a8a5..5dfa6651a3f30 100644 --- a/doc/source/whatsnew/v1.4.1.rst +++ b/doc/source/whatsnew/v1.4.1.rst @@ -23,7 +23,7 @@ Fixed regressions Bug fixes ~~~~~~~~~ -- +- Fixed segfault in :meth:``DataFrame.to_json`` when dumping tz-aware datetimes in Python 3.10 (:issue:`42130`) - .. --------------------------------------------------------------------------- diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.c b/pandas/_libs/tslibs/src/datetime/np_datetime.c index 9ad2ead5f919f..d861d2832b686 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.c +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.c @@ -378,7 +378,7 @@ int convert_pydatetime_to_datetimestruct(PyObject *dtobj, if (tmp == NULL) { return -1; } - seconds_offset = PyInt_AsLong(tmp); + seconds_offset = PyInt_AsLong(PyNumber_Long(tmp)); if (seconds_offset == -1 && PyErr_Occurred()) { Py_DECREF(tmp); return -1; diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 0b8548f98b03b..f3a6f1f80359c 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -11,7 +11,6 @@ from pandas.compat import ( IS64, - PY310, is_platform_windows, ) import pandas.util._test_decorators as td @@ -1177,7 +1176,6 @@ def test_sparse(self): expected = s.to_json() assert expected == ss.to_json() - @pytest.mark.skipif(PY310, reason="segfault GH 42130") @pytest.mark.parametrize( "ts", [ @@ -1195,7 +1193,6 @@ def test_tz_is_utc(self, ts): dt = ts.to_pydatetime() assert dumps(dt, iso_dates=True) == exp - @pytest.mark.skipif(PY310, reason="segfault GH 42130") @pytest.mark.parametrize( "tz_range", [ From c7c1983c7f5a4ece8341ab075766cff22cfa9cdc Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 23 Jan 2022 12:23:29 -0800 Subject: [PATCH 2/3] fix memory leak? --- pandas/_libs/tslibs/src/datetime/np_datetime.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.c b/pandas/_libs/tslibs/src/datetime/np_datetime.c index d861d2832b686..b56d4367a5396 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.c +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.c @@ -360,6 +360,7 @@ int convert_pydatetime_to_datetimestruct(PyObject *dtobj, Py_DECREF(tmp); } else { PyObject *offset; + PyObject *tmp_int; int seconds_offset, minutes_offset; /* The utcoffset function should return a timedelta */ @@ -378,11 +379,14 @@ int convert_pydatetime_to_datetimestruct(PyObject *dtobj, if (tmp == NULL) { return -1; } - seconds_offset = PyInt_AsLong(PyNumber_Long(tmp)); + tmp_int = PyNumber_Long(tmp); + seconds_offset = PyInt_AsLong(tmp_int); if (seconds_offset == -1 && PyErr_Occurred()) { + Py_DECREF(tmp_int); Py_DECREF(tmp); return -1; } + Py_DECREF(tmp_int); Py_DECREF(tmp); /* Convert to a minutes offset and apply it */ From 38b278f072c1ebb2475aeab8a634079f608bbb40 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 23 Jan 2022 19:26:35 -0800 Subject: [PATCH 3/3] check for null --- pandas/_libs/tslibs/src/datetime/np_datetime.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.c b/pandas/_libs/tslibs/src/datetime/np_datetime.c index b56d4367a5396..0efed210a7fdf 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.c +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.c @@ -380,6 +380,10 @@ int convert_pydatetime_to_datetimestruct(PyObject *dtobj, return -1; } tmp_int = PyNumber_Long(tmp); + if (tmp_int == NULL) { + Py_DECREF(tmp); + return -1; + } seconds_offset = PyInt_AsLong(tmp_int); if (seconds_offset == -1 && PyErr_Occurred()) { Py_DECREF(tmp_int);