From 0c035bc6cfcd522432d76cf4d6040df2cf4bbf20 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Tue, 17 Sep 2019 14:03:50 -0400 Subject: [PATCH 01/12] BUG: Raise when casting NaT to int --- pandas/core/dtypes/cast.py | 4 ++++ pandas/tests/dtypes/test_common.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index ac9b57dc8d342..48373298c3f16 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -8,6 +8,8 @@ from pandas._libs.tslibs import NaT, OutOfBoundsDatetime, Period, iNaT from pandas.util._validators import validate_bool_kwarg +import pandas as pd + from .common import ( _INT64_DTYPE, _NS_DTYPE, @@ -696,6 +698,8 @@ def astype_nansafe(arr, dtype, copy=True, skipna=False): if is_object_dtype(dtype): return tslib.ints_to_pydatetime(arr.view(np.int64)) elif dtype == np.int64: + if pd.isnull(arr).any(): + raise ValueError("Cannot convert NaT values to integer") return arr.view(dtype) # allow frequency conversions diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index db9f647e0f0c7..05441b68e9e63 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -3,6 +3,7 @@ import pandas.util._test_decorators as td +from pandas.core.dtypes.cast import astype_nansafe import pandas.core.dtypes.common as com from pandas.core.dtypes.dtypes import ( CategoricalDtype, @@ -731,3 +732,11 @@ def test__is_dtype_type_sparse(): result = np.dtype("int32") assert com._is_dtype_type(ser, lambda tipo: tipo == result) assert com._is_dtype_type(ser.dtype, lambda tipo: tipo == result) + + +def test__nansafe_nat_to_int(): + arr = np.array([np.datetime64("NaT")]) + + msg = "Cannot convert NaT values to integer" + with pytest.raises(ValueError, match=msg): + astype_nansafe(arr, dtype=np.int64) From 0d645228798519a2c36156d069b4949c9605f010 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Tue, 17 Sep 2019 23:08:11 -0400 Subject: [PATCH 02/12] Add release note --- doc/source/whatsnew/v1.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 8b2b3a09f8c87..10d86b938b373 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -136,6 +136,7 @@ Datetimelike - Bug in :class:`Timestamp` subtraction when subtracting a :class:`Timestamp` from a ``np.datetime64`` object incorrectly raising ``TypeError`` (:issue:`28286`) - Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`) - Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`) +- Bug in :meth:`Series.astype` failing to handle ``pd.NaT`` when casting to an integer dtype - From 3d91a8f93f1fd9c4fc315e83eceed883dee89506 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Tue, 17 Sep 2019 23:19:14 -0400 Subject: [PATCH 03/12] Add PR number --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 10d86b938b373..16eb2ace50661 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -136,7 +136,7 @@ Datetimelike - Bug in :class:`Timestamp` subtraction when subtracting a :class:`Timestamp` from a ``np.datetime64`` object incorrectly raising ``TypeError`` (:issue:`28286`) - Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`) - Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`) -- Bug in :meth:`Series.astype` failing to handle ``pd.NaT`` when casting to an integer dtype +- Bug in :meth:`Series.astype` failing to handle ``pd.NaT`` when casting to an integer dtype (:issue:`28492`) - From 482d629624110251ac15208e6eb8192e0c0d0d5d Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 18 Sep 2019 14:06:53 -0400 Subject: [PATCH 04/12] Use isna --- pandas/core/dtypes/cast.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 48373298c3f16..b13d41262897b 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -8,8 +8,6 @@ from pandas._libs.tslibs import NaT, OutOfBoundsDatetime, Period, iNaT from pandas.util._validators import validate_bool_kwarg -import pandas as pd - from .common import ( _INT64_DTYPE, _NS_DTYPE, @@ -698,7 +696,7 @@ def astype_nansafe(arr, dtype, copy=True, skipna=False): if is_object_dtype(dtype): return tslib.ints_to_pydatetime(arr.view(np.int64)) elif dtype == np.int64: - if pd.isnull(arr).any(): + if isna(arr).any(): raise ValueError("Cannot convert NaT values to integer") return arr.view(dtype) From 975de26d47e5696098fc1e8ecd818f50ed558121 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 18 Sep 2019 14:24:07 -0400 Subject: [PATCH 05/12] Parametrize test --- pandas/tests/dtypes/test_common.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index 05441b68e9e63..e3a0636d918ea 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -734,9 +734,11 @@ def test__is_dtype_type_sparse(): assert com._is_dtype_type(ser.dtype, lambda tipo: tipo == result) -def test__nansafe_nat_to_int(): - arr = np.array([np.datetime64("NaT")]) +@pytest.mark.parametrize("val", [np.datetime64("NaT")]) +@pytest.mark.parametrize("typ", [np.int8, np.int16, np.int32, np.int64]) +def test_astype_nansafe(val, typ): + arr = np.array([val]) msg = "Cannot convert NaT values to integer" with pytest.raises(ValueError, match=msg): - astype_nansafe(arr, dtype=np.int64) + astype_nansafe(arr, dtype=typ) From 3445dc74cbf8bf7edcafbee4ad86d00bfa4742f5 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 18 Sep 2019 14:24:21 -0400 Subject: [PATCH 06/12] Check all integer dtypes --- pandas/core/dtypes/cast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index b13d41262897b..4b135790943b9 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -695,7 +695,7 @@ def astype_nansafe(arr, dtype, copy=True, skipna=False): elif is_datetime64_dtype(arr): if is_object_dtype(dtype): return tslib.ints_to_pydatetime(arr.view(np.int64)) - elif dtype == np.int64: + elif is_integer_dtype(dtype): if isna(arr).any(): raise ValueError("Cannot convert NaT values to integer") return arr.view(dtype) From e23b9a014693800958944031ab778eb3a37714fc Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 18 Sep 2019 14:41:12 -0400 Subject: [PATCH 07/12] Fix merge --- pandas/tests/dtypes/test_common.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index 63127e34c5690..bd3b69fbbf9ce 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -709,14 +709,6 @@ def test__is_dtype_type(input_param, result): assert com._is_dtype_type(input_param, lambda tipo: tipo == result) -@ignore_sparse_warning -def test__is_dtype_type_sparse(): - ser = pd.SparseSeries([1, 2], dtype="int32") - result = np.dtype("int32") - assert com._is_dtype_type(ser, lambda tipo: tipo == result) - assert com._is_dtype_type(ser.dtype, lambda tipo: tipo == result) - - @pytest.mark.parametrize("val", [np.datetime64("NaT")]) @pytest.mark.parametrize("typ", [np.int8, np.int16, np.int32, np.int64]) def test_astype_nansafe(val, typ): From b14d752fcd88ffa0fcce2f92fba3e5e0ba2e4ee6 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 18 Sep 2019 14:43:52 -0400 Subject: [PATCH 08/12] Edit release note --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index d0564699f8bdd..75d5a3b0c4fff 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -147,7 +147,7 @@ Datetimelike - Bug in :class:`Timestamp` subtraction when subtracting a :class:`Timestamp` from a ``np.datetime64`` object incorrectly raising ``TypeError`` (:issue:`28286`) - Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`) - Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`) -- Bug in :meth:`Series.astype` failing to handle ``pd.NaT`` when casting to an integer dtype (:issue:`28492`) +- Bug in :meth:`Series.astype`, :meth:`Index.astype`, and :meth:`DataFrame.astype` failing to handle ``NaT`` when casting to an integer dtype (:issue:`28492`) - From 9419744657a1a2967f5710e3556c49b56428aca2 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 18 Sep 2019 20:08:20 -0400 Subject: [PATCH 09/12] Check for np.int64 --- pandas/core/dtypes/cast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 4b135790943b9..b13d41262897b 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -695,7 +695,7 @@ def astype_nansafe(arr, dtype, copy=True, skipna=False): elif is_datetime64_dtype(arr): if is_object_dtype(dtype): return tslib.ints_to_pydatetime(arr.view(np.int64)) - elif is_integer_dtype(dtype): + elif dtype == np.int64: if isna(arr).any(): raise ValueError("Cannot convert NaT values to integer") return arr.view(dtype) From b81f433382e5a35399a12e206178efd58ff9d88a Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 18 Sep 2019 20:51:34 -0400 Subject: [PATCH 10/12] Handle timedelta64 --- pandas/core/dtypes/cast.py | 2 ++ pandas/tests/dtypes/test_common.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index b13d41262897b..bdb84294cb50d 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -713,6 +713,8 @@ def astype_nansafe(arr, dtype, copy=True, skipna=False): if is_object_dtype(dtype): return tslibs.ints_to_pytimedelta(arr.view(np.int64)) elif dtype == np.int64: + if isna(arr).any(): + raise ValueError("Cannot convert NaT values to integer") return arr.view(dtype) if dtype not in [_INT64_DTYPE, _TD_DTYPE]: diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index bd3b69fbbf9ce..5e29a9157e4b0 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -709,8 +709,8 @@ def test__is_dtype_type(input_param, result): assert com._is_dtype_type(input_param, lambda tipo: tipo == result) -@pytest.mark.parametrize("val", [np.datetime64("NaT")]) -@pytest.mark.parametrize("typ", [np.int8, np.int16, np.int32, np.int64]) +@pytest.mark.parametrize("val", [np.datetime64("NaT"), np.timedelta64("NaT")]) +@pytest.mark.parametrize("typ", [np.int64]) def test_astype_nansafe(val, typ): arr = np.array([val]) From 18734ff9369ce7fc6ca027dbcd9df2ef4960113c Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Thu, 19 Sep 2019 13:14:46 -0400 Subject: [PATCH 11/12] Add astype_nansafe datetime tests --- pandas/tests/dtypes/test_common.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index 5e29a9157e4b0..f112762548e32 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -717,3 +717,24 @@ def test_astype_nansafe(val, typ): msg = "Cannot convert NaT values to integer" with pytest.raises(ValueError, match=msg): astype_nansafe(arr, dtype=typ) + + +@pytest.mark.parametrize("from_type", [np.datetime64, np.timedelta64]) +@pytest.mark.parametrize( + "to_type", + [ + np.uint8, + np.uint16, + np.uint32, + np.int8, + np.int16, + np.int32, + np.float16, + np.float32, + ], +) +def test_astype_datetime64_bad_dtype_raises(from_type, to_type): + arr = np.array([from_type("2018")]) + + with pytest.raises(TypeError, match="cannot astype"): + astype_nansafe(arr, dtype=to_type) From c9ba76849fe2c849297b7f7c0ace09c2c792cd3d Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 25 Sep 2019 18:02:39 -0500 Subject: [PATCH 12/12] Add test for NaT object casting --- pandas/tests/dtypes/test_common.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index f112762548e32..9cf9a6fd42bbb 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -12,6 +12,7 @@ IntervalDtype, PeriodDtype, ) +from pandas.core.dtypes.missing import isna import pandas as pd from pandas.conftest import ( @@ -738,3 +739,11 @@ def test_astype_datetime64_bad_dtype_raises(from_type, to_type): with pytest.raises(TypeError, match="cannot astype"): astype_nansafe(arr, dtype=to_type) + + +@pytest.mark.parametrize("from_type", [np.datetime64, np.timedelta64]) +def test_astype_object_preserves_datetime_na(from_type): + arr = np.array([from_type("NaT")]) + result = astype_nansafe(arr, dtype="object") + + assert isna(result)[0]