From 0f3be92e12923163390c3c07f9cf3f61269f1ff1 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 19 Sep 2017 22:28:26 -0700 Subject: [PATCH 1/6] BUG: Correctly localize naive ts strings with Series and datetimetz dtype (#17415) Add whatsnew Fix lint error Rebase and move whatsnew note Move whatsnew and add tests --- pandas/core/dtypes/cast.py | 20 +++++++++++++------- pandas/tests/series/test_constructors.py | 8 ++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 5fcb5f09dfae7..9f4a3a21f06b9 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -20,7 +20,7 @@ is_integer_dtype, is_datetime_or_timedelta_dtype, is_bool_dtype, is_scalar, - _string_dtypes, + is_string_dtype, _string_dtypes, pandas_dtype, _ensure_int8, _ensure_int16, _ensure_int32, _ensure_int64, @@ -1003,12 +1003,18 @@ def maybe_cast_to_datetime(value, dtype, errors='raise'): if is_datetime64: value = to_datetime(value, errors=errors)._values elif is_datetime64tz: - # input has to be UTC at this point, so just - # localize - value = (to_datetime(value, errors=errors) - .tz_localize('UTC') - .tz_convert(dtype.tz) - ) + # This block can be simplified once PR #17413 is + # complete + is_dt_string = is_string_dtype(value) + value = to_datetime(value, errors=errors) + if is_dt_string: + # Strings here are naive, so directly localize + value = value.tz_localize(dtype.tz) + else: + # Numeric values are UTC at this point, + # so localize and convert + value = (value.tz_localize('UTC') + .tz_convert(dtype.tz)) elif is_timedelta64: value = to_timedelta(value, errors=errors)._values except (AttributeError, ValueError, TypeError): diff --git a/pandas/tests/series/test_constructors.py b/pandas/tests/series/test_constructors.py index 5de5f1f0584f4..ad0d53462850c 100644 --- a/pandas/tests/series/test_constructors.py +++ b/pandas/tests/series/test_constructors.py @@ -707,6 +707,14 @@ def test_constructor_with_datetime_tz(self): expected = Series(pd.DatetimeIndex(['NaT', 'NaT'], tz='US/Eastern')) assert_series_equal(s, expected) + @pytest.mark.parametrize('arg, dtype', + ['2013-01-01 00:00:00', pd.NaT, np.nan, None]) + def test_constructor_with_naive_string_and_datetimetz_dtype(self, arg): + # GH 17415: With naive string + result = Series([arg], dtype='datetime64[ns, CET]') + expected = Series([pd.Timestamp(arg, tz='CET')]) + assert_series_equal(result, expected) + def test_construction_interval(self): # construction from interval & array of intervals index = IntervalIndex.from_breaks(np.arange(3), closed='right') From 735f35bcd74df4e89e45e4f069b8349f7e8a7ecb Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 10 Dec 2017 21:24:57 -0800 Subject: [PATCH 2/6] Add missing arg --- pandas/tests/series/test_constructors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/series/test_constructors.py b/pandas/tests/series/test_constructors.py index ad0d53462850c..fd6a2d8350309 100644 --- a/pandas/tests/series/test_constructors.py +++ b/pandas/tests/series/test_constructors.py @@ -709,7 +709,8 @@ def test_constructor_with_datetime_tz(self): @pytest.mark.parametrize('arg, dtype', ['2013-01-01 00:00:00', pd.NaT, np.nan, None]) - def test_constructor_with_naive_string_and_datetimetz_dtype(self, arg): + def test_constructor_with_naive_string_and_datetimetz_dtype(self, arg, + dtype): # GH 17415: With naive string result = Series([arg], dtype='datetime64[ns, CET]') expected = Series([pd.Timestamp(arg, tz='CET')]) From 18e914b3995fc5f493a53ff8879db52027ccd896 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 11 Dec 2017 22:32:53 -0800 Subject: [PATCH 3/6] fix failing test and rebase --- pandas/core/dtypes/cast.py | 2 +- pandas/tests/series/test_constructors.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 9f4a3a21f06b9..30a57a5d39442 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -1003,7 +1003,7 @@ def maybe_cast_to_datetime(value, dtype, errors='raise'): if is_datetime64: value = to_datetime(value, errors=errors)._values elif is_datetime64tz: - # This block can be simplified once PR #17413 is + # This block can be simplified once PR #17413 is # complete is_dt_string = is_string_dtype(value) value = to_datetime(value, errors=errors) diff --git a/pandas/tests/series/test_constructors.py b/pandas/tests/series/test_constructors.py index fd6a2d8350309..dd27ba725cdce 100644 --- a/pandas/tests/series/test_constructors.py +++ b/pandas/tests/series/test_constructors.py @@ -707,10 +707,9 @@ def test_constructor_with_datetime_tz(self): expected = Series(pd.DatetimeIndex(['NaT', 'NaT'], tz='US/Eastern')) assert_series_equal(s, expected) - @pytest.mark.parametrize('arg, dtype', + @pytest.mark.parametrize('arg', ['2013-01-01 00:00:00', pd.NaT, np.nan, None]) - def test_constructor_with_naive_string_and_datetimetz_dtype(self, arg, - dtype): + def test_constructor_with_naive_string_and_datetimetz_dtype(self, arg): # GH 17415: With naive string result = Series([arg], dtype='datetime64[ns, CET]') expected = Series([pd.Timestamp(arg, tz='CET')]) From bb1082860b1332b36b60940321baf693d719cde0 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 12 Dec 2017 20:42:54 -0800 Subject: [PATCH 4/6] Correct to a more appropriate test --- pandas/tests/series/test_constructors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/series/test_constructors.py b/pandas/tests/series/test_constructors.py index dd27ba725cdce..a3e40f65e922f 100644 --- a/pandas/tests/series/test_constructors.py +++ b/pandas/tests/series/test_constructors.py @@ -712,7 +712,7 @@ def test_constructor_with_datetime_tz(self): def test_constructor_with_naive_string_and_datetimetz_dtype(self, arg): # GH 17415: With naive string result = Series([arg], dtype='datetime64[ns, CET]') - expected = Series([pd.Timestamp(arg, tz='CET')]) + expected = Series(pd.Timestamp(arg)).dt.tz_localize('CET') assert_series_equal(result, expected) def test_construction_interval(self): From e66397d1c6c5d65074dd325c8651c371270af23d Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Wed, 10 Jan 2018 23:34:55 -0800 Subject: [PATCH 5/6] Rebase and move whatsnew to v0.23 --- doc/source/whatsnew/v0.23.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 92eeed89ada2a..29b09e25cb4e7 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -380,7 +380,7 @@ Conversion - Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`) - Bug in :class:`DatetimeIndex` where the repr was not showing high-precision time values at the end of a day (e.g., 23:59:59.999999999) (:issue:`19030`) - Bug where dividing a scalar timedelta-like object with :class:`TimedeltaIndex` performed the reciprocal operation (:issue:`19125`) -- +- Bug in localization of a naive, datetime string in a ``Series`` constructor with a ``datetime[ns, timezone]`` dtype (:issue:`174151`) Indexing ^^^^^^^^ From cee514f84f39d54d74d87926dabcfa3e1304575a Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Thu, 11 Jan 2018 21:10:43 -0800 Subject: [PATCH 6/6] address comments --- doc/source/whatsnew/v0.23.0.txt | 2 +- pandas/core/dtypes/cast.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 29b09e25cb4e7..ae99dfd2d7902 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -380,7 +380,7 @@ Conversion - Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`) - Bug in :class:`DatetimeIndex` where the repr was not showing high-precision time values at the end of a day (e.g., 23:59:59.999999999) (:issue:`19030`) - Bug where dividing a scalar timedelta-like object with :class:`TimedeltaIndex` performed the reciprocal operation (:issue:`19125`) -- Bug in localization of a naive, datetime string in a ``Series`` constructor with a ``datetime[ns, timezone]`` dtype (:issue:`174151`) +- Bug in localization of a naive, datetime string in a ``Series`` constructor with a ``datetime64[ns, tz]`` dtype (:issue:`174151`) Indexing ^^^^^^^^ diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 30a57a5d39442..d4e291029e67d 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -1003,8 +1003,10 @@ def maybe_cast_to_datetime(value, dtype, errors='raise'): if is_datetime64: value = to_datetime(value, errors=errors)._values elif is_datetime64tz: - # This block can be simplified once PR #17413 is - # complete + # The string check can be removed once issue #13712 + # is solved. String data that is passed with a + # datetime64tz is assumed to be naive which should + # be localized to the timezone. is_dt_string = is_string_dtype(value) value = to_datetime(value, errors=errors) if is_dt_string: