Merge pull request #11245 from llllllllll/dt64-reduce-to-nat

BUG: datetime64 series reduces to nan when empty instead of nat

Author: jreback

doc/source/whatsnew/v0.17.1.txt

@@ -28,6 +28,9 @@ Other Enhancements
 API changes
 ~~~~~~~~~~~
 
+- min and max reductions on ``datetime64`` and ``timedelta64`` dtyped series now
+  result in ``NaT`` and not ``nan`` (:issue:`11245`).
+
 .. _whatsnew_0171.deprecations:
 
 Deprecations
@@ -74,3 +77,5 @@ Bug Fixes
 
 
 - Bugs in ``to_excel`` with duplicate columns (:issue:`11007`, :issue:`10982`, :issue:`10970`)
+- Fixed a bug that prevented the construction of an empty series of dtype
+  ``datetime64[ns, tz]`` (:issue:`11245`).

pandas/core/common.py

@@ -1927,12 +1927,11 @@ def _possibly_cast_to_datetime(value, dtype, errors='raise'):
                     value = tslib.iNaT
 
                 # we have an array of datetime or timedeltas & nulls
-                elif np.prod(value.shape) and not is_dtype_equal(value.dtype, dtype):
+                elif np.prod(value.shape) or not is_dtype_equal(value.dtype, dtype):
                     try:
                         if is_datetime64:
                             value = to_datetime(value, errors=errors)._values
                         elif is_datetime64tz:
-
                             # input has to be UTC at this point, so just localize
                             value = to_datetime(value, errors=errors).tz_localize(dtype.tz)
                         elif is_timedelta64:

pandas/core/dtypes.py

@@ -138,7 +138,7 @@ class DatetimeTZDtype(ExtensionDtype):
     num = 101
     base = np.dtype('M8[ns]')
     _metadata = ['unit','tz']
-    _match = re.compile("datetime64\[(?P<unit>.+), (?P<tz>.+)\]")
+    _match = re.compile("(datetime64|M8)\[(?P<unit>.+), (?P<tz>.+)\]")
 
     def __init__(self, unit, tz=None):
         """

pandas/core/frame.py

@@ -4624,7 +4624,7 @@ def _reduce(self, op, name, axis=0, skipna=True, numeric_only=None,
                 values = self.values
             result = f(values)
 
-        if is_object_dtype(result.dtype):
+        if hasattr(result, 'dtype') and is_object_dtype(result.dtype):
             try:
                 if filter_type is None or filter_type == 'numeric':
                     result = result.astype(np.float64)

pandas/core/nanops.py

@@ -425,65 +425,34 @@ def nansem(values, axis=None, skipna=True, ddof=1):
     return np.sqrt(var) / np.sqrt(count)
 
 
-@bottleneck_switch()
-def nanmin(values, axis=None, skipna=True):
-    values, mask, dtype, dtype_max = _get_values(values, skipna,
-                                                 fill_value_typ='+inf')
-
-    # numpy 1.6.1 workaround in Python 3.x
-    if is_object_dtype(values) and compat.PY3:
-        if values.ndim > 1:
-            apply_ax = axis if axis is not None else 0
-            result = np.apply_along_axis(builtins.min, apply_ax, values)
-        else:
-            try:
-                result = builtins.min(values)
-            except:
-                result = np.nan
-    else:
+def _nanminmax(meth, fill_value_typ):
+    @bottleneck_switch()
+    def reduction(values, axis=None, skipna=True):
+        values, mask, dtype, dtype_max = _get_values(
+            values,
+            skipna,
+            fill_value_typ=fill_value_typ,
+        )
+
         if ((axis is not None and values.shape[axis] == 0)
                 or values.size == 0):
             try:
-                result = ensure_float(values.sum(axis, dtype=dtype_max))
+                result = getattr(values, meth)(axis, dtype=dtype_max)
                 result.fill(np.nan)
             except:
                 result = np.nan
         else:
-            result = values.min(axis)
+            result = getattr(values, meth)(axis)
 
-    result = _wrap_results(result, dtype)
-    return _maybe_null_out(result, axis, mask)
+        result = _wrap_results(result, dtype)
+        return _maybe_null_out(result, axis, mask)
 
+    reduction.__name__ = 'nan' + meth
+    return reduction
 
-@bottleneck_switch()
-def nanmax(values, axis=None, skipna=True):
-    values, mask, dtype, dtype_max = _get_values(values, skipna,
-                                                 fill_value_typ='-inf')
 
-    # numpy 1.6.1 workaround in Python 3.x
-    if is_object_dtype(values) and compat.PY3:
-
-        if values.ndim > 1:
-            apply_ax = axis if axis is not None else 0
-            result = np.apply_along_axis(builtins.max, apply_ax, values)
-        else:
-            try:
-                result = builtins.max(values)
-            except:
-                result = np.nan
-    else:
-        if ((axis is not None and values.shape[axis] == 0)
-                or values.size == 0):
-            try:
-                result = ensure_float(values.sum(axis, dtype=dtype_max))
-                result.fill(np.nan)
-            except:
-                result = np.nan
-        else:
-            result = values.max(axis)
-
-    result = _wrap_results(result, dtype)
-    return _maybe_null_out(result, axis, mask)
+nanmin = _nanminmax('min', fill_value_typ='+inf')
+nanmax = _nanminmax('max', fill_value_typ='-inf')
 
 
 def nanargmax(values, axis=None, skipna=True):
@@ -637,7 +606,7 @@ def _maybe_null_out(result, axis, mask):
             else:
                 result = result.astype('f8')
             result[null_mask] = np.nan
-    else:
+    elif result is not tslib.NaT:
         null_mask = mask.size - mask.sum()
         if null_mask == 0:
             result = np.nan

pandas/tests/test_dtypes.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from itertools import product
 
 import nose
 import numpy as np
@@ -148,6 +149,15 @@ def test_dst(self):
         self.assertTrue(is_datetimetz(s2))
         self.assertEqual(s1.dtype, s2.dtype)
 
+    def test_parser(self):
+        # pr #11245
+        for tz, constructor in product(('UTC', 'US/Eastern'),
+                                       ('M8', 'datetime64')):
+            self.assertEqual(
+                DatetimeTZDtype('%s[ns, %s]' % (constructor, tz)),
+                DatetimeTZDtype('ns', tz),
+            )
+
 
 
 

pandas/tests/test_series.py

@@ -7960,6 +7960,12 @@ def test_datetime_timedelta_quantiles(self):
         self.assertTrue(pd.isnull(Series([],dtype='M8[ns]').quantile(.5)))
         self.assertTrue(pd.isnull(Series([],dtype='m8[ns]').quantile(.5)))
 
+    def test_empty_timeseries_redections_return_nat(self):
+        # covers #11245
+        for dtype in ('m8[ns]', 'm8[ns]', 'M8[ns]', 'M8[ns, UTC]'):
+            self.assertIs(Series([], dtype=dtype).min(), pd.NaT)
+            self.assertIs(Series([], dtype=dtype).max(), pd.NaT)
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)