Merge pull request #2899 from jreback/timedelta_issue

BUG: Series ops with a rhs of a Timestamp raising exception (#2898)

Author: jreback

RELEASE.rst

@@ -77,6 +77,8 @@ pandas 0.11.0
       correctly
     - astype on datetimes to object are now handled (as well as NaT
       conversions to np.nan)
+    - all timedelta like objects will be correctly assigned to ``timedelta64``
+      with mixed ``NaN`` and/or ``NaT`` allowed
 
   - arguments to DataFrame.clip were inconsistent to numpy and Series clipping
     (GH2747_)
@@ -108,6 +110,16 @@ pandas 0.11.0
   - Bug showing up in applymap where some object type columns are converted (GH2909_)
     had an incorrect default in convert_objects
 
+  - TimeDeltas
+
+    - Series ops with a Timestamp on the rhs was throwing an exception (GH2898_)
+      added tests for Series ops with datetimes,timedeltas,Timestamps, and datelike 
+      Series on both lhs and rhs
+    - Series will now set its dtype automatically to ``timedelta64[ns]``
+      if all passed objects are timedelta objects
+    - Support null checking on timedelta64, representing (and formatting) with NaT
+    - Support setitem with np.nan value, converts to NaT
+
 .. _GH622: https://github.com/pydata/pandas/issues/622
 .. _GH797: https://github.com/pydata/pandas/issues/797
 .. _GH2681: https://github.com/pydata/pandas/issues/2681
@@ -121,6 +133,7 @@ pandas 0.11.0
 .. _GH2845: https://github.com/pydata/pandas/issues/2845
 .. _GH2867: https://github.com/pydata/pandas/issues/2867
 .. _GH2807: https://github.com/pydata/pandas/issues/2807
+.. _GH2898: https://github.com/pydata/pandas/issues/2898
 .. _GH2909: https://github.com/pydata/pandas/issues/2909
 
 pandas 0.10.1

doc/source/dsintro.rst

@@ -912,6 +912,8 @@ method:
    panel.to_frame()
 
 
+.. _dsintro.panel4d:
+
 Panel4D (Experimental)
 ----------------------
 

doc/source/timeseries.rst

@@ -917,3 +917,47 @@ TimeSeries, aligning the data on the UTC timestamps:
    result = eastern + berlin
    result
    result.index
+
+.. _timeseries.timedeltas:
+
+Time Deltas
+-----------
+
+Timedeltas are differences in times, expressed in difference units, e.g. days,hours,minutes,seconds.
+They can be both positive and negative.
+
+.. ipython:: python
+
+    from datetime import datetime, timedelta
+    s  = Series(date_range('2012-1-1', periods=3, freq='D'))
+    td = Series([ timedelta(days=i) for i in range(3) ])
+    df = DataFrame(dict(A = s, B = td))
+    df
+    df['C'] = df['A'] + df['B']
+    df
+    df.dtypes
+
+    s - s.max()
+    s - datetime(2011,1,1,3,5)
+    s + timedelta(minutes=5)
+
+Series of timedeltas with ``NaT`` values are supported
+
+.. ipython:: python
+
+    y = s - s.shift()
+    y
+The can be set to ``NaT`` using ``np.nan`` analagously to datetimes
+
+.. ipython:: python
+
+    y[1] = np.nan
+    y
+
+Operands can also appear in a reversed order (a singluar object operated with a Series)
+
+.. ipython:: python
+
+    s.max() - s
+    datetime(2011,1,1,3,5) - s
+    timedelta(minutes=5) + s

doc/source/v0.10.0.txt

@@ -330,7 +330,7 @@ N Dimensional Panels (Experimental)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Adding experimental support for Panel4D and factory functions to create n-dimensional named panels.
-:ref:`Docs <dsintro-panel4d>` for NDim. Here is a taste of what to expect.
+:ref:`Docs <dsintro.panel4d>` for NDim. Here is a taste of what to expect.
 
      .. ipython:: python
 

doc/source/v0.11.0.txt

@@ -140,11 +140,9 @@ Astype conversion on ``datetime64[ns]`` to ``object``, implicity converts ``NaT`
    s.dtype
 
 
-New features
+Enhancements
 ~~~~~~~~~~~~
 
-**Enhancements**
-
   - In ``HDFStore``, provide dotted attribute access to ``get`` from stores
     (e.g. store.df == store['df'])
 
@@ -178,7 +176,37 @@ New features
       price. This just obtains the data from Options.get_near_stock_price
       instead of Options.get_xxx_data().
 
-**Bug Fixes**
+Bug Fixes
+~~~~~~~~~
+
+  - Timedeltas are now fully operational (closes GH2898_)
+
+    .. ipython:: python
+
+        from datetime import datetime, timedelta
+        s  = Series(date_range('2012-1-1', periods=3, freq='D'))
+        td = Series([ timedelta(days=i) for i in range(3) ])
+        df = DataFrame(dict(A = s, B = td))
+        df
+        s - s.max()
+        s - datetime(2011,1,1,3,5)
+        s + timedelta(minutes=5)
+        df['C'] = df['A'] + df['B']
+        df
+        df.dtypes
+
+        # timedelta are representas as ``NaT``
+        y = s - s.shift()
+	y
+
+	# can be set via ``np.nan``
+	y[1] = np.nan
+	y
+
+        # works on lhs too
+   	s.max() - s
+        datetime(2011,1,1,3,5) - s
+        timedelta(minutes=5) + s
 
 See the `full release notes
 <https://github.com/pydata/pandas/blob/master/RELEASE.rst>`__ or issue tracker
@@ -187,4 +215,5 @@ on GitHub for a complete list.
 .. _GH2809: https://github.com/pydata/pandas/issues/2809
 .. _GH2810: https://github.com/pydata/pandas/issues/2810
 .. _GH2837: https://github.com/pydata/pandas/issues/2837
+.. _GH2898: https://github.com/pydata/pandas/issues/2898
 

pandas/core/common.py

@@ -148,9 +148,9 @@ def _isnull_ndarraylike(obj):
     elif values.dtype == np.dtype('M8[ns]'):
         # this is the NaT pattern
         result = values.view('i8') == tslib.iNaT
-    elif issubclass(values.dtype.type, np.timedelta64):
-        # -np.isfinite(values.view('i8'))
-        result = np.ones(values.shape, dtype=bool)
+    elif values.dtype == np.dtype('m8[ns]'):
+        # this is the NaT pattern
+        result = values.view('i8') == tslib.iNaT
     else:
         # -np.isfinite(obj)
         result = np.isnan(obj)
@@ -902,35 +902,50 @@ def _possibly_convert_platform(values):
     return values
 
 
+def _possibly_cast_to_timedelta(value):
+    """ try to cast to timedelta64 w/o coercion """
+    new_value = tslib.array_to_timedelta64(value.astype(object), coerce=False)
+    if new_value.dtype == 'i8':
+        value = np.array(new_value,dtype='timedelta64[ns]')
+    return value
+
 def _possibly_cast_to_datetime(value, dtype, coerce = False):
     """ try to cast the array/value to a datetimelike dtype, converting float nan to iNaT """
 
     if isinstance(dtype, basestring):
         dtype = np.dtype(dtype)
 
-    if dtype is not None and is_datetime64_dtype(dtype):
-        if np.isscalar(value):
-            if value == tslib.iNaT or isnull(value):
-                value = tslib.iNaT
-        else:
-            value = np.array(value)
+    if dtype is not None:
+        is_datetime64  = is_datetime64_dtype(dtype)
+        is_timedelta64 = is_timedelta64_dtype(dtype)
 
-            # have a scalar array-like (e.g. NaT)
-            if value.ndim == 0:
-                value = tslib.iNaT
+        if is_datetime64 or is_timedelta64:
 
-            # we have an array of datetime & nulls
-            elif np.prod(value.shape):
-                try:
-                    value = tslib.array_to_datetime(value, coerce = coerce)
-                except:
-                    pass
+            if np.isscalar(value):
+                if value == tslib.iNaT or isnull(value):
+                    value = tslib.iNaT
+            else:
+                value = np.array(value)
+
+                # have a scalar array-like (e.g. NaT)
+                if value.ndim == 0:
+                    value = tslib.iNaT
+
+                # we have an array of datetime or timedeltas & nulls
+                elif np.prod(value.shape) and value.dtype != dtype:
+                    try:
+                        if is_datetime64:
+                            value = tslib.array_to_datetime(value, coerce = coerce)
+                        elif is_timedelta64:
+                            value = _possibly_cast_to_timedelta(value)
+                    except:
+                        pass
 
     elif dtype is None:
         # we might have a array (or single object) that is datetime like, and no dtype is passed
         # don't change the value unless we find a datetime set
         v = value
-        if not (is_list_like(v) or hasattr(v,'len')):
+        if not is_list_like(v):
             v = [ v ]
         if len(v):
             inferred_type = lib.infer_dtype(v)
@@ -939,6 +954,8 @@ def _possibly_cast_to_datetime(value, dtype, coerce = False):
                     value = tslib.array_to_datetime(np.array(v))
                 except:
                     pass
+            elif inferred_type == 'timedelta':
+                value = _possibly_cast_to_timedelta(value)
 
     return value
 
@@ -1281,6 +1298,16 @@ def is_datetime64_dtype(arr_or_dtype):
     return issubclass(tipo, np.datetime64)
 
 
+def is_timedelta64_dtype(arr_or_dtype):
+    if isinstance(arr_or_dtype, np.dtype):
+        tipo = arr_or_dtype.type
+    elif isinstance(arr_or_dtype, type):
+        tipo = np.dtype(arr_or_dtype).type
+    else:
+        tipo = arr_or_dtype.dtype.type
+    return issubclass(tipo, np.timedelta64)
+
+
 def is_float_dtype(arr_or_dtype):
     if isinstance(arr_or_dtype, np.dtype):
         tipo = arr_or_dtype.type
@@ -1290,8 +1317,7 @@ def is_float_dtype(arr_or_dtype):
 
 
 def is_list_like(arg):
-    return hasattr(arg, '__iter__') and not isinstance(arg, basestring)
-
+    return hasattr(arg, '__iter__') and not isinstance(arg, basestring) or hasattr(arg,'len')
 
 def _is_sequence(x):
     try:

pandas/core/format.py

@@ -1012,6 +1012,8 @@ def format_array(values, formatter, float_format=None, na_rep='NaN',
         fmt_klass = IntArrayFormatter
     elif com.is_datetime64_dtype(values.dtype):
         fmt_klass = Datetime64Formatter
+    elif com.is_timedelta64_dtype(values.dtype):
+        fmt_klass = Timedelta64Formatter
     else:
         fmt_klass = GenericArrayFormatter
 
@@ -1170,7 +1172,6 @@ def get_result(self):
         fmt_values = [formatter(x) for x in self.values]
         return _make_fixed_width(fmt_values, self.justify)
 
-
 def _format_datetime64(x, tz=None):
     if isnull(x):
         return 'NaT'
@@ -1179,6 +1180,24 @@ def _format_datetime64(x, tz=None):
     return stamp._repr_base
 
 
+class Timedelta64Formatter(Datetime64Formatter):
+
+    def get_result(self):
+        if self.formatter:
+            formatter = self.formatter
+        else:
+
+            formatter = _format_timedelta64
+
+        fmt_values = [formatter(x) for x in self.values]
+        return _make_fixed_width(fmt_values, self.justify)
+
+def _format_timedelta64(x):
+    if isnull(x):
+        return 'NaT'
+
+    return lib.repr_timedelta64(x)
+
 def _make_fixed_width(strings, justify='right', minimum=None):
     if len(strings) == 0:
         return strings