BUG: apply to a Series with a timedelta (GH5458)

ENH: improved timedelta inference for non-ns dtypes

Author: jreback

pandas/core/common.py

@@ -1546,7 +1546,7 @@ def _possibly_convert_objects(values, convert_dates=True,
         if convert_timedeltas == 'coerce':
             from pandas.tseries.timedeltas import \
                  _possibly_cast_to_timedelta
-            values = _possibly_cast_to_timedelta(values)
+            values = _possibly_cast_to_timedelta(values, coerce=True)
 
             # if we are all nans then leave me alone
             if not isnull(new_values).all():
@@ -1641,7 +1641,7 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
                         elif is_timedelta64:
                             from pandas.tseries.timedeltas import \
                                 _possibly_cast_to_timedelta
-                            value = _possibly_cast_to_timedelta(value)
+                            value = _possibly_cast_to_timedelta(value, coerce=True)
                     except:
                         pass
 

pandas/src/inference.pyx

@@ -1,6 +1,8 @@
 cimport util
 from tslib import NaT
 
+iNaT = util.get_nat()
+
 _TYPE_MAP = {
     np.int8: 'integer',
     np.int16: 'integer',
@@ -61,14 +63,21 @@ def infer_dtype(object _values):
             return 'datetime64'
         elif is_timedelta_or_timedelta64_array(values):
             return 'timedelta'
+
     elif util.is_integer_object(val):
+        # a timedelta will show true here as well
+        if is_timedelta(val):
+            if is_timedelta_or_timedelta64_array(values):
+                return 'timedelta'
+
         if is_integer_array(values):
             return 'integer'
         elif is_integer_float_array(values):
             return 'mixed-integer-float'
         elif is_timedelta_or_timedelta64_array(values):
             return 'timedelta'
         return 'mixed-integer'
+
     elif is_datetime(val):
         if is_datetime_array(values):
             return 'datetime'
@@ -120,6 +129,9 @@ def infer_dtype_list(list values):
     pass
 
 
+cdef inline bint is_null_datetimelike(v):
+    return util._checknull(v) or (util.is_integer_object(v) and v == iNaT) or v is NaT
+
 cdef inline bint is_datetime(object o):
     return PyDateTime_Check(o)
 
@@ -268,7 +280,7 @@ def is_datetime_array(ndarray[object] values):
         return False
     for i in range(n):
         v = values[i]
-        if not (is_datetime(v) or util._checknull(v) or v is NaT):
+        if not (is_datetime(v) or is_null_datetimelike(v)):
             return False
     return True
 
@@ -280,7 +292,7 @@ def is_datetime64_array(ndarray values):
         return False
     for i in range(n):
         v = values[i]
-        if not (util.is_datetime64_object(v) or util._checknull(v) or v is NaT):
+        if not (util.is_datetime64_object(v) or is_null_datetimelike(v)):
             return False
     return True
 
@@ -291,19 +303,23 @@ def is_timedelta(object o):
 def is_timedelta_array(ndarray values):
     import datetime
     cdef int i, n = len(values)
+    cdef object v
     if n == 0:
         return False
     for i in range(n):
-        if not isinstance(values[i],datetime.timedelta):
+        v = values[i]
+        if not (isinstance(v,datetime.timedelta) or is_null_datetimelike(v)):
             return False
     return True
 
 def is_timedelta64_array(ndarray values):
     cdef int i, n = len(values)
+    cdef object v
     if n == 0:
         return False
     for i in range(n):
-        if not isinstance(values[i],np.timedelta64):
+        v = values[i]
+        if not (isinstance(v,np.timedelta64) or is_null_datetimelike(v)):
             return False
     return True
 
@@ -316,7 +332,8 @@ def is_timedelta_or_timedelta64_array(ndarray values):
         return False
     for i in range(n):
         v = values[i]
-        if not (isinstance(v,datetime.timedelta) or isinstance(v,np.timedelta64) or util._checknull(v) or v is NaT):
+        if not (isinstance(v,datetime.timedelta) or isinstance(v,np.timedelta64) or
+                is_null_datetimelike(v)):
             return False
     return True
 
@@ -499,7 +516,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                 break
         elif is_timedelta(val):
             if convert_timedelta:
-                itimedeltas[i] = convert_to_timedelta64(val, 'ns')
+                itimedeltas[i] = convert_to_timedelta64(val, 'ns', False)
                 seen_timedelta = 1
             else:
                 seen_object = 1

pandas/tests/test_series.py

@@ -2246,45 +2246,59 @@ def test_operators_empty_int_corner(self):
 
     def test_constructor_dtype_timedelta64(self):
 
+        # basic
         td = Series([timedelta(days=i) for i in range(3)])
         self.assert_(td.dtype == 'timedelta64[ns]')
 
-        # mixed with NaT
-        from pandas import tslib
-        td = Series([timedelta(days=i)
-                    for i in range(3)] + [tslib.NaT ], dtype='m8[ns]' )
+        td = Series([timedelta(days=1)])
+        self.assert_(td.dtype == 'timedelta64[ns]')
+
+        td = Series([timedelta(days=1),timedelta(days=2),np.timedelta64(1,'s')])
         self.assert_(td.dtype == 'timedelta64[ns]')
 
-        td = Series([timedelta(days=i)
-                    for i in range(3)] + [tslib.iNaT ], dtype='m8[ns]' )
+        # mixed with NaT
+        from pandas import tslib
+        td = Series([timedelta(days=1),tslib.NaT ], dtype='m8[ns]' )
         self.assert_(td.dtype == 'timedelta64[ns]')
 
-        td = Series([timedelta(days=i)
-                    for i in range(3)] + [np.nan ], dtype='m8[ns]' )
+        td = Series([timedelta(days=1),np.nan ], dtype='m8[ns]' )
         self.assert_(td.dtype == 'timedelta64[ns]')
 
         td = Series([np.timedelta64(300000000), pd.NaT],dtype='m8[ns]')
         self.assert_(td.dtype == 'timedelta64[ns]')
 
         # improved inference
+        # GH5689
         td = Series([np.timedelta64(300000000), pd.NaT])
         self.assert_(td.dtype == 'timedelta64[ns]')
 
+        td = Series([np.timedelta64(300000000), tslib.iNaT])
+        self.assert_(td.dtype == 'timedelta64[ns]')
+
+        td = Series([np.timedelta64(300000000), np.nan])
+        self.assert_(td.dtype == 'timedelta64[ns]')
+
         td = Series([pd.NaT, np.timedelta64(300000000)])
         self.assert_(td.dtype == 'timedelta64[ns]')
 
+        td = Series([np.timedelta64(1,'s')])
+        self.assert_(td.dtype == 'timedelta64[ns]')
+
         # these are frequency conversion astypes
         #for t in ['s', 'D', 'us', 'ms']:
         #    self.assertRaises(TypeError, td.astype, 'm8[%s]' % t)
 
         # valid astype
         td.astype('int64')
 
-        # this is an invalid casting
-        self.assertRaises(Exception, Series, [timedelta(days=i)
-                          for i in range(3)] + ['foo' ], dtype='m8[ns]' )
+        # invalid casting
         self.assertRaises(TypeError, td.astype, 'int32')
 
+        # this is an invalid casting
+        def f():
+            Series([timedelta(days=1), 'foo'],dtype='m8[ns]')
+        self.assertRaises(Exception, f)
+
         # leave as object here
         td = Series([timedelta(days=i) for i in range(3)] + ['foo'])
         self.assert_(td.dtype == 'object')

pandas/tseries/tests/test_timedeltas.py

@@ -173,6 +173,16 @@ def conv(v):
         expected = np.timedelta64(timedelta(seconds=1))
         self.assert_(result == expected)
 
+    def test_to_timedelta_via_apply(self):
+
+        # GH 5458
+        expected = Series([np.timedelta64(1,'s')])
+        result = Series(['00:00:01']).apply(to_timedelta)
+        tm.assert_series_equal(result, expected)
+
+        result = Series([to_timedelta('00:00:01')])
+        tm.assert_series_equal(result, expected)
+
     def test_timedelta_ops(self):
         _skip_if_numpy_not_friendly()
 

pandas/tslib.pxd

@@ -1,4 +1,4 @@
 from numpy cimport ndarray, int64_t
 
 cdef convert_to_tsobject(object, object, object)
-cdef convert_to_timedelta64(object, object)
+cdef convert_to_timedelta64(object, object, object)

pandas/tslib.pyx

@@ -1155,52 +1155,26 @@ def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False,
 
         return oresult
 
-def array_to_timedelta64(ndarray[object] values, coerce=True):
+def array_to_timedelta64(ndarray[object] values, coerce=False):
     """ convert an ndarray to an array of ints that are timedeltas
         force conversion if coerce = True,
-        else return an object array """
+        else will raise if cannot convert """
     cdef:
         Py_ssize_t i, n
-        object val
-        ndarray[int64_t] result
+        ndarray[int64_t] iresult
 
     n = values.shape[0]
-    result = np.empty(n, dtype='i8')
-    for i in range(n):
-        val = values[i]
-
-        # in py3 this is already an int, don't convert
-        if is_integer_object(val):
-            result[i] = val
-
-        elif isinstance(val,timedelta) or isinstance(val,np.timedelta64):
-
-             if isinstance(val, np.timedelta64):
-                 if val.dtype != 'm8[ns]':
-                      val = val.astype('m8[ns]')
-                 val = val.item()
-             else:
-                 val = _delta_to_nanoseconds(np.timedelta64(val).item())
-
-             result[i] = val
-
-        elif _checknull_with_nat(val):
-             result[i] = iNaT
-
-        else:
-
-             # just return, don't convert
-             if not coerce:
-                 return values.copy()
-
-             result[i] = iNaT
+    result = np.empty(n, dtype='m8[ns]')
+    iresult = result.view('i8')
 
-    return result
+    for i in range(n):
+        result[i] = convert_to_timedelta64(values[i], 'ns', coerce)
+    return iresult
 
-def convert_to_timedelta(object ts, object unit='ns'):
-    return convert_to_timedelta64(ts, unit)
+def convert_to_timedelta(object ts, object unit='ns', coerce=False):
+    return convert_to_timedelta64(ts, unit, coerce)
 
-cdef convert_to_timedelta64(object ts, object unit):
+cdef convert_to_timedelta64(object ts, object unit, object coerce):
     """
     Convert an incoming object to a timedelta64 if possible
 
@@ -1210,6 +1184,8 @@ cdef convert_to_timedelta64(object ts, object unit):
         - np.int64 (with unit providing a possible modifier)
         - None/NaT
 
+    if coerce, set a non-valid value to NaT
+
     Return a ns based int64
 
     # kludgy here until we have a timedelta scalar
@@ -1237,7 +1213,9 @@ cdef convert_to_timedelta64(object ts, object unit):
 
     if _np_version_under1p7:
         if not isinstance(ts, timedelta):
-            raise AssertionError("Invalid type for timedelta scalar: %s" % type(ts))
+            if coerce:
+                return np.timedelta64(iNaT)
+            raise ValueError("Invalid type for timedelta scalar: %s" % type(ts))
         if not PY2:
             # convert to microseconds in timedelta64
             ts = np.timedelta64(int(ts.total_seconds()*1e9 + ts.microseconds*1000))
@@ -1247,7 +1225,9 @@ cdef convert_to_timedelta64(object ts, object unit):
     if isinstance(ts, timedelta):
         ts = np.timedelta64(ts)
     elif not isinstance(ts, np.timedelta64):
-        raise AssertionError("Invalid type for timedelta scalar: %s" % type(ts))
+        if coerce:
+            return np.timedelta64(iNaT)
+        raise ValueError("Invalid type for timedelta scalar: %s" % type(ts))
     return ts.astype('timedelta64[ns]')
 
 def repr_timedelta64(object value, format=None):