ENH: improved inference of timedelta64[ns] and datetime64[ns] in the prescence NaT/None/NaN

     from object dtypes (GH5689)
CLN: refactor of timedelta routines to cython for inclusion in maybe_convert_objects

Author: jreback

pandas/core/common.py

@@ -1514,7 +1514,8 @@ def _values_from_object(o):
 
 
 def _possibly_convert_objects(values, convert_dates=True,
-                              convert_numeric=True):
+                              convert_numeric=True,
+                              convert_timedeltas=True):
     """ if we have an object dtype, try to coerce dates and/or numbers """
 
     # if we have passed in a list or scalar
@@ -1539,6 +1540,22 @@ def _possibly_convert_objects(values, convert_dates=True,
             values = lib.maybe_convert_objects(
                 values, convert_datetime=convert_dates)
 
+    # convert timedeltas
+    if convert_timedeltas and values.dtype == np.object_:
+
+        if convert_timedeltas == 'coerce':
+            from pandas.tseries.timedeltas import \
+                 _possibly_cast_to_timedelta
+            values = _possibly_cast_to_timedelta(values)
+
+            # if we are all nans then leave me alone
+            if not isnull(new_values).all():
+                values = new_values
+
+        else:
+            values = lib.maybe_convert_objects(
+                values, convert_timedelta=convert_timedeltas)
+
     # convert to numeric
     if values.dtype == np.object_:
         if convert_numeric:

pandas/core/frame.py

@@ -3626,7 +3626,7 @@ def append(self, other, ignore_index=False, verify_integrity=False):
             index = None if other.name is None else [other.name]
             other = other.reindex(self.columns, copy=False)
             other = DataFrame(other.values.reshape((1, len(other))),
-                              index=index, columns=self.columns)
+                              index=index, columns=self.columns).convert_objects()
         elif isinstance(other, list) and not isinstance(other[0], DataFrame):
             other = DataFrame(other)
             if (self.columns.get_indexer(other.columns) >= 0).all():

pandas/core/generic.py

@@ -1844,16 +1844,18 @@ def copy(self, deep=True):
         return self._constructor(data).__finalize__(self)
 
     def convert_objects(self, convert_dates=True, convert_numeric=False,
-                        copy=True):
+                        convert_timedeltas=True, copy=True):
         """
         Attempt to infer better dtype for object columns
 
         Parameters
         ----------
-        convert_dates : if True, attempt to soft convert_dates, if 'coerce',
+        convert_dates : if True, attempt to soft convert dates, if 'coerce',
             force conversion (and non-convertibles get NaT)
         convert_numeric : if True attempt to coerce to numbers (including
             strings), non-convertibles get NaN
+        convert_timedeltas : if True, attempt to soft convert timedeltas, if 'coerce',
+            force conversion (and non-convertibles get NaT)
         copy : Boolean, if True, return copy, default is True
 
         Returns
@@ -1863,6 +1865,7 @@ def convert_objects(self, convert_dates=True, convert_numeric=False,
         return self._constructor(
             self._data.convert(convert_dates=convert_dates,
                                convert_numeric=convert_numeric,
+                               convert_timedeltas=convert_timedeltas,
                                copy=copy)).__finalize__(self)
 
     #----------------------------------------------------------------------

pandas/core/internals.py

@@ -1315,8 +1315,8 @@ def is_bool(self):
         """
         return lib.is_bool_array(self.values.ravel())
 
-    def convert(self, convert_dates=True, convert_numeric=True, copy=True,
-                by_item=True):
+    def convert(self, convert_dates=True, convert_numeric=True, convert_timedeltas=True,
+                copy=True, by_item=True):
         """ attempt to coerce any object types to better types
             return a copy of the block (if copy = True)
             by definition we ARE an ObjectBlock!!!!!
@@ -1334,7 +1334,8 @@ def convert(self, convert_dates=True, convert_numeric=True, copy=True,
 
                 values = com._possibly_convert_objects(
                     values.ravel(), convert_dates=convert_dates,
-                    convert_numeric=convert_numeric
+                    convert_numeric=convert_numeric,
+                    convert_timedeltas=convert_timedeltas,
                 ).reshape(values.shape)
                 values = _block_shape(values, ndim=self.ndim)
                 items = self.items.take([i])

pandas/lib.pyx

@@ -31,7 +31,7 @@ from datetime import datetime as pydatetime
 # this is our tseries.pxd
 from datetime cimport *
 
-from tslib cimport convert_to_tsobject
+from tslib cimport convert_to_tsobject, convert_to_timedelta64
 import tslib
 from tslib import NaT, Timestamp, repr_timedelta64
 

pandas/src/inference.pyx

@@ -1,4 +1,5 @@
 cimport util
+from tslib import NaT
 
 _TYPE_MAP = {
     np.int8: 'integer',
@@ -55,14 +56,18 @@ def infer_dtype(object _values):
 
     val = util.get_value_1d(values, 0)
 
-    if util.is_datetime64_object(val):
+    if util.is_datetime64_object(val) or val is NaT:
         if is_datetime64_array(values):
             return 'datetime64'
+        elif is_timedelta_or_timedelta64_array(values):
+            return 'timedelta'
     elif util.is_integer_object(val):
         if is_integer_array(values):
             return 'integer'
         elif is_integer_float_array(values):
             return 'mixed-integer-float'
+        elif is_timedelta_or_timedelta64_array(values):
+            return 'timedelta'
         return 'mixed-integer'
     elif is_datetime(val):
         if is_datetime_array(values):
@@ -258,20 +263,24 @@ def is_unicode_array(ndarray values):
 
 def is_datetime_array(ndarray[object] values):
     cdef int i, n = len(values)
+    cdef object v
     if n == 0:
         return False
     for i in range(n):
-        if not is_datetime(values[i]):
+        v = values[i]
+        if not (is_datetime(v) or util._checknull(v) or v is NaT):
             return False
     return True
 
 
 def is_datetime64_array(ndarray values):
     cdef int i, n = len(values)
+    cdef object v
     if n == 0:
         return False
     for i in range(n):
-        if not util.is_datetime64_object(values[i]):
+        v = values[i]
+        if not (util.is_datetime64_object(v) or util._checknull(v) or v is NaT):
             return False
     return True
 
@@ -299,12 +308,15 @@ def is_timedelta64_array(ndarray values):
     return True
 
 def is_timedelta_or_timedelta64_array(ndarray values):
+    """ infer with timedeltas and/or nat/none """
     import datetime
     cdef int i, n = len(values)
+    cdef object v
     if n == 0:
         return False
     for i in range(n):
-        if not (isinstance(values[i],datetime.timedelta) or isinstance(values[i],np.timedelta64)):
+        v = values[i]
+        if not (isinstance(v,datetime.timedelta) or isinstance(v,np.timedelta64) or util._checknull(v) or v is NaT):
             return False
     return True
 
@@ -427,7 +439,7 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
         return ints
 
 def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
-                          bint safe=0, bint convert_datetime=0):
+                          bint safe=0, bint convert_datetime=0, bint convert_timedelta=0):
     '''
     Type inference function-- convert object array to proper dtype
     '''
@@ -438,9 +450,11 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
         ndarray[int64_t] ints
         ndarray[uint8_t] bools
         ndarray[int64_t] idatetimes
+        ndarray[int64_t] itimedeltas
         bint seen_float = 0
         bint seen_complex = 0
         bint seen_datetime = 0
+        bint seen_timedelta = 0
         bint seen_int = 0
         bint seen_bool = 0
         bint seen_object = 0
@@ -457,6 +471,8 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
     bools = np.empty(n, dtype=np.uint8)
     datetimes = np.empty(n, dtype='M8[ns]')
     idatetimes = datetimes.view(np.int64)
+    timedeltas = np.empty(n, dtype='m8[ns]')
+    itimedeltas = timedeltas.view(np.int64)
 
     onan = np.nan
     fnan = np.nan
@@ -481,9 +497,13 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                 seen_object = 1
                 # objects[i] = val.astype('O')
                 break
-        elif util.is_timedelta64_object(val):
-            seen_object = 1
-            break
+        elif is_timedelta(val):
+            if convert_timedelta:
+                itimedeltas[i] = convert_to_timedelta64(val, 'ns')
+                seen_timedelta = 1
+            else:
+                seen_object = 1
+                break
         elif util.is_integer_object(val):
             seen_int = 1
             floats[i] = <float64_t> val
@@ -523,7 +543,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
 
         if not safe:
             if seen_null:
-                if not seen_bool and not seen_datetime:
+                if not seen_bool and not seen_datetime and not seen_timedelta:
                     if seen_complex:
                         return complexes
                     elif seen_float or seen_int:
@@ -533,20 +553,23 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                     if seen_datetime:
                         if not seen_numeric:
                             return datetimes
+                    elif seen_timedelta:
+                        if not seen_numeric:
+                            return timedeltas
                     else:
                         if seen_complex:
                             return complexes
                         elif seen_float:
                             return floats
                         elif seen_int:
                             return ints
-                elif not seen_datetime and not seen_numeric:
+                elif not seen_datetime and not seen_numeric and not seen_timedelta:
                     return bools.view(np.bool_)
 
         else:
             # don't cast int to float, etc.
             if seen_null:
-                if not seen_bool and not seen_datetime:
+                if not seen_bool and not seen_datetime and not seen_timedelta:
                     if seen_complex:
                         if not seen_int:
                             return complexes
@@ -558,6 +581,9 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                     if seen_datetime:
                         if not seen_numeric:
                             return datetimes
+                    elif seen_timedelta:
+                        if not seen_numeric:
+                            return timedeltas
                     else:
                         if seen_complex:
                             if not seen_int:
@@ -567,7 +593,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                                 return floats
                         elif seen_int:
                             return ints
-                elif not seen_datetime and not seen_numeric:
+                elif not seen_datetime and not seen_numeric and not seen_timedelta:
                     return bools.view(np.bool_)
 
     return objects

pandas/tests/test_series.py

@@ -2263,6 +2263,16 @@ def test_constructor_dtype_timedelta64(self):
                     for i in range(3)] + [np.nan ], dtype='m8[ns]' )
         self.assert_(td.dtype == 'timedelta64[ns]')
 
+        td = Series([np.timedelta64(300000000), pd.NaT],dtype='m8[ns]')
+        self.assert_(td.dtype == 'timedelta64[ns]')
+
+        # improved inference
+        td = Series([np.timedelta64(300000000), pd.NaT])
+        self.assert_(td.dtype == 'timedelta64[ns]')
+
+        td = Series([pd.NaT, np.timedelta64(300000000)])
+        self.assert_(td.dtype == 'timedelta64[ns]')
+
         # these are frequency conversion astypes
         #for t in ['s', 'D', 'us', 'ms']:
         #    self.assertRaises(TypeError, td.astype, 'm8[%s]' % t)

pandas/tseries/tests/test_timedeltas.py

@@ -165,11 +165,13 @@ def conv(v):
         # single element conversion
         v = timedelta(seconds=1)
         result = to_timedelta(v,box=False)
-        expected = to_timedelta([v])
+        expected = np.timedelta64(timedelta(seconds=1))
+        self.assert_(result == expected)
 
         v = np.timedelta64(timedelta(seconds=1))
         result = to_timedelta(v,box=False)
-        expected = to_timedelta([v])
+        expected = np.timedelta64(timedelta(seconds=1))
+        self.assert_(result == expected)
 
     def test_timedelta_ops(self):
         _skip_if_numpy_not_friendly()

pandas/tseries/timedeltas.py

@@ -70,42 +70,16 @@ def _convert_listlike(arg, box):
 _whitespace = re.compile('^\s*$')
 
 def _coerce_scalar_to_timedelta_type(r, unit='ns'):
-    # kludgy here until we have a timedelta scalar
-    # handle the numpy < 1.7 case
-
-    def conv(v):
-        if _np_version_under1p7:
-            return timedelta(microseconds=v/1000.0)
-        return np.timedelta64(v)
+    """ convert strings to timedelta; coerce to np.timedelta64"""
 
     if isinstance(r, compat.string_types):
+
+        # we are already converting to nanoseconds
         converter = _get_string_converter(r, unit=unit)
         r = converter()
-        r = conv(r)
-    elif r == tslib.iNaT:
-        return r
-    elif isnull(r):
-        return np.timedelta64('NaT')
-    elif isinstance(r, np.timedelta64):
-        r = r.astype("m8[{0}]".format(unit.lower()))
-    elif is_integer(r):
-        r = tslib.cast_from_unit(r, unit)
-        r = conv(r)
+        unit='ns'
 
-    if _np_version_under1p7:
-        if not isinstance(r, timedelta):
-            raise AssertionError("Invalid type for timedelta scalar: %s" % type(r))
-        if compat.PY3:
-            # convert to microseconds in timedelta64
-            r = np.timedelta64(int(r.total_seconds()*1e9 + r.microseconds*1000))
-        else:
-            return r
-
-    if isinstance(r, timedelta):
-        r = np.timedelta64(r)
-    elif not isinstance(r, np.timedelta64):
-        raise AssertionError("Invalid type for timedelta scalar: %s" % type(r))
-    return r.astype('timedelta64[ns]')
+    return tslib.convert_to_timedelta(r,unit)
 
 def _get_string_converter(r, unit='ns'):
     """ return a string converter for r to process the timedelta format """

pandas/tslib.pxd

@@ -1,3 +1,4 @@
 from numpy cimport ndarray, int64_t
 
 cdef convert_to_tsobject(object, object, object)
+cdef convert_to_timedelta64(object, object)