BUG: timedelta64 merge/join issues (GH5695)

Author: jreback

pandas/tools/merge.py

@@ -14,7 +14,7 @@
 from pandas.core.index import (Index, MultiIndex, _get_combined_index,
                                _ensure_index, _get_consensus_names,
                                _all_indexes_same)
-from pandas.core.internals import (IntBlock, BoolBlock, BlockManager,
+from pandas.core.internals import (TimeDeltaBlock, IntBlock, BoolBlock, BlockManager,
                                    make_block, _consolidate)
 from pandas.util.decorators import cache_readonly, Appender, Substitution
 from pandas.core.common import (PandasError, ABCSeries,
@@ -816,7 +816,7 @@ def reindex_block(self, block, axis, ref_items, copy=True):
 
 def _may_need_upcasting(blocks):
     for block in blocks:
-        if isinstance(block, (IntBlock, BoolBlock)):
+        if isinstance(block, (IntBlock, BoolBlock)) and not isinstance(block, TimeDeltaBlock):
             return True
     return False
 
@@ -827,7 +827,10 @@ def _upcast_blocks(blocks):
     """
     new_blocks = []
     for block in blocks:
-        if isinstance(block, IntBlock):
+        if isinstance(block, TimeDeltaBlock):
+            # these are int blocks underlying, but are ok
+            newb = block
+        elif isinstance(block, IntBlock):
             newb = make_block(block.values.astype(float), block.items,
                               block.ref_items, placement=block._ref_locs)
         elif isinstance(block, BoolBlock):

pandas/tools/tests/test_merge.py

@@ -791,6 +791,26 @@ def test_append_dtype_coerce(self):
         result = df1.append(df2,ignore_index=True)
         assert_frame_equal(result, expected)
 
+        # timedelta64
+        # GH 5695
+        d = {'d': dt.datetime(2013, 11, 5, 5, 56), 't': dt.timedelta(0, 22500)}
+        df = DataFrame(columns=list('dt'))
+        df = df.append(d, ignore_index=True)
+        result = df.append(d, ignore_index=True)
+        expected = DataFrame({'d': [dt.datetime(2013, 11, 5, 5, 56),
+                                    dt.datetime(2013, 11, 5, 5, 56) ],
+                              't': [ dt.timedelta(0, 22500),
+                                     dt.timedelta(0, 22500) ]})
+        assert_frame_equal(result, expected)
+
+        td = np.timedelta64(300000000)
+        lhs = DataFrame(Series([td,td],index=["A","B"]))
+        rhs = DataFrame(Series([td],index=["A"]))
+
+        from pandas import NaT
+        result = lhs.join(rhs,rsuffix='r', how="left")
+        expected = DataFrame({ '0' : Series([td,td],index=list('AB')), '0r' : Series([td,NaT],index=list('AB')) })
+        assert_frame_equal(result, expected)
 
     def test_overlapping_columns_error_message(self):
         # #2649
@@ -1763,7 +1783,19 @@ def test_concat_datetime64_block(self):
         df = DataFrame({'time': rng})
 
         result = concat([df, df])
-        self.assert_((result[:10]['time'] == rng).all())
+        self.assert_((result.iloc[:10]['time'] == rng).all())
+        self.assert_((result.iloc[10:]['time'] == rng).all())
+
+    def test_concat_timedelta64_block(self):
+        from pandas import to_timedelta
+
+        rng = to_timedelta(np.arange(10),unit='s')
+
+        df = DataFrame({'time': rng})
+
+        result = concat([df, df])
+        self.assert_((result.iloc[:10]['time'] == rng).all())
+        self.assert_((result.iloc[10:]['time'] == rng).all())
 
     def test_concat_keys_with_none(self):
         # #1649