BUG: Row-wise concat of differeing dtypes failing in certain cases (GH5754)

Author: jreback

doc/source/release.rst

@@ -828,6 +828,7 @@ Bug Fixes
   - Bug in fillna with Series and a passed series/dict (:issue:`5703`)
   - Bug in groupby transform with a datetime-like grouper (:issue:`5712`)
   - Bug in multi-index selection in PY3 when using certain keys (:issue:`5725`)
+  - Row-wise concat of differeing dtypes failing in certain cases (:issue:`5754`)
 
 pandas 0.12.0
 -------------

pandas/tests/test_frame.py

@@ -6154,6 +6154,48 @@ def test_append_empty_dataframe(self):
         expected = df1.copy()
         assert_frame_equal(result, expected)
 
+    def test_append_dtypes(self):
+
+        # GH 5754
+        # row appends of different dtypes (so need to do by-item)
+        # can sometimes infer the correct type
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(5))
+        df2 = DataFrame()
+        result = df1.append(df2)
+        expected = df1.copy()
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : 'foo' }, index=lrange(1,2))
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : [ Timestamp('20130101'), 'foo' ]})
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : np.nan }, index=lrange(1,2))
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ Timestamp('20130101'), np.nan ],dtype='M8[ns]') })
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : np.nan }, index=lrange(1,2), dtype=object)
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ Timestamp('20130101'), np.nan ],dtype='M8[ns]') })
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : np.nan }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1,2))
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ np.nan, Timestamp('20130101')] ,dtype='M8[ns]') })
+        assert_frame_equal(result, expected)
+
+        df1 = DataFrame({ 'bar' : Timestamp('20130101') }, index=lrange(1))
+        df2 = DataFrame({ 'bar' : 1 }, index=lrange(1,2), dtype=object)
+        result = df1.append(df2)
+        expected = DataFrame({ 'bar' : Series([ Timestamp('20130101'), 1 ]) })
+        assert_frame_equal(result, expected)
+
     def test_asfreq(self):
         offset_monthly = self.tsframe.asfreq(datetools.bmonthEnd)
         rule_monthly = self.tsframe.asfreq('BM')

pandas/tools/merge.py

@@ -1139,52 +1139,55 @@ def _concat_blocks(self, blocks):
 
     def _concat_single_item(self, objs, item):
         # this is called if we don't have consistent dtypes in a row-wise append
-
         all_values = []
-        dtypes = set()
+        dtypes = []
+        alls = set()
 
+        # figure out the resulting dtype of the combination
         for data, orig in zip(objs, self.objs):
+            d = dict([ (t,False) for t in ['object','datetime','timedelta','other'] ])
             if item in orig:
                 values = data.get(item)
                 if hasattr(values,'to_dense'):
                     values = values.to_dense()
-                dtypes.add(values.dtype)
                 all_values.append(values)
-            else:
-                all_values.append(None)
 
-        # figure out the resulting dtype of the combination
-        alls = set()
-        seen = []
-        for dtype in dtypes:
-            d = dict([ (t,False) for t in ['object','datetime','timedelta','other'] ])
-            if issubclass(dtype.type, (np.object_, np.bool_)):
-                d['object'] = True
-                alls.add('object')
-            elif is_datetime64_dtype(dtype):
-                d['datetime'] = True
-                alls.add('datetime')
-            elif is_timedelta64_dtype(dtype):
-                d['timedelta'] = True
-                alls.add('timedelta')
+                dtype = values.dtype
+
+                if issubclass(dtype.type, (np.object_, np.bool_)):
+                    d['object'] = True
+                    alls.add('object')
+                elif is_datetime64_dtype(dtype):
+                    d['datetime'] = True
+                    alls.add('datetime')
+                elif is_timedelta64_dtype(dtype):
+                    d['timedelta'] = True
+                    alls.add('timedelta')
+                else:
+                    d['other'] = True
+                    alls.add('other')
+
             else:
+                all_values.append(None)
                 d['other'] = True
                 alls.add('other')
-            seen.append(d)
+
+            dtypes.append(d)
 
         if 'datetime' in alls or 'timedelta' in alls:
 
             if 'object' in alls or 'other' in alls:
-                for v, s in zip(all_values,seen):
-                    if s.get('datetime') or s.get('timedelta'):
+
+                for v, d in zip(all_values,dtypes):
+                    if d.get('datetime') or d.get('timedelta'):
                         pass
 
                     # if we have all null, then leave a date/time like type
                     # if we have only that type left
-                    elif isnull(v).all():
+                    elif v is None or isnull(v).all():
 
-                        alls.remove('other')
-                        alls.remove('object')
+                        alls.discard('other')
+                        alls.discard('object')
 
         # create the result
         if 'object' in alls:
@@ -1200,7 +1203,7 @@ def _concat_single_item(self, objs, item):
 
         to_concat = []
         for obj, item_values in zip(objs, all_values):
-            if item_values is None:
+            if item_values is None or isnull(item_values).all():
                 shape = obj.shape[1:]
                 missing_arr = np.empty(shape, dtype=empty_dtype)
                 missing_arr.fill(fill_value)