Merge pull request #5705 from jreback/series_fillna

BUG: Bug in fillna with Series and a passed series/dict (GH5703)

Author: jreback

doc/source/release.rst

@@ -822,6 +822,7 @@ Bug Fixes
   - Bug in groupby returning non-consistent types when user function returns a ``None``, (:issue:`5592`)
   - Work around regression in numpy 1.7.0 which erroneously raises IndexError from ``ndarray.item`` (:issue:`5666`)
   - Bug in repeated indexing of object with resultant non-unique index (:issue:`5678`)
+  - Bug in fillna with Series and a passed series/dict (:issue:`5703`)
 
 pandas 0.12.0
 -------------

pandas/core/generic.py

@@ -1905,7 +1905,16 @@ def fillna(self, value=None, method=None, axis=0, inplace=False,
 
             if len(self._get_axis(axis)) == 0:
                 return self
-            if isinstance(value, (dict, com.ABCSeries)):
+
+            if self.ndim == 1 and value is not None:
+                if isinstance(value, (dict, com.ABCSeries)):
+                    from pandas import Series
+                    value = Series(value)
+
+                new_data = self._data.fillna(value, inplace=inplace,
+                                             downcast=downcast)
+
+            elif isinstance(value, (dict, com.ABCSeries)):
                 if axis == 1:
                     raise NotImplementedError('Currently only can fill '
                                               'with dict/Series column '

pandas/core/internals.py

@@ -664,7 +664,7 @@ def putmask(self, mask, new, align=True, inplace=False):
 
         # if we are passed a scalar None, convert it here
         if not is_list_like(new) and isnull(new):
-            new = np.nan
+            new = self.fill_value
 
         if self._can_hold_element(new):
             new = self._try_cast(new)
@@ -830,7 +830,7 @@ def _interpolate(self, method=None, index=None, values=None,
             data = data.astype(np.float64)
 
         if fill_value is None:
-            fill_value = np.nan
+            fill_value = self.fill_value
 
         if method in ('krogh', 'piecewise_polynomial', 'pchip'):
             if not index.is_monotonic:
@@ -1196,6 +1196,10 @@ class TimeDeltaBlock(IntBlock):
     _can_hold_na = True
     is_numeric = False
 
+    @property
+    def fill_value(self):
+        return tslib.iNaT
+
     def _try_fill(self, value):
         """ if we are a NaT, return the actual fill value """
         if isinstance(value, type(tslib.NaT)) or isnull(value):
@@ -1532,6 +1536,10 @@ def _try_coerce_result(self, result):
             result = lib.Timestamp(result)
         return result
 
+    @property
+    def fill_value(self):
+        return tslib.iNaT
+
     def _try_fill(self, value):
         """ if we are a NaT, return the actual fill value """
         if isinstance(value, type(tslib.NaT)) or isnull(value):
@@ -3190,18 +3198,15 @@ def reindex_items(self, new_items, indexer=None, copy=True,
                     blk = blk.reindex_items_from(new_items)
                 else:
                     blk.ref_items = new_items
-                if blk is not None:
-                    new_blocks.append(blk)
+                new_blocks.extend(_valid_blocks(blk))
         else:
 
             # unique
             if self.axes[0].is_unique and new_items.is_unique:
 
                 for block in self.blocks:
-
-                    newb = block.reindex_items_from(new_items, copy=copy)
-                    if newb is not None and len(newb.items) > 0:
-                        new_blocks.append(newb)
+                    blk = block.reindex_items_from(new_items, copy=copy)
+                    new_blocks.extend(_valid_blocks(blk))
 
             # non-unique
             else:
@@ -3411,7 +3416,11 @@ def __init__(self, block, axis, do_integrity_check=False, fastpath=True):
         if fastpath:
             self.axes = [axis]
             if isinstance(block, list):
-                if len(block) != 1:
+
+                # empty block
+                if len(block) == 0:
+                    block = [np.array([])]
+                elif len(block) != 1:
                     raise ValueError('Cannot create SingleBlockManager with '
                                      'more than 1 block')
                 block = block[0]
@@ -3875,6 +3884,13 @@ def _consolidate(blocks, items):
     return new_blocks
 
 
+def _valid_blocks(newb):
+    if newb is None:
+        return []
+    if not isinstance(newb, list):
+        newb = [ newb ]
+    return [ b for b in newb if len(b.items) > 0 ]
+
 def _merge_blocks(blocks, items, dtype=None, _can_consolidate=True):
     if len(blocks) == 1:
         return blocks[0]

pandas/tests/test_series.py

@@ -2760,6 +2760,35 @@ def test_fillna(self):
         self.assertRaises(ValueError, ts.fillna)
         self.assertRaises(ValueError, self.ts.fillna, value=0, method='ffill')
 
+        # GH 5703
+        s1 = Series([np.nan])
+        s2 = Series([1])
+        result = s1.fillna(s2)
+        expected = Series([1.])
+        assert_series_equal(result,expected)
+        result = s1.fillna({})
+        assert_series_equal(result,s1)
+        result = s1.fillna(Series(()))
+        assert_series_equal(result,s1)
+        result = s2.fillna(s1)
+        assert_series_equal(result,s2)
+        result = s1.fillna({ 0 : 1})
+        assert_series_equal(result,expected)
+        result = s1.fillna({ 1 : 1})
+        assert_series_equal(result,Series([np.nan]))
+        result = s1.fillna({ 0 : 1, 1 : 1})
+        assert_series_equal(result,expected)
+        result = s1.fillna(Series({ 0 : 1, 1 : 1}))
+        assert_series_equal(result,expected)
+        result = s1.fillna(Series({ 0 : 1, 1 : 1},index=[4,5]))
+        assert_series_equal(result,s1)
+
+        s1 = Series([0, 1, 2], list('abc'))
+        s2 = Series([0, np.nan, 2], list('bac'))
+        result = s2.fillna(s1)
+        expected = Series([0,0,2.], list('bac'))
+        assert_series_equal(result,expected)
+
     def test_fillna_bug(self):
         x = Series([nan, 1., nan, 3., nan], ['z', 'a', 'b', 'c', 'd'])
         filled = x.fillna(method='ffill')