diff --git a/doc/source/release.rst b/doc/source/release.rst index 5d40cbe82e87b..0a853938f6cad 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -822,6 +822,7 @@ Bug Fixes - Bug in groupby returning non-consistent types when user function returns a ``None``, (:issue:`5592`) - Work around regression in numpy 1.7.0 which erroneously raises IndexError from ``ndarray.item`` (:issue:`5666`) - Bug in repeated indexing of object with resultant non-unique index (:issue:`5678`) + - Bug in fillna with Series and a passed series/dict (:issue:`5703`) pandas 0.12.0 ------------- diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 253136b9a11c3..5e79d05146de3 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -1905,7 +1905,16 @@ def fillna(self, value=None, method=None, axis=0, inplace=False, if len(self._get_axis(axis)) == 0: return self - if isinstance(value, (dict, com.ABCSeries)): + + if self.ndim == 1 and value is not None: + if isinstance(value, (dict, com.ABCSeries)): + from pandas import Series + value = Series(value) + + new_data = self._data.fillna(value, inplace=inplace, + downcast=downcast) + + elif isinstance(value, (dict, com.ABCSeries)): if axis == 1: raise NotImplementedError('Currently only can fill ' 'with dict/Series column ' diff --git a/pandas/core/internals.py b/pandas/core/internals.py index dbad353bab62c..6ec08fe501bcd 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -664,7 +664,7 @@ def putmask(self, mask, new, align=True, inplace=False): # if we are passed a scalar None, convert it here if not is_list_like(new) and isnull(new): - new = np.nan + new = self.fill_value if self._can_hold_element(new): new = self._try_cast(new) @@ -830,7 +830,7 @@ def _interpolate(self, method=None, index=None, values=None, data = data.astype(np.float64) if fill_value is None: - fill_value = np.nan + fill_value = self.fill_value if method in ('krogh', 'piecewise_polynomial', 'pchip'): if not index.is_monotonic: @@ -1196,6 +1196,10 @@ class TimeDeltaBlock(IntBlock): _can_hold_na = True is_numeric = False + @property + def fill_value(self): + return tslib.iNaT + def _try_fill(self, value): """ if we are a NaT, return the actual fill value """ if isinstance(value, type(tslib.NaT)) or isnull(value): @@ -1532,6 +1536,10 @@ def _try_coerce_result(self, result): result = lib.Timestamp(result) return result + @property + def fill_value(self): + return tslib.iNaT + def _try_fill(self, value): """ if we are a NaT, return the actual fill value """ if isinstance(value, type(tslib.NaT)) or isnull(value): @@ -3190,18 +3198,15 @@ def reindex_items(self, new_items, indexer=None, copy=True, blk = blk.reindex_items_from(new_items) else: blk.ref_items = new_items - if blk is not None: - new_blocks.append(blk) + new_blocks.extend(_valid_blocks(blk)) else: # unique if self.axes[0].is_unique and new_items.is_unique: for block in self.blocks: - - newb = block.reindex_items_from(new_items, copy=copy) - if newb is not None and len(newb.items) > 0: - new_blocks.append(newb) + blk = block.reindex_items_from(new_items, copy=copy) + new_blocks.extend(_valid_blocks(blk)) # non-unique else: @@ -3411,7 +3416,11 @@ def __init__(self, block, axis, do_integrity_check=False, fastpath=True): if fastpath: self.axes = [axis] if isinstance(block, list): - if len(block) != 1: + + # empty block + if len(block) == 0: + block = [np.array([])] + elif len(block) != 1: raise ValueError('Cannot create SingleBlockManager with ' 'more than 1 block') block = block[0] @@ -3875,6 +3884,13 @@ def _consolidate(blocks, items): return new_blocks +def _valid_blocks(newb): + if newb is None: + return [] + if not isinstance(newb, list): + newb = [ newb ] + return [ b for b in newb if len(b.items) > 0 ] + def _merge_blocks(blocks, items, dtype=None, _can_consolidate=True): if len(blocks) == 1: return blocks[0] diff --git a/pandas/tests/test_series.py b/pandas/tests/test_series.py index d3104cdfad062..188d61f397f5c 100644 --- a/pandas/tests/test_series.py +++ b/pandas/tests/test_series.py @@ -2760,6 +2760,35 @@ def test_fillna(self): self.assertRaises(ValueError, ts.fillna) self.assertRaises(ValueError, self.ts.fillna, value=0, method='ffill') + # GH 5703 + s1 = Series([np.nan]) + s2 = Series([1]) + result = s1.fillna(s2) + expected = Series([1.]) + assert_series_equal(result,expected) + result = s1.fillna({}) + assert_series_equal(result,s1) + result = s1.fillna(Series(())) + assert_series_equal(result,s1) + result = s2.fillna(s1) + assert_series_equal(result,s2) + result = s1.fillna({ 0 : 1}) + assert_series_equal(result,expected) + result = s1.fillna({ 1 : 1}) + assert_series_equal(result,Series([np.nan])) + result = s1.fillna({ 0 : 1, 1 : 1}) + assert_series_equal(result,expected) + result = s1.fillna(Series({ 0 : 1, 1 : 1})) + assert_series_equal(result,expected) + result = s1.fillna(Series({ 0 : 1, 1 : 1},index=[4,5])) + assert_series_equal(result,s1) + + s1 = Series([0, 1, 2], list('abc')) + s2 = Series([0, np.nan, 2], list('bac')) + result = s2.fillna(s1) + expected = Series([0,0,2.], list('bac')) + assert_series_equal(result,expected) + def test_fillna_bug(self): x = Series([nan, 1., nan, 3., nan], ['z', 'a', 'b', 'c', 'd']) filled = x.fillna(method='ffill')