merge in master

Author: AlexKirko

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 3.5
+python: 3.7
 
 # To turn off cached cython files and compiler cache
 # set NOCACHE-true

doc/source/whatsnew/v1.0.0.rst

@@ -766,6 +766,7 @@ Numeric
 - Bug in :class:`NumericIndex` construction that caused :class:`UInt64Index` to be casted to :class:`Float64Index` when integers in the ``np.uint64`` range were used to index a :class:`DataFrame` (:issue:`28279`)
 - Bug in :meth:`Series.interpolate` when using method=`index` with an unsorted index, would previously return incorrect results. (:issue:`21037`)
 - Bug in :meth:`DataFrame.round` where a :class:`DataFrame` with a :class:`CategoricalIndex` of :class:`IntervalIndex` columns would incorrectly raise a ``TypeError`` (:issue:`30063`)
+- Bug in :class:`DataFrame` cumulative operations (e.g. cumsum, cummax) incorrect casting to object-dtype (:issue:`19296`)
 
 Conversion
 ^^^^^^^^^^

pandas/core/arrays/boolean.py

@@ -524,7 +524,7 @@ def astype(self, dtype, copy=True):
             na_value = np.nan
         # coerce
         data = self._coerce_to_ndarray(na_value=na_value)
-        return astype_nansafe(data, dtype, copy=None)
+        return astype_nansafe(data, dtype, copy=False)
 
     def value_counts(self, dropna=True):
         """
@@ -755,9 +755,8 @@ def logical_method(self, other):
 
             if other_is_scalar and not (other is libmissing.NA or lib.is_bool(other)):
                 raise TypeError(
-                    "'other' should be pandas.NA or a bool. Got {} instead.".format(
-                        type(other).__name__
-                    )
+                    "'other' should be pandas.NA or a bool. "
+                    f"Got {type(other).__name__} instead."
                 )
 
             if not other_is_scalar and len(self) != len(other):
@@ -772,7 +771,7 @@ def logical_method(self, other):
 
             return BooleanArray(result, mask)
 
-        name = "__{name}__".format(name=op.__name__)
+        name = f"__{op.__name__}__"
         return set_function_name(logical_method, name, cls)
 
     @classmethod
@@ -819,7 +818,7 @@ def cmp_method(self, other):
 
             return BooleanArray(result, mask, copy=False)
 
-        name = "__{name}__".format(name=op.__name__)
+        name = f"__{op.__name__}"
         return set_function_name(cmp_method, name, cls)
 
     def _reduce(self, name, skipna=True, **kwargs):
@@ -922,7 +921,7 @@ def boolean_arithmetic_method(self, other):
 
             return self._maybe_mask_result(result, mask, other, op_name)
 
-        name = "__{name}__".format(name=op_name)
+        name = f"__{op_name}__"
         return set_function_name(boolean_arithmetic_method, name, cls)
 
 

pandas/core/arrays/integer.py

@@ -546,7 +546,7 @@ def astype(self, dtype, copy=True):
 
         # coerce
         data = self._coerce_to_ndarray()
-        return astype_nansafe(data, dtype, copy=None)
+        return astype_nansafe(data, dtype, copy=False)
 
     @property
     def _ndarray_values(self) -> np.ndarray:

pandas/core/computation/engines.py

@@ -3,6 +3,7 @@
 """
 
 import abc
+from typing import Dict, Type
 
 from pandas.core.computation.align import align_terms, reconstruct_object
 from pandas.core.computation.ops import _mathops, _reductions
@@ -53,7 +54,7 @@ def convert(self) -> str:
         """
         return printing.pprint_thing(self.expr)
 
-    def evaluate(self):
+    def evaluate(self) -> object:
         """
         Run the engine on the expression.
 
@@ -62,7 +63,7 @@ def evaluate(self):
 
         Returns
         -------
-        obj : object
+        object
             The result of the passed expression.
         """
         if not self._is_aligned:
@@ -101,12 +102,6 @@ class NumExprEngine(AbstractEngine):
 
     has_neg_frac = True
 
-    def __init__(self, expr):
-        super().__init__(expr)
-
-    def convert(self) -> str:
-        return str(super().convert())
-
     def _evaluate(self):
         import numexpr as ne
 
@@ -128,14 +123,14 @@ class PythonEngine(AbstractEngine):
 
     has_neg_frac = False
 
-    def __init__(self, expr):
-        super().__init__(expr)
-
     def evaluate(self):
         return self.expr()
 
-    def _evaluate(self):
+    def _evaluate(self) -> None:
         pass
 
 
-_engines = {"numexpr": NumExprEngine, "python": PythonEngine}
+_engines: Dict[str, Type[AbstractEngine]] = {
+    "numexpr": NumExprEngine,
+    "python": PythonEngine,
+}

pandas/core/computation/eval.py

@@ -5,6 +5,7 @@
 """
 
 import tokenize
+from typing import Optional
 import warnings
 
 from pandas._libs.lib import _no_default
@@ -17,7 +18,7 @@
 from pandas.io.formats.printing import pprint_thing
 
 
-def _check_engine(engine):
+def _check_engine(engine: Optional[str]) -> str:
     """
     Make sure a valid engine is passed.
 
@@ -168,7 +169,7 @@ def _check_for_locals(expr: str, stack_level: int, parser: str):
 def eval(
     expr,
     parser="pandas",
-    engine=None,
+    engine: Optional[str] = None,
     truediv=_no_default,
     local_dict=None,
     global_dict=None,

pandas/core/dtypes/cast.py

@@ -6,6 +6,7 @@
 
 from pandas._libs import lib, tslib, tslibs
 from pandas._libs.tslibs import NaT, OutOfBoundsDatetime, Period, iNaT
+from pandas._libs.tslibs.timezones import tz_compare
 from pandas.util._validators import validate_bool_kwarg
 
 from .common import (
@@ -409,6 +410,14 @@ def maybe_promote(dtype, fill_value=np.nan):
     elif is_datetime64tz_dtype(dtype):
         if isna(fill_value):
             fill_value = NaT
+        elif not isinstance(fill_value, datetime):
+            dtype = np.dtype(np.object_)
+        elif fill_value.tzinfo is None:
+            dtype = np.dtype(np.object_)
+        elif not tz_compare(fill_value.tzinfo, dtype.tz):
+            # TODO: sure we want to cast here?
+            dtype = np.dtype(np.object_)
+
     elif is_extension_array_dtype(dtype) and isna(fill_value):
         fill_value = dtype.na_value
 

pandas/core/frame.py

@@ -4014,16 +4014,14 @@ def fillna(
         inplace=False,
         limit=None,
         downcast=None,
-        **kwargs,
-    ):
+    ) -> Optional["DataFrame"]:
         return super().fillna(
             value=value,
             method=method,
             axis=axis,
             inplace=inplace,
             limit=limit,
             downcast=downcast,
-            **kwargs,
         )
 
     @Appender(_shared_docs["replace"] % _shared_doc_kwargs)
@@ -5148,7 +5146,7 @@ def reorder_levels(self, order, axis=0):
     # Arithmetic / combination related
 
     def _combine_frame(self, other, func, fill_value=None, level=None):
-        this, other = self.align(other, join="outer", level=level, copy=False)
+        # at this point we have `self._indexed_same(other)`
 
         if fill_value is None:
             # since _arith_op may be called in a loop, avoid function call
@@ -5164,14 +5162,15 @@ def _arith_op(left, right):
                 left, right = ops.fill_binop(left, right, fill_value)
                 return func(left, right)
 
-        if ops.should_series_dispatch(this, other, func):
+        if ops.should_series_dispatch(self, other, func):
             # iterate over columns
-            new_data = ops.dispatch_to_series(this, other, _arith_op)
+            new_data = ops.dispatch_to_series(self, other, _arith_op)
         else:
             with np.errstate(all="ignore"):
-                res_values = _arith_op(this.values, other.values)
-            new_data = dispatch_fill_zeros(func, this.values, other.values, res_values)
-        return this._construct_result(new_data)
+                res_values = _arith_op(self.values, other.values)
+            new_data = dispatch_fill_zeros(func, self.values, other.values, res_values)
+
+        return new_data
 
     def _combine_match_index(self, other, func):
         # at this point we have `self.index.equals(other.index)`

pandas/core/generic.py

@@ -11086,44 +11086,66 @@ def cum_func(self, axis=None, skipna=True, *args, **kwargs):
         else:
             axis = self._get_axis_number(axis)
 
-        y = com.values_from_object(self).copy()
-        d = self._construct_axes_dict()
-        d["copy"] = False
+        if axis == 1:
+            return cum_func(self.T, axis=0, skipna=skipna, *args, **kwargs).T
+
+        def na_accum_func(blk_values):
+            # We will be applying this function to block values
+            if blk_values.dtype.kind in ["m", "M"]:
+                # numpy 1.18 started sorting NaTs at the end instead of beginning,
+                #  so we need to work around to maintain backwards-consistency.
+                orig_dtype = blk_values.dtype
+
+                # We need to define mask before masking NaTs
+                mask = isna(blk_values)
+
+                if accum_func == np.minimum.accumulate:
+                    # Note: the accum_func comparison fails as an "is" comparison
+                    y = blk_values.view("i8")
+                    y[mask] = np.iinfo(np.int64).max
+                    changed = True
+                else:
+                    y = blk_values
+                    changed = False
+
+                result = accum_func(y.view("i8"), axis)
+                if skipna:
+                    np.putmask(result, mask, iNaT)
+                elif accum_func == np.minimum.accumulate:
+                    # Restore NaTs that we masked previously
+                    nz = (~np.asarray(mask)).nonzero()[0]
+                    if len(nz):
+                        # everything up to the first non-na entry stays NaT
+                        result[: nz[0]] = iNaT
+
+                if changed:
+                    # restore NaT elements
+                    y[mask] = iNaT  # TODO: could try/finally for this?
+
+                if isinstance(blk_values, np.ndarray):
+                    result = result.view(orig_dtype)
+                else:
+                    # DatetimeArray
+                    result = type(blk_values)._from_sequence(result, dtype=orig_dtype)
+
+            elif skipna and not issubclass(
+                blk_values.dtype.type, (np.integer, np.bool_)
+            ):
+                vals = blk_values.copy().T
+                mask = isna(vals)
+                np.putmask(vals, mask, mask_a)
+                result = accum_func(vals, axis)
+                np.putmask(result, mask, mask_b)
+            else:
+                result = accum_func(blk_values.T, axis)
 
-        if issubclass(y.dtype.type, (np.datetime64, np.timedelta64)):
-            # numpy 1.18 started sorting NaTs at the end instead of beginning,
-            #  so we need to work around to maintain backwards-consistency.
-            orig_dtype = y.dtype
-            if accum_func == np.minimum.accumulate:
-                # Note: the accum_func comparison fails as an "is" comparison
-                # Note that "y" is always a copy, so we can safely modify it
-                mask = isna(self)
-                y = y.view("i8")
-                y[mask] = np.iinfo(np.int64).max
-
-            result = accum_func(y.view("i8"), axis).view(orig_dtype)
-            if skipna:
-                mask = isna(self)
-                np.putmask(result, mask, iNaT)
-            elif accum_func == np.minimum.accumulate:
-                # Restore NaTs that we masked previously
-                nz = (~np.asarray(mask)).nonzero()[0]
-                if len(nz):
-                    # everything up to the first non-na entry stays NaT
-                    result[: nz[0]] = iNaT
+            # transpose back for ndarray, not for EA
+            return result.T if hasattr(result, "T") else result
 
-            if self.ndim == 1:
-                # restore dt64tz dtype
-                d["dtype"] = self.dtype
-
-        elif skipna and not issubclass(y.dtype.type, (np.integer, np.bool_)):
-            mask = isna(self)
-            np.putmask(y, mask, mask_a)
-            result = accum_func(y, axis)
-            np.putmask(result, mask, mask_b)
-        else:
-            result = accum_func(y, axis)
+        result = self._data.apply(na_accum_func)
 
+        d = self._construct_axes_dict()
+        d["copy"] = False
         return self._constructor(result, **d).__finalize__(self)
 
     return set_function_name(cum_func, name, cls)