Merge remote-tracking branch 'upstream/master' into numba

Author: simonjayhawkins

.pre-commit-config.yaml

@@ -138,7 +138,7 @@ repos:
         entry: python scripts/check_for_inconsistent_pandas_namespace.py
         language: python
         types: [python]
-        files: ^pandas/tests/
+        files: ^pandas/tests/frame/
     -   id: FrameOrSeriesUnion
         name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
         entry: Union\[.*(Series,.*DataFrame|DataFrame,.*Series).*\]

doc/source/whatsnew/v0.20.0.rst

@@ -374,7 +374,7 @@ For example, after running the following, ``styled.xlsx`` renders as below:
    df.iloc[0, 2] = np.nan
    df
    styled = (df.style
-             .applymap(lambda val: 'color: %s' % 'red' if val < 0 else 'black')
+             .applymap(lambda val: 'color:red;' if val < 0 else 'color:black;')
              .highlight_max())
    styled.to_excel('styled.xlsx', engine='openpyxl')
 

doc/source/whatsnew/v1.3.0.rst

@@ -65,7 +65,8 @@ Other enhancements
 - :meth:`DataFrame.plot.scatter` can now accept a categorical column as the argument to ``c`` (:issue:`12380`, :issue:`31357`)
 - :meth:`.Styler.set_tooltips` allows on hover tooltips to be added to styled HTML dataframes (:issue:`35643`, :issue:`21266`, :issue:`39317`)
 - :meth:`.Styler.set_tooltips_class` and :meth:`.Styler.set_table_styles` amended to optionally allow certain css-string input arguments (:issue:`39564`)
-- :meth:`.Styler.apply` now more consistently accepts ndarray function returns, i.e. in all cases for ``axis`` is ``0, 1 or None``. (:issue:`39359`)
+- :meth:`.Styler.apply` now more consistently accepts ndarray function returns, i.e. in all cases for ``axis`` is ``0, 1 or None`` (:issue:`39359`)
+- :meth:`.Styler.apply` and :meth:`.Styler.applymap` now raise errors if wrong format CSS is passed on render (:issue:`39660`)
 - :meth:`Series.loc.__getitem__` and :meth:`Series.loc.__setitem__` with :class:`MultiIndex` now raising helpful error message when indexer has too many dimensions (:issue:`35349`)
 - :meth:`pandas.read_stata` and :class:`StataReader` support reading data from compressed files.
 - Add support for parsing ``ISO 8601``-like timestamps with negative signs to :meth:`pandas.Timedelta` (:issue:`37172`)
@@ -324,6 +325,7 @@ Numeric
 
 Conversion
 ^^^^^^^^^^
+- Bug in :meth:`Series.to_dict` with ``orient='records'`` now returns python native types (:issue:`25969`)
 -
 -
 

pandas/core/arrays/datetimelike.py

@@ -60,6 +60,7 @@
     Substitution,
     cache_readonly,
 )
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
     is_categorical_dtype,
@@ -397,12 +398,13 @@ def astype(self, dtype, copy=True):
         elif is_integer_dtype(dtype):
             # we deliberately ignore int32 vs. int64 here.
             # See https://github.com/pandas-dev/pandas/issues/24381 for more.
+            level = find_stack_level()
             warnings.warn(
                 f"casting {self.dtype} values to int64 with .astype(...) is "
                 "deprecated and will raise in a future version. "
                 "Use .view(...) instead.",
                 FutureWarning,
-                stacklevel=3,
+                stacklevel=level,
             )
 
             values = self.asi8

pandas/core/arrays/datetimes.py

@@ -616,6 +616,10 @@ def astype(self, dtype, copy=True):
         elif is_datetime64_ns_dtype(dtype):
             return astype_dt64_to_dt64tz(self, dtype, copy, via_utc=False)
 
+        elif self.tz is None and is_datetime64_dtype(dtype) and dtype != self.dtype:
+            # unit conversion e.g. datetime64[s]
+            return self._data.astype(dtype)
+
         elif is_period_dtype(dtype):
             return self.to_period(freq=dtype.freq)
         return dtl.DatetimeLikeArrayMixin.astype(self, dtype, copy)

pandas/core/dtypes/cast.py

@@ -71,6 +71,7 @@
     is_datetime64_dtype,
     is_datetime64_ns_dtype,
     is_datetime64tz_dtype,
+    is_datetime_or_timedelta_dtype,
     is_dtype_equal,
     is_extension_array_dtype,
     is_float,
@@ -170,6 +171,29 @@ def maybe_box_datetimelike(value: Scalar, dtype: Optional[Dtype] = None) -> Scal
     return value
 
 
+def maybe_box_native(value: Scalar) -> Scalar:
+    """
+    If passed a scalar cast the scalar to a python native type.
+
+    Parameters
+    ----------
+    value : scalar or Series
+
+    Returns
+    -------
+    scalar or Series
+    """
+    if is_datetime_or_timedelta_dtype(value):
+        value = maybe_box_datetimelike(value)
+    elif is_float(value):
+        value = float(value)
+    elif is_integer(value):
+        value = int(value)
+    elif is_bool(value):
+        value = bool(value)
+    return value
+
+
 def maybe_unbox_datetimelike(value: Scalar, dtype: DtypeObj) -> Scalar:
     """
     Convert a Timedelta or Timestamp to timedelta64 or datetime64 for setting

pandas/core/frame.py

@@ -95,7 +95,7 @@
     find_common_type,
     infer_dtype_from_scalar,
     invalidate_string_dtypes,
-    maybe_box_datetimelike,
+    maybe_box_native,
     maybe_convert_platform,
     maybe_downcast_to_dtype,
     maybe_infer_to_datetimelike,
@@ -1655,15 +1655,15 @@ def to_dict(self, orient: str = "dict", into=dict):
                     (
                         "data",
                         [
-                            list(map(maybe_box_datetimelike, t))
+                            list(map(maybe_box_native, t))
                             for t in self.itertuples(index=False, name=None)
                         ],
                     ),
                 )
             )
 
         elif orient == "series":
-            return into_c((k, maybe_box_datetimelike(v)) for k, v in self.items())
+            return into_c((k, v) for k, v in self.items())
 
         elif orient == "records":
             columns = self.columns.tolist()
@@ -1672,8 +1672,7 @@ def to_dict(self, orient: str = "dict", into=dict):
                 for row in self.itertuples(index=False, name=None)
             )
             return [
-                into_c((k, maybe_box_datetimelike(v)) for k, v in row.items())
-                for row in rows
+                into_c((k, maybe_box_native(v)) for k, v in row.items()) for row in rows
             ]
 
         elif orient == "index":

pandas/core/indexes/extension.py

@@ -309,6 +309,10 @@ def astype(self, dtype, copy=True):
                 return self
             return self.copy()
 
+        if isinstance(dtype, np.dtype) and dtype.kind == "M" and dtype != "M8[ns]":
+            # For now Datetime supports this by unwrapping ndarray, but DTI doesn't
+            raise TypeError(f"Cannot cast {type(self._data).__name__} to dtype")
+
         new_values = self._data.astype(dtype, copy=copy)
 
         # pass copy=False because any copying will be done in the

pandas/core/indexing.py

@@ -2403,57 +2403,3 @@ def need_slice(obj) -> bool:
         or obj.stop is not None
         or (obj.step is not None and obj.step != 1)
     )
-
-
-def non_reducing_slice(slice_):
-    """
-    Ensure that a slice doesn't reduce to a Series or Scalar.
-
-    Any user-passed `subset` should have this called on it
-    to make sure we're always working with DataFrames.
-    """
-    # default to column slice, like DataFrame
-    # ['A', 'B'] -> IndexSlices[:, ['A', 'B']]
-    kinds = (ABCSeries, np.ndarray, Index, list, str)
-    if isinstance(slice_, kinds):
-        slice_ = IndexSlice[:, slice_]
-
-    def pred(part) -> bool:
-        """
-        Returns
-        -------
-        bool
-            True if slice does *not* reduce,
-            False if `part` is a tuple.
-        """
-        # true when slice does *not* reduce, False when part is a tuple,
-        # i.e. MultiIndex slice
-        if isinstance(part, tuple):
-            # GH#39421 check for sub-slice:
-            return any((isinstance(s, slice) or is_list_like(s)) for s in part)
-        else:
-            return isinstance(part, slice) or is_list_like(part)
-
-    if not is_list_like(slice_):
-        if not isinstance(slice_, slice):
-            # a 1-d slice, like df.loc[1]
-            slice_ = [[slice_]]
-        else:
-            # slice(a, b, c)
-            slice_ = [slice_]  # to tuplize later
-    else:
-        slice_ = [part if pred(part) else [part] for part in slice_]
-    return tuple(slice_)
-
-
-def maybe_numeric_slice(df, slice_, include_bool: bool = False):
-    """
-    Want nice defaults for background_gradient that don't break
-    with non-numeric data. But if slice_ is passed go with that.
-    """
-    if slice_ is None:
-        dtypes = [np.number]
-        if include_bool:
-            dtypes.append(bool)
-        slice_ = IndexSlice[:, df.select_dtypes(include=dtypes).columns]
-    return slice_

pandas/core/internals/__init__.py

@@ -11,7 +11,6 @@
     ObjectBlock,
     TimeDeltaBlock,
     make_block,
-    safe_reshape,
 )
 from pandas.core.internals.concat import concatenate_block_managers
 from pandas.core.internals.managers import (
@@ -31,7 +30,6 @@
     "FloatBlock",
     "ObjectBlock",
     "TimeDeltaBlock",
-    "safe_reshape",
     "make_block",
     "DataManager",
     "ArrayManager",

pandas/core/internals/blocks.py

@@ -308,7 +308,7 @@ def make_block(self, values, placement=None) -> Block:
         if placement is None:
             placement = self.mgr_locs
         if self.is_extension:
-            values = _block_shape(values, ndim=self.ndim)
+            values = ensure_block_shape(values, ndim=self.ndim)
 
         return make_block(values, placement=placement, ndim=self.ndim)
 
@@ -533,7 +533,7 @@ def make_a_block(nv, ref_loc):
             else:
                 # Put back the dimension that was taken from it and make
                 # a block out of the result.
-                nv = _block_shape(nv, ndim=self.ndim)
+                nv = ensure_block_shape(nv, ndim=self.ndim)
                 block = self.make_block(values=nv, placement=ref_loc)
             return block
 
@@ -673,6 +673,18 @@ def astype(self, dtype, copy: bool = False, errors: str = "raise"):
 
     def _astype(self, dtype: DtypeObj, copy: bool) -> ArrayLike:
         values = self.values
+        if values.dtype.kind in ["m", "M"]:
+            values = self.array_values()
+
+        if (
+            values.dtype.kind in ["m", "M"]
+            and dtype.kind in ["i", "u"]
+            and isinstance(dtype, np.dtype)
+            and dtype.itemsize != 8
+        ):
+            # TODO(2.0) remove special case once deprecation on DTA/TDA is enforced
+            msg = rf"cannot astype a datetimelike from [{values.dtype}] to [{dtype}]"
+            raise TypeError(msg)
 
         if is_datetime64tz_dtype(dtype) and is_datetime64_dtype(values.dtype):
             return astype_dt64_to_dt64tz(values, dtype, copy, via_utc=True)
@@ -1569,7 +1581,9 @@ def putmask(self, mask, new) -> List[Block]:
         if isinstance(new, (np.ndarray, ExtensionArray)) and len(new) == len(mask):
             new = new[mask]
 
-        mask = safe_reshape(mask, new_values.shape)
+        if mask.ndim == new_values.ndim + 1:
+            # TODO(EA2D): unnecessary with 2D EAs
+            mask = mask.reshape(new_values.shape)
 
         new_values[mask] = new
         return [self.make_block(values=new_values)]
@@ -2048,6 +2062,21 @@ def shift(self, periods: int, axis: int = 0, fill_value: Any = None) -> List[Blo
         new_values = values.shift(periods, fill_value=fill_value, axis=axis)
         return [self.make_block_same_class(new_values)]
 
+    def fillna(
+        self, value, limit=None, inplace: bool = False, downcast=None
+    ) -> List[Block]:
+
+        if not self._can_hold_element(value) and self.dtype.kind != "m":
+            # We support filling a DatetimeTZ with a `value` whose timezone
+            #  is different by coercing to object.
+            # TODO: don't special-case td64
+            return self.astype(object).fillna(value, limit, inplace, downcast)
+
+        values = self.array_values()
+        values = values if inplace else values.copy()
+        new_values = values.fillna(value=value, limit=limit)
+        return [self.make_block_same_class(values=new_values)]
+
 
 class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock):
     """Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock."""
@@ -2134,6 +2163,7 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeBlock):
     fill_value = NaT
     where = DatetimeBlock.where
     putmask = DatetimeLikeBlockMixin.putmask
+    fillna = DatetimeLikeBlockMixin.fillna
 
     array_values = ExtensionBlock.array_values
 
@@ -2172,19 +2202,6 @@ def external_values(self):
         # Avoid FutureWarning in .astype in casting from dt64tz to dt64
         return self.values._data
 
-    def fillna(
-        self, value, limit=None, inplace: bool = False, downcast=None
-    ) -> List[Block]:
-        # We support filling a DatetimeTZ with a `value` whose timezone
-        # is different by coercing to object.
-        if self._can_hold_element(value):
-            return super().fillna(value, limit, inplace, downcast)
-
-        # different timezones, or a non-tz
-        return self.astype(object).fillna(
-            value, limit=limit, inplace=inplace, downcast=downcast
-        )
-
 
 class TimeDeltaBlock(DatetimeLikeBlockMixin):
     __slots__ = ()
@@ -2194,14 +2211,6 @@ class TimeDeltaBlock(DatetimeLikeBlockMixin):
     fill_value = np.timedelta64("NaT", "ns")
     _dtype = fill_value.dtype
 
-    def fillna(
-        self, value, limit=None, inplace: bool = False, downcast=None
-    ) -> List[Block]:
-        values = self.array_values()
-        values = values if inplace else values.copy()
-        new_values = values.fillna(value=value, limit=limit)
-        return [self.make_block_same_class(values=new_values)]
-
 
 class ObjectBlock(Block):
     __slots__ = ()
@@ -2431,36 +2440,15 @@ def extend_blocks(result, blocks=None) -> List[Block]:
     return blocks
 
 
-def _block_shape(values: ArrayLike, ndim: int = 1) -> ArrayLike:
-    """ guarantee the shape of the values to be at least 1 d """
+def ensure_block_shape(values: ArrayLike, ndim: int = 1) -> ArrayLike:
+    """
+    Reshape if possible to have values.ndim == ndim.
+    """
     if values.ndim < ndim:
-        shape = values.shape
         if not is_extension_array_dtype(values.dtype):
             # TODO(EA2D): https://github.com/pandas-dev/pandas/issues/23023
             # block.shape is incorrect for "2D" ExtensionArrays
             # We can't, and don't need to, reshape.
 
-            # error: "ExtensionArray" has no attribute "reshape"
-            values = values.reshape(tuple((1,) + shape))  # type: ignore[attr-defined]
+            values = np.asarray(values).reshape(1, -1)
     return values
-
-
-def safe_reshape(arr: ArrayLike, new_shape: Shape) -> ArrayLike:
-    """
-    Reshape `arr` to have shape `new_shape`, unless it is an ExtensionArray,
-    in which case it will be returned unchanged (see gh-13012).
-
-    Parameters
-    ----------
-    arr : np.ndarray or ExtensionArray
-    new_shape : Tuple[int]
-
-    Returns
-    -------
-    np.ndarray or ExtensionArray
-    """
-    if not is_extension_array_dtype(arr.dtype):
-        # Note: this will include TimedeltaArray and tz-naive DatetimeArray
-        # TODO(EA2D): special case will be unnecessary with 2D EAs
-        arr = np.asarray(arr).reshape(new_shape)
-    return arr