PDEP6 implementation pt 1: block.setitem, block.putmask

doc/source/user_guide/categorical.rst

@@ -779,6 +779,7 @@ Setting values by assigning categorical data will also check that the ``categori
 Assigning a ``Categorical`` to parts of a column of other types will use the values:
 
 .. ipython:: python
+    :okwarning:
 
     df = pd.DataFrame({"a": [1, 1, 1, 1, 1], "b": ["a", "a", "a", "a", "a"]})
     df.loc[1:2, "a"] = pd.Categorical(["b", "b"], categories=["a", "b"])

doc/source/whatsnew/v0.21.0.rst

@@ -425,13 +425,12 @@ Note that this also changes the sum of an empty ``Series``. Previously this alwa
    In [1]: pd.Series([]).sum()
    Out[1]: 0
 
-but for consistency with the all-NaN case, this was changed to return NaN as well:
+but for consistency with the all-NaN case, this was changed to return 0 as well:
 
-.. ipython:: python
-   :okwarning:
-
-   pd.Series([]).sum()
+.. code-block:: ipython
 
+   In [2]: pd.Series([]).sum()
+   Out[2]: 0
 
 .. _whatsnew_0210.api_breaking.loc:
 
@@ -755,10 +754,16 @@ Previously assignments, ``.where()`` and ``.fillna()`` with a ``bool`` assignmen
 
 New behavior
 
-.. ipython:: python
+.. code-block:: ipython
 
-   s[1] = True
-   s
+   In [7]: s[1] = True
+
+   In [8]: s
+   Out[8]:
+   0       1
+   1    True
+   2       3
+   Length: 3, dtype: object
 
 Previously, as assignment to a datetimelike with a non-datetimelike would coerce the
 non-datetime-like item being assigned (:issue:`14145`).

doc/source/whatsnew/v1.3.0.rst

@@ -501,13 +501,17 @@ Consistent casting with setting into Boolean Series
 Setting non-boolean values into a :class:`Series` with ``dtype=bool`` now consistently
 casts to ``dtype=object`` (:issue:`38709`)
 
-.. ipython:: python
+.. code-block:: ipython
+
+   In [1]: orig = pd.Series([True, False])
+
+   In [2]: ser = orig.copy()
+
+   In [3]: ser.iloc[1] = np.nan
 
-   orig = pd.Series([True, False])
-   ser = orig.copy()
-   ser.iloc[1] = np.nan
-   ser2 = orig.copy()
-   ser2.iloc[1] = 2.0
+   In [4]: ser2 = orig.copy()
+
+   In [5]: ser2.iloc[1] = 2.0
 
 *Previous behavior*:
 
@@ -527,10 +531,19 @@ casts to ``dtype=object`` (:issue:`38709`)
 
 *New behavior*:
 
-.. ipython:: python
+.. code-block:: ipython
+
+   In [1]: ser
+   Out [1]:
+   0    True
+   1     NaN
+   dtype: object
 
-   ser
-   ser2
+   In [2]:ser2
+   Out [2]:
+   0    True
+   1     2.0
+   dtype: object
 
 
 .. _whatsnew_130.notable_bug_fixes.rolling_groupby_column:

pandas/core/internals/blocks.py

@@ -10,6 +10,7 @@
     cast,
     final,
 )
+import warnings
 
 import numpy as np
 
@@ -41,6 +42,7 @@
 )
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import cache_readonly
+from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core.dtypes.astype import (
@@ -441,7 +443,7 @@ def split_and_operate(self, func, *args, **kwargs) -> list[Block]:
     # Up/Down-casting
 
     @final
-    def coerce_to_target_dtype(self, other) -> Block:
+    def coerce_to_target_dtype(self, other, warn_on_upcast: bool = False) -> Block:
         """
         coerce the current block to a dtype compat for other
         we will return a block, possibly object, and not raise
@@ -450,7 +452,21 @@ def coerce_to_target_dtype(self, other) -> Block:
         and will receive the same block
         """
         new_dtype = find_result_type(self.values.dtype, other)
-
+        if warn_on_upcast:
+            warnings.warn(
+                f"Setting an item of incompatible dtype is deprecated "
+                "and will raise in a future error of pandas. "
+                f"Value '{other}' has dtype incompatible with {self.values.dtype}, "
+                "please explicitly cast to a compatible dtype first.",
+                FutureWarning,
+                stacklevel=find_stack_level(),
+            )
+        if self.dtype == new_dtype:
+            raise AssertionError(
+                f"Did not expect new dtype {new_dtype} to equal self.dtype "
+                f"{self.values.dtype}. Please report a bug at "
+                "https://github.com/pandas-dev/pandas/issues."
+            )
         return self.astype(new_dtype, copy=False)
 
     @final
@@ -1109,7 +1125,7 @@ def setitem(self, indexer, value, using_cow: bool = False) -> Block:
             casted = np_can_hold_element(values.dtype, value)
         except LossySetitemError:
             # current dtype cannot store value, coerce to common dtype
-            nb = self.coerce_to_target_dtype(value)
+            nb = self.coerce_to_target_dtype(value, warn_on_upcast=True)
             return nb.setitem(indexer, value)
         else:
             if self.dtype == _dtype_obj:
@@ -1175,7 +1191,9 @@ def putmask(self, mask, new, using_cow: bool = False) -> list[Block]:
 
                 if not is_list_like(new):
                     # using just new[indexer] can't save us the need to cast
-                    return self.coerce_to_target_dtype(new).putmask(mask, new)
+                    return self.coerce_to_target_dtype(
+                        new, warn_on_upcast=True
+                    ).putmask(mask, new)
                 else:
                     indexer = mask.nonzero()[0]
                     nb = self.setitem(indexer, new[indexer], using_cow=using_cow)

pandas/tests/copy_view/test_indexing.py

@@ -925,11 +925,20 @@ def test_column_as_series_set_with_upcast(
             s[0] = "foo"
         expected = Series([1, 2, 3], name="a")
     elif using_copy_on_write or using_array_manager:
-        s[0] = "foo"
+        with tm.assert_produces_warning(FutureWarning, match="incompatible dtype"):
+            s[0] = "foo"
         expected = Series(["foo", 2, 3], dtype=object, name="a")
     else:
         with pd.option_context("chained_assignment", "warn"):
-            with tm.assert_produces_warning(SettingWithCopyWarning):
+            msg = "|".join(
+                [
+                    "A value is trying to be set on a copy of a slice from a DataFrame",
+                    "Setting an item of incompatible dtype is deprecated",
+                ]
+            )
+            with tm.assert_produces_warning(
+                (SettingWithCopyWarning, FutureWarning), match=msg
+            ):
                 s[0] = "foo"
         expected = Series(["foo", 2, 3], dtype=object, name="a")
 
@@ -1020,7 +1029,10 @@ def test_dataframe_add_column_from_series(backend, using_copy_on_write):
     ],
 )
 def test_set_value_copy_only_necessary_column(
-    using_copy_on_write, indexer_func, indexer, val
+    using_copy_on_write,
+    indexer_func,
+    indexer,
+    val,
 ):
     # When setting inplace, only copy column that is modified instead of the whole
     # block (by splitting the block)
@@ -1029,7 +1041,13 @@ def test_set_value_copy_only_necessary_column(
     df_orig = df.copy()
     view = df[:]
 
-    indexer_func(df)[indexer] = val
+    if val == "a" and indexer[0] != slice(None):
+        with tm.assert_produces_warning(
+            FutureWarning, match="Setting an item of incompatible dtype is deprecated"
+        ):
+            indexer_func(df)[indexer] = val
+    else:
+        indexer_func(df)[indexer] = val
 
     if using_copy_on_write:
         assert np.shares_memory(get_array(df, "b"), get_array(view, "b"))

pandas/tests/copy_view/test_methods.py

@@ -1405,15 +1405,18 @@ def test_putmask_aligns_rhs_no_reference(using_copy_on_write, dtype):
         assert np.shares_memory(arr_a, get_array(df, "a"))
 
 
-@pytest.mark.parametrize("val, exp", [(5.5, True), (5, False)])
-def test_putmask_dont_copy_some_blocks(using_copy_on_write, val, exp):
+@pytest.mark.parametrize(
+    "val, exp, warn", [(5.5, True, FutureWarning), (5, False, None)]
+)
+def test_putmask_dont_copy_some_blocks(using_copy_on_write, val, exp, warn):
     df = DataFrame({"a": [1, 2], "b": 1, "c": 1.5})
     view = df[:]
     df_orig = df.copy()
     indexer = DataFrame(
         [[True, False, False], [True, False, False]], columns=list("abc")
     )
-    df[indexer] = val
+    with tm.assert_produces_warning(warn, match="incompatible dtype"):
+        df[indexer] = val
 
     if using_copy_on_write:
         assert not np.shares_memory(get_array(view, "a"), get_array(df, "a"))

pandas/tests/frame/indexing/test_coercion.py

@@ -51,19 +51,31 @@ def test_37477():
     expected = DataFrame({"A": [1, 2, 3], "B": [3, 1.2, 5]})
 
     df = orig.copy()
-    df.at[1, "B"] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.at[1, "B"] = 1.2
     tm.assert_frame_equal(df, expected)
 
     df = orig.copy()
-    df.loc[1, "B"] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.loc[1, "B"] = 1.2
     tm.assert_frame_equal(df, expected)
 
     df = orig.copy()
-    df.iat[1, 1] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.iat[1, 1] = 1.2
     tm.assert_frame_equal(df, expected)
 
     df = orig.copy()
-    df.iloc[1, 1] = 1.2
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        df.iloc[1, 1] = 1.2
     tm.assert_frame_equal(df, expected)
 
 
@@ -94,11 +106,17 @@ def test_26395(indexer_al):
     expected = DataFrame({"D": [0, 0, 2]}, index=["A", "B", "C"], dtype=np.int64)
     tm.assert_frame_equal(df, expected)
 
-    indexer_al(df)["C", "D"] = 44.5
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        indexer_al(df)["C", "D"] = 44.5
     expected = DataFrame({"D": [0, 0, 44.5]}, index=["A", "B", "C"], dtype=np.float64)
     tm.assert_frame_equal(df, expected)
 
-    indexer_al(df)["C", "D"] = "hello"
+    with tm.assert_produces_warning(
+        FutureWarning, match="Setting an item of incompatible dtype"
+    ):
+        indexer_al(df)["C", "D"] = "hello"
     expected = DataFrame({"D": [0, 0, "hello"]}, index=["A", "B", "C"], dtype=object)
     tm.assert_frame_equal(df, expected)