Merge remote-tracking branch 'upstream/main' into pt1-deprecate-ban-upcasting

Author: MarcoGorelli

doc/source/whatsnew/v2.0.0.rst

@@ -952,6 +952,7 @@ Indexing
 - Bug in :meth:`DataFrame.__setitem__` raising when indexer is a :class:`DataFrame` with ``boolean`` dtype (:issue:`47125`)
 - Bug in :meth:`DataFrame.reindex` filling with wrong values when indexing columns and index for ``uint`` dtypes (:issue:`48184`)
 - Bug in :meth:`DataFrame.loc` when setting :class:`DataFrame` with different dtypes coercing values to single dtype (:issue:`50467`)
+- Bug in :meth:`DataFrame.sort_values` where ``None`` was not returned when ``by`` is empty list and ``inplace=True`` (:issue:`50643`)
 - Bug in :meth:`DataFrame.loc` coercing dtypes when setting values with a list indexer (:issue:`49159`)
 - Bug in :meth:`Series.loc` raising error for out of bounds end of slice indexer (:issue:`50161`)
 - Bug in :meth:`DataFrame.loc` raising ``ValueError`` with ``bool`` indexer and :class:`MultiIndex` (:issue:`47687`)

pandas/core/frame.py

@@ -48,6 +48,7 @@
 from pandas._libs.hashtable import duplicated
 from pandas._libs.lib import (
     NoDefault,
+    array_equal_fast,
     no_default,
 )
 from pandas._typing import (
@@ -6345,7 +6346,7 @@ def dropna(
             raise ValueError(f"invalid how option: {how}")
 
         if np.all(mask):
-            result = self.copy()
+            result = self.copy(deep=None)
         else:
             result = self.loc(axis=axis)[mask]
 
@@ -6695,7 +6696,16 @@ def sort_values(
                 k, kind=kind, ascending=ascending, na_position=na_position, key=key
             )
         else:
-            return self.copy()
+            if inplace:
+                return self._update_inplace(self)
+            else:
+                return self.copy(deep=None)
+
+        if array_equal_fast(indexer, np.arange(0, len(indexer), dtype=indexer.dtype)):
+            if inplace:
+                return self._update_inplace(self)
+            else:
+                return self.copy(deep=None)
 
         new_data = self._mgr.take(
             indexer, axis=self._get_block_manager_axis(axis), verify=False

pandas/core/internals/managers.py

@@ -1955,9 +1955,17 @@ def _blklocs(self):
         """compat with BlockManager"""
         return None
 
-    def getitem_mgr(self, indexer: slice | npt.NDArray[np.bool_]) -> SingleBlockManager:
+    def getitem_mgr(self, indexer: slice | np.ndarray) -> SingleBlockManager:
         # similar to get_slice, but not restricted to slice indexer
         blk = self._block
+        if (
+            using_copy_on_write()
+            and isinstance(indexer, np.ndarray)
+            and len(indexer) > 0
+            and com.is_bool_indexer(indexer)
+            and indexer.all()
+        ):
+            return type(self)(blk, self.index, [weakref.ref(blk)], parent=self)
         array = blk._slice(indexer)
         if array.ndim > 1:
             # This will be caught by Series._get_values

pandas/core/nanops.py

@@ -370,9 +370,9 @@ def _wrap_results(result, dtype: np.dtype, fill_value=None):
                 result = np.nan
 
             if isna(result):
-                result = np.datetime64("NaT", "ns")
+                result = np.datetime64("NaT", "ns").astype(dtype)
             else:
-                result = np.int64(result).view("datetime64[ns]")
+                result = np.int64(result).view(dtype)
             # retain original unit
             result = result.astype(dtype, copy=False)
         else:

pandas/core/series.py

@@ -921,7 +921,7 @@ def _ixs(self, i: int, axis: AxisInt = 0) -> Any:
         """
         return self._values[i]
 
-    def _slice(self, slobj: slice, axis: Axis = 0) -> Series:
+    def _slice(self, slobj: slice | np.ndarray, axis: Axis = 0) -> Series:
         # axis kwarg is retained for compat with NDFrame method
         #  _slice is *always* positional
         return self._get_values(slobj)
@@ -3559,6 +3559,13 @@ def sort_values(
         values_to_sort = ensure_key_mapped(self, key)._values if key else self._values
         sorted_index = nargsort(values_to_sort, kind, bool(ascending), na_position)
 
+        if array_equal_fast(
+            sorted_index, np.arange(0, len(sorted_index), dtype=sorted_index.dtype)
+        ):
+            if inplace:
+                return self._update_inplace(self)
+            return self.copy(deep=None)
+
         result = self._constructor(
             self._values[sorted_index], index=self.index[sorted_index]
         )
@@ -5576,7 +5583,7 @@ def dropna(
                 return result
         else:
             if not inplace:
-                return self.copy()
+                return self.copy(deep=None)
         return None
 
     # ----------------------------------------------------------------------

pandas/tests/arrays/datetimes/test_reductions.py

@@ -10,6 +10,10 @@
 
 
 class TestReductions:
+    @pytest.fixture(params=["s", "ms", "us", "ns"])
+    def unit(self, request):
+        return request.param
+
     @pytest.fixture
     def arr1d(self, tz_naive_fixture):
         """Fixture returning DatetimeArray with parametrized timezones"""
@@ -28,17 +32,20 @@ def arr1d(self, tz_naive_fixture):
         )
         return arr
 
-    def test_min_max(self, arr1d):
+    def test_min_max(self, arr1d, unit):
         arr = arr1d
+        arr = arr.as_unit(unit)
         tz = arr.tz
 
         result = arr.min()
-        expected = pd.Timestamp("2000-01-02", tz=tz)
+        expected = pd.Timestamp("2000-01-02", tz=tz).as_unit(unit)
         assert result == expected
+        assert result.unit == expected.unit
 
         result = arr.max()
-        expected = pd.Timestamp("2000-01-05", tz=tz)
+        expected = pd.Timestamp("2000-01-05", tz=tz).as_unit(unit)
         assert result == expected
+        assert result.unit == expected.unit
 
         result = arr.min(skipna=False)
         assert result is NaT

pandas/tests/copy_view/test_internals.py

@@ -65,6 +65,7 @@ def test_clear_parent(using_copy_on_write):
     assert subset._mgr.parent is None
 
 
+@pytest.mark.single_cpu
 @td.skip_array_manager_invalid_test
 def test_switch_options():
     # ensure we can switch the value of the option within one session

pandas/tests/copy_view/test_methods.py

@@ -503,6 +503,40 @@ def test_add_suffix(using_copy_on_write):
     tm.assert_frame_equal(df, df_orig)
 
 
+@pytest.mark.parametrize("axis, val", [(0, 5.5), (1, np.nan)])
+def test_dropna(using_copy_on_write, axis, val):
+    df = DataFrame({"a": [1, 2, 3], "b": [4, val, 6], "c": "d"})
+    df_orig = df.copy()
+    df2 = df.dropna(axis=axis)
+
+    if using_copy_on_write:
+        assert np.shares_memory(get_array(df2, "a"), get_array(df, "a"))
+    else:
+        assert not np.shares_memory(get_array(df2, "a"), get_array(df, "a"))
+
+    df2.iloc[0, 0] = 0
+    if using_copy_on_write:
+        assert not np.shares_memory(get_array(df2, "a"), get_array(df, "a"))
+    tm.assert_frame_equal(df, df_orig)
+
+
+@pytest.mark.parametrize("val", [5, 5.5])
+def test_dropna_series(using_copy_on_write, val):
+    ser = Series([1, val, 4])
+    ser_orig = ser.copy()
+    ser2 = ser.dropna()
+
+    if using_copy_on_write:
+        assert np.shares_memory(ser2.values, ser.values)
+    else:
+        assert not np.shares_memory(ser2.values, ser.values)
+
+    ser2.iloc[0] = 0
+    if using_copy_on_write:
+        assert not np.shares_memory(ser2.values, ser.values)
+    tm.assert_series_equal(ser, ser_orig)
+
+
 @pytest.mark.parametrize(
     "method",
     [
@@ -644,6 +678,45 @@ def test_sort_index(using_copy_on_write):
     tm.assert_series_equal(ser, ser_orig)
 
 
+@pytest.mark.parametrize(
+    "obj, kwargs",
+    [(Series([1, 2, 3], name="a"), {}), (DataFrame({"a": [1, 2, 3]}), {"by": "a"})],
+)
+def test_sort_values(using_copy_on_write, obj, kwargs):
+    obj_orig = obj.copy()
+    obj2 = obj.sort_values(**kwargs)
+
+    if using_copy_on_write:
+        assert np.shares_memory(get_array(obj2, "a"), get_array(obj, "a"))
+    else:
+        assert not np.shares_memory(get_array(obj2, "a"), get_array(obj, "a"))
+
+    # mutating df triggers a copy-on-write for the column / block
+    obj2.iloc[0] = 0
+    assert not np.shares_memory(get_array(obj2, "a"), get_array(obj, "a"))
+    tm.assert_equal(obj, obj_orig)
+
+
+@pytest.mark.parametrize(
+    "obj, kwargs",
+    [(Series([1, 2, 3], name="a"), {}), (DataFrame({"a": [1, 2, 3]}), {"by": "a"})],
+)
+def test_sort_values_inplace(using_copy_on_write, obj, kwargs, using_array_manager):
+    obj_orig = obj.copy()
+    view = obj[:]
+    obj.sort_values(inplace=True, **kwargs)
+
+    assert np.shares_memory(get_array(obj, "a"), get_array(view, "a"))
+
+    # mutating obj triggers a copy-on-write for the column / block
+    obj.iloc[0] = 0
+    if using_copy_on_write:
+        assert not np.shares_memory(get_array(obj, "a"), get_array(view, "a"))
+        tm.assert_equal(view, obj_orig)
+    else:
+        assert np.shares_memory(get_array(obj, "a"), get_array(view, "a"))
+
+
 def test_reorder_levels(using_copy_on_write):
     index = MultiIndex.from_tuples(
         [(1, 1), (1, 2), (2, 1), (2, 2)], names=["one", "two"]

pandas/tests/frame/methods/test_sort_values.py

@@ -619,6 +619,14 @@ def test_sort_values_reshaping(self):
 
         tm.assert_frame_equal(df, expected)
 
+    def test_sort_values_no_by_inplace(self):
+        # GH#50643
+        df = DataFrame({"a": [1, 2, 3]})
+        expected = df.copy()
+        result = df.sort_values(by=[], inplace=True)
+        tm.assert_frame_equal(df, expected)
+        assert result is None
+
 
 class TestDataFrameSortKey:  # test key sorting (issue 27237)
     def test_sort_values_inplace_key(self, sort_by_key):

pandas/tests/test_nanops.py

@@ -1157,10 +1157,14 @@ def prng(self):
 
 
 class TestDatetime64NaNOps:
+    @pytest.fixture(params=["s", "ms", "us", "ns"])
+    def unit(self, request):
+        return request.param
+
     # Enabling mean changes the behavior of DataFrame.mean
     # See https://github.com/pandas-dev/pandas/issues/24752
-    def test_nanmean(self):
-        dti = pd.date_range("2016-01-01", periods=3)
+    def test_nanmean(self, unit):
+        dti = pd.date_range("2016-01-01", periods=3).as_unit(unit)
         expected = dti[1]
 
         for obj in [dti, DatetimeArray(dti), Series(dti)]:
@@ -1173,8 +1177,9 @@ def test_nanmean(self):
             result = nanops.nanmean(obj)
             assert result == expected
 
-    @pytest.mark.parametrize("dtype", ["M8[ns]", "m8[ns]"])
-    def test_nanmean_skipna_false(self, dtype):
+    @pytest.mark.parametrize("constructor", ["M8", "m8"])
+    def test_nanmean_skipna_false(self, constructor, unit):
+        dtype = f"{constructor}[{unit}]"
         arr = np.arange(12).astype(np.int64).view(dtype).reshape(4, 3)
 
         arr[-1, -1] = "NaT"