Merge branch 'main' into gh52234-nan-not-in-index

Author: ssche

doc/source/whatsnew/v2.1.0.rst

@@ -19,6 +19,7 @@ Enhancements
 Copy-on-Write improvements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- :meth:`Series.transform` not respecting Copy-on-Write when ``func`` modifies :class:`Series` inplace (:issue:`53747`)
 - Calling :meth:`Index.values` will now return a read-only NumPy array (:issue:`53704`)
 - Setting a :class:`Series` into a :class:`DataFrame` now creates a lazy instead of a deep copy (:issue:`53142`)
 - The :class:`DataFrame` constructor, when constructing a DataFrame from a dictionary
@@ -288,6 +289,7 @@ Deprecations
 - Deprecated :func:`value_counts`, use ``pd.Series(obj).value_counts()`` instead (:issue:`47862`)
 - Deprecated :meth:`Series.first` and :meth:`DataFrame.first` (please create a mask and filter using ``.loc`` instead) (:issue:`45908`)
 - Deprecated :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` for object-dtype (:issue:`53631`)
+- Deprecated :meth:`Series.last` and :meth:`DataFrame.last` (please create a mask and filter using ``.loc`` instead) (:issue:`53692`)
 - Deprecated allowing ``downcast`` keyword other than ``None``, ``False``, "infer", or a dict with these as values in :meth:`Series.fillna`, :meth:`DataFrame.fillna` (:issue:`40988`)
 - Deprecated allowing arbitrary ``fill_value`` in :class:`SparseDtype`, in a future version the ``fill_value`` will need to be compatible with the ``dtype.subtype``, either a scalar that can be held by that subtype or ``NaN`` for integer or bool subtypes (:issue:`23124`)
 - Deprecated behavior of :func:`assert_series_equal` and :func:`assert_frame_equal` considering NA-like values (e.g. ``NaN`` vs ``None`` as equivalent) (:issue:`52081`)
@@ -491,7 +493,6 @@ Reshaping
 - Bug in :meth:`DataFrame.idxmin` and :meth:`DataFrame.idxmax`, where the axis dtype would be lost for empty frames (:issue:`53265`)
 - Bug in :meth:`DataFrame.merge` not merging correctly when having ``MultiIndex`` with single level (:issue:`52331`)
 - Bug in :meth:`DataFrame.stack` losing extension dtypes when columns is a :class:`MultiIndex` and frame contains mixed dtypes (:issue:`45740`)
-- Bug in :meth:`DataFrame.stack` would incorrectly order results when ``sort=True`` and the input had :class:`MultiIndex` levels that were not sorted (:issue:`53636`)
 - Bug in :meth:`DataFrame.transpose` inferring dtype for object column (:issue:`51546`)
 - Bug in :meth:`Series.combine_first` converting ``int64`` dtype to ``float64`` and losing precision on very large integers (:issue:`51764`)
 -

pandas/conftest.py

@@ -30,6 +30,7 @@
 from decimal import Decimal
 import operator
 import os
+from pathlib import Path
 from typing import (
     Callable,
     Hashable,
@@ -1167,6 +1168,16 @@ def strict_data_files(pytestconfig):
     return pytestconfig.getoption("--strict-data-files")
 
 
+@pytest.fixture
+def tests_path() -> Path:
+    return Path(__file__).parent / "tests"
+
+
+@pytest.fixture
+def tests_io_data_path(tests_path) -> Path:
+    return tests_path / "io" / "data"
+
+
 @pytest.fixture
 def datapath(strict_data_files: str) -> Callable[..., str]:
     """

pandas/core/generic.py

@@ -9321,6 +9321,11 @@ def last(self, offset) -> Self:
         at_time : Select values at a particular time of the day.
         between_time : Select values between particular times of the day.
 
+        Notes
+        -----
+        .. deprecated:: 2.1.0
+            Please create a mask and filter using `.loc` instead
+
         Examples
         --------
         >>> i = pd.date_range('2018-04-09', periods=4, freq='2D')
@@ -9334,7 +9339,7 @@ def last(self, offset) -> Self:
 
         Get the rows for the last 3 days:
 
-        >>> ts.last('3D')
+        >>> ts.last('3D') # doctest: +SKIP
                     A
         2018-04-13  3
         2018-04-15  4
@@ -9343,6 +9348,13 @@ def last(self, offset) -> Self:
         3 observed days in the dataset, and therefore data for 2018-04-11 was
         not returned.
         """
+        warnings.warn(
+            "last is deprecated and will be removed in a future version. "
+            "Please create a mask and filter using `.loc` instead",
+            FutureWarning,
+            stacklevel=find_stack_level(),
+        )
+
         if not isinstance(self.index, DatetimeIndex):
             raise TypeError("'last' only supports a DatetimeIndex index")
 

pandas/core/reshape/reshape.py

@@ -756,16 +756,7 @@ def _convert_level_number(level_num: int, columns: Index):
     level_vals = mi_cols.levels[-1]
     level_codes = unique(mi_cols.codes[-1])
     if sort:
-        _, index, inverse = np.unique(
-            level_vals, return_index=True, return_inverse=True
-        )
-        sorted_level_vals = np.take(level_vals, index)
         level_codes = np.sort(level_codes)
-        # Take level_codes according to where level_vals get sorted to, while
-        # also allowing for NA (-1) values
-        level_codes = np.where(level_codes == -1, -1, np.take(inverse, level_codes))
-    else:
-        sorted_level_vals = level_vals
     level_vals_nan = level_vals.insert(len(level_vals), None)
 
     level_vals_used = np.take(level_vals_nan, level_codes)
@@ -827,7 +818,7 @@ def _convert_level_number(level_num: int, columns: Index):
         new_codes = [old_codes.repeat(levsize)]
         new_names = [this.index.name]  # something better?
 
-    new_levels.append(sorted_level_vals)
+    new_levels.append(level_vals)
     new_codes.append(np.tile(level_codes, N))
     new_names.append(frame.columns.names[level_num])
 

pandas/core/series.py

@@ -4499,7 +4499,8 @@ def transform(
     ) -> DataFrame | Series:
         # Validate axis argument
         self._get_axis_number(axis)
-        result = SeriesApply(self, func=func, args=args, kwargs=kwargs).transform()
+        ser = self.copy(deep=False) if using_copy_on_write() else self
+        result = SeriesApply(ser, func=func, args=args, kwargs=kwargs).transform()
         return result
 
     def apply(

pandas/io/xml.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import io
+from os import PathLike
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -326,10 +327,13 @@ def _iterparse_nodes(self, iterparse: Callable) -> list[dict[str, str | None]]:
             )
 
         if (not hasattr(self.path_or_buffer, "read")) and (
-            not isinstance(self.path_or_buffer, str)
+            not isinstance(self.path_or_buffer, (str, PathLike))
             or is_url(self.path_or_buffer)
             or is_fsspec_url(self.path_or_buffer)
-            or self.path_or_buffer.startswith(("<?xml", "<"))
+            or (
+                isinstance(self.path_or_buffer, str)
+                and self.path_or_buffer.startswith(("<?xml", "<"))
+            )
             or infer_compression(self.path_or_buffer, "infer") is not None
         ):
             raise ParserError(

pandas/tests/copy_view/test_methods.py

@@ -1764,6 +1764,32 @@ def test_transpose_ea_single_column(using_copy_on_write):
     assert not np.shares_memory(get_array(df, "a"), get_array(result, 0))
 
 
+def test_transform_frame(using_copy_on_write):
+    df = DataFrame({"a": [1, 2, 3], "b": 1})
+    df_orig = df.copy()
+
+    def func(ser):
+        ser.iloc[0] = 100
+        return ser
+
+    df.transform(func)
+    if using_copy_on_write:
+        tm.assert_frame_equal(df, df_orig)
+
+
+def test_transform_series(using_copy_on_write):
+    ser = Series([1, 2, 3])
+    ser_orig = ser.copy()
+
+    def func(ser):
+        ser.iloc[0] = 100
+        return ser
+
+    ser.transform(func)
+    if using_copy_on_write:
+        tm.assert_series_equal(ser, ser_orig)
+
+
 def test_count_read_only_array():
     df = DataFrame({"a": [1, 2], "b": 3})
     result = df.count()

pandas/tests/frame/methods/test_first_and_last.py

@@ -11,6 +11,7 @@
 import pandas._testing as tm
 
 deprecated_msg = "first is deprecated"
+last_deprecated_msg = "last is deprecated"
 
 
 class TestFirst:
@@ -55,29 +56,38 @@ def test_first_last_raises(self, frame_or_series):
             obj.first("1D")
 
         msg = "'last' only supports a DatetimeIndex index"
-        with pytest.raises(TypeError, match=msg):  # index is not a DatetimeIndex
+        with tm.assert_produces_warning(
+            FutureWarning, match=last_deprecated_msg
+        ), pytest.raises(
+            TypeError, match=msg
+        ):  # index is not a DatetimeIndex
             obj.last("1D")
 
     def test_last_subset(self, frame_or_series):
         ts = tm.makeTimeDataFrame(freq="12h")
         ts = tm.get_obj(ts, frame_or_series)
-        result = ts.last("10d")
+        with tm.assert_produces_warning(FutureWarning, match=last_deprecated_msg):
+            result = ts.last("10d")
         assert len(result) == 20
 
         ts = tm.makeTimeDataFrame(nper=30, freq="D")
         ts = tm.get_obj(ts, frame_or_series)
-        result = ts.last("10d")
+        with tm.assert_produces_warning(FutureWarning, match=last_deprecated_msg):
+            result = ts.last("10d")
         assert len(result) == 10
 
-        result = ts.last("21D")
+        with tm.assert_produces_warning(FutureWarning, match=last_deprecated_msg):
+            result = ts.last("21D")
         expected = ts["2000-01-10":]
         tm.assert_equal(result, expected)
 
-        result = ts.last("21D")
+        with tm.assert_produces_warning(FutureWarning, match=last_deprecated_msg):
+            result = ts.last("21D")
         expected = ts[-21:]
         tm.assert_equal(result, expected)
 
-        result = ts[:0].last("3M")
+        with tm.assert_produces_warning(FutureWarning, match=last_deprecated_msg):
+            result = ts[:0].last("3M")
         tm.assert_equal(result, ts[:0])
 
     @pytest.mark.parametrize("start, periods", [("2010-03-31", 1), ("2010-03-30", 2)])
@@ -104,7 +114,8 @@ def test_first_with_first_day_end_of_frq_n_greater_one(self, frame_or_series):
     def test_empty_not_input(self):
         # GH#51032
         df = DataFrame(index=pd.DatetimeIndex([]))
-        result = df.last(offset=1)
+        with tm.assert_produces_warning(FutureWarning, match=last_deprecated_msg):
+            result = df.last(offset=1)
 
         with tm.assert_produces_warning(FutureWarning, match=deprecated_msg):
             result = df.first(offset=1)

pandas/tests/frame/test_stack_unstack.py

@@ -2000,20 +2000,18 @@ def __init__(self, *args, **kwargs) -> None:
         ),
     )
     @pytest.mark.parametrize("stack_lev", range(2))
-    @pytest.mark.parametrize("sort", [True, False])
-    def test_stack_order_with_unsorted_levels(self, levels, stack_lev, sort):
+    def test_stack_order_with_unsorted_levels(self, levels, stack_lev):
         # GH#16323
         # deep check for 1-row case
         columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]])
         df = DataFrame(columns=columns, data=[range(4)])
-        df_stacked = df.stack(stack_lev, sort=sort)
-        for row in df.index:
-            for col in df.columns:
-                expected = df.loc[row, col]
-                result_row = row, col[stack_lev]
-                result_col = col[1 - stack_lev]
-                result = df_stacked.loc[result_row, result_col]
-                assert result == expected
+        df_stacked = df.stack(stack_lev)
+        assert all(
+            df.loc[row, col]
+            == df_stacked.loc[(row, col[stack_lev]), col[1 - stack_lev]]
+            for row in df.index
+            for col in df.columns
+        )
 
     def test_stack_order_with_unsorted_levels_multi_row(self):
         # GH#16323
@@ -2032,26 +2030,6 @@ def test_stack_order_with_unsorted_levels_multi_row(self):
             for col in df.columns
         )
 
-    def test_stack_order_with_unsorted_levels_multi_row_2(self):
-        # GH#53636
-        levels = ((0, 1), (1, 0))
-        stack_lev = 1
-        columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]])
-        df = DataFrame(columns=columns, data=[range(4)], index=[1, 0, 2, 3])
-        result = df.stack(stack_lev, sort=True)
-        expected_index = MultiIndex(
-            levels=[[0, 1, 2, 3], [0, 1]],
-            codes=[[1, 1, 0, 0, 2, 2, 3, 3], [1, 0, 1, 0, 1, 0, 1, 0]],
-        )
-        expected = DataFrame(
-            {
-                0: [0, 1, 0, 1, 0, 1, 0, 1],
-                1: [2, 3, 2, 3, 2, 3, 2, 3],
-            },
-            index=expected_index,
-        )
-        tm.assert_frame_equal(result, expected)
-
     def test_stack_unstack_unordered_multiindex(self):
         # GH# 18265
         values = np.arange(5)

pandas/tests/generic/test_finalize.py

@@ -395,7 +395,8 @@ def ndframe_method(request):
 
 
 @pytest.mark.filterwarnings(
-    "ignore:DataFrame.fillna with 'method' is deprecated:FutureWarning"
+    "ignore:DataFrame.fillna with 'method' is deprecated:FutureWarning",
+    "ignore:last is deprecated:FutureWarning",
 )
 def test_finalize_called(ndframe_method):
     cls, init_args, method = ndframe_method
@@ -423,6 +424,23 @@ def test_finalize_first(data):
         assert result.attrs == {"a": 1}
 
 
+@pytest.mark.parametrize(
+    "data",
+    [
+        pd.Series(1, pd.date_range("2000", periods=4)),
+        pd.DataFrame({"A": [1, 1, 1, 1]}, pd.date_range("2000", periods=4)),
+    ],
+)
+def test_finalize_last(data):
+    # GH 53710
+    deprecated_msg = "last is deprecated"
+
+    data.attrs = {"a": 1}
+    with tm.assert_produces_warning(FutureWarning, match=deprecated_msg):
+        result = data.last("3D")
+        assert result.attrs == {"a": 1}
+
+
 @not_implemented_mark
 def test_finalize_called_eval_numexpr():
     pytest.importorskip("numexpr")

pandas/tests/io/xml/conftest.py

@@ -0,0 +1,31 @@
+import pytest
+
+
+@pytest.fixture
+def xml_data_path(tests_io_data_path):
+    return tests_io_data_path / "xml"
+
+
+@pytest.fixture
+def xml_books(xml_data_path):
+    return xml_data_path / "books.xml"
+
+
+@pytest.fixture
+def xml_doc_ch_utf(xml_data_path):
+    return xml_data_path / "doc_ch_utf.xml"
+
+
+@pytest.fixture
+def xml_baby_names(xml_data_path):
+    return xml_data_path / "baby_names.xml"
+
+
+@pytest.fixture
+def kml_cta_rail_lines(xml_data_path):
+    return xml_data_path / "cta_rail_lines.kml"
+
+
+@pytest.fixture
+def xsl_flatten_doc(xml_data_path):
+    return xml_data_path / "flatten_doc.xsl"