Merge branch 'master' of https://github.com/pandas-dev/pandas into udfs_poc

Author: rhshadrach

asv_bench/benchmarks/indexing_engines.py

@@ -48,7 +48,7 @@ def setup(self, engine_and_dtype, index_type):
             "non_monotonic": np.array([1, 2, 3] * N, dtype=dtype),
         }[index_type]
 
-        self.data = engine(lambda: arr, len(arr))
+        self.data = engine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc(2)
 
@@ -70,7 +70,7 @@ def setup(self, index_type):
             "non_monotonic": np.array(list("abc") * N, dtype=object),
         }[index_type]
 
-        self.data = libindex.ObjectEngine(lambda: arr, len(arr))
+        self.data = libindex.ObjectEngine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc("b")
 

doc/source/whatsnew/v1.4.0.rst

@@ -334,7 +334,7 @@ Other Deprecations
 - Deprecated the 'include_start' and 'include_end' arguments in :meth:`DataFrame.between_time`; in a future version passing 'include_start' or 'include_end' will raise (:issue:`40245`)
 - Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`)
 - Deprecated the ``index`` argument to :class:`SparseArray` construction (:issue:`23089`)
--
+- Deprecated :meth:`.Rolling.validate`, :meth:`.Expanding.validate`, and :meth:`.ExponentialMovingWindow.validate` (:issue:`43665`)
 
 .. ---------------------------------------------------------------------------
 
@@ -486,6 +486,7 @@ Reshaping
 - Bug in :meth:`DataFrame.append` failing to retain dtypes when appended columns do not match (:issue:`43392`)
 - Bug in :func:`concat` of ``bool`` and ``boolean`` dtypes resulting in ``object`` dtype instead of ``boolean`` dtype (:issue:`42800`)
 - Bug in :func:`crosstab` when inputs are are categorical Series, there are categories that are not present in one or both of the Series, and ``margins=True``. Previously the margin value for missing categories was ``NaN``. It is now correctly reported as 0 (:issue:`43505`)
+- Bug in :func:`concat` would fail when the ``objs`` argument all had the same index and the ``keys`` argument contained duplicates (:issue:`43595`)
 
 Sparse
 ^^^^^^

pandas/_libs/index.pyi

@@ -6,7 +6,7 @@ from pandas import MultiIndex
 
 class IndexEngine:
     over_size_threshold: bool
-    def __init__(self, vgetter, n: int): ...
+    def __init__(self, values: np.ndarray): ...
     def __contains__(self, val: object) -> bool: ...
     # -> int | slice | np.ndarray[bool]
     def get_loc(self, val: object) -> int | slice | np.ndarray: ...

pandas/_libs/index.pyx

@@ -53,18 +53,18 @@ _SIZE_CUTOFF = 1_000_000
 cdef class IndexEngine:
 
     cdef readonly:
-        object vgetter
+        ndarray values
         HashTable mapping
         bint over_size_threshold
 
     cdef:
         bint unique, monotonic_inc, monotonic_dec
         bint need_monotonic_check, need_unique_check
 
-    def __init__(self, vgetter, n):
-        self.vgetter = vgetter
+    def __init__(self, ndarray values):
+        self.values = values
 
-        self.over_size_threshold = n >= _SIZE_CUTOFF
+        self.over_size_threshold = len(values) >= _SIZE_CUTOFF
         self.clear_mapping()
 
     def __contains__(self, val: object) -> bool:
@@ -214,8 +214,8 @@ cdef class IndexEngine:
             self.unique = 1
             self.need_unique_check = 0
 
-    cdef _get_index_values(self):
-        return self.vgetter()
+    cdef ndarray _get_index_values(self):
+        return self.values
 
     cdef _call_monotonic(self, values):
         return algos.is_monotonic(values, timelike=False)
@@ -438,8 +438,8 @@ cdef class DatetimeEngine(Int64Engine):
         self._ensure_mapping_populated()
         return conv in self.mapping
 
-    cdef _get_index_values(self):
-        return self.vgetter().view('i8')
+    cdef ndarray _get_index_values(self):
+        return self.values.view('i8')
 
     cdef _call_monotonic(self, values):
         return algos.is_monotonic(values, timelike=True)
@@ -537,9 +537,6 @@ cdef class PeriodEngine(Int64Engine):
 
         return Int64Engine.get_loc(self, conv)
 
-    cdef _get_index_values(self):
-        return super(PeriodEngine, self).vgetter().view("i8")
-
     cdef _call_monotonic(self, values):
         return algos.is_monotonic(values, timelike=True)
 
@@ -598,7 +595,7 @@ cdef class BaseMultiIndexCodesEngine:
 
         # Initialize underlying index (e.g. libindex.UInt64Engine) with
         # integers representing labels: we will use its get_loc and get_indexer
-        self._base.__init__(self, lambda: lab_ints, len(lab_ints))
+        self._base.__init__(self, lab_ints)
 
     def _codes_to_ints(self, ndarray[uint64_t] codes) -> np.ndarray:
         raise NotImplementedError("Implemented by subclass")

pandas/_libs/index_class_helper.pxi.in

@@ -28,8 +28,6 @@ dtypes = [('Float64', 'float64'),
 
 
 cdef class {{name}}Engine(IndexEngine):
-    # constructor-caller is responsible for ensuring that vgetter()
-    #  returns an ndarray with dtype {{dtype}}_t
 
     cdef _make_hash_table(self, Py_ssize_t n):
         return _hash.{{name}}HashTable(n)

pandas/core/indexes/base.py

@@ -819,7 +819,7 @@ def _engine(self) -> libindex.IndexEngine:
         # to avoid a reference cycle, bind `target_values` to a local variable, so
         # `self` is not passed into the lambda.
         target_values = self._get_engine_target()
-        return self._engine_type(lambda: target_values, len(self))
+        return self._engine_type(target_values)
 
     @final
     @cache_readonly

pandas/core/window/rolling.py

@@ -156,7 +156,7 @@ def __init__(
             )
 
         self._selection = selection
-        self.validate()
+        self._validate()
 
     @property
     def win_type(self):
@@ -180,6 +180,14 @@ def is_datetimelike(self) -> bool:
         return self._win_freq_i8 is not None
 
     def validate(self) -> None:
+        warnings.warn(
+            "validate is deprecated and will be removed in a future version.",
+            FutureWarning,
+            stacklevel=2,
+        )
+        return self._validate()
+
+    def _validate(self) -> None:
         if self.center is not None and not is_bool(self.center):
             raise ValueError("center must be a boolean")
         if self.min_periods is not None:
@@ -960,8 +968,8 @@ class Window(BaseWindow):
         "method",
     ]
 
-    def validate(self):
-        super().validate()
+    def _validate(self):
+        super()._validate()
 
         if not isinstance(self.win_type, str):
             raise ValueError(f"Invalid win_type {self.win_type}")
@@ -1528,8 +1536,8 @@ class Rolling(RollingAndExpandingMixin):
         "method",
     ]
 
-    def validate(self):
-        super().validate()
+    def _validate(self):
+        super()._validate()
 
         # we allow rolling on a datetimelike index
         if (

pandas/tests/indexes/test_engines.py

@@ -96,18 +96,18 @@ def test_is_monotonic(self, numeric_indexing_engine_type_and_dtype):
         arr = np.array([1] * num + [2] * num + [3] * num, dtype=dtype)
 
         # monotonic increasing
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
         assert engine.is_monotonic_increasing is True
         assert engine.is_monotonic_decreasing is False
 
         # monotonic decreasing
-        engine = engine_type(lambda: arr[::-1], len(arr))
+        engine = engine_type(arr[::-1])
         assert engine.is_monotonic_increasing is False
         assert engine.is_monotonic_decreasing is True
 
         # neither monotonic increasing or decreasing
         arr = np.array([1] * num + [2] * num + [1] * num, dtype=dtype)
-        engine = engine_type(lambda: arr[::-1], len(arr))
+        engine = engine_type(arr[::-1])
         assert engine.is_monotonic_increasing is False
         assert engine.is_monotonic_decreasing is False
 
@@ -116,31 +116,31 @@ def test_is_unique(self, numeric_indexing_engine_type_and_dtype):
 
         # unique
         arr = np.array([1, 3, 2], dtype=dtype)
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
         assert engine.is_unique is True
 
         # not unique
         arr = np.array([1, 2, 1], dtype=dtype)
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
         assert engine.is_unique is False
 
     def test_get_loc(self, numeric_indexing_engine_type_and_dtype):
         engine_type, dtype = numeric_indexing_engine_type_and_dtype
 
         # unique
         arr = np.array([1, 2, 3], dtype=dtype)
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
         assert engine.get_loc(2) == 1
 
         # monotonic
         num = 1000
         arr = np.array([1] * num + [2] * num + [3] * num, dtype=dtype)
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
         assert engine.get_loc(2) == slice(1000, 2000)
 
         # not monotonic
         arr = np.array([1, 2, 3] * num, dtype=dtype)
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
         expected = np.array([False, True, False] * num, dtype=bool)
         result = engine.get_loc(2)
         assert (result == expected).all()
@@ -149,7 +149,7 @@ def test_get_backfill_indexer(self, numeric_indexing_engine_type_and_dtype):
         engine_type, dtype = numeric_indexing_engine_type_and_dtype
 
         arr = np.array([1, 5, 10], dtype=dtype)
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
 
         new = np.arange(12, dtype=dtype)
         result = engine.get_backfill_indexer(new)
@@ -161,7 +161,7 @@ def test_get_pad_indexer(self, numeric_indexing_engine_type_and_dtype):
         engine_type, dtype = numeric_indexing_engine_type_and_dtype
 
         arr = np.array([1, 5, 10], dtype=dtype)
-        engine = engine_type(lambda: arr, len(arr))
+        engine = engine_type(arr)
 
         new = np.arange(12, dtype=dtype)
         result = engine.get_pad_indexer(new)
@@ -181,54 +181,54 @@ def test_is_monotonic(self):
         arr = np.array(["a"] * num + ["a"] * num + ["c"] * num, dtype=self.dtype)
 
         # monotonic increasing
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
         assert engine.is_monotonic_increasing is True
         assert engine.is_monotonic_decreasing is False
 
         # monotonic decreasing
-        engine = self.engine_type(lambda: arr[::-1], len(arr))
+        engine = self.engine_type(arr[::-1])
         assert engine.is_monotonic_increasing is False
         assert engine.is_monotonic_decreasing is True
 
         # neither monotonic increasing or decreasing
         arr = np.array(["a"] * num + ["b"] * num + ["a"] * num, dtype=self.dtype)
-        engine = self.engine_type(lambda: arr[::-1], len(arr))
+        engine = self.engine_type(arr[::-1])
         assert engine.is_monotonic_increasing is False
         assert engine.is_monotonic_decreasing is False
 
     def test_is_unique(self):
         # unique
         arr = np.array(self.values, dtype=self.dtype)
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
         assert engine.is_unique is True
 
         # not unique
         arr = np.array(["a", "b", "a"], dtype=self.dtype)
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
         assert engine.is_unique is False
 
     def test_get_loc(self):
         # unique
         arr = np.array(self.values, dtype=self.dtype)
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
         assert engine.get_loc("b") == 1
 
         # monotonic
         num = 1000
         arr = np.array(["a"] * num + ["b"] * num + ["c"] * num, dtype=self.dtype)
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
         assert engine.get_loc("b") == slice(1000, 2000)
 
         # not monotonic
         arr = np.array(self.values * num, dtype=self.dtype)
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
         expected = np.array([False, True, False] * num, dtype=bool)
         result = engine.get_loc("b")
         assert (result == expected).all()
 
     def test_get_backfill_indexer(self):
         arr = np.array(["a", "e", "j"], dtype=self.dtype)
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
 
         new = np.array(list("abcdefghij"), dtype=self.dtype)
         result = engine.get_backfill_indexer(new)
@@ -238,7 +238,7 @@ def test_get_backfill_indexer(self):
 
     def test_get_pad_indexer(self):
         arr = np.array(["a", "e", "j"], dtype=self.dtype)
-        engine = self.engine_type(lambda: arr, len(arr))
+        engine = self.engine_type(arr)
 
         new = np.array(list("abcdefghij"), dtype=self.dtype)
         result = engine.get_pad_indexer(new)

pandas/tests/reshape/concat/test_concat.py

@@ -3,11 +3,16 @@
     deque,
 )
 from decimal import Decimal
-from warnings import catch_warnings
+from warnings import (
+    catch_warnings,
+    simplefilter,
+)
 
 import numpy as np
 import pytest
 
+from pandas.errors import PerformanceWarning
+
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -560,6 +565,22 @@ def test_duplicate_keys(keys):
     tm.assert_frame_equal(result, expected)
 
 
+def test_duplicate_keys_same_frame():
+    # GH 43595
+    keys = ["e", "e"]
+    df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = concat([df, df], axis=1, keys=keys)
+    expected_values = [[1, 4, 1, 4], [2, 5, 2, 5], [3, 6, 3, 6]]
+    expected_columns = MultiIndex.from_tuples(
+        [(keys[0], "a"), (keys[0], "b"), (keys[1], "a"), (keys[1], "b")]
+    )
+    expected = DataFrame(expected_values, columns=expected_columns)
+    with catch_warnings():
+        # result.columns not sorted, resulting in performance warning
+        simplefilter("ignore", PerformanceWarning)
+        tm.assert_frame_equal(result, expected)
+
+
 @pytest.mark.parametrize(
     "obj",
     [

pandas/tests/window/test_api.py

@@ -339,6 +339,12 @@ def test_is_datetimelike_deprecated():
         assert not s.is_datetimelike
 
 
+def test_validate_deprecated():
+    s = Series(range(1)).rolling(1)
+    with tm.assert_produces_warning(FutureWarning):
+        assert s.validate() is None
+
+
 @pytest.mark.filterwarnings("ignore:min_periods:FutureWarning")
 def test_dont_modify_attributes_after_methods(
     arithmetic_win_operators, closed, center, min_periods