Merge branch 'main' into bug-agg-nonunique-col

Author: luke396

.github/workflows/ubuntu.yml

@@ -28,7 +28,6 @@ jobs:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml, actions-311.yaml]
         pattern: ["not single_cpu", "single_cpu"]
         pyarrow_version: ["8", "9", "10"]
-        pandas_ci: [1]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -75,7 +74,7 @@ jobs:
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
             # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
             # There are some warnings failing the build with -werror
-            pandas_ci: 0
+            pandas_ci: "0"
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "8"
@@ -99,7 +98,7 @@ jobs:
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
-      PANDAS_CI: ${{ matrix.pandas_ci }}
+      PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}

doc/source/whatsnew/v2.1.0.rst

@@ -36,6 +36,7 @@ Other enhancements
 - :class:`api.extensions.ExtensionArray` now has a :meth:`~api.extensions.ExtensionArray.map` method (:issue:`51809`)
 - Improve error message when having incompatible columns using :meth:`DataFrame.merge` (:issue:`51861`)
 - Improved error message when creating a DataFrame with empty data (0 rows), no index and an incorrect number of columns. (:issue:`52084`)
+- :meth:`arrays.SparseArray.map` now supports ``na_action`` (:issue:`52096`).
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_210.notable_bug_fixes:
@@ -236,7 +237,7 @@ Reshaping
 
 Sparse
 ^^^^^^
--
+- Bug in :meth:`arrays.SparseArray.map` allowed the fill value to be included in the sparse values (:issue:`52095`)
 -
 
 ExtensionArray

pandas/_libs/lib.pyx

@@ -752,7 +752,6 @@ cpdef ndarray[object] ensure_string_array(
             out = arr.astype(str).astype(object)
             out[arr.isna()] = na_value
             return out
-
         arr = arr.to_numpy()
     elif not util.is_array(arr):
         arr = np.array(arr, dtype="object")

pandas/core/arrays/arrow/array.py

@@ -2091,7 +2091,10 @@ def _dt_round(
         return self._round_temporally("round", freq, ambiguous, nonexistent)
 
     def _dt_to_pydatetime(self):
-        return np.array(self._pa_array.to_pylist(), dtype=object)
+        data = self._pa_array.to_pylist()
+        if self._dtype.pyarrow_dtype.unit == "ns":
+            data = [ts.to_pydatetime(warn=False) for ts in data]
+        return np.array(data, dtype=object)
 
     def _dt_tz_localize(
         self,

pandas/core/arrays/sparse/array.py

@@ -1305,23 +1305,24 @@ def map(self, mapper, na_action=None) -> Self:
         IntIndex
         Indices: array([1, 2], dtype=int32)
         """
-        if na_action is not None:
-            raise NotImplementedError
+        is_map = isinstance(mapper, (abc.Mapping, ABCSeries))
 
-        # this is used in apply.
-        # We get hit since we're an "is_extension_array_dtype" but regular extension
-        # types are not hit. This may be worth adding to the interface.
-        if isinstance(mapper, ABCSeries):
-            mapper = mapper.to_dict()
+        fill_val = self.fill_value
 
-        if isinstance(mapper, abc.Mapping):
-            fill_value = mapper.get(self.fill_value, self.fill_value)
-            sp_values = [mapper.get(x, None) for x in self.sp_values]
-        else:
-            fill_value = mapper(self.fill_value)
-            sp_values = [mapper(x) for x in self.sp_values]
+        if na_action is None or notna(fill_val):
+            fill_val = mapper.get(fill_val, fill_val) if is_map else mapper(fill_val)
+
+        def func(sp_val):
+            new_sp_val = mapper.get(sp_val, None) if is_map else mapper(sp_val)
+            # check identity and equality because nans are not equal to each other
+            if new_sp_val is fill_val or new_sp_val == fill_val:
+                msg = "fill value in the sparse values not supported"
+                raise ValueError(msg)
+            return new_sp_val
+
+        sp_values = [func(x) for x in self.sp_values]
 
-        return type(self)(sp_values, sparse_index=self.sp_index, fill_value=fill_value)
+        return type(self)(sp_values, sparse_index=self.sp_index, fill_value=fill_val)
 
     def to_dense(self) -> np.ndarray:
         """

pandas/core/arrays/string_.py

@@ -352,6 +352,9 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = Fal
             result[na_values] = libmissing.NA
 
         else:
+            if hasattr(scalars, "type"):
+                # pyarrow array
+                scalars = np.array(scalars)
             # convert non-na-likes to str, and nan-likes to StringDtype().na_value
             result = lib.ensure_string_array(scalars, na_value=libmissing.NA, copy=copy)
 

pandas/core/arrays/string_arrow.py

@@ -151,6 +151,8 @@ def _from_sequence(cls, scalars, dtype: Dtype | None = None, copy: bool = False)
             result = scalars._data
             result = lib.ensure_string_array(result, copy=copy, convert_na_value=False)
             return cls(pa.array(result, mask=na_values, type=pa.string()))
+        elif isinstance(scalars, (pa.Array, pa.ChunkedArray)):
+            return cls(pc.cast(scalars, pa.string()))
 
         # convert non-na-likes to str
         result = lib.ensure_string_array(scalars, copy=copy)

pandas/core/groupby/ops.py

@@ -257,7 +257,7 @@ def _disallow_invalid_ops(self, dtype: DtypeObj, is_numeric: bool = False):
                 #  don't go down a group-by-group path, since in the empty-groups
                 #  case that would fail to raise
                 raise TypeError(f"Cannot perform {how} with non-ordered Categorical")
-            if how not in ["rank", "any", "all"]:
+            if how not in ["rank", "any", "all", "first", "last", "min", "max"]:
                 # only "rank" is implemented in cython
                 raise NotImplementedError(f"{dtype} dtype not supported")
 
@@ -356,11 +356,17 @@ def _ea_wrap_cython_operation(
             )
 
         elif isinstance(values, Categorical):
-            assert self.how in ["rank", "any", "all"]
+            assert self.how in ["rank", "any", "all", "first", "last", "min", "max"]
             mask = values.isna()
             if self.how == "rank":
                 assert values.ordered  # checked earlier
                 npvalues = values._ndarray
+            elif self.how in ["first", "last", "min", "max"]:
+                if self.how in ["min", "max"]:
+                    assert values.ordered  # checked earlier
+                npvalues = values._ndarray
+                result_mask = np.zeros(ngroups, dtype=np.uint8)
+                kwargs["result_mask"] = result_mask
             else:
                 npvalues = values.astype(bool)
 
@@ -373,9 +379,9 @@ def _ea_wrap_cython_operation(
                 **kwargs,
             )
 
-            # If we ever have more than just "rank" here, we'll need to do
-            #  `if self.how in self.cast_blocklist` like we do for other dtypes.
-            return res_values
+            if self.how in self.cast_blocklist:
+                return res_values
+            return values._from_backing_data(res_values)
 
         npvalues = self._ea_to_cython_values(values)
 

pandas/io/sql.py

@@ -964,14 +964,16 @@ def insert_data(self) -> tuple[list[str], list[np.ndarray]]:
         data_list: list[np.ndarray] = [None] * ncols  # type: ignore[list-item]
 
         for i, (_, ser) in enumerate(temp.items()):
-            vals = ser._values
-            if vals.dtype.kind == "M":
-                d = vals.to_pydatetime()
-            elif vals.dtype.kind == "m":
+            if ser.dtype.kind == "M":
+                d = ser.dt.to_pydatetime()
+            elif ser.dtype.kind == "m":
+                vals = ser._values
+                if isinstance(vals, ArrowExtensionArray):
+                    vals = vals.to_numpy(dtype=np.dtype("m8[ns]"))
                 # store as integers, see GH#6921, GH#7076
                 d = vals.view("i8").astype(object)
             else:
-                d = vals.astype(object)
+                d = ser._values.astype(object)
 
             assert isinstance(d, np.ndarray), type(d)
 

pandas/tests/extension/test_arrow.py

@@ -2271,6 +2271,7 @@ def test_dt_to_pydatetime():
     result = ser.dt.to_pydatetime()
     expected = np.array(data, dtype=object)
     tm.assert_numpy_array_equal(result, expected)
+    assert all(type(res) is datetime for res in result)
 
     expected = ser.astype("datetime64[ns]").dt.to_pydatetime()
     tm.assert_numpy_array_equal(result, expected)
@@ -2353,6 +2354,14 @@ def test_concat_empty_arrow_backed_series(dtype):
     tm.assert_series_equal(result, expected)
 
 
+@pytest.mark.parametrize("dtype", ["string", "string[pyarrow]"])
+def test_series_from_string_array(dtype):
+    arr = pa.array("the quick brown fox".split())
+    ser = pd.Series(arr, dtype=dtype)
+    expected = pd.Series(ArrowExtensionArray(arr), dtype=dtype)
+    tm.assert_series_equal(ser, expected)
+
+
 # _data was renamed to _pa_data
 class OldArrowExtensionArray(ArrowExtensionArray):
     def __getstate__(self):

pandas/tests/extension/test_sparse.py

@@ -351,14 +351,27 @@ def test_equals(self, data, na_value, as_series, box):
         self._check_unsupported(data)
         super().test_equals(data, na_value, as_series, box)
 
+    @pytest.mark.parametrize(
+        "func, na_action, expected",
+        [
+            (lambda x: x, None, SparseArray([1.0, np.nan])),
+            (lambda x: x, "ignore", SparseArray([1.0, np.nan])),
+            (str, None, SparseArray(["1.0", "nan"], fill_value="nan")),
+            (str, "ignore", SparseArray(["1.0", np.nan])),
+        ],
+    )
+    def test_map(self, func, na_action, expected):
+        # GH52096
+        data = SparseArray([1, np.nan])
+        result = data.map(func, na_action=na_action)
+        self.assert_extension_array_equal(result, expected)
+
     @pytest.mark.parametrize("na_action", [None, "ignore"])
-    def test_map(self, data, na_action):
-        if na_action is not None:
-            with pytest.raises(NotImplementedError, match=""):
-                data.map(lambda x: x, na_action=na_action)
-        else:
-            result = data.map(lambda x: x, na_action=na_action)
-            self.assert_extension_array_equal(result, data)
+    def test_map_raises(self, data, na_action):
+        # GH52096
+        msg = "fill value in the sparse values not supported"
+        with pytest.raises(ValueError, match=msg):
+            data.map(lambda x: np.nan, na_action=na_action)
 
 
 class TestCasting(BaseSparseTests, base.BaseCastingTests):

pandas/tests/io/test_sql.py

@@ -24,6 +24,7 @@
     date,
     datetime,
     time,
+    timedelta,
 )
 from io import StringIO
 from pathlib import Path
@@ -549,6 +550,26 @@ def test_dataframe_to_sql(conn, test_frame1, request):
     test_frame1.to_sql("test", conn, if_exists="append", index=False)
 
 
+@pytest.mark.db
+@pytest.mark.parametrize("conn", all_connectable)
+def test_dataframe_to_sql_arrow_dtypes(conn, request):
+    # GH 52046
+    pytest.importorskip("pyarrow")
+    df = DataFrame(
+        {
+            "int": pd.array([1], dtype="int8[pyarrow]"),
+            "datetime": pd.array(
+                [datetime(2023, 1, 1)], dtype="timestamp[ns][pyarrow]"
+            ),
+            "timedelta": pd.array([timedelta(1)], dtype="duration[ns][pyarrow]"),
+            "string": pd.array(["a"], dtype="string[pyarrow]"),
+        }
+    )
+    conn = request.getfixturevalue(conn)
+    with tm.assert_produces_warning(UserWarning, match="the 'timedelta'"):
+        df.to_sql("test_arrow", conn, if_exists="replace", index=False)
+
+
 @pytest.mark.db
 @pytest.mark.parametrize("conn", all_connectable)
 @pytest.mark.parametrize("method", [None, "multi"])