Merge remote-tracking branch 'upstream/main' into feat/cleanup_docs

Author: mroeschke

doc/source/whatsnew/v2.1.0.rst

@@ -265,6 +265,7 @@ Other enhancements
 - Many read/to_* functions, such as :meth:`DataFrame.to_pickle` and :func:`read_csv`, support forwarding compression arguments to ``lzma.LZMAFile`` (:issue:`52979`)
 - Reductions :meth:`Series.argmax`, :meth:`Series.argmin`, :meth:`Series.idxmax`, :meth:`Series.idxmin`, :meth:`Index.argmax`, :meth:`Index.argmin`, :meth:`DataFrame.idxmax`, :meth:`DataFrame.idxmin` are now supported for object-dtype (:issue:`4279`, :issue:`18021`, :issue:`40685`, :issue:`43697`)
 - :meth:`DataFrame.to_parquet` and :func:`read_parquet` will now write and read ``attrs`` respectively (:issue:`54346`)
+- :meth:`Series.cummax`, :meth:`Series.cummin` and :meth:`Series.cumprod` are now supported for pyarrow dtypes with pyarrow version 13.0 and above (:issue:`52085`)
 - Added support for the DataFrame Consortium Standard (:issue:`54383`)
 - Performance improvement in :meth:`.DataFrameGroupBy.quantile` and :meth:`.SeriesGroupBy.quantile` (:issue:`51722`)
 

pandas/_libs/src/vendored/ujson/python/objToJSON.c

@@ -1318,6 +1318,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc,
         } else if (PyDate_Check(item) || PyDelta_Check(item)) {
             is_datetimelike = 1;
             if (PyObject_HasAttrString(item, "_value")) {
+                // pd.Timestamp object or pd.NaT
                 // see test_date_index_and_values for case with non-nano
                 i8date = get_long_attr(item, "_value");
             } else {
@@ -1471,12 +1472,12 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
                 }
                 // Currently no way to pass longVal to iso function, so use
                 // state management
-                GET_TC(tc)->longValue = longVal;
+                pc->longValue = longVal;
                 tc->type = JT_UTF8;
             } else {
                 NPY_DATETIMEUNIT base =
                     ((PyObjectEncoder *)tc->encoder)->datetimeUnit;
-                GET_TC(tc)->longValue = NpyDateTimeToEpoch(longVal, base);
+                pc->longValue = NpyDateTimeToEpoch(longVal, base);
                 tc->type = JT_LONG;
             }
         }
@@ -1497,9 +1498,9 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
     if (PyLong_Check(obj)) {
         tc->type = JT_LONG;
         int overflow = 0;
-        GET_TC(tc)->longValue = PyLong_AsLongLongAndOverflow(obj, &overflow);
+        pc->longValue = PyLong_AsLongLongAndOverflow(obj, &overflow);
         int err;
-        err = (GET_TC(tc)->longValue == -1) && PyErr_Occurred();
+        err = (pc->longValue == -1) && PyErr_Occurred();
 
         if (overflow) {
             tc->type = JT_BIGNUM;
@@ -1513,7 +1514,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
         if (npy_isnan(val) || npy_isinf(val)) {
             tc->type = JT_NULL;
         } else {
-            GET_TC(tc)->doubleValue = val;
+            pc->doubleValue = val;
             tc->type = JT_DOUBLE;
         }
         return;
@@ -1526,7 +1527,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
         tc->type = JT_UTF8;
         return;
     } else if (object_is_decimal_type(obj)) {
-        GET_TC(tc)->doubleValue = PyFloat_AsDouble(obj);
+        pc->doubleValue = PyFloat_AsDouble(obj);
         tc->type = JT_DOUBLE;
         return;
     } else if (PyDateTime_Check(obj) || PyDate_Check(obj)) {
@@ -1541,7 +1542,7 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
         } else {
             NPY_DATETIMEUNIT base =
                 ((PyObjectEncoder *)tc->encoder)->datetimeUnit;
-            GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base);
+            pc->longValue = PyDateTimeToEpoch(obj, base);
             tc->type = JT_LONG;
         }
         return;
@@ -1573,12 +1574,13 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
         } else {
             NPY_DATETIMEUNIT base =
                 ((PyObjectEncoder *)tc->encoder)->datetimeUnit;
-            GET_TC(tc)->longValue = PyDateTimeToEpoch(obj, base);
+            pc->longValue = PyDateTimeToEpoch(obj, base);
             tc->type = JT_LONG;
         }
         return;
     } else if (PyDelta_Check(obj)) {
         if (PyObject_HasAttrString(obj, "_value")) {
+            // pd.Timedelta object or pd.NaT
             value = get_long_attr(obj, "_value");
         } else {
             value = total_seconds(obj) * 1000000000LL;  // nanoseconds per sec
@@ -1604,11 +1606,11 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
 
             tc->type = JT_LONG;
         }
-        GET_TC(tc)->longValue = value;
+        pc->longValue = value;
         return;
     } else if (PyArray_IsScalar(obj, Integer)) {
         tc->type = JT_LONG;
-        PyArray_CastScalarToCtype(obj, &(GET_TC(tc)->longValue),
+        PyArray_CastScalarToCtype(obj, &(pc->longValue),
                                   PyArray_DescrFromType(NPY_INT64));
 
         exc = PyErr_Occurred();
@@ -1619,12 +1621,12 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
 
         return;
     } else if (PyArray_IsScalar(obj, Bool)) {
-        PyArray_CastScalarToCtype(obj, &(GET_TC(tc)->longValue),
+        PyArray_CastScalarToCtype(obj, &(pc->longValue),
                                   PyArray_DescrFromType(NPY_BOOL));
-        tc->type = (GET_TC(tc)->longValue) ? JT_TRUE : JT_FALSE;
+        tc->type = (pc->longValue) ? JT_TRUE : JT_FALSE;
         return;
     } else if (PyArray_IsScalar(obj, Float) || PyArray_IsScalar(obj, Double)) {
-        PyArray_CastScalarToCtype(obj, &(GET_TC(tc)->doubleValue),
+        PyArray_CastScalarToCtype(obj, &(pc->doubleValue),
                                   PyArray_DescrFromType(NPY_DOUBLE));
         tc->type = JT_DOUBLE;
         return;

pandas/conftest.py

@@ -1996,3 +1996,8 @@ def warsaw(request) -> str:
     tzinfo for Europe/Warsaw using pytz, dateutil, or zoneinfo.
     """
     return request.param
+
+
+@pytest.fixture()
+def arrow_string_storage():
+    return ("pyarrow",)

pandas/core/arrays/arrow/array.py

@@ -1389,6 +1389,9 @@ def _accumulate(
         NotImplementedError : subclass does not define accumulations
         """
         pyarrow_name = {
+            "cummax": "cumulative_max",
+            "cummin": "cumulative_min",
+            "cumprod": "cumulative_prod_checked",
             "cumsum": "cumulative_sum_checked",
         }.get(name, name)
         pyarrow_meth = getattr(pc, pyarrow_name, None)
@@ -1398,12 +1401,20 @@ def _accumulate(
         data_to_accum = self._pa_array
 
         pa_dtype = data_to_accum.type
-        if pa.types.is_duration(pa_dtype):
-            data_to_accum = data_to_accum.cast(pa.int64())
+
+        convert_to_int = (
+            pa.types.is_temporal(pa_dtype) and name in ["cummax", "cummin"]
+        ) or (pa.types.is_duration(pa_dtype) and name == "cumsum")
+
+        if convert_to_int:
+            if pa_dtype.bit_width == 32:
+                data_to_accum = data_to_accum.cast(pa.int32())
+            else:
+                data_to_accum = data_to_accum.cast(pa.int64())
 
         result = pyarrow_meth(data_to_accum, skip_nulls=skipna, **kwargs)
 
-        if pa.types.is_duration(pa_dtype):
+        if convert_to_int:
             result = result.cast(pa_dtype)
 
         return type(self)(result)

pandas/io/excel/_odswriter.py

@@ -2,6 +2,7 @@
 
 from collections import defaultdict
 import datetime
+import json
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -10,8 +11,6 @@
     overload,
 )
 
-from pandas._libs import json
-
 from pandas.io.excel._base import ExcelWriter
 from pandas.io.excel._util import (
     combine_kwargs,
@@ -257,7 +256,7 @@ def _process_style(self, style: dict[str, Any] | None) -> str | None:
 
         if style is None:
             return None
-        style_key = json.ujson_dumps(style)
+        style_key = json.dumps(style)
         if style_key in self._style_dict:
             return self._style_dict[style_key]
         name = f"pd{len(self._style_dict)+1}"

pandas/io/excel/_xlsxwriter.py

@@ -1,12 +1,11 @@
 from __future__ import annotations
 
+import json
 from typing import (
     TYPE_CHECKING,
     Any,
 )
 
-from pandas._libs import json
-
 from pandas.io.excel._base import ExcelWriter
 from pandas.io.excel._util import (
     combine_kwargs,
@@ -262,7 +261,7 @@ def _write_cells(
         for cell in cells:
             val, fmt = self._value_with_fmt(cell.val)
 
-            stylekey = json.ujson_dumps(cell.style)
+            stylekey = json.dumps(cell.style)
             if fmt:
                 stylekey += fmt
 

pandas/io/json/__init__.py

@@ -1,14 +1,14 @@
 from pandas.io.json._json import (
     read_json,
     to_json,
-    ujson_dumps as dumps,
-    ujson_loads as loads,
+    ujson_dumps,
+    ujson_loads,
 )
 from pandas.io.json._table_schema import build_table_schema
 
 __all__ = [
-    "dumps",
-    "loads",
+    "ujson_dumps",
+    "ujson_loads",
     "read_json",
     "to_json",
     "build_table_schema",

pandas/tests/arrays/string_/test_string.py

@@ -115,8 +115,8 @@ def test_add(dtype):
     tm.assert_series_equal(result, expected)
 
 
-def test_add_2d(dtype, request):
-    if dtype.storage == "pyarrow":
+def test_add_2d(dtype, request, arrow_string_storage):
+    if dtype.storage in arrow_string_storage:
         reason = "Failed: DID NOT RAISE <class 'ValueError'>"
         mark = pytest.mark.xfail(raises=None, reason=reason)
         request.node.add_marker(mark)
@@ -144,8 +144,8 @@ def test_add_sequence(dtype):
     tm.assert_extension_array_equal(result, expected)
 
 
-def test_mul(dtype, request):
-    if dtype.storage == "pyarrow":
+def test_mul(dtype, request, arrow_string_storage):
+    if dtype.storage in arrow_string_storage:
         reason = "unsupported operand type(s) for *: 'ArrowStringArray' and 'int'"
         mark = pytest.mark.xfail(raises=NotImplementedError, reason=reason)
         request.node.add_marker(mark)
@@ -369,8 +369,8 @@ def test_min_max(method, skipna, dtype, request):
 
 @pytest.mark.parametrize("method", ["min", "max"])
 @pytest.mark.parametrize("box", [pd.Series, pd.array])
-def test_min_max_numpy(method, box, dtype, request):
-    if dtype.storage == "pyarrow" and box is pd.array:
+def test_min_max_numpy(method, box, dtype, request, arrow_string_storage):
+    if dtype.storage in arrow_string_storage and box is pd.array:
         if box is pd.array:
             reason = "'<=' not supported between instances of 'str' and 'NoneType'"
         else:
@@ -384,7 +384,7 @@ def test_min_max_numpy(method, box, dtype, request):
     assert result == expected
 
 
-def test_fillna_args(dtype, request):
+def test_fillna_args(dtype, request, arrow_string_storage):
     # GH 37987
 
     arr = pd.array(["a", pd.NA], dtype=dtype)
@@ -397,7 +397,7 @@ def test_fillna_args(dtype, request):
     expected = pd.array(["a", "b"], dtype=dtype)
     tm.assert_extension_array_equal(res, expected)
 
-    if dtype.storage == "pyarrow":
+    if dtype.storage in arrow_string_storage:
         msg = "Invalid value '1' for dtype string"
     else:
         msg = "Cannot set non-string value '1' into a StringArray."
@@ -503,10 +503,10 @@ def test_use_inf_as_na(values, expected, dtype):
             tm.assert_frame_equal(result, expected)
 
 
-def test_memory_usage(dtype):
+def test_memory_usage(dtype, arrow_string_storage):
     # GH 33963
 
-    if dtype.storage == "pyarrow":
+    if dtype.storage in arrow_string_storage:
         pytest.skip(f"not applicable for {dtype.storage}")
 
     series = pd.Series(["a", "b", "c"], dtype=dtype)

pandas/tests/arrays/string_/test_string_arrow.py

@@ -49,10 +49,10 @@ def test_config_bad_storage_raises():
 @skip_if_no_pyarrow
 @pytest.mark.parametrize("chunked", [True, False])
 @pytest.mark.parametrize("array", ["numpy", "pyarrow"])
-def test_constructor_not_string_type_raises(array, chunked):
+def test_constructor_not_string_type_raises(array, chunked, arrow_string_storage):
     import pyarrow as pa
 
-    array = pa if array == "pyarrow" else np
+    array = pa if array in arrow_string_storage else np
 
     arr = array.array([1, 2, 3])
     if chunked:

pandas/tests/extension/base/ops.py

@@ -239,9 +239,23 @@ def test_compare_array(self, data, comparison_op):
 class BaseUnaryOpsTests(BaseOpsUtil):
     def test_invert(self, data):
         ser = pd.Series(data, name="name")
-        result = ~ser
-        expected = pd.Series(~data, name="name")
-        tm.assert_series_equal(result, expected)
+        try:
+            # 10 is an arbitrary choice here, just avoid iterating over
+            #  the whole array to trim test runtime
+            [~x for x in data[:10]]
+        except TypeError:
+            # scalars don't support invert -> we don't expect the vectorized
+            #  operation to succeed
+            with pytest.raises(TypeError):
+                ~ser
+            with pytest.raises(TypeError):
+                ~data
+        else:
+            # Note we do not re-use the pointwise result to construct expected
+            #  because python semantics for negating bools are weird see GH#54569
+            result = ~ser
+            expected = pd.Series(~data, name="name")
+            tm.assert_series_equal(result, expected)
 
     @pytest.mark.parametrize("ufunc", [np.positive, np.negative, np.abs])
     def test_unary_ufunc_dunder_equivalence(self, data, ufunc):