POC: infer time objects to ArrowDtype[time]

Author: jbrockmendel

pandas/_libs/lib.pyx

@@ -6,6 +6,7 @@ from typing import (
     Literal,
     _GenericAlias,
 )
+import warnings
 
 cimport cython
 from cpython.datetime cimport (
@@ -99,6 +100,8 @@ cdef extern from "pd_parser.h":
 
 PandasParser_IMPORT
 
+from pandas._config import get_option
+
 from pandas._libs cimport util
 from pandas._libs.util cimport (
     INT64_MAX,
@@ -1258,6 +1261,7 @@ cdef class Seen:
         bint datetimetz_      # seen_datetimetz
         bint period_          # seen_period
         bint interval_        # seen_interval
+        bint time_
 
     def __cinit__(self, bint coerce_numeric=False):
         """
@@ -1284,6 +1288,7 @@ cdef class Seen:
         self.datetimetz_ = False
         self.period_ = False
         self.interval_ = False
+        self.time_ = False
         self.coerce_numeric = coerce_numeric
 
     cdef bint check_uint64_conflict(self) except -1:
@@ -2567,6 +2572,12 @@ def maybe_convert_objects(ndarray[object] objects,
             else:
                 seen.object_ = True
                 break
+        elif PyTime_Check(val):
+            if convert_time:
+                seen.time_ = True
+            else:
+                seen.object_ = True
+            break
         else:
             seen.object_ = True
             break
@@ -2631,6 +2642,35 @@ def maybe_convert_objects(ndarray[object] objects,
 
         seen.object_ = True
 
+    elif seen.time_:
+        if is_time_array(objects):
+            opt = get_option("future.infer_time")
+            if opt is True:
+                import pyarrow as pa
+
+                from pandas.core.arrays.arrow import ArrowDtype
+
+                obj = pa.array(objects)
+                dtype = ArrowDtype(obj.type)
+                return dtype.construct_array_type()(obj)
+            elif opt is False:
+                # explicitly set to keep the old behavior and avoid the warning
+                pass
+            else:
+                from pandas.util._exceptions import find_stack_level
+                warnings.warn(
+                    "Pandas type inference with a sequence of `datetime.time` "
+                    "objects is deprecated. In a future version, this will give "
+                    "time32[pyarrow] dtype, which will require pyarrow to be "
+                    "installed. To opt in to the new behavior immediately set "
+                    "`pd.set_option('future.infer_time', True)`. To keep the "
+                    "old behavior pass `dtype=object`.",
+                    FutureWarning,
+                    stacklevel=find_stack_level(),
+                )
+
+        seen.object_ = True
+
     elif seen.nat_:
         if not seen.object_ and not seen.numeric_ and not seen.bool_:
             # all NaT, None, or nan (at least one NaT)

pandas/core/arrays/arrow/array.py

@@ -522,9 +522,7 @@ def _evaluate_op_method(self, other, op, arrow_funcs):
                 pa_scalar = pa.scalar(other)
             result = pc_func(self._pa_array, pa_scalar)
         else:
-            raise NotImplementedError(
-                f"{op.__name__} not implemented for {type(other)}"
-            )
+            return NotImplemented
         return type(self)(result)
 
     def _logical_method(self, other, op):

pandas/core/config_init.py

@@ -881,3 +881,14 @@ def register_converter_cb(key) -> None:
         styler_environment,
         validator=is_instance_factory([type(None), str]),
     )
+
+
+with cf.config_prefix("future"):
+    cf.register_option(
+        "future.infer_time",
+        None,
+        "Whether to infer sequence of datetime.time objects as pyarrow time "
+        "dtype, which will be the default in pandas 3.0 "
+        "(at which point this option will be deprecated).",
+        validator=is_one_of_factory([True, False, None]),
+    )

pandas/core/construction.py

@@ -14,10 +14,13 @@
     cast,
     overload,
 )
+import warnings
 
 import numpy as np
 from numpy import ma
 
+from pandas._config import get_option
+
 from pandas._libs import lib
 from pandas._libs.tslibs import (
     Period,
@@ -31,6 +34,7 @@
     DtypeObj,
     T,
 )
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.base import ExtensionDtype
 from pandas.core.dtypes.cast import (
@@ -293,6 +297,7 @@ def array(
         PeriodArray,
         TimedeltaArray,
     )
+    from pandas.core.arrays.arrow import ArrowDtype
     from pandas.core.arrays.string_ import StringDtype
 
     if lib.is_scalar(data):
@@ -360,6 +365,30 @@ def array(
         elif inferred_dtype == "boolean":
             return BooleanArray._from_sequence(data, copy=copy)
 
+        elif inferred_dtype == "time":
+            opt = get_option("future.infer_time")
+
+            if opt is True:
+                import pyarrow as pa
+
+                obj = pa.array(data)
+                dtype = ArrowDtype(obj.type)
+                return dtype.construct_array_type()(obj)
+            elif opt is False:
+                # explicitly set to keep the old behavior and avoid the warning
+                pass
+            else:
+                warnings.warn(
+                    "Pandas type inference with a sequence of `datetime.time` "
+                    "objects is deprecated. In a future version, this will give "
+                    "time32[pyarrow] dtype, which will require pyarrow to be "
+                    "installed. To opt in to the new behavior immediately set "
+                    "`pd.set_option('future.infer_time', True)`. To keep the "
+                    "old behavior pass `dtype=object`.",
+                    FutureWarning,
+                    stacklevel=find_stack_level(),
+                )
+
     # Pandas overrides NumPy for
     #   1. datetime64[ns,us,ms,s]
     #   2. timedelta64[ns,us,ms,s]

pandas/core/indexes/accessors.py

@@ -108,7 +108,9 @@ def _delegate_property_get(self, name: str):  # type: ignore[override]
         else:
             index = self._parent.index
         # return the result as a Series
-        result = Series(result, index=index, name=self.name).__finalize__(self._parent)
+        result = Series(
+            result, index=index, name=self.name, dtype=result.dtype
+        ).__finalize__(self._parent)
 
         # setting this object will show a SettingWithCopyWarning/Error
         result._is_copy = (

pandas/tests/arithmetic/test_datetime64.py

@@ -1166,7 +1166,10 @@ def test_dt64arr_add_sub_parr(
         )
         assert_invalid_addsub_type(dtarr, parr, msg)
 
-    def test_dt64arr_addsub_time_objects_raises(self, box_with_array, tz_naive_fixture):
+    @pytest.mark.parametrize("future", [True, False, None])
+    def test_dt64arr_addsub_time_objects_raises(
+        self, box_with_array, tz_naive_fixture, future
+    ):
         # https://github.com/pandas-dev/pandas/issues/10329
 
         tz = tz_naive_fixture
@@ -1175,15 +1178,23 @@ def test_dt64arr_addsub_time_objects_raises(self, box_with_array, tz_naive_fixtu
         obj2 = [time(i, i, i) for i in range(3)]
 
         obj1 = tm.box_expected(obj1, box_with_array)
-        obj2 = tm.box_expected(obj2, box_with_array)
-
-        msg = "|".join(
-            [
-                "unsupported operand",
-                "cannot subtract DatetimeArray from ndarray",
-            ]
-        )
 
+        msgs = [
+            "unsupported operand",
+            "cannot subtract DatetimeArray from ndarray",
+        ]
+        warn_msg = "Pandas type inference with a sequence of `datetime.time` objects"
+        warn = None
+        if future is True:
+            msgs.append("cannot subtract DatetimeArray from ArrowExtensionArray")
+        elif future is None:
+            warn = FutureWarning
+
+        with pd.option_context("future.infer_time", future):
+            with tm.assert_produces_warning(warn, match=warn_msg):
+                obj2 = tm.box_expected(obj2, box_with_array)
+
+        msg = "|".join(msgs)
         with warnings.catch_warnings(record=True):
             # pandas.errors.PerformanceWarning: Non-vectorized DateOffset being
             # applied to Series or DatetimeIndex

pandas/tests/dtypes/test_inference.py

@@ -1039,6 +1039,56 @@ def test_maybe_convert_objects_ea(self, idx):
         )
         tm.assert_extension_array_equal(result, idx._data)
 
+    @pytest.mark.parametrize("future", [True, False, None])
+    def test_maybe_convert_objects_time(self, future):
+        ts = Timestamp.now()
+        objs = np.array([ts.time()], dtype=object)
+
+        msg = "Pandas type inference with a sequence of `datetime.time` objects"
+        warn = None
+        if future is True:
+            pa = pytest.importorskip("pyarrow")
+            dtype = pd.ArrowDtype(pa.time64("us"))
+            exp = dtype.construct_array_type()._from_sequence(objs, dtype=dtype)
+        else:
+            if future is None:
+                warn = FutureWarning
+            exp = objs
+
+        with pd.option_context("future.infer_time", future):
+            with tm.assert_produces_warning(warn, match=msg):
+                out = lib.maybe_convert_objects(objs, convert_time=True)
+            with tm.assert_produces_warning(warn, match=msg):
+                ser = Series(objs)
+            with tm.assert_produces_warning(warn, match=msg):
+                ser2 = Series(list(objs))
+            with tm.assert_produces_warning(warn, match=msg):
+                df = DataFrame(objs)
+            with tm.assert_produces_warning(warn, match=msg):
+                df2 = DataFrame(list(objs))
+            with tm.assert_produces_warning(warn, match=msg):
+                idx = Index(objs)
+            with tm.assert_produces_warning(warn, match=msg):
+                idx2 = Index(list(objs))
+            with tm.assert_produces_warning(warn, match=msg):
+                arr = pd.array(objs)
+            with tm.assert_produces_warning(warn, match=msg):
+                arr2 = pd.array(list(objs))
+
+        tm.assert_equal(out, exp)
+        if future:
+            tm.assert_equal(arr, exp)
+            tm.assert_equal(arr2, exp)
+        else:
+            tm.assert_equal(arr, pd.core.arrays.PandasArray(exp))
+            tm.assert_equal(arr2, pd.core.arrays.PandasArray(exp))
+        tm.assert_series_equal(ser, Series(exp, dtype=exp.dtype))
+        tm.assert_series_equal(ser2, Series(exp, dtype=exp.dtype))
+        tm.assert_frame_equal(df, DataFrame(exp, dtype=exp.dtype))
+        tm.assert_frame_equal(df2, DataFrame(exp, dtype=exp.dtype))
+        tm.assert_index_equal(idx, Index(exp, dtype=exp.dtype))
+        tm.assert_index_equal(idx2, Index(exp, dtype=exp.dtype))
+
 
 class TestTypeInference:
     # Dummy class used for testing with Python objects

pandas/tests/extension/test_arrow.py

@@ -712,6 +712,28 @@ class TestBaseReshaping(base.BaseReshapingTests):
     def test_transpose(self, data):
         super().test_transpose(data)
 
+    @pytest.mark.parametrize(
+        "columns",
+        [
+            ["A", "B"],
+            pd.MultiIndex.from_tuples(
+                [("A", "a"), ("A", "b")], names=["outer", "inner"]
+            ),
+        ],
+    )
+    def test_stack(self, data, columns):
+        warn = None
+        warn_msg = "Pandas type inference with a sequence of `datetime.time` objects"
+
+        pa_dtype = data.dtype.pyarrow_dtype
+        if pa.types.is_time(pa_dtype):
+            # FIXME: need to avoid doing inference when calling frame._constructor
+            #  in _stack_multi_columns
+            warn = FutureWarning
+
+        with tm.assert_produces_warning(warn, match=warn_msg, check_stacklevel=False):
+            super().test_stack(data, columns)
+
 
 class TestBaseSetitem(base.BaseSetitemTests):
     @pytest.mark.xfail(
@@ -774,6 +796,18 @@ def test_invert(self, data, request):
 
 
 class TestBaseMethods(base.BaseMethodsTests):
+    def test_hash_pandas_object_works(self, data, as_frame):
+        pa_dtype = data.dtype.pyarrow_dtype
+        warn_msg = "Pandas type inference with a sequence of `datetime.time`"
+        warn = None
+        if pa.types.is_time(pa_dtype):
+            # TODO(#48964) This warning will be avoided by implementing
+            #  ArrowExtensionArray.hash_pandas_object
+            warn = FutureWarning
+
+        with tm.assert_produces_warning(warn, match=warn_msg, check_stacklevel=False):
+            super().test_hash_pandas_object_works(data, as_frame)
+
     @pytest.mark.parametrize("periods", [1, -2])
     def test_diff(self, data, periods, request):
         pa_dtype = data.dtype.pyarrow_dtype

pandas/tests/groupby/test_apply.py

@@ -1,6 +1,7 @@
 from datetime import (
     date,
     datetime,
+    time,
 )
 from io import StringIO
 
@@ -834,7 +835,16 @@ def test_apply_datetime_issue(group_column_dtlike):
     #   is a datetime object and the column labels are different from
     #   standard int values in range(len(num_columns))
 
-    df = DataFrame({"a": ["foo"], "b": [group_column_dtlike]})
+    warn = None
+    warn_msg = (
+        "Pandas type inference with a sequence of `datetime.time` "
+        "objects is deprecated"
+    )
+    if isinstance(group_column_dtlike, time):
+        warn = FutureWarning
+
+    with tm.assert_produces_warning(warn, match=warn_msg):
+        df = DataFrame({"a": ["foo"], "b": [group_column_dtlike]})
     result = df.groupby("a").apply(lambda x: Series(["spam"], index=[42]))
 
     expected = DataFrame(

pandas/tests/io/excel/test_readers.py

@@ -993,13 +993,17 @@ def test_reader_seconds(self, request, engine, read_ext):
                     time(16, 37, 0, 900000),
                     time(18, 20, 54),
                 ]
-            }
+            },
+            dtype=object,
         )
 
-        actual = pd.read_excel("times_1900" + read_ext, sheet_name="Sheet1")
+        warn_msg = "Pandas type inference with a sequence of `datetime.time` objects"
+        with tm.assert_produces_warning(FutureWarning, match=warn_msg):
+            actual = pd.read_excel("times_1900" + read_ext, sheet_name="Sheet1")
         tm.assert_frame_equal(actual, expected)
 
-        actual = pd.read_excel("times_1904" + read_ext, sheet_name="Sheet1")
+        with tm.assert_produces_warning(FutureWarning, match=warn_msg):
+            actual = pd.read_excel("times_1904" + read_ext, sheet_name="Sheet1")
         tm.assert_frame_equal(actual, expected)
 
     def test_read_excel_multiindex(self, request, read_ext):