allow format="mixed"

Author: MarcoGorelli

doc/source/user_guide/io.rst

@@ -1001,17 +1001,17 @@ way to parse dates is to explicitly set ``format=``.
    )
    df
 
-In the case that you have mixed datetime formats within the same column, you'll need to
-first read it in as an object dtype and then apply :func:`to_datetime` to each element.
+In the case that you have mixed datetime formats within the same column, you can
+pass  ``format='mixed'``
 
 .. ipython:: python
 
    data = io.StringIO("date\n12 Jan 2000\n2000-01-13\n")
    df = pd.read_csv(data)
-   df['date'] = df['date'].apply(pd.to_datetime)
+   df['date'] = pd.to_datetime(df['date'], format='mixed')
    df
 
-or, if your datetime formats are all ISO8601 (but possibly not identically-formatted):
+or, if your datetime formats are all ISO8601 (possibly not identically-formatted):
 
 .. ipython:: python
 

doc/source/whatsnew/v2.0.0.rst

@@ -188,6 +188,7 @@ Other enhancements
 - Added :meth:`DatetimeIndex.as_unit` and :meth:`TimedeltaIndex.as_unit` to convert to different resolutions; supported resolutions are "s", "ms", "us", and "ns" (:issue:`50616`)
 - Added new argument ``dtype`` to :func:`read_sql` to be consistent with :func:`read_sql_query` (:issue:`50797`)
 - :func:`to_datetime` now accepts ``"ISO8601"`` as an argument to ``format``, which will match any ISO8601 string (but possibly not identically-formatted) (:issue:`50411`)
+- :func:`to_datetime` now accepts ``"mixed"`` as an argument to ``format``, which will infer the format for each element individually (:issue:`50972`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -573,11 +574,11 @@ In the past, :func:`to_datetime` guessed the format for each element independent
 
 Note that this affects :func:`read_csv` as well.
 
-If you still need to parse dates with inconsistent formats, you'll need to apply :func:`to_datetime`
-to each element individually, e.g. ::
+If you still need to parse dates with inconsistent formats, you can use
+``format='mixed`` (preferably alongside ``dayfirst``) ::
 
      ser = pd.Series(['13-01-2000', '12 January 2000'])
-     ser.apply(pd.to_datetime)
+     pd.to_datetime(ser, format='mixed', dayfirst=True)
 
 or, if your formats are all ISO8601 (but possibly not identically-formatted) ::
 

pandas/_libs/tslibs/strptime.pyi

@@ -5,7 +5,6 @@ from pandas._typing import npt
 def array_strptime(
     values: npt.NDArray[np.object_],
     fmt: str | None,
-    fmt_inferred: bool = ...,
     exact: bool = ...,
     errors: str = ...,
     utc: bool = ...,

pandas/_libs/tslibs/strptime.pyx

@@ -152,7 +152,6 @@ cdef dict _parse_code_table = {"y": 0,
 def array_strptime(
     ndarray[object] values,
     str fmt,
-    bint fmt_inferred=False,
     bint exact=True,
     errors="raise",
     bint utc=False,
@@ -349,40 +348,22 @@ def array_strptime(
             if exact:
                 found = format_regex.match(val)
                 if not found:
-                    if fmt_inferred:
-                        raise ValueError(
-                            f"time data \"{val}\" doesn't "
-                            f"match (inferred) format \"{fmt}\""
-                        )
-                    else:
-                        raise ValueError(
-                            f"time data \"{val}\" doesn't match format \"{fmt}\""
-                        )
+                    raise ValueError(
+                        f"time data \"{val}\" doesn't match format \"{fmt}\""
+                    )
                 if len(val) != found.end():
-                    if fmt_inferred:
-                        raise ValueError(
-                            "unconverted data remains when parsing with "
-                            f"(inferred) format \"{fmt}\": \"{val[found.end():]}\""
-                        )
-                    else:
-                        raise ValueError(
-                            "unconverted data remains when parsing with "
-                            f"format \"{fmt}\": \"{val[found.end():]}\""
-                        )
+                    raise ValueError(
+                        "unconverted data remains when parsing with "
+                        f"format \"{fmt}\": \"{val[found.end():]}\""
+                    )
 
             # search
             else:
                 found = format_regex.search(val)
                 if not found:
-                    if fmt_inferred:
-                        raise ValueError(
-                            f"time data \"{val}\" doesn't match "
-                            f"(inferred) format \"{fmt}\""
-                        )
-                    else:
-                        raise ValueError(
-                            f"time data \"{val}\" doesn't match format \"{fmt}\""
-                        )
+                    raise ValueError(
+                        f"time data \"{val}\" doesn't match format \"{fmt}\""
+                    )
 
             iso_year = -1
             year = 1900
@@ -533,12 +514,14 @@ def array_strptime(
             result_timezone[i] = tz
 
         except (ValueError, OutOfBoundsDatetime) as ex:
-            if iso_format:
-                ex.args = (f"{str(ex)}, at position {i}. If your time strings "
-                           "are all (not-necessarily-identically-formatted) ISO8601, "
-                           "you could try passing 'format=\"ISO8601\"'",)
-            else:
-                ex.args = (f"{str(ex)}, at position {i}",)
+            ex.args = (
+                f"{str(ex)}, at position {i}. You might want to try:\n"
+                "    - passing ``format='ISO8601'`` if your strings are "
+                "all ISO8601 but not necessarily in exactly the same format;\n"
+                "    - passing ``format='mixed'``, and the format will be "
+                "inferred for each element individually. "
+                "You might want to use ``dayfirst`` alongside this.",
+            )
             if is_coerce:
                 iresult[i] = NPY_NAT
                 continue

pandas/core/tools/datetimes.py

@@ -442,15 +442,12 @@ def _convert_listlike_datetimes(
 
     arg = ensure_object(arg)
 
-    format_inferred = False
-    if format is None:
+    if format is None and format != "mixed":
         format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
-        format_inferred = True
 
-    if format is not None:
-        return _array_strptime_with_fallback(
-            arg, name, utc, format, format_inferred, exact, errors
-        )
+    # `format` could not be inferred, or user asked for mixed-format parsing.
+    if format is not None and format != "mixed":
+        return _array_strptime_with_fallback(arg, name, utc, format, exact, errors)
 
     result, tz_parsed = objects_to_datetime64ns(
         arg,
@@ -475,16 +472,13 @@ def _array_strptime_with_fallback(
     name,
     utc: bool,
     fmt: str,
-    fmt_inferred: bool,
     exact: bool,
     errors: str,
 ) -> Index:
     """
     Call array_strptime, with fallback behavior depending on 'errors'.
     """
-    result, timezones = array_strptime(
-        arg, fmt, fmt_inferred=fmt_inferred, exact=exact, errors=errors, utc=utc
-    )
+    result, timezones = array_strptime(arg, fmt, exact=exact, errors=errors, utc=utc)
     if any(tz is not None for tz in timezones):
         return _return_parsed_timezone_results(result, timezones, utc, name)
 
@@ -694,7 +688,7 @@ def to_datetime(
     yearfirst: bool = False,
     utc: bool = False,
     format: str | None = None,
-    exact: bool = True,
+    exact: bool | lib.NoDefault = lib.no_default,
     unit: str | None = None,
     infer_datetime_format: lib.NoDefault | bool = lib.no_default,
     origin: str = "unix",
@@ -766,15 +760,20 @@ def to_datetime(
         <https://docs.python.org/3/library/datetime.html
         #strftime-and-strptime-behavior>`_ for more information on choices, though
         note that :const:`"%f"` will parse all the way up to nanoseconds.
-        You can also pass "ISO8601" to parse any ISO8601 time string.
+        You can also pass:
+
+        - "ISO8601", to parse any ISO8601 time string (not necessarily in exactly the
+          same format);
+        - "mixed", to infer the format for each element individually. This is risky,
+          and you should probably use it along with `dayfirst`.
     exact : bool, default True
         Control how `format` is used:
 
         - If :const:`True`, require an exact `format` match.
         - If :const:`False`, allow the `format` to match anywhere in the target
           string.
 
-        Note that if ``format='ISO8601'`` then `exact` has no effect.
+        Cannot be used alongside ``format='ISO8601'`` or ``format='mixed'``.
     unit : str, default 'ns'
         The unit of the arg (D,s,ms,us,ns) denote the unit, which is an
         integer or float number. This will be based off the origin.
@@ -1006,6 +1005,8 @@ def to_datetime(
     DatetimeIndex(['2018-10-26 12:00:00+00:00', '2020-01-01 18:00:00+00:00'],
                   dtype='datetime64[ns, UTC]', freq=None)
     """
+    if exact is not lib.no_default and format in {"mixed", "ISO8601"}:
+        raise ValueError("Cannot use 'exact' when 'format' is 'mixed' or 'ISO8601'")
     if infer_datetime_format is not lib.no_default:
         warnings.warn(
             "The argument 'infer_datetime_format' is deprecated and will "

pandas/tests/io/parser/test_parse_dates.py

@@ -1719,8 +1719,8 @@ def test_parse_multiple_delimited_dates_with_swap_warnings():
     with pytest.raises(
         ValueError,
         match=(
-            r'^time data "31/05/2000" doesn\'t match \(inferred\) format "%m/%d/%Y", '
-            r"at position 1$"
+            r'^time data "31/05/2000" doesn\'t match format "%m/%d/%Y", '
+            r"at position 1. You might want to try:\n    - passing ``format='ISO8601'``"
         ),
     ):
         pd.to_datetime(["01/01/2000", "31/05/2000", "31/05/2001", "01/02/2000"])