debug

Author: Marco Gorelli

pandas/_libs/tslib.pyx

@@ -570,6 +570,15 @@ cpdef array_to_datetime(
                     iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)
 
                 elif is_integer_object(val) or is_float_object(val):
+                    if require_iso8601:
+                        if is_coerce:
+                            iresult[i] = NPY_NAT
+                            continue
+                        elif is_raise:
+                            raise ValueError(
+                                f"time data \"{val}\" at position {i} doesn't match format \"{format}\""
+                            )
+                        return values, tz_out
                     # these must be ns unit by-definition
                     seen_integer = True
 
@@ -615,7 +624,7 @@ cpdef array_to_datetime(
                                 continue
                             elif is_raise:
                                 raise ValueError(
-                                    f"time data \"{val}\" at position {i} doesn't match format specified"
+                                    f"time data \"{val}\" at position {i} doesn't match \"{format}\""
                                 )
                             return values, tz_out
 
@@ -646,6 +655,16 @@ cpdef array_to_datetime(
                         _ts = convert_datetime_to_tsobject(py_dt, None)
                         iresult[i] = _ts.value
                     if not string_to_dts_failed:
+                        if require_iso8601:
+                            if is_coerce:
+                                iresult[i] = NPY_NAT
+                                continue
+                            elif is_raise:
+                                raise ValueError(
+                                    f"time data \"{val}\" at position {i} doesn't "
+                                    f"match format \"{format}\""
+                                )
+                            return values, tz_out
                         # No error reported by string_to_dts, pick back up
                         # where we left off
                         value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)

pandas/_libs/tslibs/src/datetime/np_datetime_strings.c

@@ -29,6 +29,7 @@ This file implements string parsing and creation for NumPy datetime.
 #include <Python.h>
 
 #include <time.h>
+#include <stdio.h>
 
 #include <numpy/arrayobject.h>
 #include <numpy/arrayscalars.h>
@@ -68,22 +69,26 @@ This file implements string parsing and creation for NumPy datetime.
  */
 
 #define FORMAT_STARTSWITH(ch) \
+  if (format_len > 0){   \
   if (*format != ch) {   \
     goto parse_error;    \
   }                      \
   ++format;              \
+  } \
 
 int parse_iso_8601_datetime(const char *str, int len, int want_exc,
                             npy_datetimestruct *out,
                             NPY_DATETIMEUNIT *out_bestunit,
                             int *out_local, int *out_tzoffset,
                             const char* format, int exact) {
+    printf("entering\n");
     int year_leap = 0;
     int i, numdigits;
     const char *substr;
     int sublen;
     NPY_DATETIMEUNIT bestunit = NPY_FR_GENERIC;
     int format_len = strlen(format);
+    printf("format len: %d", format_len);
 
     /* If year-month-day are separated by a valid separator,
      * months/days without leading zeroes will be parsed

pandas/tests/tools/test_to_datetime.py

@@ -1751,6 +1751,104 @@ def test_to_datetime_iso8601(self, cache, arg, exp_str):
         exp = Timestamp(exp_str)
         assert result[0] == exp
 
+    @pytest.mark.parametrize(
+        "input, format",
+        [
+            ("2012", "%Y-%m"),
+            ("2012-01", "%Y-%m-%d"),
+            ("2012-01-01", "%Y-%m-%d %H"),
+            ("2012-01-01 10", "%Y-%m-%d %H:%M"),
+            ("2012-01-01 10:00", "%Y-%m-%d %H:%M:%S"),
+            (0, "%Y-%m-%d"),
+        ],
+    )
+    @pytest.mark.parametrize("exact", [True, False])
+    def test_to_datetime_iso8601_fails(self, input, format, exact):
+        # https://github.com/pandas-dev/pandas/issues/12649
+        with pytest.raises(
+            ValueError,
+            match=rf"time data \"{input}\" at position 0 doesn't match format {format}",
+        ):
+            to_datetime(input, format=format, exact=exact)
+
+    @pytest.mark.parametrize(
+        "input, format",
+        [
+            ("2012-01-01", "%Y-%m"),
+            ("2012-01-01 10", "%Y-%m-%d"),
+            ("2012-01-01 10:00", "%Y-%m-%d %H"),
+            ("2012-01-01 10:00:00", "%Y-%m-%d %H:%M"),
+            (0, "%Y-%m-%d"),
+        ],
+    )
+    def test_to_datetime_iso8601_exact_fails(self, input, format):
+        # https://github.com/pandas-dev/pandas/issues/12649
+        with pytest.raises(
+            ValueError,
+            match=rf"time data \"{input}\" at position 0 doesn't match format {format}",
+        ):
+            to_datetime(input, format=format)
+
+    @pytest.mark.parametrize(
+        "input, format",
+        [
+            ("2012-01-01", "%Y-%m"),
+            ("2012-01-01 00", "%Y-%m-%d"),
+            ("2012-01-01 00:00", "%Y-%m-%d %H"),
+            ("2012-01-01 00:00:00", "%Y-%m-%d %H:%M"),
+        ],
+    )
+    def test_to_datetime_iso8601_non_exact(self, input, format):
+        # https://github.com/pandas-dev/pandas/issues/12649
+        expected = Timestamp(2012, 1, 1)
+        result = to_datetime(input, format=format, exact=False)
+        assert result == expected
+
+    @pytest.mark.parametrize(
+        "input, format",
+        [
+            ("2020-01", "%Y/%m"),
+            ("2020-01-01", "%Y/%m/%d"),
+            ("2020-01-01 00", "%Y/%m/%dT%H"),
+            ("2020-01-01T00", "%Y/%m/%d %H"),
+            ("2020-01-01 00:00", "%Y/%m/%dT%H:%M"),
+            ("2020-01-01T00:00", "%Y/%m/%d %H:%M"),
+            ("2020-01-01 00:00:00", "%Y/%m/%dT%H:%M:%S"),
+            ("2020-01-01T00:00:00", "%Y/%m/%d %H:%M:%S"),
+        ],
+    )
+    def test_to_datetime_iso8601_separator(self, input, format):
+        # https://github.com/pandas-dev/pandas/issues/12649
+        with pytest.raises(
+            ValueError,
+            match=(
+                rf"time data \"{input}\" at position 0 doesn\'t match format {format}"
+            ),
+        ):
+            to_datetime(input, format=format)
+
+    @pytest.mark.parametrize(
+        "input, format",
+        [
+            ("2020-01", "%Y-%m"),
+            ("2020-01-01", "%Y-%m-%d"),
+            ("2020-01-01 00", "%Y-%m-%d %H"),
+            ("2020-01-01T00", "%Y-%m-%dT%H"),
+            ("2020-01-01 00:00", "%Y-%m-%d %H:%M"),
+            ("2020-01-01T00:00", "%Y-%m-%dT%H:%M"),
+            ("2020-01-01 00:00:00", "%Y-%m-%d %H:%M:%S"),
+            ("2020-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S"),
+            ("2020-01-01T00:00:00.000", "%Y-%m-%dT%H:%M:%S.%f"),
+            ("2020-01-01T00:00:00.000000", "%Y-%m-%dT%H:%M:%S.%f"),
+            ("2020-01-01T00:00:00.000000000", "%Y-%m-%dT%H:%M:%S.%f"),
+        ],
+    )
+    def test_to_datetime_iso8601_valid(self, input, format):
+        # https://github.com/pandas-dev/pandas/issues/12649
+        expected = Timestamp(2020, 1, 1)
+        result = to_datetime(input, format=format)
+        assert result == expected
+
     def test_to_datetime_default(self, cache):
         rs = to_datetime("2001", cache=cache)
         xp = datetime(2001, 1, 1)
@@ -2264,7 +2362,7 @@ def test_day_not_in_month_raise(self, cache):
 
     @pytest.mark.parametrize("arg", ["2015-02-29", "2015-02-32", "2015-04-31"])
     def test_day_not_in_month_raise_value(self, cache, arg):
-        msg = f'time data "{arg}" at position 0 doesn\'t match format specified'
+        msg = f'time data "{arg}" at position 0 doesn\'t match format \"%Y-%m-%d\"'
         with pytest.raises(ValueError, match=msg):
             to_datetime(arg, errors="raise", format="%Y-%m-%d", cache=cache)