Merge branch 'main' into DEPR-to_datetime-mixed-offsets-with-utc=False

Author: natmokval

.circleci/config.yml

@@ -47,8 +47,8 @@ jobs:
       - run:
           name: Build aarch64 wheels
           command: |
-            pip3 install cibuildwheel==2.12.1
-            cibuildwheel --output-dir wheelhouse
+            pip3 install cibuildwheel==2.14.1
+            cibuildwheel --prerelease-pythons --output-dir wheelhouse
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
 
@@ -91,4 +91,5 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"]
+              # TODO: Enable Python 3.12 wheels when numpy releases a version that supports Python 3.12
+              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"]#, "cp312-manylinux_aarch64"]

.github/workflows/unit-tests.yml

@@ -311,12 +311,16 @@ jobs:
     #    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
-    if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    #if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macOS-latest, windows-latest]
+        # TODO: Disable macOS for now, Github Actions bug where python is not
+        # symlinked correctly to 3.12
+        # xref https://github.com/actions/setup-python/issues/701
+        #os: [ubuntu-22.04, macOS-latest, windows-latest]
+        os: [ubuntu-22.04, windows-latest]
 
     timeout-minutes: 180
 
@@ -340,21 +344,21 @@ jobs:
       - name: Set up Python Dev Version
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11-dev'
+          python-version: '3.12-dev'
 
       - name: Install dependencies
         run: |
           python --version
-          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.0.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install git+https://github.com/nedbat/coveragepy.git
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
           python -m pip list
 
       - name: Build Pandas
         run: |
-          python -m pip install -e . --no-build-isolation --no-index
+          python -m pip install -ve . --no-build-isolation --no-index
 
       - name: Build Version
         run: |

.github/workflows/wheels.yml

@@ -93,7 +93,8 @@ jobs:
         - [macos-12, macosx_*]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
-        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]
+        # TODO: Enable Python 3.12 wheels when numpy releases a version that supports Python 3.12
+        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]#, ["cp312", "3.12"]]
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -117,6 +118,7 @@ jobs:
         #with:
         #  package-dir: ./dist/${{ needs.build_sdist.outputs.sdist_file }}
         env:
+          CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
       - name: Set up Python

doc/source/whatsnew/v2.1.0.rst

@@ -571,6 +571,7 @@ I/O
 - Bug in :func:`read_sql` when reading multiple timezone aware columns with the same column name (:issue:`44421`)
 - Bug in :func:`read_xml` stripping whitespace in string data (:issue:`53811`)
 - Bug in :meth:`DataFrame.to_html` where ``colspace`` was incorrectly applied in case of multi index columns (:issue:`53885`)
+- Bug in :meth:`DataFrame.to_json` where :class:`DateTimeArray`/:class:`DateTimeIndex` with non nanosecond precision could not be serialized correctly (:issue:`53686`)
 - Bug when writing and reading empty Stata dta files where dtype information was lost (:issue:`46240`)
 - Bug where ``bz2`` was treated as a hard requirement (:issue:`53857`)
 

meson.build

@@ -27,6 +27,13 @@ versioneer = files('generate_version.py')
 add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'c')
 add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp')
 
+# Allow supporting older numpys than the version compiled against
+# Set the define to the min supported version of numpy for pandas
+# e.g. right now this is targeting numpy 1.21+
+add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'c')
+add_project_arguments('-DNPY_TARGET_VERSION=NPY_1_21_API_VERSION', language : 'cpp')
+
+
 if fs.exists('_version_meson.py')
     py.install_sources('_version_meson.py', pure: false, subdir: 'pandas')
 else

pandas/_libs/include/pandas/datetime/date_conversions.h

@@ -18,7 +18,10 @@ int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit);
 // up to precision `base` e.g. base="s" yields 2020-01-03T00:00:00Z
 // while base="ns" yields "2020-01-01T00:00:00.000000000Z"
 // len is mutated to save the length of the returned string
-char *int64ToIso(int64_t value, NPY_DATETIMEUNIT base, size_t *len);
+char *int64ToIso(int64_t value,
+                 NPY_DATETIMEUNIT valueUnit,
+                 NPY_DATETIMEUNIT base,
+                 size_t *len);
 
 // TODO(username): this function doesn't do a lot; should augment or
 // replace with scaleNanosecToUnit

pandas/_libs/include/pandas/datetime/pd_datetime.h

@@ -34,7 +34,7 @@ typedef struct {
   npy_datetime (*npy_datetimestruct_to_datetime)(NPY_DATETIMEUNIT,
                                                  const npy_datetimestruct *);
   int (*scaleNanosecToUnit)(npy_int64 *, NPY_DATETIMEUNIT);
-  char *(*int64ToIso)(int64_t, NPY_DATETIMEUNIT, size_t *);
+  char *(*int64ToIso)(int64_t, NPY_DATETIMEUNIT, NPY_DATETIMEUNIT, size_t *);
   npy_datetime (*NpyDateTimeToEpoch)(npy_datetime, NPY_DATETIMEUNIT);
   char *(*PyDateTimeToIso)(PyObject *, NPY_DATETIMEUNIT, size_t *);
   npy_datetime (*PyDateTimeToEpoch)(PyObject *, NPY_DATETIMEUNIT);
@@ -73,8 +73,8 @@ static PandasDateTime_CAPI *PandasDateTimeAPI = NULL;
                                                     (npy_datetimestruct))
 #define scaleNanosecToUnit(value, unit)                                        \
   PandasDateTimeAPI->scaleNanosecToUnit((value), (unit))
-#define int64ToIso(value, base, len)                                           \
-  PandasDateTimeAPI->int64ToIso((value), (base), (len))
+#define int64ToIso(value, valueUnit, base, len)                                \
+  PandasDateTimeAPI->int64ToIso((value), (valueUnit), (base), (len))
 #define NpyDateTimeToEpoch(dt, base)                                           \
   PandasDateTimeAPI->NpyDateTimeToEpoch((dt), (base))
 #define PyDateTimeToIso(obj, base, len)                                        \

pandas/_libs/src/datetime/date_conversions.c

@@ -41,11 +41,14 @@ int scaleNanosecToUnit(npy_int64 *value, NPY_DATETIMEUNIT unit) {
 }
 
 /* Converts the int64_t representation of a datetime to ISO; mutates len */
-char *int64ToIso(int64_t value, NPY_DATETIMEUNIT base, size_t *len) {
+char *int64ToIso(int64_t value,
+                 NPY_DATETIMEUNIT valueUnit,
+                 NPY_DATETIMEUNIT base,
+                 size_t *len) {
     npy_datetimestruct dts;
     int ret_code;
 
-    pandas_datetime_to_datetimestruct(value, NPY_FR_ns, &dts);
+    pandas_datetime_to_datetimestruct(value, valueUnit, &dts);
 
     *len = (size_t)get_datetime_iso_8601_strlen(0, base);
     char *result = PyObject_Malloc(*len);

pandas/_libs/src/vendored/ujson/python/objToJSON.c

@@ -131,6 +131,7 @@ typedef struct __PyObjectEncoder {
 
     int datetimeIso;
     NPY_DATETIMEUNIT datetimeUnit;
+    NPY_DATETIMEUNIT valueUnit;
 
     // output format style for pandas data types
     int outputFormat;
@@ -350,7 +351,8 @@ static char *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc,
 static char *NpyDateTimeToIsoCallback(JSOBJ Py_UNUSED(unused),
                                       JSONTypeContext *tc, size_t *len) {
     NPY_DATETIMEUNIT base = ((PyObjectEncoder *)tc->encoder)->datetimeUnit;
-    GET_TC(tc)->cStr = int64ToIso(GET_TC(tc)->longValue, base, len);
+    NPY_DATETIMEUNIT valueUnit = ((PyObjectEncoder *)tc->encoder)->valueUnit;
+    GET_TC(tc)->cStr = int64ToIso(GET_TC(tc)->longValue, valueUnit, base, len);
     return GET_TC(tc)->cStr;
 }
 
@@ -364,8 +366,9 @@ static char *NpyTimeDeltaToIsoCallback(JSOBJ Py_UNUSED(unused),
 /* JSON callback */
 static char *PyDateTimeToIsoCallback(JSOBJ obj, JSONTypeContext *tc,
                                      size_t *len) {
-    if (!PyDate_Check(obj)) {
-        PyErr_SetString(PyExc_TypeError, "Expected date object");
+    if (!PyDate_Check(obj) && !PyDateTime_Check(obj)) {
+        PyErr_SetString(PyExc_TypeError, "Expected date or datetime object");
+        ((JSONObjectEncoder *)tc->encoder)->errorMsg = "";
         return NULL;
     }
 
@@ -502,6 +505,10 @@ int NpyArr_iterNextItem(JSOBJ obj, JSONTypeContext *tc) {
         GET_TC(tc)->itemValue = obj;
         Py_INCREF(obj);
         ((PyObjectEncoder *)tc->encoder)->npyType = PyArray_TYPE(npyarr->array);
+        // Also write the resolution (unit) of the ndarray
+        PyArray_Descr *dtype = PyArray_DESCR(npyarr->array);
+        ((PyObjectEncoder *)tc->encoder)->valueUnit =
+            get_datetime_metadata_from_dtype(dtype).base;
         ((PyObjectEncoder *)tc->encoder)->npyValue = npyarr->dataptr;
         ((PyObjectEncoder *)tc->encoder)->npyCtxtPassthru = npyarr;
     } else {
@@ -1255,6 +1262,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc,
     char **ret;
     char *dataptr, *cLabel;
     int type_num;
+    PyArray_Descr *dtype;
     NPY_DATETIMEUNIT base = enc->datetimeUnit;
 
     if (!labels) {
@@ -1283,6 +1291,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc,
     stride = PyArray_STRIDE(labels, 0);
     dataptr = PyArray_DATA(labels);
     type_num = PyArray_TYPE(labels);
+    dtype = PyArray_DESCR(labels);
 
     for (i = 0; i < num; i++) {
         item = PyArray_GETITEM(labels, dataptr);
@@ -1293,7 +1302,8 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc,
         }
 
         int is_datetimelike = 0;
-        npy_int64 nanosecVal;
+        npy_int64 i8date;
+        NPY_DATETIMEUNIT dateUnit = NPY_FR_ns;
         if (PyTypeNum_ISDATETIME(type_num)) {
             is_datetimelike = 1;
             PyArray_VectorUnaryFunc *castfunc =
@@ -1303,35 +1313,37 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc,
                              "Cannot cast numpy dtype %d to long",
                              enc->npyType);
             }
-            castfunc(dataptr, &nanosecVal, 1, NULL, NULL);
+            castfunc(dataptr, &i8date, 1, NULL, NULL);
+            dateUnit = get_datetime_metadata_from_dtype(dtype).base;
         } else if (PyDate_Check(item) || PyDelta_Check(item)) {
             is_datetimelike = 1;
             if (PyObject_HasAttrString(item, "_value")) {
                 // see test_date_index_and_values for case with non-nano
-                nanosecVal = get_long_attr(item, "_value");
+                i8date = get_long_attr(item, "_value");
             } else {
                 if (PyDelta_Check(item)) {
-                    nanosecVal = total_seconds(item) *
+                    i8date = total_seconds(item) *
                                  1000000000LL;  // nanoseconds per second
                 } else {
                     // datetime.* objects don't follow above rules
-                    nanosecVal = PyDateTimeToEpoch(item, NPY_FR_ns);
+                    i8date = PyDateTimeToEpoch(item, NPY_FR_ns);
                 }
             }
         }
 
         if (is_datetimelike) {
-            if (nanosecVal == get_nat()) {
+            if (i8date == get_nat()) {
                 len = 4;
                 cLabel = PyObject_Malloc(len + 1);
                 strncpy(cLabel, "null", len + 1);
             } else {
                 if (enc->datetimeIso) {
                     if ((type_num == NPY_TIMEDELTA) || (PyDelta_Check(item))) {
-                        cLabel = int64ToIsoDuration(nanosecVal, &len);
+                        // TODO(username): non-nano timedelta support?
+                        cLabel = int64ToIsoDuration(i8date, &len);
                     } else {
                         if (type_num == NPY_DATETIME) {
-                            cLabel = int64ToIso(nanosecVal, base, &len);
+                            cLabel = int64ToIso(i8date, dateUnit, base, &len);
                         } else {
                             cLabel = PyDateTimeToIso(item, base, &len);
                         }
@@ -1346,7 +1358,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc,
                     int size_of_cLabel = 21;  // 21 chars for int 64
                     cLabel = PyObject_Malloc(size_of_cLabel);
                     snprintf(cLabel, size_of_cLabel, "%" NPY_DATETIME_FMT,
-                            NpyDateTimeToEpoch(nanosecVal, base));
+                            NpyDateTimeToEpoch(i8date, base));
                     len = strlen(cLabel);
                 }
             }
@@ -1538,13 +1550,25 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
         tc->type = JT_UTF8;
         return;
     } else if (PyArray_IsScalar(obj, Datetime)) {
+        npy_int64 longVal;
         if (((PyDatetimeScalarObject *)obj)->obval == get_nat()) {
             tc->type = JT_NULL;
             return;
         }
+        PyArray_Descr *dtype = PyArray_DescrFromScalar(obj);
+        if (!PyTypeNum_ISDATETIME(dtype->type_num)) {
+            PyErr_Format(PyExc_ValueError, "Could not get resolution of datetime");
+            return;
+        }
+
+        PyArray_Descr *outcode = PyArray_DescrFromType(NPY_INT64);
+        PyArray_CastScalarToCtype(obj, &longVal, outcode);
+        Py_DECREF(outcode);
 
         if (enc->datetimeIso) {
-            pc->PyTypeToUTF8 = PyDateTimeToIsoCallback;
+            GET_TC(tc)->longValue = longVal;
+            pc->PyTypeToUTF8 = NpyDateTimeToIsoCallback;
+            enc->valueUnit = get_datetime_metadata_from_dtype(dtype).base;
             tc->type = JT_UTF8;
         } else {
             NPY_DATETIMEUNIT base =

pandas/compat/__init__.py

@@ -19,6 +19,7 @@
     ISMUSL,
     PY310,
     PY311,
+    PY312,
     PYPY,
 )
 import pandas.compat.compressors
@@ -189,5 +190,6 @@ def get_bz2_file() -> type[pandas.compat.compressors.BZ2File]:
     "ISMUSL",
     "PY310",
     "PY311",
+    "PY312",
     "PYPY",
 ]

pandas/compat/_constants.py

@@ -15,6 +15,7 @@
 
 PY310 = sys.version_info >= (3, 10)
 PY311 = sys.version_info >= (3, 11)
+PY312 = sys.version_info >= (3, 12)
 PYPY = platform.python_implementation() == "PyPy"
 ISMUSL = "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or "")
 REF_COUNT = 2 if PY311 else 3
@@ -24,5 +25,6 @@
     "ISMUSL",
     "PY310",
     "PY311",
+    "PY312",
     "PYPY",
 ]

pandas/core/arrays/_mixins.py

@@ -296,7 +296,9 @@ def _fill_mask_inplace(
         func(self._ndarray.T, limit=limit, mask=mask.T)
 
     @doc(ExtensionArray.fillna)
-    def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
+    def fillna(
+        self, value=None, method=None, limit: int | None = None, copy: bool = True
+    ) -> Self:
         value, method = validate_fillna_kwargs(
             value, method, validate_scalar_dict_value=False
         )
@@ -313,7 +315,9 @@ def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
                 # TODO: check value is None
                 # (for now) when self.ndim == 2, we assume axis=0
                 func = missing.get_fill_func(method, ndim=self.ndim)
-                npvalues = self._ndarray.T.copy()
+                npvalues = self._ndarray.T
+                if copy:
+                    npvalues = npvalues.copy()
                 func(npvalues, limit=limit, mask=mask.T)
                 npvalues = npvalues.T
 
@@ -322,14 +326,20 @@ def fillna(self, value=None, method=None, limit: int | None = None) -> Self:
                 new_values = self._from_backing_data(npvalues)
             else:
                 # fill with value
-                new_values = self.copy()
+                if copy:
+                    new_values = self.copy()
+                else:
+                    new_values = self[:]
                 new_values[mask] = value
         else:
             # We validate the fill_value even if there is nothing to fill
             if value is not None:
                 self._validate_setitem_value(value)
 
-            new_values = self.copy()
+            if not copy:
+                new_values = self[:]
+            else:
+                new_values = self.copy()
         return new_values
 
     # ------------------------------------------------------------------------