Merge branch 'master' into read-tar-archives

Author: Skn0tt

.github/workflows/posix.yml

@@ -31,6 +31,7 @@ jobs:
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-39-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
           [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
           [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
         ]

ci/deps/actions-pypy-38.yaml

@@ -0,0 +1,20 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  # TODO: Add the rest of the dependencies in here
+  # once the other plentiful failures/segfaults
+  # with base pandas has been dealt with
+  - python=3.8
+
+  # tools
+  - cython>=0.29.24
+  - pytest>=6.0
+  - pytest-cov
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
+
+  # required
+  - numpy
+  - python-dateutil
+  - pytz

doc/source/whatsnew/v1.3.5.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_135:
 
-What's new in 1.3.5 (November ??, 2021)
+What's new in 1.3.5 (December 12, 2021)
 ---------------------------------------
 
 These are the changes in pandas 1.3.5. See :ref:`release` for a full changelog
@@ -16,29 +16,13 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed regression in :meth:`Series.equals` when comparing floats with dtype object to None (:issue:`44190`)
 - Fixed regression in :func:`merge_asof` raising error when array was supplied as join key (:issue:`42844`)
+- Fixed regression when resampling :class:`DataFrame` with :class:`DateTimeIndex` with empty groups and ``uint8``, ``uint16`` or ``uint32`` columns incorrectly raising ``RuntimeError`` (:issue:`43329`)
 - Fixed regression in creating a :class:`DataFrame` from a timezone-aware :class:`Timestamp` scalar near a Daylight Savings Time transition (:issue:`42505`)
 - Fixed performance regression in :func:`read_csv` (:issue:`44106`)
 - Fixed regression in :meth:`Series.duplicated` and :meth:`Series.drop_duplicates` when Series has :class:`Categorical` dtype with boolean categories (:issue:`44351`)
 - Fixed regression in :meth:`.GroupBy.sum` with ``timedelta64[ns]`` dtype containing ``NaT`` failing to treat that value as NA (:issue:`42659`)
--
+- Fixed regression in :meth:`.RollingGroupby.cov` and :meth:`.RollingGroupby.corr` when ``other`` had the same shape as each group would incorrectly return superfluous groups in the result (:issue:`42915`)
 
-.. ---------------------------------------------------------------------------
-
-.. _whatsnew_135.bug_fixes:
-
-Bug fixes
-~~~~~~~~~
--
--
-
-.. ---------------------------------------------------------------------------
-
-.. _whatsnew_135.other:
-
-Other
-~~~~~
--
--
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.4.0.rst

@@ -219,7 +219,7 @@ Other enhancements
 - :meth:`DataFrame.dropna` now accepts a single label as ``subset`` along with array-like (:issue:`41021`)
 - :class:`ExcelWriter` argument ``if_sheet_exists="overlay"`` option added (:issue:`40231`)
 - :meth:`read_excel` now accepts a ``decimal`` argument that allow the user to specify the decimal point when parsing string columns to numeric (:issue:`14403`)
-- :meth:`.GroupBy.mean` now supports `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`43731`)
+- :meth:`.GroupBy.mean`, :meth:`.GroupBy.std`, and :meth:`.GroupBy.var` now supports `Numba <http://numba.pydata.org/>`_ execution with the ``engine`` keyword (:issue:`43731`, :issue:`44862`)
 - :meth:`Timestamp.isoformat`, now handles the ``timespec`` argument from the base :class:``datetime`` class (:issue:`26131`)
 - :meth:`NaT.to_numpy` ``dtype`` argument is now respected, so ``np.timedelta64`` can be returned (:issue:`44460`)
 - New option ``display.max_dir_items`` customizes the number of columns added to :meth:`Dataframe.__dir__` and suggested for tab completion (:issue:`37996`)
@@ -229,6 +229,8 @@ Other enhancements
 - :meth:`Series.info` has been added, for compatibility with :meth:`DataFrame.info` (:issue:`5167`)
 - Implemented :meth:`IntervalArray.min`, :meth:`IntervalArray.max`, as a result of which ``min`` and ``max`` now work for :class:`IntervalIndex`, :class:`Series` and :class:`DataFrame` with ``IntervalDtype`` (:issue:`44746`)
 - :meth:`UInt64Index.map` now retains ``dtype`` where possible (:issue:`44609`)
+- :meth:`read_json` can now parse unsigned long long integers (:issue:`26068`)
+- :meth:`DataFrame.take` now raises a ``TypeError`` when passed a scalar for the indexer (:issue:`42875`)
 -
 
 
@@ -547,7 +549,7 @@ Performance improvements
 - Performance improvement in :meth:`.GroupBy.sample`, especially when ``weights`` argument provided (:issue:`34483`)
 - Performance improvement when converting non-string arrays to string arrays (:issue:`34483`)
 - Performance improvement in :meth:`.GroupBy.transform` for user-defined functions (:issue:`41598`)
-- Performance improvement in constructing :class:`DataFrame` objects (:issue:`42631`, :issue:`43142`, :issue:`43147`, :issue:`43307`, :issue:`43144`)
+- Performance improvement in constructing :class:`DataFrame` objects (:issue:`42631`, :issue:`43142`, :issue:`43147`, :issue:`43307`, :issue:`43144`, :issue:`44826`)
 - Performance improvement in :meth:`GroupBy.shift` when ``fill_value`` argument is provided (:issue:`26615`)
 - Performance improvement in :meth:`DataFrame.corr` for ``method=pearson`` on data without missing values (:issue:`40956`)
 - Performance improvement in some :meth:`GroupBy.apply` operations (:issue:`42992`, :issue:`43578`)
@@ -644,6 +646,7 @@ Numeric
 - Bug in arithmetic operations involving :class:`RangeIndex` where the result would have the incorrect ``name`` (:issue:`43962`)
 - Bug in arithmetic operations involving :class:`Series` where the result could have the incorrect ``name`` when the operands having matching NA or matching tuple names (:issue:`44459`)
 - Bug in division with ``IntegerDtype`` or ``BooleanDtype`` array and NA scalar incorrectly raising (:issue:`44685`)
+- Bug in multiplying a :class:`Series` with ``FloatingDtype`` with a timedelta-like scalar incorrectly raising (:issue:`44772`)
 -
 
 Conversion
@@ -657,7 +660,7 @@ Conversion
 
 Strings
 ^^^^^^^
--
+- Fixed bug in checking for ``string[pyarrow]`` dtype incorrectly raising an ImportError when pyarrow is not installed (:issue:`44327`)
 -
 
 Interval
@@ -696,8 +699,10 @@ Indexing
 - Bug in :meth:`DataFrame.loc.__getitem__` incorrectly raising ``KeyError`` when selecting a single column with a boolean key (:issue:`44322`).
 - Bug in setting :meth:`DataFrame.iloc` with a single ``ExtensionDtype`` column and setting 2D values e.g. ``df.iloc[:] = df.values`` incorrectly raising (:issue:`44514`)
 - Bug in indexing on columns with ``loc`` or ``iloc`` using a slice with a negative step with ``ExtensionDtype`` columns incorrectly raising (:issue:`44551`)
+- Bug in :meth:`DataFrame.loc.__setitem__` changing dtype when indexer was completely ``False`` (:issue:`37550`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` returning boolean mask instead of array of integers for a non unique and non monotonic index (:issue:`44084`)
 - Bug in :meth:`IntervalIndex.get_indexer_non_unique` not handling targets of ``dtype`` 'object' with NaNs correctly (:issue:`44482`)
+- Fixed regression where a single column ``np.matrix`` was no longer coerced to a 1d ``np.ndarray`` when added to a :class:`DataFrame` (:issue:`42376`)
 -
 
 Missing
@@ -706,13 +711,15 @@ Missing
 - Bug in :meth:`DataFrame.fillna` not replacing missing values when using a dict-like ``value`` and duplicate column names (:issue:`43476`)
 - Bug in constructing a :class:`DataFrame` with a dictionary ``np.datetime64`` as a value and ``dtype='timedelta64[ns]'``, or vice-versa, incorrectly casting instead of raising (:issue:`??`)
 - Bug in :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` with ``inplace=True`` not writing to the underlying array(s) in-place (:issue:`44749`)
+- Bug in :meth:`Index.fillna` incorrectly returning an un-filled :class:`Index` when NA values are present and ``downcast`` argument is specified. This now raises ``NotImplementedError`` instead; do not pass ``downcast`` argument (:issue:`44873`)
 -
 
 MultiIndex
 ^^^^^^^^^^
 - Bug in :meth:`MultiIndex.get_loc` where the first level is a :class:`DatetimeIndex` and a string key is passed (:issue:`42465`)
 - Bug in :meth:`MultiIndex.reindex` when passing a ``level`` that corresponds to an ``ExtensionDtype`` level (:issue:`42043`)
 - Bug in :meth:`MultiIndex.get_loc` raising ``TypeError`` instead of ``KeyError`` on nested tuple (:issue:`42440`)
+- Bug in :meth:`MultiIndex.union` setting wrong ``sortorder`` causing errors in subsequent indexing operations with slices (:issue:`44752`)
 - Bug in :meth:`MultiIndex.putmask` where the other value was also a :class:`MultiIndex` (:issue:`43212`)
 -
 
@@ -747,6 +754,9 @@ I/O
 - Bug in :func:`read_csv` raising ``AttributeError`` when attempting to read a .csv file and infer index column dtype from an nullable integer type (:issue:`44079`)
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` with ``compression`` set to ``'zip'`` no longer create a zip file containing a file ending with ".zip". Instead, they try to infer the inner file name more smartly. (:issue:`39465`)
 - Bug in :func:`read_csv` when passing simultaneously a parser in ``date_parser`` and ``parse_dates=False``, the parsing was still called (:issue:`44366`)
+- Bug in :func:`read_csv` silently ignoring errors when failling to create a memory-mapped file (:issue:`44766`)
+- Bug in :func:`read_csv` when passing a ``tempfile.SpooledTemporaryFile`` opened in binary mode (:issue:`44748`)
+-
 
 Period
 ^^^^^^

pandas/_libs/interval.pyx

@@ -516,9 +516,9 @@ def intervals_to_interval_bounds(ndarray intervals, bint validate_closed=True):
 
     Returns
     -------
-    tuple of tuples
-        left : (ndarray, object, array)
-        right : (ndarray, object, array)
+    tuple of
+        left : ndarray
+        right : ndarray
         closed: str
     """
     cdef:

pandas/_libs/join.pyi

@@ -55,39 +55,39 @@ def asof_join_backward_on_X_by_Y(
     left_by_values: np.ndarray,  # by_t[:]
     right_by_values: np.ndarray,  # by_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_forward_on_X_by_Y(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     left_by_values: np.ndarray,  # by_t[:]
     right_by_values: np.ndarray,  # by_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_nearest_on_X_by_Y(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     left_by_values: np.ndarray,  # by_t[:]
     right_by_values: np.ndarray,  # by_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_backward(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_forward(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
 def asof_join_nearest(
     left_values: np.ndarray,  # asof_t[:]
     right_values: np.ndarray,  # asof_t[:]
     allow_exact_matches: bool = ...,
-    tolerance=...,
+    tolerance: np.number | int | float | None = ...,
 ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...

pandas/_libs/missing.pyi

@@ -0,0 +1,15 @@
+import numpy as np
+from numpy import typing as npt
+
+class NAType: ...
+
+NA: NAType
+
+def is_matching_na(
+    left: object, right: object, nan_matches_none: bool = ...
+) -> bool: ...
+def isposinf_scalar(val: object) -> bool: ...
+def isneginf_scalar(val: object) -> bool: ...
+def checknull(val: object, inf_as_na: bool = ...) -> bool: ...
+def isnaobj(arr: np.ndarray, inf_as_na: bool = ...) -> npt.NDArray[np.bool_]: ...
+def is_numeric_na(values: np.ndarray) -> npt.NDArray[np.bool_]: ...

pandas/_libs/ops.pyi

@@ -1,6 +1,7 @@
 from typing import (
     Any,
     Callable,
+    Iterable,
     Literal,
     overload,
 )
@@ -35,15 +36,15 @@ def vec_binop(
 @overload
 def maybe_convert_bool(
     arr: npt.NDArray[np.object_],
-    true_values=...,
-    false_values=...,
+    true_values: Iterable = ...,
+    false_values: Iterable = ...,
     convert_to_masked_nullable: Literal[False] = ...,
 ) -> tuple[np.ndarray, None]: ...
 @overload
 def maybe_convert_bool(
     arr: npt.NDArray[np.object_],
-    true_values=...,
-    false_values=...,
+    true_values: Iterable = ...,
+    false_values: Iterable = ...,
     *,
     convert_to_masked_nullable: Literal[True],
 ) -> tuple[np.ndarray, np.ndarray]: ...

pandas/_libs/ops_dispatch.pyx

@@ -34,7 +34,7 @@ UFUNC_ALIASES = {
     "true_divide": "truediv",
     "power": "pow",
     "remainder": "mod",
-    "divide": "div",
+    "divide": "truediv",
     "equal": "eq",
     "not_equal": "ne",
     "less": "lt",

pandas/_libs/src/ujson/lib/ultrajson.h

@@ -297,6 +297,7 @@ typedef struct __JSONObjectDecoder {
   JSOBJ (*endArray)(void *prv, JSOBJ obj);
   JSOBJ (*newInt)(void *prv, JSINT32 value);
   JSOBJ (*newLong)(void *prv, JSINT64 value);
+  JSOBJ (*newUnsignedLong)(void *prv, JSUINT64 value);
   JSOBJ (*newDouble)(void *prv, double value);
   void (*releaseObject)(void *prv, JSOBJ obj, void *decoder);
   JSPFN_MALLOC malloc;

pandas/_libs/src/ujson/lib/ultrajsondec.c

@@ -116,8 +116,8 @@ JSOBJ FASTCALL_MSVC decodePreciseFloat(struct DecoderState *ds) {
 
 JSOBJ FASTCALL_MSVC decode_numeric(struct DecoderState *ds) {
     int intNeg = 1;
-    int mantSize = 0;
     JSUINT64 intValue;
+    JSUINT64 prevIntValue;
     int chr;
     int decimalCount = 0;
     double frcValue = 0.0;
@@ -134,10 +134,10 @@ JSOBJ FASTCALL_MSVC decode_numeric(struct DecoderState *ds) {
     } else if (*(offset) == '-') {
         offset++;
         intNeg = -1;
+        overflowLimit  = LLONG_MIN;
         if (*(offset) == 'I') {
           goto DECODE_INF;
         }
-        overflowLimit = LLONG_MIN;
     }
 
     // Scan integer part
@@ -157,19 +157,18 @@ JSOBJ FASTCALL_MSVC decode_numeric(struct DecoderState *ds) {
             case '7':
             case '8':
             case '9': {
-                // FIXME: Check for arithmetic overflow here
-                // PERF: Don't do 64-bit arithmetic here unless we know we have
-                // to
-                intValue = intValue * 10ULL + (JSLONG)(chr - 48);
-
-                if (intValue > overflowLimit) {
-                    return SetError(ds, -1, overflowLimit == LLONG_MAX
-                                                ? "Value is too big"
-                                                : "Value is too small");
+                // PERF: Don't do 64-bit arithmetic here unless we have to
+                prevIntValue = intValue;
+                intValue = intValue * 10ULL + (JSLONG) (chr - 48);
+
+                if (intNeg == 1 && prevIntValue > intValue) {
+                    return SetError(ds, -1, "Value is too big!");
+                } else if (intNeg == -1 && intValue > overflowLimit) {
+                    return SetError(ds, -1, overflowLimit == LLONG_MAX ?
+                                    "Value is too big!" : "Value is too small");
                 }
 
                 offset++;
-                mantSize++;
                 break;
             }
             case '.': {
@@ -196,11 +195,12 @@ JSOBJ FASTCALL_MSVC decode_numeric(struct DecoderState *ds) {
     ds->lastType = JT_INT;
     ds->start = offset;
 
-    if ((intValue >> 31)) {
+    if (intNeg == 1 && (intValue & 0x8000000000000000ULL) != 0)
+        return ds->dec->newUnsignedLong(ds->prv, intValue);
+    else if ((intValue >> 31))
         return ds->dec->newLong(ds->prv, (JSINT64)(intValue * (JSINT64)intNeg));
-    } else {
+    else
         return ds->dec->newInt(ds->prv, (JSINT32)(intValue * intNeg));
-    }
 
 DECODE_FRACTION:
 

pandas/_libs/src/ujson/python/JSONtoObj.c

@@ -479,6 +479,10 @@ JSOBJ Object_newLong(void *prv, JSINT64 value) {
     return PyLong_FromLongLong(value);
 }
 
+JSOBJ Object_newUnsignedLong(void *prv, JSUINT64 value) {
+    return PyLong_FromUnsignedLongLong(value);
+}
+
 JSOBJ Object_newDouble(void *prv, double value) {
     return PyFloat_FromDouble(value);
 }
@@ -508,7 +512,8 @@ PyObject *JSONToObj(PyObject *self, PyObject *args, PyObject *kwargs) {
         Object_newTrue,   Object_newFalse,      Object_newNull,
         Object_newPosInf, Object_newNegInf,     Object_newObject,
         Object_endObject,     Object_newArray,  Object_endArray,
-        Object_newInteger,    Object_newLong,   Object_newDouble,
+        Object_newInteger,    Object_newLong,   Object_newUnsignedLong,
+        Object_newDouble,
         Object_releaseObject, PyObject_Malloc, PyObject_Free,
         PyObject_Realloc};