Merge branch 'master' of https://github.com/pandas-dev/pandas into sty-private

Author: jbrockmendel

doc/source/whatsnew/v1.1.2.rst

@@ -17,10 +17,12 @@ Fixed regressions
 - Regression in :meth:`DatetimeIndex.intersection` incorrectly raising ``AssertionError`` when intersecting against a list (:issue:`35876`)
 - Fix regression in updating a column inplace (e.g. using ``df['col'].fillna(.., inplace=True)``) (:issue:`35731`)
 - Performance regression for :meth:`RangeIndex.format` (:issue:`35712`)
+- Fix regression in invalid cache after an indexing operation; this can manifest when setting which does not update the data (:issue:`35521`)
 - Regression in :meth:`DataFrame.replace` where a ``TypeError`` would be raised when attempting to replace elements of type :class:`Interval` (:issue:`35931`)
+- Fix regression in pickle roundtrip of the ``closed`` attribute of :class:`IntervalIndex` (:issue:`35658`)
+- Fixed regression in :meth:`DataFrameGroupBy.agg` where a ``ValueError: buffer source array is read-only`` would be raised when the underlying array is read-only (:issue:`36014`)
 -
 
-
 .. ---------------------------------------------------------------------------
 
 .. _whatsnew_112.bug_fixes:
@@ -32,6 +34,7 @@ Bug fixes
 - Bug in :meth:`DataFrame.apply` with ``result_type="reduce"`` returning with incorrect index (:issue:`35683`)
 - Bug in :meth:`DateTimeIndex.format` and :meth:`PeriodIndex.format` with ``name=True`` setting the first item to ``"None"`` where it should be ``""`` (:issue:`35712`)
 - Bug in :meth:`Float64Index.__contains__` incorrectly raising ``TypeError`` instead of returning ``False`` (:issue:`35788`)
+- Bug in :meth:`Series.dt.isocalendar` and :meth:`DatetimeIndex.isocalendar` that returned incorrect year for certain dates (:issue:`36032`)
 - Bug in :class:`DataFrame` indexing returning an incorrect :class:`Series` in some cases when the series has been altered and a cache not invalidated (:issue:`33675`)
 
 .. ---------------------------------------------------------------------------

pandas/_libs/groupby.pyx

@@ -229,7 +229,7 @@ def group_cumprod_float64(float64_t[:, :] out,
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_cumsum(numeric[:, :] out,
-                 numeric[:, :] values,
+                 ndarray[numeric, ndim=2] values,
                  const int64_t[:] labels,
                  int ngroups,
                  is_datetimelike,
@@ -472,7 +472,7 @@ ctypedef fused complexfloating_t:
 @cython.boundscheck(False)
 def _group_add(complexfloating_t[:, :] out,
                int64_t[:] counts,
-               complexfloating_t[:, :] values,
+               ndarray[complexfloating_t, ndim=2] values,
                const int64_t[:] labels,
                Py_ssize_t min_count=0):
     """
@@ -483,8 +483,9 @@ def _group_add(complexfloating_t[:, :] out,
         complexfloating_t val, count
         complexfloating_t[:, :] sumx
         int64_t[:, :] nobs
+        Py_ssize_t len_values = len(values), len_labels = len(labels)
 
-    if len(values) != len(labels):
+    if len_values != len_labels:
         raise ValueError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -530,7 +531,7 @@ group_add_complex128 = _group_add['double complex']
 @cython.boundscheck(False)
 def _group_prod(floating[:, :] out,
                 int64_t[:] counts,
-                floating[:, :] values,
+                ndarray[floating, ndim=2] values,
                 const int64_t[:] labels,
                 Py_ssize_t min_count=0):
     """
@@ -541,8 +542,9 @@ def _group_prod(floating[:, :] out,
         floating val, count
         floating[:, :] prodx
         int64_t[:, :] nobs
+        Py_ssize_t len_values = len(values), len_labels = len(labels)
 
-    if not len(values) == len(labels):
+    if len_values != len_labels:
         raise ValueError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -582,7 +584,7 @@ group_prod_float64 = _group_prod['double']
 @cython.cdivision(True)
 def _group_var(floating[:, :] out,
                int64_t[:] counts,
-               floating[:, :] values,
+               ndarray[floating, ndim=2] values,
                const int64_t[:] labels,
                Py_ssize_t min_count=-1,
                int64_t ddof=1):
@@ -591,10 +593,11 @@ def _group_var(floating[:, :] out,
         floating val, ct, oldmean
         floating[:, :] mean
         int64_t[:, :] nobs
+        Py_ssize_t len_values = len(values), len_labels = len(labels)
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    if len_values != len_labels:
         raise ValueError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -639,18 +642,19 @@ group_var_float64 = _group_var['double']
 @cython.boundscheck(False)
 def _group_mean(floating[:, :] out,
                 int64_t[:] counts,
-                floating[:, :] values,
+                ndarray[floating, ndim=2] values,
                 const int64_t[:] labels,
                 Py_ssize_t min_count=-1):
     cdef:
         Py_ssize_t i, j, N, K, lab, ncounts = len(counts)
         floating val, count
         floating[:, :] sumx
         int64_t[:, :] nobs
+        Py_ssize_t len_values = len(values), len_labels = len(labels)
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    if len_values != len_labels:
         raise ValueError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -689,7 +693,7 @@ group_mean_float64 = _group_mean['double']
 @cython.boundscheck(False)
 def _group_ohlc(floating[:, :] out,
                 int64_t[:] counts,
-                floating[:, :] values,
+                ndarray[floating, ndim=2] values,
                 const int64_t[:] labels,
                 Py_ssize_t min_count=-1):
     """
@@ -740,7 +744,7 @@ group_ohlc_float64 = _group_ohlc['double']
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_quantile(ndarray[float64_t] out,
-                   numeric[:] values,
+                   ndarray[numeric, ndim=1] values,
                    ndarray[int64_t] labels,
                    ndarray[uint8_t] mask,
                    float64_t q,
@@ -1072,7 +1076,7 @@ def group_nth(rank_t[:, :] out,
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_rank(float64_t[:, :] out,
-               rank_t[:, :] values,
+               ndarray[rank_t, ndim=2] values,
                const int64_t[:] labels,
                int ngroups,
                bint is_datetimelike, object ties_method="average",
@@ -1424,7 +1428,7 @@ def group_min(groupby_t[:, :] out,
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_cummin(groupby_t[:, :] out,
-                 groupby_t[:, :] values,
+                 ndarray[groupby_t, ndim=2] values,
                  const int64_t[:] labels,
                  int ngroups,
                  bint is_datetimelike):
@@ -1484,7 +1488,7 @@ def group_cummin(groupby_t[:, :] out,
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def group_cummax(groupby_t[:, :] out,
-                 groupby_t[:, :] values,
+                 ndarray[groupby_t, ndim=2] values,
                  const int64_t[:] labels,
                  int ngroups,
                  bint is_datetimelike):

pandas/_libs/indexing.pyx

@@ -1,4 +1,4 @@
-cdef class _NDFrameIndexerBase:
+cdef class NDFrameIndexerBase:
     """
     A base class for _NDFrameIndexer for fast instantiation and attribute access.
     """

pandas/_libs/missing.pyx

@@ -18,15 +18,15 @@ from pandas._libs.tslibs.nattype cimport (
 from pandas._libs.tslibs.np_datetime cimport get_datetime64_value, get_timedelta64_value
 
 from pandas._libs.ops_dispatch import maybe_dispatch_ufunc_to_dunder_op
-from pandas.compat import is_platform_32bit
+from pandas.compat import IS64
 
 cdef:
     float64_t INF = <float64_t>np.inf
     float64_t NEGINF = -INF
 
     int64_t NPY_NAT = util.get_nat()
 
-    bint is_32bit = is_platform_32bit()
+    bint is_32bit = not IS64
 
 
 cpdef bint checknull(object val):

pandas/_libs/tslibs/ccalendar.pyx

@@ -201,10 +201,10 @@ cpdef iso_calendar_t get_iso_calendar(int year, int month, int day) nogil:
             iso_week = 1
 
     iso_year = year
-    if iso_week == 1 and doy > 7:
+    if iso_week == 1 and month == 12:
         iso_year += 1
 
-    elif iso_week >= 52 and doy < 7:
+    elif iso_week >= 52 and month == 1:
         iso_year -= 1
 
     return iso_year, iso_week, dow + 1

pandas/_libs/tslibs/parsing.pyx

@@ -771,7 +771,7 @@ class _timelex:
 _DATEUTIL_LEXER_SPLIT = _timelex.split
 
 
-def _format_is_iso(f) -> bint:
+def format_is_iso(f: str) -> bint:
     """
     Does format match the iso8601 set that can be handled by the C parser?
     Generally of form YYYY-MM-DDTHH:MM:SS - date separator can be different
@@ -789,7 +789,7 @@ def _format_is_iso(f) -> bint:
     return False
 
 
-def _guess_datetime_format(
+def guess_datetime_format(
     dt_str,
     bint dayfirst=False,
     dt_str_parse=du_parse,

pandas/compat/__init__.py

@@ -8,7 +8,6 @@
 * platform checker
 """
 import platform
-import struct
 import sys
 import warnings
 
@@ -20,14 +19,6 @@
 IS64 = sys.maxsize > 2 ** 32
 
 
-# ----------------------------------------------------------------------------
-# functions largely based / taken from the six module
-
-# Much of the code in this module comes from Benjamin Peterson's six library.
-# The license for this library can be found in LICENSES/SIX and the code can be
-# found at https://bitbucket.org/gutworth/six
-
-
 def set_function_name(f: F, name: str, cls) -> F:
     """
     Bind the name/qualname attributes of the function.
@@ -38,7 +29,6 @@ def set_function_name(f: F, name: str, cls) -> F:
     return f
 
 
-# https://github.com/pandas-dev/pandas/pull/9123
 def is_platform_little_endian() -> bool:
     """
     Checking if the running platform is little endian.
@@ -72,7 +62,7 @@ def is_platform_linux() -> bool:
     bool
         True if the running platform is linux.
     """
-    return sys.platform == "linux2"
+    return sys.platform == "linux"
 
 
 def is_platform_mac() -> bool:
@@ -87,18 +77,6 @@ def is_platform_mac() -> bool:
     return sys.platform == "darwin"
 
 
-def is_platform_32bit() -> bool:
-    """
-    Checking if the running platform is 32-bit.
-
-    Returns
-    -------
-    bool
-        True if the running platform is 32-bit.
-    """
-    return struct.calcsize("P") * 8 < 64
-
-
 def _import_lzma():
     """
     Importing the `lzma` module.

pandas/core/arrays/datetimes.py

@@ -602,9 +602,9 @@ def astype(self, dtype, copy=True):
     # Rendering Methods
 
     def _format_native_types(self, na_rep="NaT", date_format=None, **kwargs):
-        from pandas.io.formats.format import _get_format_datetime64_from_values
+        from pandas.io.formats.format import get_format_datetime64_from_values
 
-        fmt = _get_format_datetime64_from_values(self, date_format)
+        fmt = get_format_datetime64_from_values(self, date_format)
 
         return tslib.format_array_from_datetime(
             self.asi8.ravel(), tz=self.tz, format=fmt, na_rep=na_rep

pandas/core/arrays/timedeltas.py

@@ -379,14 +379,14 @@ def median(
     # Rendering Methods
 
     def _formatter(self, boxed=False):
-        from pandas.io.formats.format import _get_format_timedelta64
+        from pandas.io.formats.format import get_format_timedelta64
 
-        return _get_format_timedelta64(self, box=True)
+        return get_format_timedelta64(self, box=True)
 
     def _format_native_types(self, na_rep="NaT", date_format=None, **kwargs):
-        from pandas.io.formats.format import _get_format_timedelta64
+        from pandas.io.formats.format import get_format_timedelta64
 
-        formatter = _get_format_timedelta64(self._data, na_rep)
+        formatter = get_format_timedelta64(self._data, na_rep)
         return np.array([formatter(x) for x in self._data.ravel()]).reshape(self.shape)
 
     # ----------------------------------------------------------------

pandas/core/common.py

@@ -31,7 +31,7 @@
     ABCIndexClass,
     ABCSeries,
 )
-from pandas.core.dtypes.inference import _iterable_not_string
+from pandas.core.dtypes.inference import iterable_not_string
 from pandas.core.dtypes.missing import isna, isnull, notnull  # noqa
 
 
@@ -61,7 +61,7 @@ def flatten(l):
     flattened : generator
     """
     for el in l:
-        if _iterable_not_string(el):
+        if iterable_not_string(el):
             for s in flatten(el):
                 yield s
         else:

pandas/core/computation/common.py

@@ -5,7 +5,7 @@
 from pandas._config import get_option
 
 
-def _ensure_decoded(s):
+def ensure_decoded(s):
     """
     If we have bytes, decode them to unicode.
     """

pandas/core/computation/ops.py

@@ -15,7 +15,7 @@
 from pandas.core.dtypes.common import is_list_like, is_scalar
 
 import pandas.core.common as com
-from pandas.core.computation.common import _ensure_decoded, result_type_many
+from pandas.core.computation.common import ensure_decoded, result_type_many
 from pandas.core.computation.scope import _DEFAULT_GLOBALS
 
 from pandas.io.formats.printing import pprint_thing, pprint_thing_encoded
@@ -466,7 +466,7 @@ def stringify(value):
             v = rhs.value
             if isinstance(v, (int, float)):
                 v = stringify(v)
-            v = Timestamp(_ensure_decoded(v))
+            v = Timestamp(ensure_decoded(v))
             if v.tz is not None:
                 v = v.tz_convert("UTC")
             self.rhs.update(v)
@@ -475,7 +475,7 @@ def stringify(value):
             v = lhs.value
             if isinstance(v, (int, float)):
                 v = stringify(v)
-            v = Timestamp(_ensure_decoded(v))
+            v = Timestamp(ensure_decoded(v))
             if v.tz is not None:
                 v = v.tz_convert("UTC")
             self.lhs.update(v)

pandas/core/computation/pytables.py

@@ -14,7 +14,7 @@
 import pandas as pd
 import pandas.core.common as com
 from pandas.core.computation import expr, ops, scope as _scope
-from pandas.core.computation.common import _ensure_decoded
+from pandas.core.computation.common import ensure_decoded
 from pandas.core.computation.expr import BaseExprVisitor
 from pandas.core.computation.ops import UndefinedVariableError, is_term
 from pandas.core.construction import extract_array
@@ -189,12 +189,12 @@ def stringify(value):
                 encoder = pprint_thing
             return encoder(value)
 
-        kind = _ensure_decoded(self.kind)
-        meta = _ensure_decoded(self.meta)
+        kind = ensure_decoded(self.kind)
+        meta = ensure_decoded(self.meta)
         if kind == "datetime64" or kind == "datetime":
             if isinstance(v, (int, float)):
                 v = stringify(v)
-            v = _ensure_decoded(v)
+            v = ensure_decoded(v)
             v = Timestamp(v)
             if v.tz is not None:
                 v = v.tz_convert("UTC")

pandas/core/dtypes/cast.py

@@ -651,7 +651,7 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> Tuple[DtypeObj,
         If False, scalar belongs to pandas extension types is inferred as
         object
     """
-    dtype = np.dtype(object)
+    dtype: DtypeObj = np.dtype(object)
 
     # a 1-element ndarray
     if isinstance(val, np.ndarray):