Merge remote-tracking branch 'upstream/main' into td-construction

Author: patrickmckenna

.github/workflows/posix.yml

@@ -155,24 +155,11 @@ jobs:
         channel-priority: flexible
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
-      if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support
 
     - name: Upgrade Arrow version
       run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }}
       if: ${{ matrix.pyarrow_version }}
 
-    - name: Setup PyPy
-      uses: actions/setup-python@v3
-      with:
-        python-version: "pypy-3.8"
-      if: ${{ env.IS_PYPY == 'true' }}
-
-    - name: Setup PyPy dependencies
-      run: |
-        # TODO: re-enable cov, its slowing the tests down though
-        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio>=0.17 hypothesis>=5.5.3
-      if: ${{ env.IS_PYPY == 'true' }}
-
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 

asv_bench/benchmarks/strings.py

@@ -268,7 +268,7 @@ def time_get_dummies(self, dtype):
 
 class Encode:
     def setup(self):
-        self.ser = Series(tm.makeUnicodeIndex())
+        self.ser = Series(tm.makeStringIndex())
 
     def time_encode_decode(self):
         self.ser.str.encode("utf-8").str.decode("utf-8")

ci/deps/actions-38-minimum_versions.yaml

@@ -20,38 +20,37 @@ dependencies:
   - numpy=1.18.5
   - pytz=2020.1
 
-  # optional dependencies, markupsafe for jinja2
-  - beautifulsoup4=4.8.2
-  - blosc=1.20.1
-  - bottleneck=1.3.1
+  # optional dependencies
+  - beautifulsoup4=4.9.3
+  - blosc=1.21.0
+  - bottleneck=1.3.2
   - brotlipy=0.7.0
   - fastparquet=0.4.0
-  - fsspec=0.7.4
+  - fsspec=2021.05.0
   - html5lib=1.1
-  - hypothesis=5.5.3
-  - gcsfs=0.6.0
-  - jinja2=2.11
-  - lxml=4.5.0
-  - markupsafe=2.0.1
+  - hypothesis=6.13.0
+  - gcsfs=2021.05.0
+  - jinja2=3.0.0
+  - lxml=4.6.3
   - matplotlib=3.3.2
-  - numba=0.50.1
-  - numexpr=2.7.1
+  - numba=0.53.1
+  - numexpr=2.7.3
   - odfpy=1.4.1
-  - openpyxl=3.0.3
-  - pandas-gbq=0.14.0
-  - psycopg2=2.8.4
+  - openpyxl=3.0.7
+  - pandas-gbq=0.15.0
+  - psycopg2=2.8.6
   - pyarrow=1.0.1
-  - pymysql=0.10.1
-  - pyreadstat=1.1.0
+  - pymysql=1.0.2
+  - pyreadstat=1.1.2
   - pytables=3.6.1
   - python-snappy=0.6.0
-  - pyxlsb=1.0.6
-  - s3fs=0.4.0
-  - scipy=1.4.1
-  - sqlalchemy=1.4.0
-  - tabulate=0.8.7
-  - xarray=0.15.1
+  - pyxlsb=1.0.8
+  - s3fs=2021.05.0
+  - scipy=1.7.1
+  - sqlalchemy=1.4.16
+  - tabulate=0.8.9
+  - xarray=0.19.0
   - xlrd=2.0.1
-  - xlsxwriter=1.2.2
+  - xlsxwriter=1.4.3
   - xlwt=1.3.0
   - zstandard=0.15.2

ci/deps/actions-pypy-38.yaml

@@ -11,6 +11,7 @@ dependencies:
   - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
+  - pytest-asyncio
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
 

doc/source/whatsnew/v1.4.3.rst

@@ -16,7 +16,8 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 - Fixed regression in :meth:`DataFrame.nsmallest` led to wrong results when ``np.nan`` in the sorting column (:issue:`46589`)
 - Fixed regression in :func:`read_fwf` raising ``ValueError`` when ``widths`` was specified with ``usecols`` (:issue:`46580`)
--
+- Fixed regression in :meth:`.Groupby.transform` and :meth:`.Groupby.agg` failing with ``engine="numba"`` when the index was a :class:`MultiIndex` (:issue:`46867`)
+- Fixed regression is :meth:`.Styler.to_latex` and :meth:`.Styler.to_html` where ``buf`` failed in combination with ``encoding`` (:issue:`47053`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.5.0.rst

@@ -150,6 +150,7 @@ Other enhancements
 - Added ``validate`` argument to :meth:`DataFrame.join` (:issue:`46622`)
 - A :class:`errors.PerformanceWarning` is now thrown when using ``string[pyarrow]`` dtype with methods that don't dispatch to ``pyarrow.compute`` methods (:issue:`42613`)
 - Added ``numeric_only`` argument to :meth:`Resampler.sum`, :meth:`Resampler.prod`, :meth:`Resampler.min`, :meth:`Resampler.max`, :meth:`Resampler.first`, and :meth:`Resampler.last` (:issue:`46442`)
+- ``times`` argument in :class:`.ExponentialMovingWindow` now accepts ``np.timedelta64`` (:issue:`47003`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -362,7 +363,50 @@ If installed, we now require:
 +=================+=================+==========+=========+
 | mypy (dev)      | 0.950           |          |    X    |
 +-----------------+-----------------+----------+---------+
-
+| beautifulsoup4  | 4.9.3           |          |    X    |
++-----------------+-----------------+----------+---------+
+| blosc           | 1.21.0          |          |    X    |
++-----------------+-----------------+----------+---------+
+| bottleneck      | 1.3.2           |          |    X    |
++-----------------+-----------------+----------+---------+
+| fsspec          | 2021.05.0       |          |    X    |
++-----------------+-----------------+----------+---------+
+| hypothesis      | 6.13.0          |          |    X    |
++-----------------+-----------------+----------+---------+
+| gcsfs           | 2021.05.0       |          |    X    |
++-----------------+-----------------+----------+---------+
+| jinja2          | 3.0.0           |          |    X    |
++-----------------+-----------------+----------+---------+
+| lxml            | 4.6.3           |          |    X    |
++-----------------+-----------------+----------+---------+
+| numba           | 0.53.1          |          |    X    |
++-----------------+-----------------+----------+---------+
+| numexpr         | 2.7.3           |          |    X    |
++-----------------+-----------------+----------+---------+
+| openpyxl        | 3.0.7           |          |    X    |
++-----------------+-----------------+----------+---------+
+| pandas-gbq      | 0.15.0          |          |    X    |
++-----------------+-----------------+----------+---------+
+| psycopg2        | 2.8.6           |          |    X    |
++-----------------+-----------------+----------+---------+
+| pymysql         | 1.0.2           |          |    X    |
++-----------------+-----------------+----------+---------+
+| pyreadstat      | 1.1.2           |          |    X    |
++-----------------+-----------------+----------+---------+
+| pyxlsb          | 1.0.8           |          |    X    |
++-----------------+-----------------+----------+---------+
+| s3fs            | 2021.05.0       |          |    X    |
++-----------------+-----------------+----------+---------+
+| scipy           | 1.7.1           |          |    X    |
++-----------------+-----------------+----------+---------+
+| sqlalchemy      | 1.4.16          |          |    X    |
++-----------------+-----------------+----------+---------+
+| tabulate        | 0.8.9           |          |    X    |
++-----------------+-----------------+----------+---------+
+| xarray          | 0.19.0          |          |    X    |
++-----------------+-----------------+----------+---------+
+| xlsxwriter      | 1.4.3           |          |    X    |
++-----------------+-----------------+----------+---------+
 
 For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
@@ -493,7 +537,8 @@ retained by specifying ``group_keys=False``.
 ``numeric_only`` default value
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Across the DataFrame operations such as ``min``, ``sum``, and ``idxmax``, the default
+Across the DataFrame and DataFrameGroupBy operations such as
+``min``, ``sum``, and ``idxmax``, the default
 value of the ``numeric_only`` argument, if it exists at all, was inconsistent.
 Furthermore, operations with the default value ``None`` can lead to surprising
 results. (:issue:`46560`)
@@ -523,6 +568,8 @@ gained the ``numeric_only`` argument.
 - :meth:`DataFrame.cov`
 - :meth:`DataFrame.idxmin`
 - :meth:`DataFrame.idxmax`
+- :meth:`.DataFrameGroupBy.cummin`
+- :meth:`.DataFrameGroupBy.cummax`
 - :meth:`.DataFrameGroupBy.idxmin`
 - :meth:`.DataFrameGroupBy.idxmax`
 - :meth:`.GroupBy.var`
@@ -553,6 +600,8 @@ Other Deprecations
 - Deprecated the ``closed`` argument in :meth:`interval_range` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the methods :meth:`DataFrame.mad`, :meth:`Series.mad`, and the corresponding groupby methods (:issue:`11787`)
 - Deprecated positional arguments to :meth:`Index.join` except for ``other``, use keyword-only arguments instead of positional arguments (:issue:`46518`)
+- Deprecated indexing on a timezone-naive :class:`DatetimeIndex` using a string representing a timezone-aware datetime (:issue:`46903`, :issue:`36148`)
+-
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.performance:
@@ -594,6 +643,7 @@ Datetimelike
 - Bug in :meth:`Index.astype` when casting from object dtype to ``timedelta64[ns]`` dtype incorrectly casting ``np.datetime64("NaT")`` values to ``np.timedelta64("NaT")`` instead of raising (:issue:`45722`)
 - Bug in :meth:`SeriesGroupBy.value_counts` index when passing categorical column (:issue:`44324`)
 - Bug in :meth:`DatetimeIndex.tz_localize` localizing to UTC failing to make a copy of the underlying data (:issue:`46460`)
+- Bug in :meth:`DatetimeIndex.resolution` incorrectly returning "day" instead of "nanosecond" for nanosecond-resolution indexes (:issue:`46903`)
 -
 
 Timedelta
@@ -745,6 +795,7 @@ Reshaping
 - Bug in concanenation with ``IntegerDtype``, or ``FloatingDtype`` arrays where the resulting dtype did not mirror the behavior of the non-nullable dtypes (:issue:`46379`)
 - Bug in :func:`concat` with identical key leads to error when indexing :class:`MultiIndex` (:issue:`46519`)
 - Bug in :meth:`DataFrame.join` with a list when using suffixes to join DataFrames with duplicate column names (:issue:`46396`)
+- Bug in :meth:`DataFrame.pivot_table` with ``sort=False`` results in sorted index (:issue:`17041`)
 -
 
 Sparse

pandas/_libs/src/ujson/lib/ultrajson.h

@@ -29,7 +29,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 https://github.com/client9/stringencoders
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
  * Copyright (c) 1988-1993 The Regents of the University of California.
  * Copyright (c) 1994 Sun Microsystems, Inc.

pandas/_libs/src/ujson/lib/ultrajsondec.c

@@ -32,7 +32,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights
 reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
 * Copyright (c) 1988-1993 The Regents of the University of California.
 * Copyright (c) 1994 Sun Microsystems, Inc.

pandas/_libs/src/ujson/lib/ultrajsonenc.c

@@ -32,7 +32,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights
 reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
  * Copyright (c) 1988-1993 The Regents of the University of California.
  * Copyright (c) 1994 Sun Microsystems, Inc.

pandas/_libs/src/ujson/python/JSONtoObj.c

@@ -29,7 +29,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 https://github.com/client9/stringencoders
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
  * Copyright (c) 1988-1993 The Regents of the University of California.
  * Copyright (c) 1994 Sun Microsystems, Inc.

pandas/_libs/src/ujson/python/objToJSON.c

@@ -30,7 +30,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights
 reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
 * Copyright (c) 1988-1993 The Regents of the University of California.
 * Copyright (c) 1994 Sun Microsystems, Inc.

pandas/_libs/src/ujson/python/ujson.c

@@ -29,7 +29,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 https://github.com/client9/stringencoders
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
 * Copyright (c) 1988-1993 The Regents of the University of California.
 * Copyright (c) 1994 Sun Microsystems, Inc.

pandas/_libs/src/ujson/python/version.h

@@ -29,7 +29,7 @@ Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc)
 https://github.com/client9/stringencoders
 Copyright (c) 2007  Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved.
 
-Numeric decoder derived from from TCL library
+Numeric decoder derived from TCL library
 https://www.opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
  * Copyright (c) 1988-1993 The Regents of the University of California.
  * Copyright (c) 1994 Sun Microsystems, Inc.

pandas/_libs/tslib.pyx

@@ -60,6 +60,10 @@ from pandas._libs.tslibs.nattype cimport (
 )
 from pandas._libs.tslibs.timestamps cimport _Timestamp
 
+from pandas._libs.tslibs import (
+    Resolution,
+    get_resolution,
+)
 from pandas._libs.tslibs.timestamps import Timestamp
 
 # Note: this is the only non-tslibs intra-pandas dependency here
@@ -122,11 +126,11 @@ def format_array_from_datetime(
     """
     cdef:
         int64_t val, ns, N = len(values)
-        ndarray[int64_t] consider_values
         bint show_ms = False, show_us = False, show_ns = False
         bint basic_format = False
         ndarray[object] result = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0)
-        object ts, res
+        _Timestamp ts
+        str res
         npy_datetimestruct dts
 
     if na_rep is None:
@@ -136,16 +140,10 @@ def format_array_from_datetime(
     # a format based on precision
     basic_format = format is None and tz is None
     if basic_format:
-        consider_values = values[values != NPY_NAT]
-        show_ns = (consider_values % 1000).any()
-
-        if not show_ns:
-            consider_values //= 1000
-            show_us = (consider_values % 1000).any()
-
-            if not show_ms:
-                consider_values //= 1000
-                show_ms = (consider_values % 1000).any()
+        reso_obj = get_resolution(values)
+        show_ns = reso_obj == Resolution.RESO_NS
+        show_us = reso_obj == Resolution.RESO_US
+        show_ms = reso_obj == Resolution.RESO_MS
 
     for i in range(N):
         val = values[i]
@@ -178,6 +176,7 @@ def format_array_from_datetime(
                 # invalid format string
                 # requires dates > 1900
                 try:
+                    # Note: dispatches to pydatetime
                     result[i] = ts.strftime(format)
                 except ValueError:
                     result[i] = str(ts)

pandas/_libs/tslibs/fields.pyi

@@ -22,6 +22,7 @@ def get_start_end_field(
 def get_date_field(
     dtindex: npt.NDArray[np.int64],  # const int64_t[:]
     field: str,
+    reso: int = ...,  # NPY_DATETIMEUNIT
 ) -> npt.NDArray[np.int32]: ...
 def get_timedelta_field(
     tdindex: npt.NDArray[np.int64],  # const int64_t[:]
@@ -32,6 +33,7 @@ def isleapyear_arr(
 ) -> npt.NDArray[np.bool_]: ...
 def build_isocalendar_sarray(
     dtindex: npt.NDArray[np.int64],  # const int64_t[:]
+    reso: int = ...,  # NPY_DATETIMEUNIT
 ) -> np.ndarray: ...
 def _get_locale_names(name_type: str, locale: str | None = ...): ...