Merge branch 'master' of https://github.com/pandas-dev/pandas into bug-cdt-eq

Author: jbrockmendel

.github/workflows/ci.yml

@@ -74,14 +74,10 @@ jobs:
         asv check -E existing
         git remote add upstream https://github.com/pandas-dev/pandas.git
         git fetch upstream
-        if git diff upstream/master --name-only | grep -q "^asv_bench/"; then
-            asv machine --yes
-            asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
-            if grep "failed" benchmarks.log > /dev/null ; then
-                exit 1
-            fi
-        else
-            echo "Benchmarks did not run, no changes detected"
+        asv machine --yes
+        asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
+        if grep "failed" benchmarks.log > /dev/null ; then
+            exit 1
         fi
       if: always()
 

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+minimum_pre_commit_version: '2.9.2'
 repos:
 -   repo: https://github.com/python/black
     rev: 20.8b1
@@ -21,10 +22,8 @@ repos:
     rev: 5.6.4
     hooks:
     -   id: isort
-        name: isort (python)
-    -   id: isort
-        name: isort (cython)
-        types: [cython]
+        types: [text] # overwrite upstream `types: [python]`
+        types_or: [python, cython]
 -   repo: https://github.com/asottile/pyupgrade
     rev: v2.7.4
     hooks:
@@ -96,17 +95,17 @@ repos:
         name: Check for incorrect code block or IPython directives
         language: pygrep
         entry: (\.\. code-block ::|\.\. ipython ::)
-        files: \.(py|pyx|rst)$
+        types_or: [python, cython, rst]
     -   id: unwanted-patterns-strings-to-concatenate
         name: Check for use of not concatenated strings
         language: python
         entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
-        files: \.(py|pyx|pxd|pxi)$
+        types_or: [python, cython]
     -   id: unwanted-patterns-strings-with-wrong-placed-whitespace
         name: Check for strings with wrong placed spaces
         language: python
         entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace"
-        files: \.(py|pyx|pxd|pxi)$
+        types_or: [python, cython]
     -   id: unwanted-patterns-private-import-across-module
         name: Check for import of private attributes across modules
         language: python

README.md

@@ -87,7 +87,7 @@ The source code is currently hosted on GitHub at:
 https://github.com/pandas-dev/pandas
 
 Binary installers for the latest released version are available at the [Python
-package index](https://pypi.org/project/pandas) and on conda.
+Package Index (PyPI)](https://pypi.org/project/pandas) and on [Conda](https://docs.conda.io/en/latest/).
 
 ```sh
 # conda
@@ -100,15 +100,15 @@ pip install pandas
 ```
 
 ## Dependencies
-- [NumPy](https://www.numpy.org)
-- [python-dateutil](https://labix.org/python-dateutil)
-- [pytz](https://pythonhosted.org/pytz)
+- [NumPy - Adds support for large, multi-dimensional arrays, matrices and high-level mathematical functions to operate on these arrays](https://www.numpy.org)
+- [python-dateutil - Provides powerful extensions to the standard datetime module](https://labix.org/python-dateutil)
+- [pytz - Brings the Olson tz database into Python which allows accurate and cross platform timezone calculations](https://pythonhosted.org/pytz)
 
 See the [full installation instructions](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies) for minimum supported versions of required, recommended and optional dependencies.
 
 ## Installation from sources
-To install pandas from source you need Cython in addition to the normal
-dependencies above. Cython can be installed from pypi:
+To install pandas from source you need [Cython](https://cython.org/) in addition to the normal
+dependencies above. Cython can be installed from PyPI:
 
 ```sh
 pip install cython
@@ -145,7 +145,7 @@ See the full instructions for [installing from source](https://pandas.pydata.org
 The official documentation is hosted on PyData.org: https://pandas.pydata.org/pandas-docs/stable
 
 ## Background
-Work on ``pandas`` started at AQR (a quantitative hedge fund) in 2008 and
+Work on ``pandas`` started at [AQR](https://www.aqr.com/) (a quantitative hedge fund) in 2008 and
 has been under active development since then.
 
 ## Getting Help

asv_bench/benchmarks/indexing.py

@@ -3,6 +3,7 @@
 lower-level methods directly on Index and subclasses, see index_object.py,
 indexing_engine.py, and index_cached.py
 """
+import string
 import warnings
 
 import numpy as np
@@ -255,6 +256,7 @@ def setup(self, index):
             "non_monotonic": CategoricalIndex(list("abc" * N)),
         }
         self.data = indices[index]
+        self.data_unique = CategoricalIndex(list(string.printable))
 
         self.int_scalar = 10000
         self.int_list = list(range(10000))
@@ -281,7 +283,7 @@ def time_get_loc_scalar(self, index):
         self.data.get_loc(self.cat_scalar)
 
     def time_get_indexer_list(self, index):
-        self.data.get_indexer(self.cat_list)
+        self.data_unique.get_indexer(self.cat_list)
 
 
 class MethodLookup:

ci/deps/azure-37-slow.yaml

@@ -31,7 +31,7 @@ dependencies:
   - moto>=1.3.14
   - scipy
   - sqlalchemy
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt
   - moto

ci/deps/azure-38-locale.yaml

@@ -30,7 +30,7 @@ dependencies:
   - pytz
   - scipy
   - xarray
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt
   - moto

ci/deps/azure-macos-37.yaml

@@ -26,7 +26,7 @@ dependencies:
   - python-dateutil==2.7.3
   - pytz
   - xarray
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt
   - pip

ci/deps/azure-windows-37.yaml

@@ -33,7 +33,7 @@ dependencies:
   - s3fs>=0.4.2
   - scipy
   - sqlalchemy
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt
   - pyreadstat

ci/deps/azure-windows-38.yaml

@@ -31,6 +31,6 @@ dependencies:
   - pytz
   - s3fs>=0.4.0
   - scipy
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt

ci/deps/travis-37-cov.yaml

@@ -43,7 +43,7 @@ dependencies:
   - sqlalchemy
   - statsmodels
   - xarray
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt
   - pip

ci/deps/travis-37-locale.yaml

@@ -35,7 +35,7 @@ dependencies:
   - pytables>=3.5.1
   - scipy
   - xarray=0.12.3
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt
   - moto

ci/deps/travis-38-slow.yaml

@@ -30,7 +30,7 @@ dependencies:
   - moto>=1.3.14
   - scipy
   - sqlalchemy
-  - xlrd
+  - xlrd<2.0
   - xlsxwriter
   - xlwt
   - moto

environment.yml

@@ -24,7 +24,7 @@ dependencies:
   - flake8-comprehensions>=3.1.0  # used by flake8, linting of unnecessary comprehensions
   - isort>=5.2.1  # check that imports are in the right order
   - mypy=0.782
-  - pre-commit
+  - pre-commit>=2.9.2
   - pycodestyle  # used by flake8
   - pyupgrade
 

pandas/_libs/lib.pyx

@@ -1313,7 +1313,7 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
     'boolean'
 
     >>> infer_dtype([True, False, np.nan])
-    'mixed'
+    'boolean'
 
     >>> infer_dtype([pd.Timestamp('20130101')])
     'datetime'

pandas/_typing.py

@@ -95,6 +95,7 @@
 IndexLabel = Union[Label, Sequence[Label]]
 Level = Union[Label, int]
 Shape = Tuple[int, ...]
+Suffixes = Tuple[str, str]
 Ordered = Optional[bool]
 JSONSerializable = Optional[Union[PythonScalar, List, Dict]]
 Axes = Collection

pandas/core/arrays/timedeltas.py

@@ -24,6 +24,7 @@
 )
 from pandas.compat.numpy import function as nv
 
+from pandas.core.dtypes.cast import astype_td64_unit_conversion
 from pandas.core.dtypes.common import (
     DT64NS_DTYPE,
     TD64NS_DTYPE,
@@ -35,7 +36,6 @@
     is_scalar,
     is_string_dtype,
     is_timedelta64_dtype,
-    is_timedelta64_ns_dtype,
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
@@ -324,22 +324,14 @@ def astype(self, dtype, copy: bool = True):
         # DatetimeLikeArrayMixin super call handles other cases
         dtype = pandas_dtype(dtype)
 
-        if is_timedelta64_dtype(dtype) and not is_timedelta64_ns_dtype(dtype):
-            # by pandas convention, converting to non-nano timedelta64
-            #  returns an int64-dtyped array with ints representing multiples
-            #  of the desired timedelta unit.  This is essentially division
-            if self._hasnans:
-                # avoid double-copying
-                result = self._data.astype(dtype, copy=False)
-                return self._maybe_mask_results(
-                    result, fill_value=None, convert="float64"
-                )
-            result = self._data.astype(dtype, copy=copy)
-            return result.astype("i8")
-        elif is_timedelta64_ns_dtype(dtype):
+        if is_dtype_equal(dtype, self.dtype):
             if copy:
                 return self.copy()
             return self
+
+        elif dtype.kind == "m":
+            return astype_td64_unit_conversion(self._data, dtype, copy=copy)
+
         return dtl.DatetimeLikeArrayMixin.astype(self, dtype, copy=copy)
 
     def __iter__(self):