BUG: TypeError when doing equality comparison with Extension Dtype

.github/workflows/ci.yml

@@ -23,8 +23,9 @@ jobs:
         shell: bash -l {0}
 
     concurrency:
-      group: ${{ github.ref }}-checks
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-checks
+      cancel-in-progress: true
 
     steps:
     - name: Checkout
@@ -102,7 +103,8 @@ jobs:
     runs-on: ubuntu-latest
 
     concurrency:
-      group: ${{ github.ref }}-web-docs
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs
       cancel-in-progress: true
 
     steps:
@@ -160,11 +162,20 @@ jobs:
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
     strategy:
       matrix:
         pattern: ["not slow and not network and not clipboard", "slow"]
     concurrency:
-      group: ${{ github.ref }}-data_manager-${{ matrix.pattern }}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager-${{ matrix.pattern }}
       cancel-in-progress: true
 
     steps:

.github/workflows/database.yml

@@ -31,8 +31,9 @@ jobs:
       fail-fast: false
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.ENV_FILE }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.ENV_FILE }}
+      cancel-in-progress: true
 
     services:
       mysql:
@@ -62,6 +63,14 @@ jobs:
         ports:
           - 5432:5432
 
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
+
     steps:
     - name: Checkout
       uses: actions/checkout@v2

.github/workflows/posix.yml

@@ -27,12 +27,12 @@ jobs:
         settings: [
           [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
+          [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-39-slow.yaml, "slow", "", "", "", "", ""],
           [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-39.yaml, "not slow and not network and not clipboard", "", "", "", "", ""]
+          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
         ]
       fail-fast: false
     env:
@@ -46,8 +46,17 @@ jobs:
       TEST_ARGS: ${{ matrix.settings[6] }}
       PYTEST_TARGET:  pandas
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.settings[0] }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}
+      cancel-in-progress: true
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
 
     steps:
     - name: Checkout

.github/workflows/pre-commit.yml

@@ -11,8 +11,9 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     concurrency:
-      group: ${{ github.ref }}-pre-commit
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
+      cancel-in-progress: true
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.github/workflows/python-dev.yml

@@ -31,8 +31,9 @@ jobs:
     timeout-minutes: 80
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      #https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/sdist.yml

@@ -25,8 +25,9 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
     concurrency:
-      group: ${{github.ref}}-${{matrix.python-version}}-sdist
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/python/black
-    rev: 21.10b0
+    rev: 21.11b1
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -55,11 +55,11 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.0
+    rev: v2.29.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -111,6 +111,8 @@ repos:
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
+            # directive should not have a space before ::
+            |\.\.\ \w+\ ::
 
             # Check for deprecated messages without sphinx directive
             |(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)

asv_bench/benchmarks/frame_ctor.py

@@ -19,7 +19,10 @@
     )
 except ImportError:
     # For compatibility with older versions
-    from pandas.core.datetools import *  # noqa
+    from pandas.core.datetools import (
+        Hour,
+        Nano,
+    )
 
 
 class FromDicts:

asv_bench/benchmarks/frame_methods.py

@@ -76,7 +76,7 @@ def time_reindex_axis1_missing(self):
         self.df.reindex(columns=self.idx)
 
     def time_reindex_both_axes(self):
-        self.df.reindex(index=self.idx, columns=self.idx)
+        self.df.reindex(index=self.idx, columns=self.idx_cols)
 
     def time_reindex_upcast(self):
         self.df2.reindex(np.random.permutation(range(1200)))
@@ -731,17 +731,6 @@ def time_dataframe_describe(self):
         self.df.describe()
 
 
-class SelectDtypes:
-    params = [100, 1000]
-    param_names = ["n"]
-
-    def setup(self, n):
-        self.df = DataFrame(np.random.randn(10, n))
-
-    def time_select_dtypes(self, n):
-        self.df.select_dtypes(include="int")
-
-
 class MemoryUsage:
     def setup(self):
         self.df = DataFrame(np.random.randn(100000, 2), columns=list("AB"))

asv_bench/benchmarks/groupby.py

@@ -443,7 +443,7 @@ class GroupByMethods:
             "var",
         ],
         ["direct", "transformation"],
-        [1, 2, 5, 10],
+        [1, 5],
     ]
 
     def setup(self, dtype, method, application, ncols):
@@ -455,6 +455,7 @@ def setup(self, dtype, method, application, ncols):
             raise NotImplementedError
 
         if application == "transformation" and method in [
+            "describe",
             "head",
             "tail",
             "unique",
@@ -464,7 +465,12 @@ def setup(self, dtype, method, application, ncols):
             # DataFrameGroupBy doesn't have these methods
             raise NotImplementedError
 
-        ngroups = 1000
+        if method == "describe":
+            ngroups = 20
+        elif method in ["mad", "skew"]:
+            ngroups = 100
+        else:
+            ngroups = 1000
         size = ngroups * 2
         rng = np.arange(ngroups).reshape(-1, 1)
         rng = np.broadcast_to(rng, (len(rng), ncols))
@@ -491,9 +497,6 @@ def setup(self, dtype, method, application, ncols):
             cols = cols[0]
 
         if application == "transformation":
-            if method == "describe":
-                raise NotImplementedError
-
             self.as_group_method = lambda: df.groupby("key")[cols].transform(method)
             self.as_field_method = lambda: df.groupby(cols)["key"].transform(method)
         else:

asv_bench/benchmarks/io/csv.py

@@ -525,4 +525,14 @@ def time_to_datetime_format_DD_MM_YYYY(self, cache_dates):
         to_datetime(df["date"], cache=cache_dates, format="%d-%m-%Y")
 
 
+class ReadCSVIndexCol(StringIORewind):
+    def setup(self):
+        count_elem = 100_000
+        data = "a,b\n" + "1,2\n" * count_elem
+        self.StringIO_input = StringIO(data)
+
+    def time_read_csv_index_col(self):
+        read_csv(self.StringIO_input, index_col="a")
+
+
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/rolling.py

@@ -53,7 +53,7 @@ class NumbaEngineMethods:
         ["DataFrame", "Series"],
         ["int", "float"],
         [("rolling", {"window": 10}), ("expanding", {})],
-        ["sum", "max", "min", "median", "mean"],
+        ["sum", "max", "min", "median", "mean", "var", "std"],
         [True, False],
         [None, 100],
     )

ci/azure/posix.yml

@@ -12,12 +12,12 @@ jobs:
         py38_macos_1:
           ENV_FILE: ci/deps/azure-macos-38.yaml
           CONDA_PY: "38"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow"
           PYTEST_TARGET: "pandas/tests/[a-h]*"
         py38_macos_2:
           ENV_FILE: ci/deps/azure-macos-38.yaml
           CONDA_PY: "38"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow"
           PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:

ci/azure/windows.yml

@@ -11,28 +11,28 @@ jobs:
       py38_np18_1:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
-        PATTERN: "not slow and not network"
+        PATTERN: "not slow"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[a-h]*"
 
       py38_np18_2:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
-        PATTERN: "not slow and not network"
+        PATTERN: "not slow"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[i-z]*"
 
       py39_1:
         ENV_FILE: ci/deps/azure-windows-39.yaml
         CONDA_PY: "39"
-        PATTERN: "not slow and not network and not high_memory"
+        PATTERN: "not slow and not high_memory"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[a-h]*"
 
       py39_2:
         ENV_FILE: ci/deps/azure-windows-39.yaml
         CONDA_PY: "39"
-        PATTERN: "not slow and not network and not high_memory"
+        PATTERN: "not slow and not high_memory"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[i-z]*"
 

ci/code_checks.sh

@@ -93,8 +93,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS01, SS02, SS03, SS04, SS05, PR03, PR04, PR05, PRO9, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS03,SS04,SS05,PR03,PR04,PR05,PR09,PR10,EX04,RT01,RT04,RT05,SA02,SA03
+    MSG='Validate docstrings (GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS01, SS02, SS03, SS04, SS05, PR03, PR04, PR05, PR06, PR08, PR09, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS03,SS04,SS05,PR03,PR04,PR05,PR06,PR08,PR09,PR10,EX04,RT01,RT04,RT05,SA02,SA03
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/actions-38-db-min.yaml

@@ -38,10 +38,8 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
-  - flask
 
   # sql
   - psycopg2=2.8.4
   - pymysql=0.10.1
-  - sqlalchemy=1.3.11
+  - sqlalchemy=1.4.0

ci/deps/actions-38-db.yaml

@@ -14,6 +14,7 @@ dependencies:
   # pandas dependencies
   - aiobotocore<2.0.0
   - beautifulsoup4
+  - boto3
   - botocore>=1.11
   - dask
   - fastparquet>=0.4.0
@@ -22,8 +23,6 @@ dependencies:
   - geopandas
   - html5lib
   - matplotlib
-  - moto>=1.3.14
-  - flask
   - nomkl
   - numexpr
   - numpy=1.18

ci/deps/actions-38-locale.yaml

@@ -14,14 +14,12 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - flask
   - html5lib
   - ipython
   - jinja2
   - jedi
   - lxml
   - matplotlib<3.3.0
-  - moto
   - nomkl
   - numexpr
   - numpy<1.20  # GH#39541 compat with pyarrow<3
@@ -34,7 +32,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - moto
   - pyarrow=1.0.1
   - pip
   - pip:

ci/deps/actions-38-locale_slow.yaml

@@ -23,7 +23,7 @@ dependencies:
   - python-blosc
   - pytz=2020.1
   - scipy
-  - sqlalchemy=1.3.11
+  - sqlalchemy=1.4.0
   - xlrd=2.0.1
   - xlsxwriter=1.2.2
   - xlwt=1.3.0