Merge remote-tracking branch 'upstream/master' into factorize

Author: simonjayhawkins

.github/actions/build_pandas/action.yml

@@ -0,0 +1,17 @@
+name: Build pandas
+description: Rebuilds the C extensions and installs pandas
+runs:
+  using: composite
+  steps:
+
+    - name: Environment Detail
+      run: |
+        conda info
+        conda list
+      shell: bash -l {0}
+
+    - name: Build Pandas
+      run: |
+        python setup.py build_ext -j 2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
+      shell: bash -l {0}

.github/actions/setup/action.yml

@@ -0,0 +1,12 @@
+name: Set up pandas
+description: Runs all the setup steps required to have a built pandas ready to use
+runs:
+  using: composite
+  steps:
+    - name: Setting conda path
+      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
+      shell: bash -l {0}
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+      shell: bash -l {0}

.github/workflows/ci.yml

@@ -41,15 +41,8 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Environment Detail
-      run: |
-        conda info
-        conda list
-
     - name: Build Pandas
-      run: |
-        python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+      uses: ./.github/actions/build_pandas
 
     - name: Linting
       run: ci/code_checks.sh lint
@@ -100,14 +93,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-
     - name: Checkout
       uses: actions/checkout@v1
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
+    - name: Set up pandas
+      uses: ./.github/actions/setup
 
     - name: Build website
       run: |
@@ -139,19 +129,25 @@ jobs:
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
       if: github.event_name == 'push'
 
+    - name: Move docs into site directory
+      run: mv doc/build/html web/build/docs
+    - name: Save website as an artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: website
+        path: web/build
+        retention-days: 14
+
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
     steps:
 
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-
     - name: Checkout
       uses: actions/checkout@v1
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
+    - name: Set up pandas
+      uses: ./.github/actions/setup
 
     - name: Run tests
       run: |

.github/workflows/comment_bot.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Install-pre-commit
         run: python -m pip install --upgrade pre-commit
       - name: Run pre-commit
-        run: pre-commit run --all-files || (exit 0)
+        run: pre-commit run --from-ref=origin/master --to-ref=HEAD --all-files || (exit 0)
       - name: Commit results
         run: |
           git config user.name "$(git log -1 --pretty=format:%an)"

.github/workflows/database.yml

@@ -72,15 +72,8 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Environment Detail
-      run: |
-        conda info
-        conda list
-
     - name: Build Pandas
-      run: |
-        python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+      uses: ./.github/actions/build_pandas
 
     - name: Test
       run: ci/run_tests.sh
@@ -158,15 +151,8 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Environment Detail
-      run: |
-        conda info
-        conda list
-
     - name: Build Pandas
-      run: |
-        python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+      uses: ./.github/actions/build_pandas
 
     - name: Test
       run: ci/run_tests.sh

.pre-commit-config.yaml

@@ -24,12 +24,12 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.4
+    rev: v2.10.0
     hooks:
     -   id: pyupgrade
-        args: [--py37-plus]
+        args: [--py37-plus, --keep-runtime-typing]
 -   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.7.0
+    rev: v1.7.1
     hooks:
       - id: rst-backticks
       - id: rst-directive-colons
@@ -127,6 +127,12 @@ repos:
         types: [python]
         files: ^pandas/tests/
         exclude: ^pandas/tests/extension/
+    -   id: unwanted-patters-pytest-xfail
+        name: Check for use of pytest.xfail
+        entry: pytest\.xfail
+        language: pygrep
+        types: [python]
+        files: ^pandas/tests/
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace in tests'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py
@@ -192,6 +198,11 @@ repos:
         files: ^pandas/
         exclude: ^pandas/tests/
 -   repo: https://github.com/MarcoGorelli/no-string-hints
-    rev: v0.1.6
+    rev: v0.1.7
     hooks:
     -   id: no-string-hints
+-   repo: https://github.com/MarcoGorelli/abs-imports
+    rev: v0.1.2
+    hooks:
+    -   id: abs-imports
+        files: ^pandas/

asv_bench/benchmarks/categoricals.py

@@ -118,12 +118,29 @@ def setup(self):
         self.a = pd.Categorical(list("aabbcd") * N)
         self.b = pd.Categorical(list("bbcdjk") * N)
 
+        self.idx_a = pd.CategoricalIndex(range(N), range(N))
+        self.idx_b = pd.CategoricalIndex(range(N + 1), range(N + 1))
+        self.df_a = pd.DataFrame(range(N), columns=["a"], index=self.idx_a)
+        self.df_b = pd.DataFrame(range(N + 1), columns=["a"], index=self.idx_b)
+
     def time_concat(self):
         pd.concat([self.s, self.s])
 
     def time_union(self):
         union_categoricals([self.a, self.b])
 
+    def time_append_overlapping_index(self):
+        self.idx_a.append(self.idx_a)
+
+    def time_append_non_overlapping_index(self):
+        self.idx_a.append(self.idx_b)
+
+    def time_concat_overlapping_index(self):
+        pd.concat([self.df_a, self.df_a])
+
+    def time_concat_non_overlapping_index(self):
+        pd.concat([self.df_a, self.df_b])
+
 
 class ValueCounts:
 
@@ -306,7 +323,7 @@ def time_get_loc(self):
         self.index.get_loc(self.category)
 
     def time_shallow_copy(self):
-        self.index._shallow_copy()
+        self.index._view()
 
     def time_align(self):
         pd.DataFrame({"a": self.series, "b": self.series[:500]})

asv_bench/benchmarks/dtypes.py

@@ -2,9 +2,10 @@
 
 import numpy as np
 
+import pandas as pd
 from pandas import DataFrame
 import pandas._testing as tm
-from pandas.api.types import pandas_dtype
+from pandas.api.types import is_extension_array_dtype, pandas_dtype
 
 from .pandas_vb_common import (
     datetime_dtypes,
@@ -119,4 +120,16 @@ def time_select_dtype_string_exclude(self, dtype):
         self.df_string.select_dtypes(exclude=dtype)
 
 
+class CheckDtypes:
+    def setup(self):
+        self.ext_dtype = pd.Int64Dtype()
+        self.np_dtype = np.dtype("int64")
+
+    def time_is_extension_array_dtype_true(self):
+        is_extension_array_dtype(self.ext_dtype)
+
+    def time_is_extension_array_dtype_false(self):
+        is_extension_array_dtype(self.np_dtype)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/gil.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 from pandas import DataFrame, Series, date_range, factorize, read_csv
-from pandas.core.algorithms import take_1d
+from pandas.core.algorithms import take_nd
 
 from .pandas_vb_common import tm
 
@@ -110,7 +110,7 @@ def setup(self, dtype):
 
         @test_parallel(num_threads=2)
         def parallel_take1d():
-            take_1d(df["col"].values, indexer)
+            take_nd(df["col"].values, indexer)
 
         self.parallel_take1d = parallel_take1d
 

asv_bench/benchmarks/hash_functions.py

@@ -25,6 +25,15 @@ def time_isin_outside(self, dtype, exponent):
         self.s.isin(self.values_outside)
 
 
+class UniqueForLargePyObjectInts:
+    def setup(self):
+        lst = [x << 32 for x in range(5000)]
+        self.arr = np.array(lst, dtype=np.object_)
+
+    def time_unique(self):
+        pd.unique(self.arr)
+
+
 class IsinWithRandomFloat:
     params = [
         [np.float64, np.object],

asv_bench/benchmarks/io/csv.py

@@ -84,8 +84,8 @@ class ToCSVIndexes(BaseIO):
     def _create_df(rows, cols):
         index_cols = {
             "index1": np.random.randint(0, rows, rows),
-            "index2": np.full(rows, 1, dtype=np.int),
-            "index3": np.full(rows, 1, dtype=np.int),
+            "index2": np.full(rows, 1, dtype=int),
+            "index3": np.full(rows, 1, dtype=int),
         }
         data_cols = {
             f"col{i}": np.random.uniform(0, 100000.0, rows) for i in range(cols)

asv_bench/benchmarks/period.py

@@ -86,7 +86,7 @@ def time_get_loc(self):
         self.index.get_loc(self.period)
 
     def time_shallow_copy(self):
-        self.index._shallow_copy()
+        self.index._view()
 
     def time_series_loc(self):
         self.series.loc[self.period]

asv_bench/benchmarks/rolling.py

@@ -140,8 +140,11 @@ class Pairwise:
 
     def setup(self, window, method, pairwise):
         N = 10 ** 4
+        n_groups = 20
+        groups = [i for _ in range(N // n_groups) for i in range(n_groups)]
         arr = np.random.random(N)
         self.df = pd.DataFrame(arr)
+        self.df_group = pd.DataFrame({"A": groups, "B": arr}).groupby("A")
 
     def time_pairwise(self, window, method, pairwise):
         if window is None:
@@ -150,6 +153,13 @@ def time_pairwise(self, window, method, pairwise):
             r = self.df.rolling(window=window)
         getattr(r, method)(self.df, pairwise=pairwise)
 
+    def time_groupby(self, window, method, pairwise):
+        if window is None:
+            r = self.df_group.expanding()
+        else:
+            r = self.df_group.rolling(window=window)
+        getattr(r, method)(self.df, pairwise=pairwise)
+
 
 class Quantile:
     params = (
@@ -245,6 +255,19 @@ def time_rolling_multiindex_creation(self):
 
 class GroupbyEWM:
 
+    params = ["var", "std", "cov", "corr"]
+    param_names = ["method"]
+
+    def setup(self, method):
+        df = pd.DataFrame({"A": range(50), "B": range(50)})
+        self.gb_ewm = df.groupby("A").ewm(com=1.0)
+
+    def time_groupby_method(self, method):
+        getattr(self.gb_ewm, method)()
+
+
+class GroupbyEWMEngine:
+
     params = ["cython", "numba"]
     param_names = ["engine"]
 

asv_bench/benchmarks/series_methods.py

@@ -108,8 +108,8 @@ def setup(self):
         self.vals_short = np.arange(2).astype(object)
         self.vals_long = np.arange(10 ** 5).astype(object)
         # because of nans floats are special:
-        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float)).astype(object)
-        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float).astype(object)
+        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float_)).astype(object)
+        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float_).astype(object)
 
     def time_isin_nans(self):
         # if nan-objects are different objects,

asv_bench/benchmarks/timedelta.py

@@ -74,7 +74,7 @@ def time_get_loc(self):
         self.index.get_loc(self.timedelta)
 
     def time_shallow_copy(self):
-        self.index._shallow_copy()
+        self.index._view()
 
     def time_series_loc(self):
         self.series.loc[self.timedelta]

ci/deps/actions-37-locale.yaml

@@ -11,7 +11,7 @@ dependencies:
   - hypothesis>=3.58.0
 
   # required
-  - numpy
+  - numpy<1.20  # GH#39541 compat for pyarrow<3
   - python-dateutil
   - pytz
 

ci/deps/azure-37.yaml

@@ -18,7 +18,7 @@ dependencies:
   - numpy
   - python-dateutil
   - nomkl
-  - pyarrow
+  - pyarrow=0.15.1
   - pytz
   - s3fs>=0.4.0
   - moto>=1.3.14

ci/deps/azure-38-locale.yaml

@@ -24,7 +24,7 @@ dependencies:
   - moto
   - nomkl
   - numexpr
-  - numpy
+  - numpy<1.20  # GH#39541 compat with pyarrow<3
   - openpyxl
   - pytables
   - python-dateutil