Merge remote-tracking branch 'upstream/main'

Author: Dr-Irv

.github/workflows/code-checks.yml

@@ -74,7 +74,7 @@ jobs:
 
     - name: Install pyright
       # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g pyright@1.1.202
+      run: npm install -g pyright@1.1.212
 
     - name: Build Pandas
       id: build

.github/workflows/datamanger.yml

@@ -9,6 +9,8 @@ on:
     branches:
       - main
       - 1.4.x
+    paths-ignore:
+      - "doc/**"
 
 env:
   ENV_FILE: environment.yml

.github/workflows/posix.yml

@@ -28,14 +28,16 @@ jobs:
           [actions-38-downstream_compat.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-minimum_versions.yaml, "slow", "", "", "", "", ""],
           [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
+          [actions-38.yaml, "not slow and not network", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
+          [actions-38.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
-          [actions-38-slow.yaml, "slow", "", "", "", "", ""],
-          [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-39-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-38.yaml, "slow", "", "", "", "", ""],
           [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", "--max-worker-restart 0"],
-          [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
+          [actions-39.yaml, "slow", "", "", "", "", ""],
+          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""],
+          [actions-310-numpydev.yaml, "not slow and not network", "xclip", "", "", "deprecate", "-W error"],
+          [actions-310.yaml, "not slow and not clipboard", "", "", "", "", ""],
+          [actions-310.yaml, "slow", "", "", "", "", ""],
         ]
       fail-fast: false
     env:

.github/workflows/python-dev.yml

@@ -1,3 +1,11 @@
+# This file is purposely frozen(does not run). DO NOT DELETE IT
+# Unfreeze(by commentingthe if: false() condition) once the
+# next Python Dev version has released beta 1 and both Cython and numpy support it
+# After that Python has released, migrate the workflows to the
+# posix GHA workflows/Azure pipelines and "freeze" this file by
+# uncommenting the if: false() condition
+# Feel free to modify this comment as necessary.
+
 name: Python Dev
 
 on:
@@ -21,13 +29,14 @@ env:
 
 jobs:
   build:
+    if: false # Comment this line out to "unfreeze"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
-    name: actions-310-dev
+    name: actions-311-dev
     timeout-minutes: 80
 
     concurrency:
@@ -43,7 +52,7 @@ jobs:
     - name: Set up Python Dev Version
       uses: actions/setup-python@v2
       with:
-        python-version: '3.10-dev'
+        python-version: '3.11-dev'
 
     # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install dependencies

.github/workflows/stale-pr.yml

@@ -14,7 +14,8 @@ jobs:
         stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please [update](https://pandas.pydata.org/pandas-docs/stable/development/contributing.html#updating-your-pull-request) and respond to this comment if you're still interested in working on this."
         stale-pr-label: "Stale"
         exempt-pr-labels: "Needs Review,Blocked,Needs Discussion"
-        days-before-stale: 30
+        days-before-issue-stale: -1
+        days-before-pr-stale: 30
         days-before-close: -1
         remove-stale-when-updated: false
         debug-only: false

.pre-commit-config.yaml

@@ -84,8 +84,8 @@ repos:
         pass_filenames: false
         types: [python]
         stages: [manual]
-        # note: keep version in sync with .github/workflows/ci.yml
-        additional_dependencies: ['pyright@1.1.202']
+        # note: keep version in sync with .github/workflows/code-checks.yml
+        additional_dependencies: ['pyright@1.1.212']
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -188,3 +188,8 @@ repos:
         entry: 'pg8000'
         files: ^ci/deps
         types: [yaml]
+    -   id: validate-min-versions-in-sync
+        name: Check minimum version of dependencies are aligned
+        entry: python scripts/validate_min_versions_in_sync.py
+        language: python
+        files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$

asv_bench/asv.conf.json

@@ -13,6 +13,10 @@
     // benchmarked
     "repo": "..",
 
+    // List of branches to benchmark. If not provided, defaults to "master"
+    // (for git) or "default" (for mercurial).
+    "branches": ["main"],
+
     // The tool to use to create environments.  May be "conda",
     // "virtualenv" or other value depending on the plugins in use.
     // If missing or the empty string, the tool will be automatically
@@ -25,7 +29,6 @@
 
     // The Pythons you'd like to test against.  If not provided, defaults
     // to the current version of Python used to run `asv`.
-    // "pythons": ["2.7", "3.4"],
     "pythons": ["3.8"],
 
     // The matrix of dependencies to test.  Each key is the name of a

asv_bench/benchmarks/frame_methods.py

@@ -611,6 +611,9 @@ def time_frame_duplicated(self):
     def time_frame_duplicated_wide(self):
         self.df2.duplicated()
 
+    def time_frame_duplicated_subset(self):
+        self.df.duplicated(subset=["a"])
+
 
 class XS:
 

asv_bench/benchmarks/groupby.py

@@ -735,6 +735,12 @@ def setup(self):
         data = DataFrame(arr, index=index, columns=["col1", "col20", "col3"])
         self.df = data
 
+        n = 1000
+        self.df_wide = DataFrame(
+            np.random.randn(n, n),
+            index=np.random.choice(range(10), n),
+        )
+
         n = 20000
         self.df1 = DataFrame(
             np.random.randint(1, n, (n, 3)), columns=["jim", "joe", "jolie"]
@@ -754,6 +760,9 @@ def time_transform_lambda_max(self):
     def time_transform_ufunc_max(self):
         self.df.groupby(level="lev1").transform(np.max)
 
+    def time_transform_lambda_max_wide(self):
+        self.df_wide.groupby(level=0).transform(lambda x: np.max(x, axis=0))
+
     def time_transform_multi_key1(self):
         self.df1.groupby(["jim", "joe"])["jolie"].transform("max")
 

asv_bench/benchmarks/index_cached_properties.py

@@ -56,9 +56,6 @@ def time_values(self, index_type):
     def time_shape(self, index_type):
         self.idx.shape
 
-    def time_is_monotonic(self, index_type):
-        self.idx.is_monotonic
-
     def time_is_monotonic_decreasing(self, index_type):
         self.idx.is_monotonic_decreasing
 

asv_bench/benchmarks/multiindex_object.py

@@ -112,7 +112,7 @@ def time_get_indexer_and_pad(self):
         self.mi_int.get_indexer(self.other_mi_many_mismatches, method="pad")
 
     def time_is_monotonic(self):
-        self.mi_int.is_monotonic
+        self.mi_int.is_monotonic_increasing
 
 
 class Duplicated:

asv_bench/benchmarks/series_methods.py

@@ -3,6 +3,7 @@
 import numpy as np
 
 from pandas import (
+    Index,
     NaT,
     Series,
     date_range,
@@ -12,20 +13,23 @@
 
 
 class SeriesConstructor:
-
-    params = [None, "dict"]
-    param_names = ["data"]
-
-    def setup(self, data):
+    def setup(self):
         self.idx = date_range(
             start=datetime(2015, 10, 26), end=datetime(2016, 1, 1), freq="50s"
         )
-        dict_data = dict(zip(self.idx, range(len(self.idx))))
-        self.data = None if data is None else dict_data
+        self.data = dict(zip(self.idx, range(len(self.idx))))
+        self.array = np.array([1, 2, 3])
+        self.idx2 = Index(["a", "b", "c"])
 
-    def time_constructor(self, data):
+    def time_constructor_dict(self):
         Series(data=self.data, index=self.idx)
 
+    def time_constructor_no_data(self):
+        Series(data=None, index=self.idx)
+
+    def time_constructor_fastpath(self):
+        Series(self.array, index=self.idx2, name="name", fastpath=True)
+
 
 class ToFrame:
     params = [["int64", "datetime64[ns]", "category", "Int64"], [None, "foo"]]

asv_bench/benchmarks/strings.py

@@ -3,10 +3,12 @@
 import numpy as np
 
 from pandas import (
+    NA,
     Categorical,
     DataFrame,
     Series,
 )
+from pandas.arrays import StringArray
 
 from .pandas_vb_common import tm
 
@@ -285,3 +287,18 @@ class Iter(Dtypes):
     def time_iter(self, dtype):
         for i in self.s:
             pass
+
+
+class StringArrayConstruction:
+    def setup(self):
+        self.series_arr = tm.rands_array(nchars=10, size=10 ** 5)
+        self.series_arr_nan = np.concatenate([self.series_arr, np.array([NA] * 1000)])
+
+    def time_string_array_construction(self):
+        StringArray(self.series_arr)
+
+    def time_string_array_with_nan_construction(self):
+        StringArray(self.series_arr_nan)
+
+    def peakmem_stringarray_construction(self):
+        StringArray(self.series_arr)

azure-pipelines.yml

@@ -6,7 +6,7 @@ trigger:
     - 1.4.x
   paths:
     exclude:
-    - 'doc/*'
+    - 'doc/**'
 
 pr:
   autoCancel: true

ci/azure/posix.yml

@@ -18,6 +18,26 @@ jobs:
         CONDA_PY: "38"
         PATTERN: "not slow"
         PYTEST_TARGET: "pandas/tests/[i-z]*"
+      py39_macos_1:
+        ENV_FILE: ci/deps/azure-macos-39.yaml
+        CONDA_PY: "39"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[a-h]*"
+      py39_macos_2:
+        ENV_FILE: ci/deps/azure-macos-39.yaml
+        CONDA_PY: "39"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[i-z]*"
+      py310_macos_1:
+        ENV_FILE: ci/deps/azure-macos-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[a-h]*"
+      py310_macos_2:
+        ENV_FILE: ci/deps/azure-macos-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:
     - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'

ci/azure/windows.yml

@@ -36,6 +36,20 @@ jobs:
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[i-z]*"
 
+      py310_1:
+        ENV_FILE: ci/deps/azure-windows-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow and not high_memory"
+        PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[a-h]*"
+
+      py310_2:
+        ENV_FILE: ci/deps/azure-windows-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow and not high_memory"
+        PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[i-z]*"
+
   steps:
     - powershell: |
         Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"

ci/deps/actions-39-numpydev.yaml renamed to ci/deps/actions-310-numpydev.yaml

@@ -2,7 +2,7 @@ name: pandas-dev
 channels:
   - defaults
 dependencies:
-  - python=3.9
+  - python=3.10
 
   # tools
   - pytest>=6.0

ci/deps/actions-310.yaml

@@ -0,0 +1,51 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.9
+
+  # test dependencies
+  - cython=0.29.24
+  - pytest>=6.0
+  - pytest-cov
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
+  - psutil
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+  - pytz
+
+  # optional dependencies
+  - beautifulsoup4
+  - blosc
+  - bottleneck
+  - fastparquet
+  - fsspec
+  - html5lib
+  - gcsfs
+  - jinja2
+  - lxml
+  - matplotlib
+  # TODO: uncomment after numba supports py310
+  #- numba
+  - numexpr
+  - openpyxl
+  - odfpy
+  - pandas-gbq
+  - psycopg2
+  - pymysql
+  - pytables
+  - pyarrow
+  - pyreadstat
+  - pyxlsb
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - tabulate
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - zstandard