Merge remote-tracking branch 'upstream/main' into gh-49298

Author: MarcoGorelli

.github/workflows/32-bit-linux.yml

@@ -38,7 +38,8 @@ jobs:
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python -m pip install versioneer[toml] && \
+          python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
           python setup.py build_ext -q -j1 && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
           python -m pip list && \

.github/workflows/code-checks.yml

@@ -79,6 +79,10 @@ jobs:
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' && always() }}
 
+    - name: Run check of documentation notebooks
+      run: ci/code_checks.sh notebooks
+      if: ${{ steps.build.outcome == 'success' && always() }}
+
     - name: Use existing environment for type checking
       run: |
         echo $PATH >> $GITHUB_PATH

.github/workflows/docbuild-and-upload.yml

@@ -54,11 +54,6 @@ jobs:
     - name: Build documentation zip
       run: doc/make.py zip_html
 
-    - name: Build the interactive terminal
-      run: |
-        cd web/interactive_terminal
-        jupyter lite build
-
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh

.github/workflows/package-checks.yml

@@ -43,6 +43,7 @@ jobs:
       - name: Install required dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+          python -m pip install versioneer[toml]
         shell: bash -el {0}
 
       - name: Pip install with extra

.github/workflows/python-dev.yml

@@ -75,6 +75,7 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
+        python -m pip install versioneer[toml]
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 

.github/workflows/sdist.yml

@@ -47,6 +47,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel
+        python -m pip install versioneer[toml]
 
         # GH 39416
         pip install numpy

.pre-commit-config.yaml

@@ -26,6 +26,7 @@ repos:
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
+        additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
     rev: v0.2.1
     hooks:

asv_bench/benchmarks/frame_ctor.py

@@ -205,21 +205,4 @@ def time_frame_from_arrays_sparse(self):
         )
 
 
-class From3rdParty:
-    # GH#44616
-
-    def setup(self):
-        try:
-            import torch
-        except ImportError:
-            raise NotImplementedError
-
-        row = 700000
-        col = 64
-        self.val_tensor = torch.randn(row, col)
-
-    def time_from_torch(self):
-        DataFrame(self.val_tensor)
-
-
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/indexing.py

@@ -3,8 +3,6 @@
 lower-level methods directly on Index and subclasses, see index_object.py,
 indexing_engine.py, and index_cached.py
 """
-import itertools
-import string
 import warnings
 
 import numpy as np
@@ -353,15 +351,13 @@ def setup(self, index):
             "non_monotonic": CategoricalIndex(list("abc" * N)),
         }
         self.data = indices[index]
-        self.data_unique = CategoricalIndex(
-            ["".join(perm) for perm in itertools.permutations(string.printable, 3)]
-        )
+        self.data_unique = CategoricalIndex([str(i) for i in range(N * 3)])
 
         self.int_scalar = 10000
         self.int_list = list(range(10000))
 
         self.cat_scalar = "b"
-        self.cat_list = ["a", "c"]
+        self.cat_list = ["1", "3"]
 
     def time_getitem_scalar(self, index):
         self.data[self.int_scalar]

ci/code_checks.sh

@@ -12,9 +12,10 @@
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh single-docs   # check single-page docs build warning-free
+#   $ ./ci/code_checks.sh notebooks     # check execution of documentation notebooks
 
-[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" ]] || \
-    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings]"; exit 9999; }
+[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" || "$1" == "notebooks" ]] || \
+    { echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|single-docs|notebooks]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -84,6 +85,15 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
 fi
 
+### DOCUMENTATION NOTEBOOKS ###
+if [[ -z "$CHECK" || "$CHECK" == "notebooks" ]]; then
+
+    MSG='Notebooks' ; echo $MSG
+    jupyter nbconvert --execute $(find doc/source -name '*.ipynb') --to notebook
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+fi
+
 ### SINGLE-PAGE DOCS ###
 if [[ -z "$CHECK" || "$CHECK" == "single-docs" ]]; then
     python doc/make.py --warnings-are-errors --single pandas.Series.value_counts

ci/deps/actions-310-numpydev.yaml

@@ -4,7 +4,10 @@ channels:
 dependencies:
   - python=3.10
 
-  # tools
+  # build dependencies
+  - versioneer[toml]
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-310.yaml

@@ -4,8 +4,11 @@ channels:
 dependencies:
   - python=3.10
 
-  # test dependencies
+  # build dependencies
+  - versioneer[toml]
   - cython>=0.29.32
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-downstream_compat.yaml

@@ -5,8 +5,11 @@ channels:
 dependencies:
   - python=3.8
 
-  # test dependencies
+  # build dependencies
+  - versioneer[toml]
   - cython>=0.29.32
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
@@ -67,4 +70,3 @@ dependencies:
   - pandas-datareader
   - pyyaml
   - py
-  - pytorch

ci/deps/actions-38-minimum_versions.yaml

@@ -6,8 +6,11 @@ channels:
 dependencies:
   - python=3.8.0
 
-  # test dependencies
+  # build dependencies
+  - versioneer[toml]
   - cython>=0.29.32
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38.yaml

@@ -4,8 +4,11 @@ channels:
 dependencies:
   - python=3.8
 
-  # test dependencies
+  # build dependencies
+  - versioneer[toml]
   - cython>=0.29.32
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-39.yaml

@@ -4,8 +4,11 @@ channels:
 dependencies:
   - python=3.9
 
-  # test dependencies
+  # build dependencies
+  - versioneer[toml]
   - cython>=0.29.32
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-pypy-38.yaml

@@ -7,8 +7,11 @@ dependencies:
   # with base pandas has been dealt with
   - python=3.8[build=*_pypy] # TODO: use this once pypy3.8 is available
 
-  # tools
+  # build dependencies
+  - versioneer[toml]
   - cython>=0.29.32
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-asyncio

ci/deps/circle-38-arm64.yaml

@@ -4,8 +4,11 @@ channels:
 dependencies:
   - python=3.8
 
-  # test dependencies
+  # build dependencies
+  - versioneer[toml]
   - cython>=0.29.32
+
+  # test dependencies
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

doc/source/development/contributing_codebase.rst

@@ -777,6 +777,14 @@ install pandas) by typing::
 
     pytest pandas
 
+.. note::
+
+    If a handful of tests don't pass, it may not be an issue with your pandas installation.
+    Some tests (e.g. some SQLAlchemy ones) require additional setup, others might start
+    failing because a non-pinned library released a new version, and others might be flaky
+    if run in parallel. As long as you can import pandas from your locally built version,
+    your installation is probably fine and you can start contributing!
+
 Often it is worth running only a subset of tests first around your changes before running the
 entire suite.