Merge remote-tracking branch 'upstream/main' into issue49649

Author: Dr-Irv

.devcontainer.json

@@ -9,8 +9,7 @@
 	// You can edit these settings after create using File > Preferences > Settings > Remote.
 	"settings": {
 		"terminal.integrated.shell.linux": "/bin/bash",
-		"python.condaPath": "/opt/conda/bin/conda",
-		"python.pythonPath": "/opt/conda/bin/python",
+		"python.pythonPath": "/usr/local/bin/python",
 		"python.formatting.provider": "black",
 		"python.linting.enabled": true,
 		"python.linting.flake8Enabled": true,

.github/actions/setup-conda/action.yml

@@ -18,7 +18,7 @@ runs:
     - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
       run: |
         grep -q '  - pyarrow' ${{ inputs.environment-file }}
-        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow<10/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
         cat ${{ inputs.environment-file }}
       shell: bash
       if: ${{ inputs.pyarrow-version }}

.github/workflows/32-bit-linux.yml

@@ -17,7 +17,7 @@ permissions:
 
 jobs:
   pytest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -38,7 +38,8 @@ jobs:
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python -m pip install versioneer[toml] && \
+          python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
           python setup.py build_ext -q -j1 && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
           python -m pip list && \

.github/workflows/assign.yml

@@ -11,7 +11,7 @@ jobs:
     permissions:
       issues: write
       pull-requests: write
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - if: github.event.comment.body == 'take'
       run: |

.github/workflows/asv-bot.yml

@@ -21,7 +21,7 @@ jobs:
     name: "Run benchmarks"
     # TODO: Support more benchmarking options later, against different branches, against self, etc
     if: startsWith(github.event.comment.body, '@github-actions benchmark')
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash -el {0}

.github/workflows/autoupdate-pre-commit-config.yml

@@ -15,10 +15,10 @@ jobs:
       pull-requests: write  # for technote-space/create-pr-action to create a PR
     if: github.repository_owner == 'pandas-dev'
     name: Autoupdate pre-commit config
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
       - name: Cache multiple paths
         uses: actions/cache@v3
         with:

.github/workflows/code-checks.yml

@@ -20,7 +20,7 @@ permissions:
 jobs:
   pre_commit:
     name: pre-commit
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
@@ -30,16 +30,18 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Install Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
-        python-version: '3.9.7'
+        python-version: '3.9'
 
     - name: Run pre-commit
       uses: pre-commit/action@v2.0.3
+      with:
+        extra_args: --verbose --all-files
 
-  typing_and_docstring_validation:
-    name: Docstring and typing validation
-    runs-on: ubuntu-latest
+  docstring_typing_pylint:
+    name: Docstring validation, typing, and pylint
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash -el {0}
@@ -79,6 +81,10 @@ jobs:
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' && always() }}
 
+    - name: Run check of documentation notebooks
+      run: ci/code_checks.sh notebooks
+      if: ${{ steps.build.outcome == 'success' && always() }}
+
     - name: Use existing environment for type checking
       run: |
         echo $PATH >> $GITHUB_PATH
@@ -89,7 +95,7 @@ jobs:
     - name: Typing + pylint
       uses: pre-commit/action@v2.0.3
       with:
-        extra_args: --hook-stage manual --all-files
+        extra_args: --verbose --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' && always() }}
 
     - name: Run docstring validation script tests
@@ -98,7 +104,7 @@ jobs:
 
   asv-benchmarks:
     name: ASV Benchmarks
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash -el {0}
@@ -129,7 +135,7 @@ jobs:
 
   build_docker_dev_environment:
     name: Build Docker Dev Environment
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash -el {0}
@@ -152,11 +158,11 @@ jobs:
         run: docker build --pull --no-cache --tag pandas-dev-env .
 
       - name: Show environment
-        run: docker run -w /home/pandas pandas-dev-env mamba run -n pandas-dev python -c "import pandas as pd; print(pd.show_versions())"
+        run: docker run --rm pandas-dev-env python -c "import pandas as pd; print(pd.show_versions())"
 
   requirements-dev-text-installable:
     name: Test install requirements-dev.txt
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
@@ -171,7 +177,7 @@ jobs:
 
       - name: Setup Python
         id: setup_python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
           cache: 'pip'

.github/workflows/codeql.yml

@@ -10,7 +10,7 @@ concurrency:
 
 jobs:
   analyze:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       actions: read
       contents: read

.github/workflows/docbuild-and-upload.yml

@@ -22,7 +22,7 @@ permissions:
 jobs:
   web_and_docs:
     name: Doc Build and Upload
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
@@ -54,11 +54,6 @@ jobs:
     - name: Build documentation zip
       run: doc/make.py zip_html
 
-    - name: Build the interactive terminal
-      run: |
-        cd web/interactive_terminal
-        jupyter lite build
-
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh

.github/workflows/package-checks.yml

@@ -9,13 +9,15 @@ on:
     branches:
       - main
       - 1.5.x
+    types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
   contents: read
 
 jobs:
   pip:
-    runs-on: ubuntu-latest
+    if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         extra: ["test", "performance", "timezone", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"]
@@ -34,20 +36,14 @@ jobs:
 
       - name: Setup Python
         id: setup_python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
 
-      # Hacky patch to disable building cython extensions.
-      # This job should only check that the extras successfully install.
-      - name: Disable building ext_modules
-        run: |
-          sed -i '/ext_modules=/d' setup.py
-        shell: bash -el {0}
-
       - name: Install required dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+          python -m pip install versioneer[toml]
         shell: bash -el {0}
 
       - name: Pip install with extra

.github/workflows/python-dev.yml

@@ -49,7 +49,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
+        os: [ubuntu-22.04, macOS-latest, windows-latest]
 
     name: actions-311-dev
     timeout-minutes: 120
@@ -75,6 +75,7 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
+        python -m pip install versioneer[toml]
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 

.github/workflows/scorecards.yml

@@ -13,7 +13,7 @@ permissions: read-all
 jobs:
   analysis:
     name: Scorecards analysis
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       # Needed to upload the results to code-scanning dashboard.
       security-events: write

.github/workflows/sdist.yml

@@ -19,7 +19,7 @@ permissions:
 jobs:
   build:
     if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     defaults:
       run:
@@ -47,6 +47,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel
+        python -m pip install versioneer[toml]
 
         # GH 39416
         pip install numpy

.github/workflows/stale-pr.yml

@@ -11,7 +11,7 @@ jobs:
   stale:
     permissions:
       pull-requests: write
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/stale@v4
       with:

.github/workflows/ubuntu.yml

@@ -20,7 +20,7 @@ permissions:
 
 jobs:
   pytest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash -el {0}

.github/workflows/wheels.yml

@@ -127,7 +127,7 @@ jobs:
       (github.event_name == 'pull_request' &&
       contains(github.event.pull_request.labels.*.name, 'Build')) ||
       (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0')))
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}

.gitpod.yml

@@ -0,0 +1,59 @@
+# Building pandas on init
+# Might delegate this later to prebuild with Q2 improvements on gitpod
+# https://www.gitpod.io/docs/config-start-tasks/#configuring-the-terminal
+# -------------------------------------------------------------------------
+
+# assuming we use dockerhub: name of the docker user, docker image, tag, e.g. https://hub.docker.com/r/pandas/pandas-gitpod/tags
+image: pythonpandas/pandas-gitpod:latest
+tasks:
+  - name: Prepare development environment
+    init: |
+      mkdir -p .vscode
+      cp gitpod/settings.json .vscode/settings.json
+      conda activate pandas-dev
+      git pull --unshallow  # need to force this else the prebuild fails
+      git fetch --tags
+      python setup.py build_ext -j 4
+      python -m pip install -e . --no-build-isolation
+      echo "🛠 Completed rebuilding Pandas!! 🛠 "
+      echo "✨ Pre-build complete! You can close this terminal ✨ "
+
+# --------------------------------------------------------
+# exposing ports for liveserve
+ports:
+  - port: 5500
+    onOpen: notify
+
+# --------------------------------------------------------
+# some useful extensions to have
+vscode:
+  extensions:
+    - ms-python.python
+    - yzhang.markdown-all-in-one
+    - eamodio.gitlens
+    - lextudio.restructuredtext
+    - ritwickdey.liveserver
+    # add or remove what you think is generally useful to most contributors
+    # avoid adding too many. they each open a pop-up window
+
+# --------------------------------------------------------
+# using prebuilds for the container
+# With this configuration the prebuild will happen on push to main
+github:
+  prebuilds:
+    # enable for main/default branch
+    main: true
+    # enable for other branches (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: false
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: false
+    # add a check to pull requests (defaults to true)
+    addCheck: false
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
+    addComment: false
+    # add a "Review in Gitpod" button to the pull request's description (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: false