Merge remote-tracking branch 'upstream/main' into pdep5

Author: MarcoGorelli

.devcontainer.json

@@ -9,8 +9,7 @@
 	// You can edit these settings after create using File > Preferences > Settings > Remote.
 	"settings": {
 		"terminal.integrated.shell.linux": "/bin/bash",
-		"python.condaPath": "/opt/conda/bin/conda",
-		"python.pythonPath": "/opt/conda/bin/python",
+		"python.pythonPath": "/usr/local/bin/python",
 		"python.formatting.provider": "black",
 		"python.linting.enabled": true,
 		"python.linting.flake8Enabled": true,

.github/workflows/32-bit-linux.yml

@@ -38,7 +38,8 @@ jobs:
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python -m pip install versioneer[toml] && \
+          python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
           python setup.py build_ext -q -j1 && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
           python -m pip list && \

.github/workflows/code-checks.yml

@@ -36,9 +36,11 @@ jobs:
 
     - name: Run pre-commit
       uses: pre-commit/action@v2.0.3
+      with:
+        extra_args: --verbose --all-files
 
-  typing_and_docstring_validation:
-    name: Docstring and typing validation
+  docstring_typing_pylint:
+    name: Docstring validation, typing, and pylint
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -79,6 +81,10 @@ jobs:
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' && always() }}
 
+    - name: Run check of documentation notebooks
+      run: ci/code_checks.sh notebooks
+      if: ${{ steps.build.outcome == 'success' && always() }}
+
     - name: Use existing environment for type checking
       run: |
         echo $PATH >> $GITHUB_PATH
@@ -89,7 +95,7 @@ jobs:
     - name: Typing + pylint
       uses: pre-commit/action@v2.0.3
       with:
-        extra_args: --hook-stage manual --all-files
+        extra_args: --verbose --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' && always() }}
 
     - name: Run docstring validation script tests
@@ -152,7 +158,7 @@ jobs:
         run: docker build --pull --no-cache --tag pandas-dev-env .
 
       - name: Show environment
-        run: docker run -w /home/pandas pandas-dev-env mamba run -n pandas-dev python -c "import pandas as pd; print(pd.show_versions())"
+        run: docker run --rm pandas-dev-env python -c "import pandas as pd; print(pd.show_versions())"
 
   requirements-dev-text-installable:
     name: Test install requirements-dev.txt

.github/workflows/docbuild-and-upload.yml

@@ -54,11 +54,6 @@ jobs:
     - name: Build documentation zip
       run: doc/make.py zip_html
 
-    - name: Build the interactive terminal
-      run: |
-        cd web/interactive_terminal
-        jupyter lite build
-
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh

.github/workflows/package-checks.yml

@@ -9,12 +9,14 @@ on:
     branches:
       - main
       - 1.5.x
+    types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
   contents: read
 
 jobs:
   pip:
+    if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -38,16 +40,10 @@ jobs:
         with:
           python-version: '3.8'
 
-      # Hacky patch to disable building cython extensions.
-      # This job should only check that the extras successfully install.
-      - name: Disable building ext_modules
-        run: |
-          sed -i '/ext_modules=/d' setup.py
-        shell: bash -el {0}
-
       - name: Install required dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+          python -m pip install versioneer[toml]
         shell: bash -el {0}
 
       - name: Pip install with extra

.github/workflows/python-dev.yml

@@ -75,6 +75,7 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
+        python -m pip install versioneer[toml]
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 

.github/workflows/sdist.yml

@@ -47,6 +47,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel
+        python -m pip install versioneer[toml]
 
         # GH 39416
         pip install numpy

.gitpod.yml

@@ -0,0 +1,59 @@
+# Building pandas on init
+# Might delegate this later to prebuild with Q2 improvements on gitpod
+# https://www.gitpod.io/docs/config-start-tasks/#configuring-the-terminal
+# -------------------------------------------------------------------------
+
+# assuming we use dockerhub: name of the docker user, docker image, tag, e.g. https://hub.docker.com/r/pandas/pandas-gitpod/tags
+image: pythonpandas/pandas-gitpod:latest
+tasks:
+  - name: Prepare development environment
+    init: |
+      mkdir -p .vscode
+      cp gitpod/settings.json .vscode/settings.json
+      conda activate pandas-dev
+      git pull --unshallow  # need to force this else the prebuild fails
+      git fetch --tags
+      python setup.py build_ext -j 4
+      python -m pip install -e . --no-build-isolation
+      echo "🛠 Completed rebuilding Pandas!! 🛠 "
+      echo "✨ Pre-build complete! You can close this terminal ✨ "
+
+# --------------------------------------------------------
+# exposing ports for liveserve
+ports:
+  - port: 5500
+    onOpen: notify
+
+# --------------------------------------------------------
+# some useful extensions to have
+vscode:
+  extensions:
+    - ms-python.python
+    - yzhang.markdown-all-in-one
+    - eamodio.gitlens
+    - lextudio.restructuredtext
+    - ritwickdey.liveserver
+    # add or remove what you think is generally useful to most contributors
+    # avoid adding too many. they each open a pop-up window
+
+# --------------------------------------------------------
+# using prebuilds for the container
+# With this configuration the prebuild will happen on push to main
+github:
+  prebuilds:
+    # enable for main/default branch
+    main: true
+    # enable for other branches (defaults to false)
+    branches: false
+    # enable for pull requests coming from this repo (defaults to true)
+    pullRequests: false
+    # enable for pull requests coming from forks (defaults to false)
+    pullRequestsFromForks: false
+    # add a check to pull requests (defaults to true)
+    addCheck: false
+    # add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
+    addComment: false
+    # add a "Review in Gitpod" button to the pull request's description (defaults to false)
+    addBadge: false
+    # add a label once the prebuild is ready to pull requests (defaults to false)
+    addLabel: false

.pre-commit-config.yaml

@@ -17,21 +17,19 @@ repos:
         entry: python scripts/run_vulture.py
         pass_filenames: true
         require_serial: false
--   repo: https://github.com/python/black
-    rev: 22.10.0
-    hooks:
-    -   id: black
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.2.2
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
+        additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.2.1
+    rev: v0.9.1
     hooks:
     -   id: cython-lint
+    -   id: double-quote-cython-strings
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
     -   id: debug-statements
     -   id: end-of-file-fixer
@@ -50,26 +48,43 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
     -   id: flake8
         # Need to patch os.remove rule in pandas-dev-flaker
         exclude: ^ci/fix_wheels.py
         additional_dependencies: &flake8_dependencies
-        - flake8==5.0.4
+        - flake8==6.0.0
         - flake8-bugbear==22.7.1
         - pandas-dev-flaker==0.5.0
 -   repo: https://github.com/pycqa/pylint
-    rev: v2.15.5
+    rev: v2.15.6
+    hooks:
+    -   id: pylint
+        stages: [manual]
+-   repo: https://github.com/pycqa/pylint
+    rev: v2.15.6
     hooks:
     -   id: pylint
+        alias: redefined-outer-name
+        name: Redefining name from outer scope
+        files: ^pandas/
+        exclude: |
+            (?x)
+            ^pandas/tests  # keep excluded
+            |/_testing/  # keep excluded
+            |^pandas/util/_test_decorators\.py  # keep excluded
+            |^pandas/_version\.py  # keep excluded
+            |^pandas/conftest\.py  # keep excluded
+            |^pandas/core/generic\.py
+        args: [--disable=all, --enable=redefined-outer-name]
         stages: [manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.2.0
+    rev: v3.2.2
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -94,6 +109,16 @@ repos:
         additional_dependencies: *flake8_dependencies
 -   repo: local
     hooks:
+    # NOTE: we make `black` a local hook because if it's installed from
+    # PyPI (rather than from source) then it'll run twice as fast thanks to mypyc
+    -   id: black
+        name: black
+        description: "Black: The uncompromising Python code formatter"
+        entry: black
+        language: python
+        require_serial: true
+        types_or: [python, pyi]
+        additional_dependencies: [black==22.10.0]
     -   id: pyright
         # note: assumes python env is setup and activated
         name: pyright
@@ -201,7 +226,7 @@ repos:
         entry: python scripts/sync_flake8_versions.py
         files: ^(\.pre-commit-config\.yaml|environment\.yml)$
         pass_filenames: false
-        additional_dependencies: [pyyaml]
+        additional_dependencies: [pyyaml, toml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py
@@ -250,6 +275,7 @@ repos:
         entry: python scripts/validate_min_versions_in_sync.py
         language: python
         files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$
+        additional_dependencies: [tomli]
     -   id: validate-errors-locations
         name: Validate errors locations
         description: Validate errors are in appropriate locations.
@@ -287,5 +313,4 @@ repos:
         language: python
         additional_dependencies:
         - autotyping==22.9.0
-        - black==22.6.0
         - libcst==0.4.7

Dockerfile

@@ -1,42 +1,13 @@
-FROM quay.io/condaforge/mambaforge
+FROM python:3.10.8
+WORKDIR /home/pandas
 
-# if you forked pandas, you can pass in your own GitHub username to use your fork
-# i.e. gh_username=myname
-ARG gh_username=pandas-dev
-ARG pandas_home="/home/pandas"
+RUN apt-get update && apt-get -y upgrade
+RUN apt-get install -y build-essential
 
-# Avoid warnings by switching to noninteractive
-ENV DEBIAN_FRONTEND=noninteractive
+# hdf5 needed for pytables installation
+RUN apt-get install -y libhdf5-dev
 
-# Configure apt and install packages
-RUN apt-get update \
-    && apt-get -y install --no-install-recommends apt-utils git tzdata dialog 2>&1 \
-    #
-    # Configure timezone (fix for tests which try to read from "/etc/localtime")
-    && ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime \
-    && dpkg-reconfigure -f noninteractive tzdata \
-    #
-    # cleanup
-    && apt-get autoremove -y \
-    && apt-get clean -y \
-    && rm -rf /var/lib/apt/lists/*
-
-# Switch back to dialog for any ad-hoc use of apt-get
-ENV DEBIAN_FRONTEND=dialog
-
-# Clone pandas repo
-RUN mkdir "$pandas_home" \
-    && git clone "https://github.com/$gh_username/pandas.git" "$pandas_home" \
-    && cd "$pandas_home" \
-    && git remote add upstream "https://github.com/pandas-dev/pandas.git" \
-    && git pull upstream main
-
-# Set up environment
-RUN mamba env create -f "$pandas_home/environment.yml"
-
-# Build C extensions and pandas
-SHELL ["mamba", "run", "--no-capture-output", "-n", "pandas-dev", "/bin/bash", "-c"]
-RUN cd "$pandas_home" \
-    && export \
-    && python setup.py build_ext -j 4 \
-    && python -m pip install --no-build-isolation -e .
+RUN python -m pip install --upgrade pip
+RUN python -m pip install --use-deprecated=legacy-resolver \
+    -r https://raw.githubusercontent.com/pandas-dev/pandas/main/requirements-dev.txt
+CMD ["/bin/bash"]

asv_bench/asv.conf.json

@@ -57,7 +57,7 @@
         "odfpy": [],
         "jinja2": [],
     },
-    "conda_channels": ["defaults", "conda-forge"],
+    "conda_channels": ["conda-forge"],
     // Combinations of libraries/python versions can be excluded/included
     // from the set to test. Each entry is a dictionary containing additional
     // key-value pairs to include/exclude.
@@ -125,6 +125,7 @@
     "regression_thresholds": {
     },
     "build_command":
-    ["python setup.py build -j4",
+    ["python -m pip install versioneer[toml]",
+     "python setup.py build -j4",
      "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"],
 }

asv_bench/benchmarks/arithmetic.py

@@ -106,6 +106,10 @@ def time_frame_op_with_series_axis0(self, opname):
     def time_frame_op_with_series_axis1(self, opname):
         getattr(operator, opname)(self.df, self.ser)
 
+    # exclude comparisons from the params for time_frame_op_with_series_axis1
+    #  since they do not do alignment so raise
+    time_frame_op_with_series_axis1.params = [params[0][6:]]
+
 
 class FrameWithFrameWide:
     # Many-columns, mixed dtypes