Merge branch 'main' into gha-win-mac

Author: jonashaag

.github/actions/build_pandas/action.yml

@@ -28,3 +28,6 @@ runs:
     - name: Build Version
       run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
       shell: bash -el {0}
+      env:
+        # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
+        N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}

.github/workflows/code-checks.yml

@@ -74,12 +74,16 @@ jobs:
 
     - name: Install pyright
       # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g pyright@1.1.230
+      run: npm install -g pyright@1.1.245
 
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
 
+    - name: Check for no warnings when building single-page docs
+      run: ci/code_checks.sh single-docs
+      if: ${{ steps.build.outcome == 'success' }}
+
     - name: Run checks on imported code
       run: ci/code_checks.sh code
       if: ${{ steps.build.outcome == 'success' }}
@@ -140,22 +144,8 @@ jobs:
     - name: Run ASV benchmarks
       run: |
         cd asv_bench
-        asv check -E existing
-        git remote add upstream https://github.com/pandas-dev/pandas.git
-        git fetch upstream
         asv machine --yes
-        asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
-        if grep "failed" benchmarks.log > /dev/null ; then
-            exit 1
-        fi
-      if: ${{ steps.build.outcome == 'success' }}
-
-    - name: Publish benchmarks artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: Benchmarks log
-        path: asv_bench/benchmarks.log
-      if: failure()
+        asv run --quick --dry-run --strict --durations=30 --python=same
 
   build_docker_dev_environment:
     name: Build Docker Dev Environment

.github/workflows/docbuild-and-upload.yml

@@ -37,6 +37,7 @@ jobs:
       run: |
         source activate pandas-dev
         python web/pandas_web.py web/pandas --target-path=web/build
+
     - name: Build documentation
       run: |
         source activate pandas-dev

.github/workflows/posix.yml

@@ -28,7 +28,7 @@ jobs:
         pattern: ["not single_cpu", "single_cpu"]
         # Don't test pyarrow v2/3: Causes timeouts in read_csv engine
         # even if tests are skipped/xfailed
-        pyarrow_version: ["5", "7"]
+        pyarrow_version: ["5", "6", "7"]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -61,7 +61,16 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_testing_mode: "deprecate"
-            test_args: "-W error"
+            test_args: "-W error::DeprecationWarning:numpy"
+        exclude:
+          - env_file: actions-39.yaml
+            pyarrow_version: "6"
+          - env_file: actions-39.yaml
+            pyarrow_version: "7"
+          - env_file: actions-310.yaml
+            pyarrow_version: "6"
+          - env_file: actions-310.yaml
+            pyarrow_version: "7"
       fail-fast: false
     name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }}
     env:
@@ -171,10 +180,9 @@ jobs:
       run: ci/run_tests.sh
       # TODO: Don't continue on error for PyPy
       continue-on-error: ${{ env.IS_PYPY == 'true' }}
-      if: always()
 
     - name: Build Version
-      run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+      run: conda list
 
     - name: Publish test results
       uses: actions/upload-artifact@v3

.github/workflows/windows.yml

@@ -0,0 +1,75 @@
+name: Windows
+
+on:
+  push:
+    branches:
+      - main
+      - 1.4.x
+  pull_request:
+    branches:
+      - main
+      - 1.4.x
+    paths-ignore:
+      - "doc/**"
+
+env:
+  PANDAS_CI: 1
+  PYTEST_TARGET: pandas
+  PYTEST_WORKERS: auto
+  PATTERN: "not slow and not db and not network and not single_cpu"
+
+
+jobs:
+  pytest:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+      fail-fast: false
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-windows
+      cancel-in-progress: true
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Install Dependencies
+      uses: conda-incubator/setup-miniconda@v2.1.1
+      with:
+        mamba-version: "*"
+        channels: conda-forge
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ci/deps/${{ matrix.env_file }}
+        use-only-tar-bz2: true
+
+    - name: Build Pandas
+      uses: ./.github/actions/build_pandas
+
+    - name: Test
+      run: ci/run_tests.sh
+
+    - name: Build Version
+      run: conda list
+
+    - name: Publish test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: false

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         types_or: [python, rst, markdown]
         files: ^(pandas|doc)/
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.2.0
     hooks:
     -   id: debug-statements
     -   id: end-of-file-fixer
@@ -56,7 +56,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.1
+    rev: v2.32.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -70,6 +70,10 @@ repos:
       - id: rst-inline-touching-normal
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
+-   repo: https://github.com/sphinx-contrib/sphinx-lint
+    rev: v0.4.1
+    hooks:
+    - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
     rev: v1.3.0
     hooks:
@@ -85,7 +89,7 @@ repos:
         types: [python]
         stages: [manual]
         # note: keep version in sync with .github/workflows/code-checks.yml
-        additional_dependencies: ['pyright@1.1.230']
+        additional_dependencies: ['pyright@1.1.245']
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -172,6 +176,13 @@ repos:
         files: ^pandas/core/
         exclude: ^pandas/core/api\.py$
         types: [python]
+    -   id: use-io-common-urlopen
+        name: Use pandas.io.common.urlopen instead of urllib.request.urlopen
+        language: python
+        entry: python scripts/use_io_common_urlopen.py
+        files: ^pandas/
+        exclude: ^pandas/tests/
+        types: [python]
     -   id: no-bool-in-core-generic
         name: Use bool_t instead of bool in pandas/core/generic.py
         entry: python scripts/no_bool_in_generic.py

LICENSES/KLIB_LICENSE

@@ -0,0 +1,23 @@
+The MIT License
+
+Copyright (c) 2008-     Attractive Chaos <attractor@live.co.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSES/OTHER

@@ -1,8 +1,3 @@
-numpydoc license
-----------------
-
-The numpydoc license is in pandas/doc/sphinxext/LICENSE.txt
-
 Bottleneck license
 ------------------
 
@@ -77,4 +72,4 @@ DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -1,4 +1,5 @@
 include RELEASE.md
+include versioneer.py
 
 graft doc
 prune doc/build
@@ -54,9 +55,6 @@ global-exclude *.pxi
 # exclude the whole directory to avoid running related tests in sdist
 prune pandas/tests/io/parser/data
 
-include versioneer.py
-include pandas/_version.py
-include pandas/io/formats/templates/*.tpl
-
+# Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src
 graft pandas/_libs/tslibs/src

asv_bench/benchmarks/tslibs/tz_convert.py

@@ -11,10 +11,14 @@
 
 try:
     old_sig = False
-    from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    from pandas._libs.tslibs import tz_convert_from_utc
 except ImportError:
-    old_sig = True
-    from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
+    try:
+        old_sig = False
+        from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    except ImportError:
+        old_sig = True
+        from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
 
 
 class TimeTZConvert:

azure-pipelines.yml

@@ -27,11 +27,6 @@ jobs:
     name: macOS
     vmImage: macOS-10.15
 
-- template: ci/azure/windows.yml
-  parameters:
-    name: Windows
-    vmImage: windows-2019
-
 - job: py38_32bit
   pool:
     vmImage: ubuntu-18.04
@@ -48,6 +43,7 @@ jobs:
         pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
+        export PANDAS_CI=1 && \
         pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'