From 344e3c5e05b89b417642719af1daccbbce9be9c2 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 17 May 2022 17:38:38 -0700 Subject: [PATCH 1/6] CI: Debug CI Windows recurssion error --- .github/workflows/32-bit-linux.yml | 43 ---- .github/workflows/assign.yml | 13 -- .github/workflows/asv-bot.yml | 81 -------- .../autoupdate-pre-commit-config.yml | 33 ---- .github/workflows/code-checks.yml | 172 ---------------- .github/workflows/comment_bot.yml | 40 ---- .github/workflows/docbuild-and-upload.yml | 73 ------- .github/workflows/posix.yml | 186 ------------------ .github/workflows/python-dev.yml | 97 --------- .github/workflows/sdist.yml | 91 --------- .github/workflows/stale-pr.yml | 21 -- ci/run_tests.sh | 2 +- 12 files changed, 1 insertion(+), 851 deletions(-) delete mode 100644 .github/workflows/32-bit-linux.yml delete mode 100644 .github/workflows/assign.yml delete mode 100644 .github/workflows/asv-bot.yml delete mode 100644 .github/workflows/autoupdate-pre-commit-config.yml delete mode 100644 .github/workflows/code-checks.yml delete mode 100644 .github/workflows/comment_bot.yml delete mode 100644 .github/workflows/docbuild-and-upload.yml delete mode 100644 .github/workflows/posix.yml delete mode 100644 .github/workflows/python-dev.yml delete mode 100644 .github/workflows/sdist.yml delete mode 100644 .github/workflows/stale-pr.yml diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml deleted file mode 100644 index 500e800a082d9..0000000000000 --- a/.github/workflows/32-bit-linux.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: 32 Bit Linux - -on: - push: - branches: - - main - - 1.4.x - pull_request: - branches: - - main - - 1.4.x - paths-ignore: - - "doc/**" - -jobs: - pytest: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Run 32-bit manylinux2014 Docker Build / Tests - run: | - docker pull quay.io/pypa/manylinux2014_i686 - docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - /bin/bash -xc "cd pandas && \ - /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ - . ~/virtualenvs/pandas-dev/bin/activate && \ - python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ - python setup.py build_ext -q -j2 && \ - python -m pip install --no-build-isolation --no-use-pep517 -e . && \ - export PANDAS_CI=1 && \ - pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" - - - name: Publish test results for Python 3.8-32 bit full Linux - uses: actions/upload-artifact@v3 - with: - name: Test results - path: test-data.xml - if: failure() diff --git a/.github/workflows/assign.yml b/.github/workflows/assign.yml deleted file mode 100644 index a1812843b1a8f..0000000000000 --- a/.github/workflows/assign.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Assign -on: - issue_comment: - types: created - -jobs: - issue_assign: - runs-on: ubuntu-latest - steps: - - if: github.event.comment.body == 'take' - run: | - echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml deleted file mode 100644 index 78c224b84d5d9..0000000000000 --- a/.github/workflows/asv-bot.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: "ASV Bot" - -on: - issue_comment: # Pull requests are issues - types: - - created - -env: - ENV_FILE: environment.yml - COMMENT: ${{github.event.comment.body}} - -jobs: - autotune: - name: "Run benchmarks" - # TODO: Support more benchmarking options later, against different branches, against self, etc - if: startsWith(github.event.comment.body, '@github-actions benchmark') - runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} - - concurrency: - # Set concurrency to prevent abuse(full runs are ~5.5 hours !!!) - # each user can only run one concurrent benchmark bot at a time - # We don't cancel in progress jobs, but if you want to benchmark multiple PRs, you're gonna have - # to wait - group: ${{ github.actor }}-asv - cancel-in-progress: false - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v3 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - # Although asv sets up its own env, deps are still needed - # during discovery process - - uses: conda-incubator/setup-miniconda@v2.1.1 - with: - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - - - name: Run benchmarks - id: bench - continue-on-error: true # This is a fake failure, asv will exit code 1 for regressions - run: | - # extracting the regex, see https://stackoverflow.com/a/36798723 - REGEX=$(echo "$COMMENT" | sed -n "s/^.*-b\s*\(\S*\).*$/\1/p") - cd asv_bench - asv check -E existing - git remote add upstream https://github.com/pandas-dev/pandas.git - git fetch upstream - asv machine --yes - asv continuous -f 1.1 -b $REGEX upstream/main HEAD - echo 'BENCH_OUTPUT<> $GITHUB_ENV - asv compare -f 1.1 upstream/main HEAD >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - echo "REGEX=$REGEX" >> $GITHUB_ENV - - - uses: actions/github-script@v6 - env: - BENCH_OUTPUT: ${{env.BENCH_OUTPUT}} - REGEX: ${{env.REGEX}} - with: - script: | - const ENV_VARS = process.env - const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '\nBenchmarks completed. View runner logs here.' + run_url + '\nRegex used: '+ 'regex ' + ENV_VARS["REGEX"] + '\n' + ENV_VARS["BENCH_OUTPUT"] - }) diff --git a/.github/workflows/autoupdate-pre-commit-config.yml b/.github/workflows/autoupdate-pre-commit-config.yml deleted file mode 100644 index d2eac234ca361..0000000000000 --- a/.github/workflows/autoupdate-pre-commit-config.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: "Update pre-commit config" - -on: - schedule: - - cron: "0 7 1 * *" # At 07:00 on 1st of every month. - workflow_dispatch: - -jobs: - update-pre-commit: - if: github.repository_owner == 'pandas-dev' - name: Autoupdate pre-commit config - runs-on: ubuntu-latest - steps: - - name: Set up Python - uses: actions/setup-python@v3 - - name: Cache multiple paths - uses: actions/cache@v3 - with: - path: | - ~/.cache/pre-commit - ~/.cache/pip - key: pre-commit-autoupdate-${{ runner.os }}-build - - name: Update pre-commit config packages - uses: technote-space/create-pr-action@v2 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - EXECUTE_COMMANDS: | - pip install pre-commit - pre-commit autoupdate || (exit 0); - pre-commit run -a || (exit 0); - COMMIT_MESSAGE: "⬆️ UPGRADE: Autoupdate pre-commit config" - PR_BRANCH_NAME: "pre-commit-config-update-${PR_ID}" - PR_TITLE: "⬆️ UPGRADE: Autoupdate pre-commit config" diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml deleted file mode 100644 index 7d24b26f5538b..0000000000000 --- a/.github/workflows/code-checks.yml +++ /dev/null @@ -1,172 +0,0 @@ -name: Code Checks - -on: - push: - branches: - - main - - 1.4.x - pull_request: - branches: - - main - - 1.4.x - -env: - ENV_FILE: environment.yml - PANDAS_CI: 1 - -jobs: - pre_commit: - name: pre-commit - runs-on: ubuntu-latest - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit - cancel-in-progress: true - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.9.7' - - - name: Run pre-commit - uses: pre-commit/action@v2.0.3 - - typing_and_docstring_validation: - name: Docstring and typing validation - runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-code-checks - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v3 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2.1.1 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - - - name: Install node.js (for pyright) - uses: actions/setup-node@v3 - with: - node-version: "16" - - - name: Install pyright - # note: keep version in sync with .pre-commit-config.yaml - run: npm install -g pyright@1.1.247 - - - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas - - - name: Check for no warnings when building single-page docs - run: ci/code_checks.sh single-docs - if: ${{ steps.build.outcome == 'success' }} - - - name: Run checks on imported code - run: ci/code_checks.sh code - if: ${{ steps.build.outcome == 'success' }} - - - name: Run doctests - run: ci/code_checks.sh doctests - if: ${{ steps.build.outcome == 'success' }} - - - name: Run docstring validation - run: ci/code_checks.sh docstrings - if: ${{ steps.build.outcome == 'success' }} - - - name: Run typing validation - run: ci/code_checks.sh typing - if: ${{ steps.build.outcome == 'success' }} - - - name: Run docstring validation script tests - run: pytest scripts - if: ${{ steps.build.outcome == 'success' }} - - asv-benchmarks: - name: ASV Benchmarks - runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-asv-benchmarks - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v3 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2.1.1 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - - - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas - - - name: Run ASV benchmarks - run: | - cd asv_bench - asv machine --yes - asv run --quick --dry-run --strict --durations=30 --python=same - - build_docker_dev_environment: - name: Build Docker Dev Environment - runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-build_docker_dev_environment - cancel-in-progress: true - - steps: - - name: Clean up dangling images - run: docker image prune -f - - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Build image - run: docker build --pull --no-cache --tag pandas-dev-env . diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml deleted file mode 100644 index 3824e015e8336..0000000000000 --- a/.github/workflows/comment_bot.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Comment-bot - -on: - issue_comment: - types: - - created - - edited - -jobs: - autotune: - name: "Fixup pre-commit formatting" - if: startsWith(github.event.comment.body, '@github-actions pre-commit') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: r-lib/actions/pr-fetch@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache multiple paths - uses: actions/cache@v3 - with: - path: | - ~/.cache/pre-commit - ~/.cache/pip - key: pre-commit-dispatched-${{ runner.os }}-build - - uses: actions/setup-python@v3 - with: - python-version: 3.8 - - name: Install-pre-commit - run: python -m pip install --upgrade pre-commit - - name: Run pre-commit - run: pre-commit run --from-ref=origin/main --to-ref=HEAD --all-files || (exit 0) - - name: Commit results - run: | - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git commit -a -m 'Fixes from pre-commit [automated commit]' || echo "No changes to commit" - - uses: r-lib/actions/pr-push@v2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml deleted file mode 100644 index 8c2c86dc693a9..0000000000000 --- a/.github/workflows/docbuild-and-upload.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Doc Build and Upload - -on: - push: - branches: - - main - - 1.4.x - pull_request: - branches: - - main - - 1.4.x - -env: - ENV_FILE: environment.yml - PANDAS_CI: 1 - -jobs: - web_and_docs: - name: Doc Build and Upload - runs-on: ubuntu-latest - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up pandas - uses: ./.github/actions/setup - - - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build - - - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors - - - name: Install ssh key - run: | - mkdir -m 700 -p ~/.ssh - echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} - - - name: Copy cheatsheets into site directory - run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/ - - - name: Upload web - run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} - - - name: Upload dev docs - run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} - - - name: Move docs into site directory - run: mv doc/build/html web/build/docs - - - name: Save website as an artifact - uses: actions/upload-artifact@v3 - with: - name: website - path: web/build - retention-days: 14 diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml deleted file mode 100644 index 35c40f2a4aa54..0000000000000 --- a/.github/workflows/posix.yml +++ /dev/null @@ -1,186 +0,0 @@ -name: Posix - -on: - push: - branches: - - main - - 1.4.x - pull_request: - branches: - - main - - 1.4.x - paths-ignore: - - "doc/**" - -env: - PANDAS_CI: 1 - -jobs: - pytest: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} - timeout-minutes: 120 - strategy: - matrix: - env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml] - pattern: ["not single_cpu", "single_cpu"] - # Don't test pyarrow v2/3: Causes timeouts in read_csv engine - # even if tests are skipped/xfailed - pyarrow_version: ["5", "6", "7"] - include: - - name: "Downstream Compat" - env_file: actions-38-downstream_compat.yaml - pattern: "not slow and not network and not single_cpu" - pytest_target: "pandas/tests/test_downstream.py" - - name: "Minimum Versions" - env_file: actions-38-minimum_versions.yaml - pattern: "not slow and not network and not single_cpu" - - name: "Locale: it_IT.utf8" - env_file: actions-38.yaml - pattern: "not slow and not network and not single_cpu" - extra_apt: "language-pack-it" - lang: "it_IT.utf8" - lc_all: "it_IT.utf8" - - name: "Locale: zh_CN.utf8" - env_file: actions-38.yaml - pattern: "not slow and not network and not single_cpu" - extra_apt: "language-pack-zh-hans" - lang: "zh_CN.utf8" - lc_all: "zh_CN.utf8" - - name: "Data Manager" - env_file: actions-38.yaml - pattern: "not slow and not network and not single_cpu" - pandas_data_manager: "array" - - name: "Pypy" - env_file: actions-pypy-38.yaml - pattern: "not slow and not network and not single_cpu" - test_args: "--max-worker-restart 0" - - name: "Numpy Dev" - env_file: actions-310-numpydev.yaml - pattern: "not slow and not network and not single_cpu" - pandas_testing_mode: "deprecate" - test_args: "-W error::DeprecationWarning:numpy" - exclude: - - env_file: actions-39.yaml - pyarrow_version: "6" - - env_file: actions-39.yaml - pyarrow_version: "7" - - env_file: actions-310.yaml - pyarrow_version: "6" - - env_file: actions-310.yaml - pyarrow_version: "7" - fail-fast: false - name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} - env: - ENV_FILE: ci/deps/${{ matrix.env_file }} - PATTERN: ${{ matrix.pattern }} - EXTRA_APT: ${{ matrix.extra_apt || '' }} - LANG: ${{ matrix.lang || '' }} - LC_ALL: ${{ matrix.lc_all || '' }} - PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }} - PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }} - TEST_ARGS: ${{ matrix.test_args || '' }} - PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }} - PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }} - IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }} - # TODO: re-enable coverage on pypy, its slow - COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }} - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }} - cancel-in-progress: true - - services: - mysql: - image: mysql - env: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: pandas - options: >- - --health-cmd "mysqladmin ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 3306:3306 - - postgres: - image: postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: pandas - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - moto: - image: motoserver/moto - env: - AWS_ACCESS_KEY_ID: foobar_key - AWS_SECRET_ACCESS_KEY: foobar_secret - ports: - - 5000:5000 - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v3 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} - - - name: Extra installs - # xsel for clipboard tests - run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }} - - - uses: conda-incubator/setup-miniconda@v2.1.1 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: flexible - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - - - name: Upgrade Arrow version - run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} - if: ${{ matrix.pyarrow_version }} - - - name: Build Pandas - uses: ./.github/actions/build_pandas - - - name: Test - run: ci/run_tests.sh - # TODO: Don't continue on error for PyPy - continue-on-error: ${{ env.IS_PYPY == 'true' }} - - - name: Build Version - run: conda list - - - name: Publish test results - uses: actions/upload-artifact@v3 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: false diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml deleted file mode 100644 index 23a48e567dfe9..0000000000000 --- a/.github/workflows/python-dev.yml +++ /dev/null @@ -1,97 +0,0 @@ -# This file is purposely frozen(does not run). DO NOT DELETE IT -# Unfreeze(by commentingthe if: false() condition) once the -# next Python Dev version has released beta 1 and both Cython and numpy support it -# After that Python has released, migrate the workflows to the -# posix GHA workflows and "freeze" this file by -# uncommenting the if: false() condition -# Feel free to modify this comment as necessary. - -name: Python Dev - -on: - push: - branches: - - main - - 1.4.x - pull_request: - branches: - - main - - 1.4.x - paths-ignore: - - "doc/**" - -env: - PYTEST_WORKERS: "auto" - PANDAS_CI: 1 - PATTERN: "not slow and not network and not clipboard and not single_cpu" - COVERAGE: true - PYTEST_TARGET: pandas - -jobs: - build: - if: false # Comment this line out to "unfreeze" - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] - - name: actions-311-dev - timeout-minutes: 80 - - concurrency: - #https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev - cancel-in-progress: true - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Python Dev Version - uses: actions/setup-python@v3 - with: - python-version: '3.11-dev' - - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - name: Install dependencies - shell: bash -el {0} - run: | - python -m pip install --upgrade pip "setuptools<60.0.0" wheel - pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - pip install git+https://github.com/nedbat/coveragepy.git - pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list - - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 - - - name: Build Version - run: | - python -c "import pandas; pandas.show_versions();" - - - name: Test with pytest - shell: bash -el {0} - run: | - ci/run_tests.sh - - - name: Publish test results - uses: actions/upload-artifact@v3 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Report Coverage - run: | - coverage report -m - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: true diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml deleted file mode 100644 index cd19eb7641c8c..0000000000000 --- a/.github/workflows/sdist.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: sdist - -on: - push: - branches: - - main - - 1.4.x - pull_request: - branches: - - main - - 1.4.x - types: [labeled, opened, synchronize, reopened] - paths-ignore: - - "doc/**" - -jobs: - build: - if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}} - runs-on: ubuntu-latest - timeout-minutes: 60 - defaults: - run: - shell: bash -el {0} - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist - cancel-in-progress: true - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - name: Install dependencies - run: | - python -m pip install --upgrade pip "setuptools<60.0.0" wheel - - # GH 39416 - pip install numpy - - - name: Build pandas sdist - run: | - pip list - python setup.py sdist --formats=gztar - - - name: Upload sdist artifact - uses: actions/upload-artifact@v3 - with: - name: ${{matrix.python-version}}-sdist.gz - path: dist/*.gz - - - uses: conda-incubator/setup-miniconda@v2.1.1 - with: - activate-environment: pandas-sdist - channels: conda-forge - python-version: '${{ matrix.python-version }}' - - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - name: Install pandas from sdist - run: | - python -m pip install --upgrade "setuptools<60.0.0" - pip list - python -m pip install dist/*.gz - - - name: Force oldest supported NumPy - run: | - case "${{matrix.python-version}}" in - 3.8) - pip install numpy==1.18.5 ;; - 3.9) - pip install numpy==1.19.3 ;; - 3.10) - pip install numpy==1.21.2 ;; - esac - - - name: Import pandas - run: | - cd .. - conda list - python -c "import pandas; pandas.show_versions();" diff --git a/.github/workflows/stale-pr.yml b/.github/workflows/stale-pr.yml deleted file mode 100644 index b97b60717a2b8..0000000000000 --- a/.github/workflows/stale-pr.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Stale PRs" -on: - schedule: - # * is a special character in YAML so you have to quote this string - - cron: "0 0 * * *" - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please [update](https://pandas.pydata.org/pandas-docs/stable/development/contributing.html#updating-your-pull-request) and respond to this comment if you're still interested in working on this." - stale-pr-label: "Stale" - exempt-pr-labels: "Needs Review,Blocked,Needs Discussion" - days-before-issue-stale: -1 - days-before-pr-stale: 30 - days-before-close: -1 - remove-stale-when-updated: false - debug-only: false diff --git a/ci/run_tests.sh b/ci/run_tests.sh index e6de5caf955fc..5bff8bb412947 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -24,7 +24,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 20f5bf54ea173fb7dc0a0aaea0db0e28497e23a2 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 17 May 2022 20:02:38 -0700 Subject: [PATCH 2/6] Trigger CI From 35391522782ca53092cd4a4d9cd63c6e84c75903 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 17 May 2022 21:08:29 -0700 Subject: [PATCH 3/6] Trigger CI From 18f8336c17a17a1d3b4f21c16f4c8e1d6d65e52d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 17 May 2022 23:05:37 -0700 Subject: [PATCH 4/6] Trigger CI From 613a284664ba152179b86c8abae9aee523b09b82 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 18 May 2022 10:10:13 -0700 Subject: [PATCH 5/6] Fix offending test --- pandas/tests/extension/json/test_json.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json/test_json.py index f1e8e82d23691..f3129a89124f4 100644 --- a/pandas/tests/extension/json/test_json.py +++ b/pandas/tests/extension/json/test_json.py @@ -186,7 +186,13 @@ def test_series_constructor_scalar_na_with_index(self, dtype, na_value): @pytest.mark.xfail(reason="collection as scalar, GH-33901") def test_series_constructor_scalar_with_index(self, data, dtype): # TypeError: All values must be of type - super().test_series_constructor_scalar_with_index(data, dtype) + rec_limit = sys.getrecursionlimit() + try: + # Limit to avoid stack overflow on Windows CI + sys.setrecursionlimit(100) + super().test_series_constructor_scalar_with_index(data, dtype) + finally: + sys.setrecursionlimit(rec_limit) class TestReshaping(BaseJSON, base.BaseReshapingTests): From 39ea1f20c4920d5013417d42b92c322d0d7c4871 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 18 May 2022 10:11:24 -0700 Subject: [PATCH 6/6] Reset changes --- .github/workflows/32-bit-linux.yml | 43 ++++ .github/workflows/assign.yml | 13 ++ .github/workflows/asv-bot.yml | 81 ++++++++ .../autoupdate-pre-commit-config.yml | 33 ++++ .github/workflows/code-checks.yml | 172 ++++++++++++++++ .github/workflows/comment_bot.yml | 40 ++++ .github/workflows/docbuild-and-upload.yml | 73 +++++++ .github/workflows/posix.yml | 186 ++++++++++++++++++ .github/workflows/python-dev.yml | 97 +++++++++ .github/workflows/sdist.yml | 91 +++++++++ .github/workflows/stale-pr.yml | 21 ++ ci/run_tests.sh | 2 +- 12 files changed, 851 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/32-bit-linux.yml create mode 100644 .github/workflows/assign.yml create mode 100644 .github/workflows/asv-bot.yml create mode 100644 .github/workflows/autoupdate-pre-commit-config.yml create mode 100644 .github/workflows/code-checks.yml create mode 100644 .github/workflows/comment_bot.yml create mode 100644 .github/workflows/docbuild-and-upload.yml create mode 100644 .github/workflows/posix.yml create mode 100644 .github/workflows/python-dev.yml create mode 100644 .github/workflows/sdist.yml create mode 100644 .github/workflows/stale-pr.yml diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml new file mode 100644 index 0000000000000..500e800a082d9 --- /dev/null +++ b/.github/workflows/32-bit-linux.yml @@ -0,0 +1,43 @@ +name: 32 Bit Linux + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + paths-ignore: + - "doc/**" + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run 32-bit manylinux2014 Docker Build / Tests + run: | + docker pull quay.io/pypa/manylinux2014_i686 + docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ + /bin/bash -xc "cd pandas && \ + /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ + . ~/virtualenvs/pandas-dev/bin/activate && \ + python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ + pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ + python setup.py build_ext -q -j2 && \ + python -m pip install --no-build-isolation --no-use-pep517 -e . && \ + export PANDAS_CI=1 && \ + pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" + + - name: Publish test results for Python 3.8-32 bit full Linux + uses: actions/upload-artifact@v3 + with: + name: Test results + path: test-data.xml + if: failure() diff --git a/.github/workflows/assign.yml b/.github/workflows/assign.yml new file mode 100644 index 0000000000000..a1812843b1a8f --- /dev/null +++ b/.github/workflows/assign.yml @@ -0,0 +1,13 @@ +name: Assign +on: + issue_comment: + types: created + +jobs: + issue_assign: + runs-on: ubuntu-latest + steps: + - if: github.event.comment.body == 'take' + run: | + echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml new file mode 100644 index 0000000000000..78c224b84d5d9 --- /dev/null +++ b/.github/workflows/asv-bot.yml @@ -0,0 +1,81 @@ +name: "ASV Bot" + +on: + issue_comment: # Pull requests are issues + types: + - created + +env: + ENV_FILE: environment.yml + COMMENT: ${{github.event.comment.body}} + +jobs: + autotune: + name: "Run benchmarks" + # TODO: Support more benchmarking options later, against different branches, against self, etc + if: startsWith(github.event.comment.body, '@github-actions benchmark') + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + concurrency: + # Set concurrency to prevent abuse(full runs are ~5.5 hours !!!) + # each user can only run one concurrent benchmark bot at a time + # We don't cancel in progress jobs, but if you want to benchmark multiple PRs, you're gonna have + # to wait + group: ${{ github.actor }}-asv + cancel-in-progress: false + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v3 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + + # Although asv sets up its own env, deps are still needed + # during discovery process + - uses: conda-incubator/setup-miniconda@v2.1.1 + with: + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Run benchmarks + id: bench + continue-on-error: true # This is a fake failure, asv will exit code 1 for regressions + run: | + # extracting the regex, see https://stackoverflow.com/a/36798723 + REGEX=$(echo "$COMMENT" | sed -n "s/^.*-b\s*\(\S*\).*$/\1/p") + cd asv_bench + asv check -E existing + git remote add upstream https://github.com/pandas-dev/pandas.git + git fetch upstream + asv machine --yes + asv continuous -f 1.1 -b $REGEX upstream/main HEAD + echo 'BENCH_OUTPUT<> $GITHUB_ENV + asv compare -f 1.1 upstream/main HEAD >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + echo "REGEX=$REGEX" >> $GITHUB_ENV + + - uses: actions/github-script@v6 + env: + BENCH_OUTPUT: ${{env.BENCH_OUTPUT}} + REGEX: ${{env.REGEX}} + with: + script: | + const ENV_VARS = process.env + const run_url = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '\nBenchmarks completed. View runner logs here.' + run_url + '\nRegex used: '+ 'regex ' + ENV_VARS["REGEX"] + '\n' + ENV_VARS["BENCH_OUTPUT"] + }) diff --git a/.github/workflows/autoupdate-pre-commit-config.yml b/.github/workflows/autoupdate-pre-commit-config.yml new file mode 100644 index 0000000000000..d2eac234ca361 --- /dev/null +++ b/.github/workflows/autoupdate-pre-commit-config.yml @@ -0,0 +1,33 @@ +name: "Update pre-commit config" + +on: + schedule: + - cron: "0 7 1 * *" # At 07:00 on 1st of every month. + workflow_dispatch: + +jobs: + update-pre-commit: + if: github.repository_owner == 'pandas-dev' + name: Autoupdate pre-commit config + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v3 + - name: Cache multiple paths + uses: actions/cache@v3 + with: + path: | + ~/.cache/pre-commit + ~/.cache/pip + key: pre-commit-autoupdate-${{ runner.os }}-build + - name: Update pre-commit config packages + uses: technote-space/create-pr-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXECUTE_COMMANDS: | + pip install pre-commit + pre-commit autoupdate || (exit 0); + pre-commit run -a || (exit 0); + COMMIT_MESSAGE: "⬆️ UPGRADE: Autoupdate pre-commit config" + PR_BRANCH_NAME: "pre-commit-config-update-${PR_ID}" + PR_TITLE: "⬆️ UPGRADE: Autoupdate pre-commit config" diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml new file mode 100644 index 0000000000000..7d24b26f5538b --- /dev/null +++ b/.github/workflows/code-checks.yml @@ -0,0 +1,172 @@ +name: Code Checks + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + +env: + ENV_FILE: environment.yml + PANDAS_CI: 1 + +jobs: + pre_commit: + name: pre-commit + runs-on: ubuntu-latest + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.9.7' + + - name: Run pre-commit + uses: pre-commit/action@v2.0.3 + + typing_and_docstring_validation: + name: Docstring and typing validation + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-code-checks + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v3 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + + - uses: conda-incubator/setup-miniconda@v2.1.1 + with: + mamba-version: "*" + channels: conda-forge + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Install node.js (for pyright) + uses: actions/setup-node@v3 + with: + node-version: "16" + + - name: Install pyright + # note: keep version in sync with .pre-commit-config.yaml + run: npm install -g pyright@1.1.247 + + - name: Build Pandas + id: build + uses: ./.github/actions/build_pandas + + - name: Check for no warnings when building single-page docs + run: ci/code_checks.sh single-docs + if: ${{ steps.build.outcome == 'success' }} + + - name: Run checks on imported code + run: ci/code_checks.sh code + if: ${{ steps.build.outcome == 'success' }} + + - name: Run doctests + run: ci/code_checks.sh doctests + if: ${{ steps.build.outcome == 'success' }} + + - name: Run docstring validation + run: ci/code_checks.sh docstrings + if: ${{ steps.build.outcome == 'success' }} + + - name: Run typing validation + run: ci/code_checks.sh typing + if: ${{ steps.build.outcome == 'success' }} + + - name: Run docstring validation script tests + run: pytest scripts + if: ${{ steps.build.outcome == 'success' }} + + asv-benchmarks: + name: ASV Benchmarks + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-asv-benchmarks + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v3 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + + - uses: conda-incubator/setup-miniconda@v2.1.1 + with: + mamba-version: "*" + channels: conda-forge + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Build Pandas + id: build + uses: ./.github/actions/build_pandas + + - name: Run ASV benchmarks + run: | + cd asv_bench + asv machine --yes + asv run --quick --dry-run --strict --durations=30 --python=same + + build_docker_dev_environment: + name: Build Docker Dev Environment + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-build_docker_dev_environment + cancel-in-progress: true + + steps: + - name: Clean up dangling images + run: docker image prune -f + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build image + run: docker build --pull --no-cache --tag pandas-dev-env . diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml new file mode 100644 index 0000000000000..3824e015e8336 --- /dev/null +++ b/.github/workflows/comment_bot.yml @@ -0,0 +1,40 @@ +name: Comment-bot + +on: + issue_comment: + types: + - created + - edited + +jobs: + autotune: + name: "Fixup pre-commit formatting" + if: startsWith(github.event.comment.body, '@github-actions pre-commit') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: r-lib/actions/pr-fetch@v2 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Cache multiple paths + uses: actions/cache@v3 + with: + path: | + ~/.cache/pre-commit + ~/.cache/pip + key: pre-commit-dispatched-${{ runner.os }}-build + - uses: actions/setup-python@v3 + with: + python-version: 3.8 + - name: Install-pre-commit + run: python -m pip install --upgrade pre-commit + - name: Run pre-commit + run: pre-commit run --from-ref=origin/main --to-ref=HEAD --all-files || (exit 0) + - name: Commit results + run: | + git config user.name "$(git log -1 --pretty=format:%an)" + git config user.email "$(git log -1 --pretty=format:%ae)" + git commit -a -m 'Fixes from pre-commit [automated commit]' || echo "No changes to commit" + - uses: r-lib/actions/pr-push@v2 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml new file mode 100644 index 0000000000000..8c2c86dc693a9 --- /dev/null +++ b/.github/workflows/docbuild-and-upload.yml @@ -0,0 +1,73 @@ +name: Doc Build and Upload + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + +env: + ENV_FILE: environment.yml + PANDAS_CI: 1 + +jobs: + web_and_docs: + name: Doc Build and Upload + runs-on: ubuntu-latest + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up pandas + uses: ./.github/actions/setup + + - name: Build website + run: | + source activate pandas-dev + python web/pandas_web.py web/pandas --target-path=web/build + + - name: Build documentation + run: | + source activate pandas-dev + doc/make.py --warnings-are-errors + + - name: Install ssh key + run: | + mkdir -m 700 -p ~/.ssh + echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts + if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} + + - name: Copy cheatsheets into site directory + run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/ + + - name: Upload web + run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas + if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} + + - name: Upload dev docs + run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev + if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} + + - name: Move docs into site directory + run: mv doc/build/html web/build/docs + + - name: Save website as an artifact + uses: actions/upload-artifact@v3 + with: + name: website + path: web/build + retention-days: 14 diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml new file mode 100644 index 0000000000000..35c40f2a4aa54 --- /dev/null +++ b/.github/workflows/posix.yml @@ -0,0 +1,186 @@ +name: Posix + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + paths-ignore: + - "doc/**" + +env: + PANDAS_CI: 1 + +jobs: + pytest: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + timeout-minutes: 120 + strategy: + matrix: + env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml] + pattern: ["not single_cpu", "single_cpu"] + # Don't test pyarrow v2/3: Causes timeouts in read_csv engine + # even if tests are skipped/xfailed + pyarrow_version: ["5", "6", "7"] + include: + - name: "Downstream Compat" + env_file: actions-38-downstream_compat.yaml + pattern: "not slow and not network and not single_cpu" + pytest_target: "pandas/tests/test_downstream.py" + - name: "Minimum Versions" + env_file: actions-38-minimum_versions.yaml + pattern: "not slow and not network and not single_cpu" + - name: "Locale: it_IT.utf8" + env_file: actions-38.yaml + pattern: "not slow and not network and not single_cpu" + extra_apt: "language-pack-it" + lang: "it_IT.utf8" + lc_all: "it_IT.utf8" + - name: "Locale: zh_CN.utf8" + env_file: actions-38.yaml + pattern: "not slow and not network and not single_cpu" + extra_apt: "language-pack-zh-hans" + lang: "zh_CN.utf8" + lc_all: "zh_CN.utf8" + - name: "Data Manager" + env_file: actions-38.yaml + pattern: "not slow and not network and not single_cpu" + pandas_data_manager: "array" + - name: "Pypy" + env_file: actions-pypy-38.yaml + pattern: "not slow and not network and not single_cpu" + test_args: "--max-worker-restart 0" + - name: "Numpy Dev" + env_file: actions-310-numpydev.yaml + pattern: "not slow and not network and not single_cpu" + pandas_testing_mode: "deprecate" + test_args: "-W error::DeprecationWarning:numpy" + exclude: + - env_file: actions-39.yaml + pyarrow_version: "6" + - env_file: actions-39.yaml + pyarrow_version: "7" + - env_file: actions-310.yaml + pyarrow_version: "6" + - env_file: actions-310.yaml + pyarrow_version: "7" + fail-fast: false + name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} + env: + ENV_FILE: ci/deps/${{ matrix.env_file }} + PATTERN: ${{ matrix.pattern }} + EXTRA_APT: ${{ matrix.extra_apt || '' }} + LANG: ${{ matrix.lang || '' }} + LC_ALL: ${{ matrix.lc_all || '' }} + PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }} + PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }} + TEST_ARGS: ${{ matrix.test_args || '' }} + PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }} + PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }} + IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }} + # TODO: re-enable coverage on pypy, its slow + COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }} + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }} + cancel-in-progress: true + + services: + mysql: + image: mysql + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: pandas + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 3306:3306 + + postgres: + image: postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: pandas + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + moto: + image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret + ports: + - 5000:5000 + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v3 + env: + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('${{ env.ENV_FILE }}') }} + + - name: Extra installs + # xsel for clipboard tests + run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }} + + - uses: conda-incubator/setup-miniconda@v2.1.1 + with: + mamba-version: "*" + channels: conda-forge + activate-environment: pandas-dev + channel-priority: flexible + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Upgrade Arrow version + run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} + if: ${{ matrix.pyarrow_version }} + + - name: Build Pandas + uses: ./.github/actions/build_pandas + + - name: Test + run: ci/run_tests.sh + # TODO: Don't continue on error for PyPy + continue-on-error: ${{ env.IS_PYPY == 'true' }} + + - name: Build Version + run: conda list + + - name: Publish test results + uses: actions/upload-artifact@v3 + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unittests + name: codecov-pandas + fail_ci_if_error: false diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml new file mode 100644 index 0000000000000..23a48e567dfe9 --- /dev/null +++ b/.github/workflows/python-dev.yml @@ -0,0 +1,97 @@ +# This file is purposely frozen(does not run). DO NOT DELETE IT +# Unfreeze(by commentingthe if: false() condition) once the +# next Python Dev version has released beta 1 and both Cython and numpy support it +# After that Python has released, migrate the workflows to the +# posix GHA workflows and "freeze" this file by +# uncommenting the if: false() condition +# Feel free to modify this comment as necessary. + +name: Python Dev + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + paths-ignore: + - "doc/**" + +env: + PYTEST_WORKERS: "auto" + PANDAS_CI: 1 + PATTERN: "not slow and not network and not clipboard and not single_cpu" + COVERAGE: true + PYTEST_TARGET: pandas + +jobs: + build: + if: false # Comment this line out to "unfreeze" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + + name: actions-311-dev + timeout-minutes: 80 + + concurrency: + #https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev + cancel-in-progress: true + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python Dev Version + uses: actions/setup-python@v3 + with: + python-version: '3.11-dev' + + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + - name: Install dependencies + shell: bash -el {0} + run: | + python -m pip install --upgrade pip "setuptools<60.0.0" wheel + pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + pip install git+https://github.com/nedbat/coveragepy.git + pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov + pip list + + - name: Build Pandas + run: | + python setup.py build_ext -q -j2 + python -m pip install -e . --no-build-isolation --no-use-pep517 + + - name: Build Version + run: | + python -c "import pandas; pandas.show_versions();" + + - name: Test with pytest + shell: bash -el {0} + run: | + ci/run_tests.sh + + - name: Publish test results + uses: actions/upload-artifact@v3 + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Report Coverage + run: | + coverage report -m + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unittests + name: codecov-pandas + fail_ci_if_error: true diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml new file mode 100644 index 0000000000000..cd19eb7641c8c --- /dev/null +++ b/.github/workflows/sdist.yml @@ -0,0 +1,91 @@ +name: sdist + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + types: [labeled, opened, synchronize, reopened] + paths-ignore: + - "doc/**" + +jobs: + build: + if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}} + runs-on: ubuntu-latest + timeout-minutes: 60 + defaults: + run: + shell: bash -el {0} + + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist + cancel-in-progress: true + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + - name: Install dependencies + run: | + python -m pip install --upgrade pip "setuptools<60.0.0" wheel + + # GH 39416 + pip install numpy + + - name: Build pandas sdist + run: | + pip list + python setup.py sdist --formats=gztar + + - name: Upload sdist artifact + uses: actions/upload-artifact@v3 + with: + name: ${{matrix.python-version}}-sdist.gz + path: dist/*.gz + + - uses: conda-incubator/setup-miniconda@v2.1.1 + with: + activate-environment: pandas-sdist + channels: conda-forge + python-version: '${{ matrix.python-version }}' + + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + - name: Install pandas from sdist + run: | + python -m pip install --upgrade "setuptools<60.0.0" + pip list + python -m pip install dist/*.gz + + - name: Force oldest supported NumPy + run: | + case "${{matrix.python-version}}" in + 3.8) + pip install numpy==1.18.5 ;; + 3.9) + pip install numpy==1.19.3 ;; + 3.10) + pip install numpy==1.21.2 ;; + esac + + - name: Import pandas + run: | + cd .. + conda list + python -c "import pandas; pandas.show_versions();" diff --git a/.github/workflows/stale-pr.yml b/.github/workflows/stale-pr.yml new file mode 100644 index 0000000000000..b97b60717a2b8 --- /dev/null +++ b/.github/workflows/stale-pr.yml @@ -0,0 +1,21 @@ +name: "Stale PRs" +on: + schedule: + # * is a special character in YAML so you have to quote this string + - cron: "0 0 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please [update](https://pandas.pydata.org/pandas-docs/stable/development/contributing.html#updating-your-pull-request) and respond to this comment if you're still interested in working on this." + stale-pr-label: "Stale" + exempt-pr-labels: "Needs Review,Blocked,Needs Discussion" + days-before-issue-stale: -1 + days-before-pr-stale: 30 + days-before-close: -1 + remove-stale-when-updated: false + debug-only: false diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 5bff8bb412947..e6de5caf955fc 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -24,7 +24,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""