From 437105439e2a15860ae263c4d9a550515871eebb Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 28 Jun 2024 11:09:46 +0200 Subject: [PATCH 01/18] Upload free-threaded nightly wheels on Linux and macOS --- .github/workflows/wheels.yml | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b92588d81f4ed..099c8b3ac020d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -199,3 +199,110 @@ jobs: source ci/upload_wheels.sh set_upload_vars upload_wheels + + build_free_threaded_wheels: + needs: build_sdist + name: Build wheel for ${{ matrix.python }}-${{ matrix.buildplat[1] }} + if: >- + (github.event_name == 'schedule') || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'Build')) + runs-on: ${{ matrix.buildplat[0] }} + strategy: + fail-fast: false + matrix: + # GitHub Actions doesn't support pairing matrix values together, let's improvise + # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 + buildplat: + - [ubuntu-22.04, manylinux_x86_64] + - [ubuntu-22.04, musllinux_x86_64] + - [macos-13, macosx_x86_64] + - [macos-14, macosx_arm64] + python: ["cp313t"] + env: + IS_PUSH: false + IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Checkout pandas + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # TODO: Build wheels from sdist again + # There's some sort of weird race condition? + # within Github that makes the sdist be missing files + + # We need to build wheels from the sdist since the sdist + # removes unnecessary files from the release + - name: Download sdist (not macOS) + #if: ${{ matrix.buildplat[1] != 'macosx_*' }} + uses: actions/download-artifact@v4 + with: + name: sdist + path: ./dist + + - name: Output sdist name (macOS) + id: save-path + shell: bash -el {0} + run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_ENV" + + # Python version used to build sdist doesn't matter + # wheel will be built from sdist with the correct version + - name: Unzip sdist (macOS) + if: ${{ startsWith(matrix.buildplat[1], 'macosx') }} + run: | + tar -xzf ./dist/${{ env.sdist_name }} -C ./dist + + - name: Output sdist name (macOS) + id: save-path2 + shell: bash -el {0} + run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV" + + - name: Build wheels + uses: pypa/cibuildwheel@v2.19.1 + with: + package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} + env: + CIBW_PRERELEASE_PYTHONS: True + CIBW_FREE_THREADED_SUPPORT: True + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }} + + - name: Set up Python + uses: mamba-org/setup-micromamba@v1 + with: + environment-name: wheel-env + # Use a fixed Python, since we might have an unreleased Python not + # yet present on conda-forge + create-args: >- + python=3.11 + anaconda-client + wheel + cache-downloads: true + cache-environment: true + + - name: Validate wheel RECORD + shell: bash -el {0} + run: for whl in $(ls wheelhouse); do wheel unpack wheelhouse/$whl -d /tmp; done + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.python }}-${{ matrix.buildplat[1] }} + path: ./wheelhouse/*.whl + + - name: Upload wheels & sdist + if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }} + shell: bash -el {0} + env: + PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }} + PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }} + # trigger an upload to + # https://anaconda.org/scientific-python-nightly-wheels/pandas + # for cron jobs or "Run workflow" (restricted to main branch). + # Tags will upload to + # https://anaconda.org/multibuild-wheels-staging/pandas + # The tokens were originally generated at anaconda.org + run: | + source ci/upload_wheels.sh + set_upload_vars + upload_wheels From 2cb8bbccbd06c826e3ecb3b40242998740badc63 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 2 Jul 2024 13:12:49 +0200 Subject: [PATCH 02/18] Consolidate jobs into one --- .github/workflows/wheels.yml | 117 +++-------------------------------- 1 file changed, 7 insertions(+), 110 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 099c8b3ac020d..3be5746113ca1 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -99,7 +99,10 @@ jobs: - [macos-14, macosx_arm64] - [windows-2022, win_amd64] # TODO: support PyPy? - python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]] + python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]] + exclude: + - buildplat: [windows-2022, win_amd64] + python: ["cp313t", "3.13"] env: IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} @@ -145,6 +148,7 @@ jobs: package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} env: CIBW_PRERELEASE_PYTHONS: True + CIBW_FREE_THREADED_SUPPORT: True CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} - name: Set up Python @@ -175,8 +179,8 @@ jobs: python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`'; "@ # add rc to the end of the image name if the Python version is unreleased - docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} - docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD + docker pull python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} + docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD - uses: actions/upload-artifact@v4 with: @@ -199,110 +203,3 @@ jobs: source ci/upload_wheels.sh set_upload_vars upload_wheels - - build_free_threaded_wheels: - needs: build_sdist - name: Build wheel for ${{ matrix.python }}-${{ matrix.buildplat[1] }} - if: >- - (github.event_name == 'schedule') || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && - contains(github.event.pull_request.labels.*.name, 'Build')) - runs-on: ${{ matrix.buildplat[0] }} - strategy: - fail-fast: false - matrix: - # GitHub Actions doesn't support pairing matrix values together, let's improvise - # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 - buildplat: - - [ubuntu-22.04, manylinux_x86_64] - - [ubuntu-22.04, musllinux_x86_64] - - [macos-13, macosx_x86_64] - - [macos-14, macosx_arm64] - python: ["cp313t"] - env: - IS_PUSH: false - IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - steps: - - name: Checkout pandas - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # TODO: Build wheels from sdist again - # There's some sort of weird race condition? - # within Github that makes the sdist be missing files - - # We need to build wheels from the sdist since the sdist - # removes unnecessary files from the release - - name: Download sdist (not macOS) - #if: ${{ matrix.buildplat[1] != 'macosx_*' }} - uses: actions/download-artifact@v4 - with: - name: sdist - path: ./dist - - - name: Output sdist name (macOS) - id: save-path - shell: bash -el {0} - run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_ENV" - - # Python version used to build sdist doesn't matter - # wheel will be built from sdist with the correct version - - name: Unzip sdist (macOS) - if: ${{ startsWith(matrix.buildplat[1], 'macosx') }} - run: | - tar -xzf ./dist/${{ env.sdist_name }} -C ./dist - - - name: Output sdist name (macOS) - id: save-path2 - shell: bash -el {0} - run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV" - - - name: Build wheels - uses: pypa/cibuildwheel@v2.19.1 - with: - package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} - env: - CIBW_PRERELEASE_PYTHONS: True - CIBW_FREE_THREADED_SUPPORT: True - CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }} - - - name: Set up Python - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: wheel-env - # Use a fixed Python, since we might have an unreleased Python not - # yet present on conda-forge - create-args: >- - python=3.11 - anaconda-client - wheel - cache-downloads: true - cache-environment: true - - - name: Validate wheel RECORD - shell: bash -el {0} - run: for whl in $(ls wheelhouse); do wheel unpack wheelhouse/$whl -d /tmp; done - - - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.python }}-${{ matrix.buildplat[1] }} - path: ./wheelhouse/*.whl - - - name: Upload wheels & sdist - if: ${{ success() && (env.IS_SCHEDULE_DISPATCH == 'true' || env.IS_PUSH == 'true') }} - shell: bash -el {0} - env: - PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }} - PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }} - # trigger an upload to - # https://anaconda.org/scientific-python-nightly-wheels/pandas - # for cron jobs or "Run workflow" (restricted to main branch). - # Tags will upload to - # https://anaconda.org/multibuild-wheels-staging/pandas - # The tokens were originally generated at anaconda.org - run: | - source ci/upload_wheels.sh - set_upload_vars - upload_wheels From 5f54ed7c3bd032bcb591f66c9aa12dfccb9601dc Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 10 Jul 2024 14:22:07 +0200 Subject: [PATCH 03/18] Install build dependencies in before-build and pass --no-build-isolation --- .github/workflows/wheels.yml | 7 +++++++ pyproject.toml | 6 ++++++ scripts/cibw_before_build.sh | 9 +++++++++ 3 files changed, 22 insertions(+) create mode 100644 scripts/cibw_before_build.sh diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3be5746113ca1..18511d3c7c5df 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -142,6 +142,13 @@ jobs: shell: bash -el {0} run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV" + # TODO: Remove this and installing build deps in cibw_before_build.sh + # after NumPy can be built with a released Cython + - name: Define CIBW_BUILD_FRONTEND environment variable + if: ${{ matrix.python[0] == 'cp313t' }} + run: | + echo "CIBW_BUILD_FRONTEND='pip; args: --no-build-isolation'" >> "$GITHUB_ENV" + - name: Build wheels uses: pypa/cibuildwheel@v2.19.1 with: diff --git a/pyproject.toml b/pyproject.toml index 661e8efbb95fc..914ecf074bee7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,12 @@ test-command = """ before-build = "pip install delvewheel" repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" +[tool.cibuildwheel.linux] +before-build = "bash {project}/scripts/cibw_before_build.sh" + +[tool.cibuildwheel.macos] +before-build = "bash {project}/scripts/cibw_before_build.sh" + [[tool.cibuildwheel.overrides]] select = "*-musllinux*" before-test = "apk update && apk add musl-locales" diff --git a/scripts/cibw_before_build.sh b/scripts/cibw_before_build.sh new file mode 100644 index 0000000000000..b84d4d8fce182 --- /dev/null +++ b/scripts/cibw_before_build.sh @@ -0,0 +1,9 @@ +# TODO: delete along with enabling build isolation by unsetting +# CIBW_BUILD_FRONTEND when scipy is buildable under free-threaded +# python with a released version of cython +FREE_THREADED_BUILD="$(python -c "import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + python -m pip install -U pip + python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython + python -m pip install ninja meson-python +fi From 8bee3509cb192462f0ac8f5eef0ce2ad05af45d7 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 10 Jul 2024 18:41:04 +0200 Subject: [PATCH 04/18] Fix {project} placeholder in cibuildwheel config --- pyproject.toml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 914ecf074bee7..2741055960b48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,17 +154,12 @@ test-command = """ pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "-n 2", "--no-strict-data-files"]); \ pd.test(extra_args=["-m not clipboard and single_cpu and not slow and not network and not db", "--no-strict-data-files"]);' \ """ +before-build = "bash scripts/cibw_before_build.sh" [tool.cibuildwheel.windows] before-build = "pip install delvewheel" repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" -[tool.cibuildwheel.linux] -before-build = "bash {project}/scripts/cibw_before_build.sh" - -[tool.cibuildwheel.macos] -before-build = "bash {project}/scripts/cibw_before_build.sh" - [[tool.cibuildwheel.overrides]] select = "*-musllinux*" before-test = "apk update && apk add musl-locales" From 0a842b2013aa9c55faef9da5d9fb137e0f20cee4 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 10 Jul 2024 19:41:36 +0200 Subject: [PATCH 05/18] Correctly quote echo CIBW_BUILD_FRONTEND command --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 18511d3c7c5df..d51edfaba7dc7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -147,7 +147,7 @@ jobs: - name: Define CIBW_BUILD_FRONTEND environment variable if: ${{ matrix.python[0] == 'cp313t' }} run: | - echo "CIBW_BUILD_FRONTEND='pip; args: --no-build-isolation'" >> "$GITHUB_ENV" + echo 'CIBW_BUILD_FRONTEND="pip; args: --no-build-isolation"' >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.19.1 From c2bbdcb0e4c0cecf54a9d551467ca459518e7821 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 10 Jul 2024 19:45:13 +0200 Subject: [PATCH 06/18] Run echo -e --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d51edfaba7dc7..7002833286801 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -147,7 +147,7 @@ jobs: - name: Define CIBW_BUILD_FRONTEND environment variable if: ${{ matrix.python[0] == 'cp313t' }} run: | - echo 'CIBW_BUILD_FRONTEND="pip; args: --no-build-isolation"' >> "$GITHUB_ENV" + echo -e 'CIBW_BUILD_FRONTEND="pip; args: --no-build-isolation"' >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.19.1 From 7e879664ad9900ad6a012470c3f4598a68ad101e Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 10 Jul 2024 19:49:12 +0200 Subject: [PATCH 07/18] Add {package} to before-build --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2741055960b48..79a61481ced0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,7 +154,7 @@ test-command = """ pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "-n 2", "--no-strict-data-files"]); \ pd.test(extra_args=["-m not clipboard and single_cpu and not slow and not network and not db", "--no-strict-data-files"]);' \ """ -before-build = "bash scripts/cibw_before_build.sh" +before-build = "bash {package}/scripts/cibw_before_build.sh" [tool.cibuildwheel.windows] before-build = "pip install delvewheel" From c36d905f282b314064992ef389533286814c6bcf Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 10:50:46 +0200 Subject: [PATCH 08/18] Include cibw script in sdist & add matrix value for build frontend --- .github/workflows/wheels.yml | 15 ++++++++------- pyproject.toml | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 7002833286801..0f81a0806ca3c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -100,6 +100,13 @@ jobs: - [windows-2022, win_amd64] # TODO: support PyPy? python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]] + include: + # 'pip' is the default + - cibw_build_frontend: pip + # TODO: Remove this and installing build deps in cibw_before_build.sh + # after NumPy can be built with a released Cython + - python: ["cp313t", "3.13"] + cibw_build_frontend: 'pip; args: --no-build-isolation' exclude: - buildplat: [windows-2022, win_amd64] python: ["cp313t", "3.13"] @@ -142,13 +149,6 @@ jobs: shell: bash -el {0} run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV" - # TODO: Remove this and installing build deps in cibw_before_build.sh - # after NumPy can be built with a released Cython - - name: Define CIBW_BUILD_FRONTEND environment variable - if: ${{ matrix.python[0] == 'cp313t' }} - run: | - echo -e 'CIBW_BUILD_FRONTEND="pip; args: --no-build-isolation"' >> "$GITHUB_ENV" - - name: Build wheels uses: pypa/cibuildwheel@v2.19.1 with: @@ -157,6 +157,7 @@ jobs: CIBW_PRERELEASE_PYTHONS: True CIBW_FREE_THREADED_SUPPORT: True CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} + CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend }} - name: Set up Python uses: mamba-org/setup-micromamba@v1 diff --git a/pyproject.toml b/pyproject.toml index 79a61481ced0d..ff3168677d934 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,7 @@ all = ['adbc-driver-postgresql>=0.10.0', # TODO: Remove after setuptools support is dropped. [tool.setuptools] include-package-data = true +script-files = ["scripts/cibw_before_build.sh"] # Needed for building wheels [tool.setuptools.packages.find] include = ["pandas", "pandas.*"] From f11120d780a9b213878317176407753125ec2f7b Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 11:51:39 +0200 Subject: [PATCH 09/18] Change manifest and gitattributes --- .gitattributes | 3 +++ MANIFEST.in | 3 +++ pyproject.toml | 1 - 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 19c6fd2fd1d47..500a7f5df3477 100644 --- a/.gitattributes +++ b/.gitattributes @@ -82,3 +82,6 @@ setup.py export-ignore # csv_dir_path fixture checks the existence of the directory # exclude the whole directory to avoid running related tests in sdist pandas/tests/io/parser/data export-ignore + +# Include cibw script in sdist since it's needed for building wheels +scripts/cibw_before_build.sh -export-ignore diff --git a/MANIFEST.in b/MANIFEST.in index 9894381ed6252..a7d7d7eb4e062 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -62,3 +62,6 @@ prune pandas/tests/io/parser/data # Selectively re-add *.cxx files that were excluded above graft pandas/_libs/src graft pandas/_libs/include + +# Include cibw script in sdist since it's needed for building wheels +include scripts/cibw_before_build.sh diff --git a/pyproject.toml b/pyproject.toml index ff3168677d934..79a61481ced0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,6 @@ all = ['adbc-driver-postgresql>=0.10.0', # TODO: Remove after setuptools support is dropped. [tool.setuptools] include-package-data = true -script-files = ["scripts/cibw_before_build.sh"] # Needed for building wheels [tool.setuptools.packages.find] include = ["pandas", "pandas.*"] From 0b700253eb7ec82b7696d09c969c28718775925e Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 12:07:38 +0200 Subject: [PATCH 10/18] Change gitattributes --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 500a7f5df3477..2655d0d018d4f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -68,7 +68,7 @@ ci export-ignore doc export-ignore gitpod export-ignore MANIFEST.in export-ignore -scripts export-ignore +scripts/** export-ignore typings export-ignore web export-ignore CITATION.cff export-ignore From 812527e3a49bb83bbc03425e779636a5cd935aff Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 12:19:13 +0200 Subject: [PATCH 11/18] Install verioneer in before-build --- scripts/cibw_before_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cibw_before_build.sh b/scripts/cibw_before_build.sh index b84d4d8fce182..4bd37c47e6b4b 100644 --- a/scripts/cibw_before_build.sh +++ b/scripts/cibw_before_build.sh @@ -5,5 +5,5 @@ FREE_THREADED_BUILD="$(python -c "import sysconfig; print(bool(sysconfig.get_con if [[ $FREE_THREADED_BUILD == "True" ]]; then python -m pip install -U pip python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython - python -m pip install ninja meson-python + python -m pip install ninja meson-python versioneer[toml] fi From e46055a4f685fc442d2fa9a549c564c0d138ee74 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 13:34:51 +0200 Subject: [PATCH 12/18] Add cibw_before_test to install nightly NumPy --- .gitattributes | 1 + .github/workflows/wheels.yml | 1 - MANIFEST.in | 1 + pyproject.toml | 2 ++ scripts/cibw_before_test.sh | 8 ++++++++ 5 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 scripts/cibw_before_test.sh diff --git a/.gitattributes b/.gitattributes index 2655d0d018d4f..b3d70ca8b24fb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -85,3 +85,4 @@ pandas/tests/io/parser/data export-ignore # Include cibw script in sdist since it's needed for building wheels scripts/cibw_before_build.sh -export-ignore +scripts/cibw_before_test.sh -export-ignore diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0f81a0806ca3c..392b1aec56a6f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -155,7 +155,6 @@ jobs: package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} env: CIBW_PRERELEASE_PYTHONS: True - CIBW_FREE_THREADED_SUPPORT: True CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend }} diff --git a/MANIFEST.in b/MANIFEST.in index a7d7d7eb4e062..f586d457eaaf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -65,3 +65,4 @@ graft pandas/_libs/include # Include cibw script in sdist since it's needed for building wheels include scripts/cibw_before_build.sh +include scripts/cibw_before_test.sh diff --git a/pyproject.toml b/pyproject.toml index 79a61481ced0d..82ad5fa4c71a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,7 +154,9 @@ test-command = """ pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "-n 2", "--no-strict-data-files"]); \ pd.test(extra_args=["-m not clipboard and single_cpu and not slow and not network and not db", "--no-strict-data-files"]);' \ """ +free-threaded-support = true before-build = "bash {package}/scripts/cibw_before_build.sh" +before-test = "bash {package}/scripts/cibw_before_test.sh" [tool.cibuildwheel.windows] before-build = "pip install delvewheel" diff --git a/scripts/cibw_before_test.sh b/scripts/cibw_before_test.sh new file mode 100644 index 0000000000000..0863377ab9b78 --- /dev/null +++ b/scripts/cibw_before_test.sh @@ -0,0 +1,8 @@ +# TODO: delete along with enabling build isolation by unsetting +# CIBW_BUILD_FRONTEND when scipy is buildable under free-threaded +# python with a released version of cython +FREE_THREADED_BUILD="$(python -c "import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + python -m pip install -U pip + python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy +fi From c4879983d864bffb9bbeaf3b98753dcf6f914439 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 14:18:22 +0200 Subject: [PATCH 13/18] Expand before-test to musl --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 82ad5fa4c71a9..0fcee507c4a09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,7 +164,7 @@ repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" [[tool.cibuildwheel.overrides]] select = "*-musllinux*" -before-test = "apk update && apk add musl-locales" +before-test = "apk update && apk add musl-locales && bash {package}/scripts/cibw_before_test.sh" [[tool.cibuildwheel.overrides]] select = "*-win*" From 2258d64e1fdfc513099c5712a23dc6c630849e56 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 14:51:58 +0200 Subject: [PATCH 14/18] Better comments plus always run before-build/before-test on 3.13 --- .github/workflows/wheels.yml | 5 +++-- pyproject.toml | 2 +- scripts/cibw_before_build.sh | 10 +++++----- scripts/cibw_before_test.sh | 10 +++++----- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 392b1aec56a6f..f3894eb1fdaba 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -103,10 +103,11 @@ jobs: include: # 'pip' is the default - cibw_build_frontend: pip - # TODO: Remove this and installing build deps in cibw_before_build.sh - # after NumPy can be built with a released Cython + # TODO: Remove this plus installing build deps in cibw_before_build.sh + # and test deps in cibw_before_test.sh after pandas can be built with a released NumPy/Cython - python: ["cp313t", "3.13"] cibw_build_frontend: 'pip; args: --no-build-isolation' + # TODO: Build free-threaded wheels for Windows exclude: - buildplat: [windows-2022, win_amd64] python: ["cp313t", "3.13"] diff --git a/pyproject.toml b/pyproject.toml index 0fcee507c4a09..245e1d7c467a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,7 +159,7 @@ before-build = "bash {package}/scripts/cibw_before_build.sh" before-test = "bash {package}/scripts/cibw_before_test.sh" [tool.cibuildwheel.windows] -before-build = "pip install delvewheel" +before-build = "pip install delvewheel && bash {package}/scripts/cibw_before_build.sh" repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" [[tool.cibuildwheel.overrides]] diff --git a/scripts/cibw_before_build.sh b/scripts/cibw_before_build.sh index 4bd37c47e6b4b..f3049b27ed5d1 100644 --- a/scripts/cibw_before_build.sh +++ b/scripts/cibw_before_build.sh @@ -1,8 +1,8 @@ -# TODO: delete along with enabling build isolation by unsetting -# CIBW_BUILD_FRONTEND when scipy is buildable under free-threaded -# python with a released version of cython -FREE_THREADED_BUILD="$(python -c "import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" -if [[ $FREE_THREADED_BUILD == "True" ]]; then +# TODO: Delete when there's PyPI NumPy/Cython releases the support Python 3.13. +# If free-threading support is not included in those releases, this script will have +# to whether this runs for a free-threaded build instead. +PYTHON_VERSION="$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")" +if [[ $PYTHON_VERSION == "313" ]]; then python -m pip install -U pip python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython python -m pip install ninja meson-python versioneer[toml] diff --git a/scripts/cibw_before_test.sh b/scripts/cibw_before_test.sh index 0863377ab9b78..7d1b143881ced 100644 --- a/scripts/cibw_before_test.sh +++ b/scripts/cibw_before_test.sh @@ -1,8 +1,8 @@ -# TODO: delete along with enabling build isolation by unsetting -# CIBW_BUILD_FRONTEND when scipy is buildable under free-threaded -# python with a released version of cython -FREE_THREADED_BUILD="$(python -c "import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" -if [[ $FREE_THREADED_BUILD == "True" ]]; then +# TODO: Delete when there's PyPI NumPy/Cython releases the support Python 3.13. +# If free-threading support is not included in those releases, this script will have +# to whether this runs for a free-threaded build instead. +PYTHON_VERSION="$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")" +if [[ $PYTHON_VERSION == "313" ]]; then python -m pip install -U pip python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy fi From 0c4753a64c8e6fcef7ad966c07c88a0e5303671c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 15:18:54 +0200 Subject: [PATCH 15/18] Add --no-build-isolation in 3.13 as well --- .github/workflows/wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index f3894eb1fdaba..2a1382fd8a61a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -105,6 +105,8 @@ jobs: - cibw_build_frontend: pip # TODO: Remove this plus installing build deps in cibw_before_build.sh # and test deps in cibw_before_test.sh after pandas can be built with a released NumPy/Cython + - python: ["cp313", "3.13"] + cibw_build_frontend: 'pip; args: --no-build-isolation' - python: ["cp313t", "3.13"] cibw_build_frontend: 'pip; args: --no-build-isolation' # TODO: Build free-threaded wheels for Windows From 15d0c8d809f0db9b6ca96a66ab51ca0e9e42ca72 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 11 Jul 2024 17:10:59 +0200 Subject: [PATCH 16/18] Install nightly numpy before windows tests --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2a1382fd8a61a..d6d49f4e7106e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -185,6 +185,7 @@ jobs: run: | $TST_CMD = @" python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0; + ${{ matrix.python[1] == '3.13' && 'python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy;' }} python -m pip install `$(Get-Item pandas\wheelhouse\*.whl); python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`'; "@ From 2397f4c6f475dfa0c3d520af8d8cf79aaebcad41 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 20 Jul 2024 00:13:52 +0200 Subject: [PATCH 17/18] Address feedback; add todo for NumPy nightly and move default outside matrix --- .github/workflows/wheels.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2e2689e0db5bf..0e1eab3f320eb 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -101,8 +101,6 @@ jobs: # TODO: support PyPy? python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]] include: - # 'pip' is the default - - cibw_build_frontend: pip # TODO: Remove this plus installing build deps in cibw_before_build.sh # and test deps in cibw_before_test.sh after pandas can be built with a released NumPy/Cython - python: ["cp313", "3.13"] @@ -165,7 +163,7 @@ jobs: env: CIBW_PRERELEASE_PYTHONS: True CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} - CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend }} + CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }} CIBW_PLATFORM: ${{ matrix.buildplat[1] == 'pyodide_wasm32' && 'pyodide' || 'auto' }} - name: Set up Python @@ -189,6 +187,7 @@ jobs: - name: Test Windows Wheels if: ${{ matrix.buildplat[1] == 'win_amd64' }} shell: pwsh + # TODO: Remove NumPy nightly install when there's a 3.13 wheel on PyPI run: | $TST_CMD = @" python -m pip install hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0; From 778a60ae5c52a599ca57ef9f516478403d662014 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 22 Jul 2024 20:13:43 +0200 Subject: [PATCH 18/18] Set build_frontend to 'build' in pyodide build --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0e1eab3f320eb..9f07648b254dd 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -112,6 +112,7 @@ jobs: # that it uses cibuildwheel instead of a standard Pyodide xbuildenv setup. - buildplat: [ubuntu-22.04, pyodide_wasm32] python: ["cp312", "3.12"] + cibw_build_frontend: 'build' # TODO: Build free-threaded wheels for Windows exclude: - buildplat: [windows-2022, win_amd64]