From 557a93def5c8b6f062e9db6d3be81b069d002414 Mon Sep 17 00:00:00 2001 From: Gyorgy Sarvari Date: Mon, 10 Mar 2025 13:52:11 +0100 Subject: [PATCH] BLD: use meson's NumPy resolution mechanism for building Instead of querying the include folder's location from NumPy during building, use Meson's built-in dependency resolution for NumPy. With this change during build-time Meson first tries to query the dependency details using numpy-config (which, in turn essentially uses the same method as the original code this commit replaces), and in case that fails for some reason, it tries to discover NumPy resources using pkg-config - which, beside being a good fail-over mechanism, has the added benefit of somewhat simpler cross-compiling, as querying the include folder location from NumPy module is only usable for cross-compiling only in some corner cases, while pkg-config is a bit more universal. Signed-off-by: Gyorgy Sarvari --- .github/actions/build_pandas/action.yml | 6 ++---- .github/workflows/code-checks.yml | 1 + .github/workflows/docbuild-and-upload.yml | 7 +++++++ .github/workflows/unit-tests.yml | 19 +++++++++++-------- .github/workflows/wheels.yml | 8 ++++++++ ci/deps/actions-310-minimum_versions.yaml | 2 +- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311-downstream_compat.yaml | 2 +- ci/deps/actions-311-numpydev.yaml | 2 +- ci/deps/actions-311-pyarrownightly.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-312.yaml | 2 +- ci/deps/actions-pypy-39.yaml | 2 +- doc/source/whatsnew/v3.0.0.rst | 1 + environment.yml | 2 +- meson.build | 4 ++-- pandas/_libs/meson.build | 9 +++++++-- pandas/_libs/tslibs/meson.build | 9 +++++++-- pandas/_libs/window/meson.build | 6 ++++-- pandas/meson.build | 21 --------------------- pyproject.toml | 2 +- requirements-dev.txt | 2 +- 22 files changed, 61 insertions(+), 52 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index b92bacd1a537c..3ee10efaaf96f 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -25,10 +25,8 @@ runs: - name: Build Pandas run: | if [[ ${{ inputs.editable }} == "true" ]]; then - pip install -e . --no-build-isolation -v --no-deps \ - -Csetup-args="--werror" + pip install -e . --no-build-isolation -v --no-deps else - pip install . --no-build-isolation -v --no-deps \ - -Csetup-args="--werror" + pip install . --no-build-isolation -v --no-deps fi shell: bash -el {0} diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index e1d2d1ea846b8..6acb8634beff4 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -13,6 +13,7 @@ on: env: ENV_FILE: environment.yml PANDAS_CI: 1 + PKG_CONFIG_PATH: "${{env.PYTHONHOME}}/numpy/_core/lib/pkgconfig" permissions: contents: read diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 294334ca1d54b..db3fa8d17b764 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -40,11 +40,18 @@ jobs: with: fetch-depth: 0 + - name: Determine NumPy pkg-config location + run: | + echo "PKG_CONFIG_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/numpy/_core/lib/pkgconfig" >> $GITHUB_ENV + - name: Set up Conda uses: ./.github/actions/setup-conda - name: Build Pandas uses: ./.github/actions/build_pandas + env: + CIBW_ENVIRONMENT: "PKG_CONFIG_PATH=${{env.PKG_CONFIG_PATH}}" + PKG_CONFIG_PATH: ${{env.PKG_CONFIG_PATH}} - name: Extra installs # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 08c41a1eeb21f..dfd5272fd578f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -20,6 +20,9 @@ defaults: run: shell: bash -el {0} +env: + PKG_CONFIG_PATH: "${{env.PYTHONHOME}}/numpy/_core/lib/pkgconfig" + jobs: ubuntu: runs-on: ${{ matrix.platform }} @@ -244,10 +247,10 @@ jobs: run: | /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate - python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.4.0 meson-python==0.13.1 python -m pip install numpy -Csetup-args="-Dallow-noblas=true" python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0 - python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror" + python -m pip install --no-cache-dir --no-build-isolation -e . python -m pip list --no-cache-dir PANDAS_CI=1 python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml concurrency: @@ -282,9 +285,9 @@ jobs: run: | /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev . ~/virtualenvs/pandas-dev/bin/activate - python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1 + python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.4.0 python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0 - python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror" + python -m pip install --no-cache-dir --no-build-isolation -e . python -m pip list --no-cache-dir - name: Run Tests @@ -353,10 +356,10 @@ jobs: - name: Build Environment run: | python --version - python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.4.0 meson-python==0.13.1 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy python -m pip install versioneer[toml] python-dateutil tzdata cython hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov - python -m pip install -ve . --no-build-isolation --no-index --no-deps -Csetup-args="--werror" + python -m pip install -ve . --no-build-isolation --no-index --no-deps python -m pip list - name: Run Tests @@ -395,10 +398,10 @@ jobs: - name: Build Environment run: | python --version - python -m pip install --upgrade pip setuptools wheel numpy meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --upgrade pip setuptools wheel numpy meson[ninja]==1.4.0 meson-python==0.13.1 python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython python -m pip install versioneer[toml] python-dateutil pytz tzdata hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov - python -m pip install -ve . --no-build-isolation --no-index --no-deps -Csetup-args="--werror" + python -m pip install -ve . --no-build-isolation --no-index --no-deps python -m pip list - name: Run Tests diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4da2b0b84ac83..eb0aced6572f9 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -57,6 +57,10 @@ jobs: with: python-version: '3.11' + - name: Determine NumPy pkg-config location + run: | + echo "PKG_CONFIG_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/numpy/_core/lib/pkgconfig" >> $GITHUB_ENV + - name: Build sdist run: | python -m pip install build @@ -140,6 +144,10 @@ jobs: shell: bash -el {0} run: echo "sdist_name=$(ls ./dist)" >> "$GITHUB_ENV" + - name: Determine NumPy pkg-config location + run: | + echo "PKG_CONFIG_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/numpy/_core/lib/pkgconfig" >> $GITHUB_ENV + # Python version used to build sdist doesn't matter # wheel will be built from sdist with the correct version - name: Unzip sdist (macOS) diff --git a/ci/deps/actions-310-minimum_versions.yaml b/ci/deps/actions-310-minimum_versions.yaml index c7c72828db481..530c5b5eef7ba 100644 --- a/ci/deps/actions-310-minimum_versions.yaml +++ b/ci/deps/actions-310-minimum_versions.yaml @@ -9,7 +9,7 @@ dependencies: # build dependencies - versioneer - cython>=0.29.33 - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 74cab4e0970dc..88de56a7c388c 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -7,7 +7,7 @@ dependencies: # build dependencies - versioneer - cython>=0.29.33 - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml index 092ca18d61259..b448fdc573893 100644 --- a/ci/deps/actions-311-downstream_compat.yaml +++ b/ci/deps/actions-311-downstream_compat.yaml @@ -8,7 +8,7 @@ dependencies: # build dependencies - versioneer - cython>=0.29.33 - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-311-numpydev.yaml b/ci/deps/actions-311-numpydev.yaml index 325a6d45d74fd..9db3d638dfe7e 100644 --- a/ci/deps/actions-311-numpydev.yaml +++ b/ci/deps/actions-311-numpydev.yaml @@ -6,7 +6,7 @@ dependencies: # build dependencies - versioneer - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 - cython>=0.29.33 diff --git a/ci/deps/actions-311-pyarrownightly.yaml b/ci/deps/actions-311-pyarrownightly.yaml index 2d3d11c294e12..9ba1bb6162eb3 100644 --- a/ci/deps/actions-311-pyarrownightly.yaml +++ b/ci/deps/actions-311-pyarrownightly.yaml @@ -6,7 +6,7 @@ dependencies: # build dependencies - versioneer - - meson=1.2.1 + - meson=1.4.0 - cython>=0.29.33 - meson-python=0.13.1 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index b6f515dceaea9..561e13834a273 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -7,7 +7,7 @@ dependencies: # build dependencies - versioneer - cython>=0.29.33 - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml index bc66f8a5382c9..cb683765475d7 100644 --- a/ci/deps/actions-312.yaml +++ b/ci/deps/actions-312.yaml @@ -7,7 +7,7 @@ dependencies: # build dependencies - versioneer - cython>=0.29.33 - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 # test dependencies diff --git a/ci/deps/actions-pypy-39.yaml b/ci/deps/actions-pypy-39.yaml index 90933b24b88db..da36cff31435d 100644 --- a/ci/deps/actions-pypy-39.yaml +++ b/ci/deps/actions-pypy-39.yaml @@ -10,7 +10,7 @@ dependencies: # build dependencies - versioneer - cython>=0.29.33 - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 # test dependencies diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index ad086c4d636d5..780f973eed354 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -81,6 +81,7 @@ Other enhancements - Support passing a :class:`Iterable[Hashable]` input to :meth:`DataFrame.drop_duplicates` (:issue:`59237`) - Support reading Stata 102-format (Stata 1) dta files (:issue:`58978`) - Support reading Stata 110-format (Stata 7) dta files (:issue:`47176`) +- Use Meson's native NumPy dependency resolution method for building (:issue:`61095`) .. --------------------------------------------------------------------------- .. _whatsnew_300.notable_bug_fixes: diff --git a/environment.yml b/environment.yml index a8c8b20e20fe4..5b421c8150021 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: # build dependencies - versioneer - cython~=3.0.5 - - meson=1.2.1 + - meson=1.4.0 - meson-python=0.13.1 # test dependencies diff --git a/meson.build b/meson.build index 66583095a6e77..e9a76a375bc5c 100644 --- a/meson.build +++ b/meson.build @@ -6,7 +6,7 @@ project( 'cython', version: run_command(['generate_version.py', '--print'], check: true).stdout().strip(), license: 'BSD-3', - meson_version: '>=1.2.1', + meson_version: '>=1.4.0', default_options: ['buildtype=release', 'c_std=c11', 'warning_level=2'], ) @@ -14,7 +14,7 @@ fs = import('fs') py = import('python').find_installation(pure: false) tempita = files('generate_pxi.py') versioneer = files('generate_version.py') - +numpy_dep = dependency('numpy', method: 'auto') add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language: 'c') add_project_arguments('-DNPY_NO_DEPRECATED_API=0', language: 'cpp') diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build index a50976767928a..bd036607ded6a 100644 --- a/pandas/_libs/meson.build +++ b/pandas/_libs/meson.build @@ -149,12 +149,17 @@ if get_option('buildtype') == 'debug' endif foreach ext_name, ext_dict : libs_sources + dependencies = [numpy_dep] + if ext_dict.has_key('deps') + dependencies += ext_dict.get('deps') + endif + py.extension_module( ext_name, ext_dict.get('sources'), cython_args: cython_args, - include_directories: [inc_np, inc_pd], - dependencies: ext_dict.get('deps', ''), + include_directories: [inc_pd], + dependencies: dependencies, subdir: 'pandas/_libs', install: true, ) diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build index 052a8568b76af..2989ae710c3c0 100644 --- a/pandas/_libs/tslibs/meson.build +++ b/pandas/_libs/tslibs/meson.build @@ -29,12 +29,17 @@ if get_option('buildtype') == 'debug' endif foreach ext_name, ext_dict : tslibs_sources + dependencies = [numpy_dep] + if ext_dict.has_key('deps') + dependencies += ext_dict.get('deps') + endif + py.extension_module( ext_name, ext_dict.get('sources'), cython_args: cython_args, - include_directories: [inc_np, inc_pd], - dependencies: ext_dict.get('deps', ''), + include_directories: [inc_pd], + dependencies: dependencies, subdir: 'pandas/_libs/tslibs', install: true, ) diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build index 1d49bba47e139..bb594222f520b 100644 --- a/pandas/_libs/window/meson.build +++ b/pandas/_libs/window/meson.build @@ -2,7 +2,8 @@ py.extension_module( 'aggregations', ['aggregations.pyx'], cython_args: ['-X always_allow_keywords=true'], - include_directories: [inc_np, inc_pd], + include_directories: [inc_pd], + dependencies: [numpy_dep], subdir: 'pandas/_libs/window', override_options: ['cython_language=cpp'], install: true, @@ -12,7 +13,8 @@ py.extension_module( 'indexers', ['indexers.pyx'], cython_args: ['-X always_allow_keywords=true'], - include_directories: [inc_np, inc_pd], + include_directories: [inc_pd], + dependencies: [numpy_dep], subdir: 'pandas/_libs/window', install: true, ) diff --git a/pandas/meson.build b/pandas/meson.build index 840ac257bba09..f18d0bd911309 100644 --- a/pandas/meson.build +++ b/pandas/meson.build @@ -1,24 +1,3 @@ -incdir_numpy = run_command( - py, - [ - '-c', - ''' -import os -import numpy as np -try: - # Check if include directory is inside the pandas dir - # e.g. a venv created inside the pandas dir - # If so, convert it to a relative path - incdir = os.path.relpath(np.get_include()) -except Exception: - incdir = np.get_include() -print(incdir) - ''', - ], - check: true, -).stdout().strip() - -inc_np = include_directories(incdir_numpy) inc_pd = include_directories('_libs/include') fs.copyfile('__init__.py') diff --git a/pyproject.toml b/pyproject.toml index b7d53b0d8934a..8da6c93501f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ "meson-python>=0.13.1", - "meson>=1.2.1,<2", + "meson>=1.4.0,<2", "wheel", "Cython~=3.0.5", # Note: sync with setup.py, environment.yml and asv.conf.json # Force numpy higher than 2.0rc1, so that built wheels are compatible diff --git a/requirements-dev.txt b/requirements-dev.txt index 990901958cd9e..b386d91e5c88f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ pip versioneer[toml] cython~=3.0.5 -meson[ninja]==1.2.1 +meson[ninja]==1.4.0 meson-python==0.13.1 pytest>=7.3.2 pytest-cov