From 134ca2c1f222686849e9a89e19bd77f604059624 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:49:01 -0700 Subject: [PATCH 01/17] CI/TST: pip extras install --- .github/workflows/package-checks.yml | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/package-checks.yml diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml new file mode 100644 index 0000000000000..2f4f281788b79 --- /dev/null +++ b/.github/workflows/package-checks.yml @@ -0,0 +1,50 @@ +name: Package Checks + +on: + push: + branches: + - main + - 1.5.x + pull_request: + branches: + - main + - 1.5.x + +permissions: + contents: read + +jobs: + pip: + runs-on: ubuntu-latest + strategy: + matrix: + extra: ["test", "performance", "timezone", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"] + fail-fast: false + name: Install Extras - ${{ matrix.extra }} + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.extra }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python + id: setup_python + uses: actions/setup-python@v3 + with: + python-version: '3.8' + cache: 'pip' + + - name: Build pandas + run: | + python setup.py build_ext -j 1 + shell: bash -el {0} + + - name: Install with extra + run: | + python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --no-use-pep517 + shell: bash -el {0} From 94630044bbfbf7fe4d012d83d13b8148beb6d6e9 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:50:56 -0700 Subject: [PATCH 02/17] Better concurrency key --- .github/workflows/package-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 2f4f281788b79..170d3f8df8502 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -23,7 +23,7 @@ jobs: name: Install Extras - ${{ matrix.extra }} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.extra }} + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pip-extras-${{ matrix.extra }} cancel-in-progress: true steps: From 333a7ac6992b2b376edd9df40a4dc24e6e36a31b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:09:08 -0700 Subject: [PATCH 03/17] Don't cache - no file --- .github/workflows/package-checks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 170d3f8df8502..5780eb323ce8e 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -37,7 +37,6 @@ jobs: uses: actions/setup-python@v3 with: python-version: '3.8' - cache: 'pip' - name: Build pandas run: | From 15c033531bf524e3ded156c43f3606019db5d80a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 25 Oct 2022 13:37:25 -0700 Subject: [PATCH 04/17] Install required deps --- .github/workflows/package-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 5780eb323ce8e..2d8b43d96ab3b 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -40,6 +40,7 @@ jobs: - name: Build pandas run: | + python -m pip install python-dateutil pytz numpy cython python setup.py build_ext -j 1 shell: bash -el {0} From 09b2865cb37d8e23aca6f802f9cd8466f5f28a88 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 28 Oct 2022 13:20:40 -0700 Subject: [PATCH 05/17] Note blosc only for conda --- doc/source/getting_started/install.rst | 2 +- setup.cfg | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index ca45540b637ba..e28c0e7c08e42 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -415,7 +415,7 @@ depending on required compatibility. Dependency Minimum Version optional_extra Notes ========================= ================== ================ ============================================================= PyTables 3.6.1 hdf5 HDF5-based reading / writing -blosc 1.21.0 hdf5 Compression for HDF5 +blosc 1.21.0 hdf5 Compression for HDF5; only available on ``conda`` zlib hdf5 Compression for HDF5 fastparquet 0.4.0 - Parquet reading / writing (pyarrow is default) pyarrow 6.0.0 parquet, feather Parquet, ORC, and feather reading / writing diff --git a/setup.cfg b/setup.cfg index d80d09725d18b..378541cce70e0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,7 +84,8 @@ parquet = feather = pyarrow>=6.0.0 hdf5 = - blosc>=1.20.1 + # blosc only available on conda + # blosc>=1.20.1 tables>=3.6.1 spss = pyreadstat>=1.1.2 @@ -120,7 +121,8 @@ compression = # `all ` should be kept as the complete set of pandas optional dependencies for general use. all = beautifulsoup4>=4.9.3 - blosc>=1.21.0 + # blosc only available on conda + # blosc>=1.21.0 bottleneck>=1.3.1 boto3>=1.22.7 brotlipy>=0.7.0 From 1344b9e8d4da69c375fee7db64733c1d5598dc9c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 31 Oct 2022 12:16:52 -0700 Subject: [PATCH 06/17] Update setup.cfg Co-authored-by: JMBurley --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 378541cce70e0..28efbf2479fbd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,7 +84,7 @@ parquet = feather = pyarrow>=6.0.0 hdf5 = - # blosc only available on conda + # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) # blosc>=1.20.1 tables>=3.6.1 spss = From 7f9732123a520f926d3314ce7face022db6af45f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 31 Oct 2022 12:16:59 -0700 Subject: [PATCH 07/17] Update setup.cfg Co-authored-by: JMBurley --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 28efbf2479fbd..fa93e6085a793 100644 --- a/setup.cfg +++ b/setup.cfg @@ -121,7 +121,7 @@ compression = # `all ` should be kept as the complete set of pandas optional dependencies for general use. all = beautifulsoup4>=4.9.3 - # blosc only available on conda + # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) # blosc>=1.21.0 bottleneck>=1.3.1 boto3>=1.22.7 From 976b60ed287cd2a6dbd5a14a4575c9f128a36477 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 3 Nov 2022 14:58:07 -0700 Subject: [PATCH 08/17] try installing without building --- .github/workflows/package-checks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 2d8b43d96ab3b..a0b2fb890ec1a 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -38,10 +38,9 @@ jobs: with: python-version: '3.8' - - name: Build pandas + - name: Install dependencies run: | python -m pip install python-dateutil pytz numpy cython - python setup.py build_ext -j 1 shell: bash -el {0} - name: Install with extra From fd9e35ec97d48874a6873516c2eed77e8c76af9c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:47:22 -0700 Subject: [PATCH 09/17] Try to see if we can skip cython --- .github/workflows/package-checks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index a0b2fb890ec1a..a2fd879317302 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -38,12 +38,12 @@ jobs: with: python-version: '3.8' - - name: Install dependencies + - name: Install required dependencies run: | - python -m pip install python-dateutil pytz numpy cython + python -m pip install python-dateutil pytz numpy shell: bash -el {0} - - name: Install with extra + - name: Pip install with extra run: | python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --no-use-pep517 shell: bash -el {0} From 5b068e0f0a29405b7de66e84e4c8b8ac3157fd76 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 7 Nov 2022 13:11:25 -0800 Subject: [PATCH 10/17] Try passing clean --- .github/workflows/package-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index a2fd879317302..49eab6f50fad1 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -45,5 +45,5 @@ jobs: - name: Pip install with extra run: | - python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --no-use-pep517 + python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --no-use-pep517 --install-option="clean" shell: bash -el {0} From d75435f5d367c354efff39358a7887abeb6b59a0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 7 Nov 2022 20:05:42 -0800 Subject: [PATCH 11/17] Reinstall cython --- .github/workflows/package-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 49eab6f50fad1..976227ee21f1d 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -40,7 +40,7 @@ jobs: - name: Install required dependencies run: | - python -m pip install python-dateutil pytz numpy + python -m pip install python-dateutil pytz numpy cython shell: bash -el {0} - name: Pip install with extra From 213d1f321e026b34bded7576da135d2ce0f02e05 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Nov 2022 12:09:02 -0800 Subject: [PATCH 12/17] pep517 --- .github/workflows/package-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 976227ee21f1d..75af05b51dfd1 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -45,5 +45,5 @@ jobs: - name: Pip install with extra run: | - python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --no-use-pep517 --install-option="clean" + python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --install-option="clean" shell: bash -el {0} From 0fb1e103476d0893f4ffae3b8746f7f636ade7c5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Nov 2022 14:52:54 -0800 Subject: [PATCH 13/17] Try forcefully disabling extensions --- .github/workflows/package-checks.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 75af05b51dfd1..2df03e4322ce6 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -38,6 +38,13 @@ jobs: with: python-version: '3.8' + # Can take a significant amount of time while this should only check that + # the extras are installed + - name: Disable building ext_modules + run: | + sed -i '/ext_modules=/d' setup.py + shell: bash -el {0} + - name: Install required dependencies run: | python -m pip install python-dateutil pytz numpy cython From d2fb7ebdda44228670a664ed8e89b071936b7e3e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Nov 2022 15:45:40 -0800 Subject: [PATCH 14/17] Remove install option --- .github/workflows/package-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 2df03e4322ce6..7d9853c5d7b2b 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -52,5 +52,5 @@ jobs: - name: Pip install with extra run: | - python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --install-option="clean" + python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation shell: bash -el {0} From 05b6ef7e5fd84fccd135a7e1e06981476b0b6b1a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 9 Nov 2022 13:12:13 -0800 Subject: [PATCH 15/17] Install wheel --- .github/workflows/package-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index 7d9853c5d7b2b..fd90b10d42ce2 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -47,7 +47,7 @@ jobs: - name: Install required dependencies run: | - python -m pip install python-dateutil pytz numpy cython + python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython shell: bash -el {0} - name: Pip install with extra From 89c9f25858e8518c07a47c5d63859ef4fd2dee6d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 9 Nov 2022 17:27:36 -0800 Subject: [PATCH 16/17] Try less hacky way --- .github/workflows/package-checks.yml | 14 +++++++------- setup.py | 4 ++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index fd90b10d42ce2..d8303a3535bd9 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -38,12 +38,12 @@ jobs: with: python-version: '3.8' - # Can take a significant amount of time while this should only check that - # the extras are installed - - name: Disable building ext_modules - run: | - sed -i '/ext_modules=/d' setup.py - shell: bash -el {0} +# # Can take a significant amount of time while this should only check that +# # the extras are installed +# - name: Disable building ext_modules +# run: | +# sed -i '/ext_modules=/d' setup.py +# shell: bash -el {0} - name: Install required dependencies run: | @@ -52,5 +52,5 @@ jobs: - name: Pip install with extra run: | - python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation + python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --install-option="no-extensions" shell: bash -el {0} diff --git a/setup.py b/setup.py index 05d4407c7e959..ce06990ae5b77 100755 --- a/setup.py +++ b/setup.py @@ -407,6 +407,10 @@ def maybe_cythonize(extensions, *args, **kwargs): # See https://github.com/cython/cython/issues/1495 return extensions + elif "no-extensions" in sys.argv: + # For pip install extras CI job in .github/workflows/package-checks.yml + return [] + elif not _CYTHON_INSTALLED: # GH#28836 raise a helfpul error message if _CYTHON_VERSION: From fe4cacabdafd21a1651d01b2436d25841928d50c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 10 Nov 2022 10:09:53 -0800 Subject: [PATCH 17/17] Didn't work --- .github/workflows/package-checks.yml | 14 +++++++------- scripts/validate_min_versions_in_sync.py | 9 ++++++--- setup.py | 4 ---- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml index d8303a3535bd9..762cb509be136 100644 --- a/.github/workflows/package-checks.yml +++ b/.github/workflows/package-checks.yml @@ -38,12 +38,12 @@ jobs: with: python-version: '3.8' -# # Can take a significant amount of time while this should only check that -# # the extras are installed -# - name: Disable building ext_modules -# run: | -# sed -i '/ext_modules=/d' setup.py -# shell: bash -el {0} + # Hacky patch to disable building cython extensions. + # This job should only check that the extras successfully install. + - name: Disable building ext_modules + run: | + sed -i '/ext_modules=/d' setup.py + shell: bash -el {0} - name: Install required dependencies run: | @@ -52,5 +52,5 @@ jobs: - name: Pip install with extra run: | - python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation --install-option="no-extensions" + python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation shell: bash -el {0} diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index ad0375a4320a2..a69bdb95c0f9b 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -24,7 +24,7 @@ ) CODE_PATH = pathlib.Path("pandas/compat/_optional.py").resolve() SETUP_PATH = pathlib.Path("setup.cfg").resolve() -EXCLUDE_DEPS = {"tzdata"} +EXCLUDE_DEPS = {"tzdata", "blosc"} # pandas package is not available # in pre-commit environment sys.path.append("pandas/compat") @@ -38,10 +38,11 @@ def get_versions_from_code() -> dict[str, str]: + """Min versions for checking within pandas code.""" install_map = _optional.INSTALL_MAPPING versions = _optional.VERSIONS for item in EXCLUDE_DEPS: - versions.pop(item) + versions.pop(item, None) return { install_map.get(k, k).casefold(): v for k, v in versions.items() @@ -50,6 +51,7 @@ def get_versions_from_code() -> dict[str, str]: def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, str]]: + """Min versions in CI job for testing all optional dependencies.""" # Don't parse with pyyaml because it ignores comments we're looking for seen_required = False seen_optional = False @@ -79,6 +81,7 @@ def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, def get_versions_from_setup() -> dict[str, str]: + """Min versions in setup.cfg for pip install pandas[extra].""" install_map = _optional.INSTALL_MAPPING optional_dependencies = {} @@ -99,7 +102,7 @@ def get_versions_from_setup() -> dict[str, str]: optional_dependencies[install_map.get(package, package).casefold()] = version for item in EXCLUDE_DEPS: - optional_dependencies.pop(item) + optional_dependencies.pop(item, None) return optional_dependencies diff --git a/setup.py b/setup.py index ce06990ae5b77..05d4407c7e959 100755 --- a/setup.py +++ b/setup.py @@ -407,10 +407,6 @@ def maybe_cythonize(extensions, *args, **kwargs): # See https://github.com/cython/cython/issues/1495 return extensions - elif "no-extensions" in sys.argv: - # For pip install extras CI job in .github/workflows/package-checks.yml - return [] - elif not _CYTHON_INSTALLED: # GH#28836 raise a helfpul error message if _CYTHON_VERSION: