From 50b536a2330b53ef87d818c06c6ef60fbf7a0bcb Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Fri, 1 Dec 2023 17:32:09 +0600 Subject: [PATCH 1/2] DEP: update python-calamine to 0.1.7 --- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311-downstream_compat.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-39-minimum_versions.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/circle-310-arm64.yaml | 2 +- doc/source/getting_started/install.rst | 2 +- environment.yml | 2 +- pandas/compat/_optional.py | 2 +- pandas/io/excel/_calamine.py | 13 ++++++------- pandas/tests/io/excel/test_readers.py | 4 ---- pyproject.toml | 4 ++-- requirements-dev.txt | 2 +- scripts/tests/data/deps_expected_random.yaml | 2 +- scripts/tests/data/deps_minimum.toml | 4 ++-- scripts/tests/data/deps_unmodified_random.yaml | 2 +- 16 files changed, 22 insertions(+), 27 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 7d08f2df5a8ca..4b62ecc79e4ef 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -46,7 +46,7 @@ dependencies: - pyqt>=5.15.9 - pyreadstat>=1.2.0 - pytables>=3.8.0 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 - scipy>=1.10.0 diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml index 7a2bbe442cde7..95c0319d6f5b8 100644 --- a/ci/deps/actions-311-downstream_compat.yaml +++ b/ci/deps/actions-311-downstream_compat.yaml @@ -47,7 +47,7 @@ dependencies: - pyqt>=5.15.9 - pyreadstat>=1.2.0 - pytables>=3.8.0 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 - scipy>=1.10.0 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 681345a34513d..52074ae00ea18 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -46,7 +46,7 @@ dependencies: - pymysql>=1.0.2 - pyreadstat>=1.2.0 - pytables>=3.8.0 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 - scipy>=1.10.0 diff --git a/ci/deps/actions-39-minimum_versions.yaml b/ci/deps/actions-39-minimum_versions.yaml index 2e3b03f63eb99..fd71315d2e7ac 100644 --- a/ci/deps/actions-39-minimum_versions.yaml +++ b/ci/deps/actions-39-minimum_versions.yaml @@ -48,7 +48,7 @@ dependencies: - pyqt=5.15.9 - pyreadstat=1.2.0 - pytables=3.8.0 - - python-calamine=0.1.6 + - python-calamine=0.1.7 - pyxlsb=1.0.10 - s3fs=2022.11.0 - scipy=1.10.0 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index d3e545b636de9..cbe8f77c15730 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -46,7 +46,7 @@ dependencies: - pyqt>=5.15.9 - pyreadstat>=1.2.0 - pytables>=3.8.0 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 - scipy>=1.10.0 diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml index 0098406037876..8e106445cd4e0 100644 --- a/ci/deps/circle-310-arm64.yaml +++ b/ci/deps/circle-310-arm64.yaml @@ -46,7 +46,7 @@ dependencies: - pyqt>=5.15.9 - pyreadstat>=1.2.0 - pytables>=3.8.0 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 - scipy>=1.10.0 diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index c048c71613b57..1d7eca5223544 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -281,7 +281,7 @@ xlrd 2.0.1 excel Reading Excel xlsxwriter 3.0.5 excel Writing Excel openpyxl 3.1.0 excel Reading / writing for xlsx files pyxlsb 1.0.10 excel Reading for xlsb files -python-calamine 0.1.6 excel Reading for xls/xlsx/xlsb/ods files +python-calamine 0.1.7 excel Reading for xls/xlsx/xlsb/ods files ========================= ================== =============== ============================================================= HTML diff --git a/environment.yml b/environment.yml index 4aa8073e93f6d..3c95831593723 100644 --- a/environment.yml +++ b/environment.yml @@ -48,7 +48,7 @@ dependencies: - pymysql>=1.0.2 - pyreadstat>=1.2.0 - pytables>=3.8.0 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 - scipy>=1.10.0 diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index 1354c0a6db7c5..9d04d7c0a1216 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -39,7 +39,7 @@ "pyarrow": "10.0.1", "pyreadstat": "1.2.0", "pytest": "7.3.2", - "python-calamine": "0.1.6", + "python-calamine": "0.1.7", "pyxlsb": "1.0.10", "s3fs": "2022.11.0", "scipy": "1.10.0", diff --git a/pandas/io/excel/_calamine.py b/pandas/io/excel/_calamine.py index d61a9fc664164..1b30d0f759634 100644 --- a/pandas/io/excel/_calamine.py +++ b/pandas/io/excel/_calamine.py @@ -116,12 +116,11 @@ def _convert_cell(value: _CellValueT) -> Scalar: return value - rows: list[list[_CellValueT]] = sheet.to_python(skip_empty_area=False) - data: list[list[Scalar]] = [] - - for row in rows: - data.append([_convert_cell(cell) for cell in row]) - if file_rows_needed is not None and len(data) >= file_rows_needed: - break + rows: list[list[_CellValueT]] = sheet.to_python( + skip_empty_area=False, nrows=file_rows_needed + ) + data: list[list[Scalar]] = [ + [_convert_cell(cell) for cell in row] for row in rows + ] return data diff --git a/pandas/tests/io/excel/test_readers.py b/pandas/tests/io/excel/test_readers.py index 98d82f10375b4..a9e83f6ef87a3 100644 --- a/pandas/tests/io/excel/test_readers.py +++ b/pandas/tests/io/excel/test_readers.py @@ -1010,10 +1010,6 @@ def test_read_excel_multiindex(self, request, engine, read_ext): # see gh-4679 xfail_datetimes_with_pyxlsb(engine, request) - # https://github.com/tafia/calamine/issues/354 - if engine == "calamine" and read_ext == ".ods": - request.applymarker(pytest.mark.xfail(reason="Last test fails in calamine")) - mi = MultiIndex.from_product([["foo", "bar"], ["a", "b"]]) mi_file = "testmultiindex" + read_ext diff --git a/pyproject.toml b/pyproject.toml index 65c647e439cb1..1ac471c62ea2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ computation = ['scipy>=1.10.0', 'xarray>=2022.12.0'] fss = ['fsspec>=2022.11.0'] aws = ['s3fs>=2022.11.0'] gcp = ['gcsfs>=2022.11.0', 'pandas-gbq>=0.19.0'] -excel = ['odfpy>=1.4.1', 'openpyxl>=3.1.0', 'python-calamine>=0.1.6', 'pyxlsb>=1.0.10', 'xlrd>=2.0.1', 'xlsxwriter>=3.0.5'] +excel = ['odfpy>=1.4.1', 'openpyxl>=3.1.0', 'python-calamine>=0.1.7', 'pyxlsb>=1.0.10', 'xlrd>=2.0.1', 'xlsxwriter>=3.0.5'] parquet = ['pyarrow>=10.0.1'] feather = ['pyarrow>=10.0.1'] hdf5 = [# blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) @@ -113,7 +113,7 @@ all = ['adbc-driver-postgresql>=0.8.0', 'pyreadstat>=1.2.0', 'pytest>=7.3.2', 'pytest-xdist>=2.2.0', - 'python-calamine>=0.1.6', + 'python-calamine>=0.1.7', 'pyxlsb>=1.0.10', 'qtpy>=2.3.0', 'scipy>=1.10.0', diff --git a/requirements-dev.txt b/requirements-dev.txt index fa3ef1c214a14..57e55ff31aa47 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -37,7 +37,7 @@ pyarrow>=10.0.1 pymysql>=1.0.2 pyreadstat>=1.2.0 tables>=3.8.0 -python-calamine>=0.1.6 +python-calamine>=0.1.7 pyxlsb>=1.0.10 s3fs>=2022.11.0 scipy>=1.10.0 diff --git a/scripts/tests/data/deps_expected_random.yaml b/scripts/tests/data/deps_expected_random.yaml index 8fed09d88612d..7bb95d05afb45 100644 --- a/scripts/tests/data/deps_expected_random.yaml +++ b/scripts/tests/data/deps_expected_random.yaml @@ -42,7 +42,7 @@ dependencies: - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 diff --git a/scripts/tests/data/deps_minimum.toml b/scripts/tests/data/deps_minimum.toml index c74ad3d17a4a9..046dccb470daa 100644 --- a/scripts/tests/data/deps_minimum.toml +++ b/scripts/tests/data/deps_minimum.toml @@ -62,7 +62,7 @@ computation = ['scipy>=1.7.1', 'xarray>=0.21.0'] fss = ['fsspec>=2021.07.0'] aws = ['s3fs>=2021.08.0'] gcp = ['gcsfs>=2021.07.0'] -excel = ['odfpy>=1.4.1', 'openpyxl>=3.0.7', 'python-calamine>=0.1.6', 'pyxlsb>=1.0.8', 'xlrd>=2.0.1', 'xlsxwriter>=1.4.3'] +excel = ['odfpy>=1.4.1', 'openpyxl>=3.0.7', 'python-calamine>=0.1.7', 'pyxlsb>=1.0.8', 'xlrd>=2.0.1', 'xlsxwriter>=1.4.3'] parquet = ['pyarrow>=7.0.0'] feather = ['pyarrow>=7.0.0'] hdf5 = [# blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) @@ -101,7 +101,7 @@ all = ['beautifulsoup4>=5.9.3', 'pyreadstat>=1.1.2', 'pytest>=7.3.2', 'pytest-xdist>=2.2.0', - 'python-calamine>=0.1.6', + 'python-calamine>=0.1.7', 'pyxlsb>=1.0.8', 'qtpy>=2.2.0', 'scipy>=1.7.1', diff --git a/scripts/tests/data/deps_unmodified_random.yaml b/scripts/tests/data/deps_unmodified_random.yaml index e560dd50c41d4..49299aa078ce4 100644 --- a/scripts/tests/data/deps_unmodified_random.yaml +++ b/scripts/tests/data/deps_unmodified_random.yaml @@ -42,7 +42,7 @@ dependencies: - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 - - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 From e80ee881992330bec14bb50b7c6e9d96e0cf772a Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Fri, 1 Dec 2023 18:37:36 +0600 Subject: [PATCH 2/2] uncomment python-calamine>=0.1.7 in python3.12 test --- ci/deps/actions-312.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml index 394b65525c791..4c51e9e6029e3 100644 --- a/ci/deps/actions-312.yaml +++ b/ci/deps/actions-312.yaml @@ -46,7 +46,7 @@ dependencies: - pymysql>=1.0.2 - pyreadstat>=1.2.0 # - pytables>=3.8.0 - # - python-calamine>=0.1.6 + - python-calamine>=0.1.7 - pyxlsb>=1.0.10 - s3fs>=2022.11.0 - scipy>=1.10.0