bump pytables to 3.5.1 #24839 (#36683)

Author: fangchenli

ci/deps/azure-37-minimum_versions.yaml

@@ -20,7 +20,7 @@ dependencies:
   - numexpr=2.6.8
   - numpy=1.16.5
   - openpyxl=2.6.0
-  - pytables=3.4.4
+  - pytables=3.5.1
   - python-dateutil=2.7.3
   - pytz=2017.3
   - pyarrow=0.15

ci/deps/travis-37-locale.yaml

@@ -13,7 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - blosc=1.14.3
+  - blosc=1.15.0
   - python-blosc
   - fastparquet=0.3.2
   - html5lib
@@ -30,7 +30,7 @@ dependencies:
   - pyarrow>=0.17
   - psycopg2=2.7
   - pymysql=0.7.11
-  - pytables
+  - pytables>=3.5.1
   - python-dateutil
   - pytz
   - scipy

doc/source/getting_started/install.rst

@@ -266,7 +266,7 @@ PyTables                  3.4.4              HDF5-based reading / writing
 SQLAlchemy                1.2.8              SQL support for databases other than sqlite
 SciPy                     1.12.0             Miscellaneous statistical functions
 xlsxwriter                1.0.2              Excel writing
-blosc                     1.14.3             Compression for HDF5
+blosc                     1.15.0             Compression for HDF5
 fsspec                    0.7.4              Handling files aside from local and HTTP
 fastparquet               0.3.2              Parquet reading / writing
 gcsfs                     0.6.0              Google Cloud Storage access
@@ -280,7 +280,7 @@ psycopg2                  2.7                PostgreSQL engine for sqlalchemy
 pyarrow                   0.15.0             Parquet, ORC, and feather reading / writing
 pymysql                   0.7.11             MySQL engine for sqlalchemy
 pyreadstat                                   SPSS files (.sav) reading
-pytables                  3.4.4              HDF5 reading / writing
+pytables                  3.5.1              HDF5 reading / writing
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
 s3fs                      0.4.0              Amazon S3 access

doc/source/whatsnew/v1.2.0.rst

@@ -181,7 +181,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | pymysql         | 0.7.11          |    X    |
 +-----------------+-----------------+---------+
-| pytables        | 3.4.4           |    X    |
+| pytables        | 3.5.1           |    X    |
 +-----------------+-----------------+---------+
 | s3fs            | 0.4.0           |         |
 +-----------------+-----------------+---------+
@@ -331,6 +331,7 @@ I/O
 - Bug in :func:`LongTableBuilder.middle_separator` was duplicating LaTeX longtable entries in the List of Tables of a LaTeX document (:issue:`34360`)
 - Bug in :meth:`read_csv` with ``engine='python'`` truncating data if multiple items present in first row and first element started with BOM (:issue:`36343`)
 - Removed ``private_key`` and ``verbose`` from :func:`read_gbq` as they are no longer supported in ``pandas-gbq`` (:issue:`34654`, :issue:`30200`)
+- Bumped minimum pytables version to 3.5.1 to avoid a ``ValueError`` in :meth:`read_hdf` (:issue:`24839`)
 
 Plotting
 ^^^^^^^^

environment.yml

@@ -100,7 +100,7 @@ dependencies:
   - python-snappy  # required by pyarrow
 
   - pyqt>=5.9.2  # pandas.read_clipboard
-  - pytables>=3.4.4  # pandas.read_hdf, DataFrame.to_hdf
+  - pytables>=3.5.1  # pandas.read_hdf, DataFrame.to_hdf
   - s3fs>=0.4.0  # file IO when using 's3://...' path
   - fsspec>=0.7.4  # for generic remote file operations
   - gcsfs>=0.6.0  # file IO when using 'gcs://...' path

pandas/tests/io/pytables/test_complex.py

@@ -3,17 +3,13 @@
 import numpy as np
 import pytest
 
-import pandas.util._test_decorators as td
-
 import pandas as pd
 from pandas import DataFrame, Series
 import pandas._testing as tm
 from pandas.tests.io.pytables.common import ensure_clean_path, ensure_clean_store
 
 from pandas.io.pytables import read_hdf
 
-# GH10447
-
 
 def test_complex_fixed(setup_path):
     df = DataFrame(
@@ -62,7 +58,6 @@ def test_complex_table(setup_path):
         tm.assert_frame_equal(df, reread)
 
 
-@td.xfail_non_writeable
 def test_complex_mixed_fixed(setup_path):
     complex64 = np.array(
         [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex64

pandas/tests/io/pytables/test_store.py

@@ -909,7 +909,6 @@ def test_put_integer(self, setup_path):
         df = DataFrame(np.random.randn(50, 100))
         self._check_roundtrip(df, tm.assert_frame_equal, setup_path)
 
-    @td.xfail_non_writeable
     def test_put_mixed_type(self, setup_path):
         df = tm.makeTimeDataFrame()
         df["obj1"] = "foo"
@@ -1518,9 +1517,7 @@ def test_to_hdf_with_min_itemsize(self, setup_path):
                 pd.read_hdf(path, "ss4"), pd.concat([df["B"], df2["B"]])
             )
 
-    @pytest.mark.parametrize(
-        "format", [pytest.param("fixed", marks=td.xfail_non_writeable), "table"]
-    )
+    @pytest.mark.parametrize("format", ["fixed", "table"])
     def test_to_hdf_errors(self, format, setup_path):
 
         data = ["\ud800foo"]
@@ -1956,7 +1953,6 @@ def test_pass_spec_to_storer(self, setup_path):
             with pytest.raises(TypeError):
                 store.select("df", where=[("columns=A")])
 
-    @td.xfail_non_writeable
     def test_append_misc(self, setup_path):
 
         with ensure_clean_store(setup_path) as store:
@@ -2164,14 +2160,6 @@ def test_unimplemented_dtypes_table_columns(self, setup_path):
             with pytest.raises(TypeError):
                 store.append("df_unimplemented", df)
 
-    @td.xfail_non_writeable
-    @pytest.mark.skipif(
-        LooseVersion(np.__version__) == LooseVersion("1.15.0"),
-        reason=(
-            "Skipping  pytables test when numpy version is "
-            "exactly equal to 1.15.0: gh-22098"
-        ),
-    )
     def test_calendar_roundtrip_issue(self, setup_path):
 
         # 8591
@@ -2405,7 +2393,6 @@ def test_float_index(self, setup_path):
         s = Series(np.random.randn(10), index=index)
         self._check_roundtrip(s, tm.assert_series_equal, path=setup_path)
 
-    @td.xfail_non_writeable
     def test_tuple_index(self, setup_path):
 
         # GH #492
@@ -2418,7 +2405,6 @@ def test_tuple_index(self, setup_path):
             simplefilter("ignore", pd.errors.PerformanceWarning)
             self._check_roundtrip(DF, tm.assert_frame_equal, path=setup_path)
 
-    @td.xfail_non_writeable
     @pytest.mark.filterwarnings("ignore::pandas.errors.PerformanceWarning")
     def test_index_types(self, setup_path):
 
@@ -2480,7 +2466,6 @@ def test_timeseries_preepoch(self, setup_path):
         except OverflowError:
             pytest.skip("known failer on some windows platforms")
 
-    @td.xfail_non_writeable
     @pytest.mark.parametrize(
         "compression", [False, pytest.param(True, marks=td.skip_if_windows_python_3)]
     )
@@ -2514,7 +2499,6 @@ def test_frame(self, compression, setup_path):
         # empty
         self._check_roundtrip(df[:0], tm.assert_frame_equal, path=setup_path)
 
-    @td.xfail_non_writeable
     def test_empty_series_frame(self, setup_path):
         s0 = Series(dtype=object)
         s1 = Series(name="myseries", dtype=object)
@@ -2528,7 +2512,6 @@ def test_empty_series_frame(self, setup_path):
         self._check_roundtrip(df1, tm.assert_frame_equal, path=setup_path)
         self._check_roundtrip(df2, tm.assert_frame_equal, path=setup_path)
 
-    @td.xfail_non_writeable
     @pytest.mark.parametrize(
         "dtype", [np.int64, np.float64, object, "m8[ns]", "M8[ns]"]
     )
@@ -2614,7 +2597,6 @@ def test_store_series_name(self, setup_path):
             recons = store["series"]
             tm.assert_series_equal(recons, series)
 
-    @td.xfail_non_writeable
     @pytest.mark.parametrize(
         "compression", [False, pytest.param(True, marks=td.skip_if_windows_python_3)]
     )
@@ -4182,7 +4164,6 @@ def test_pytables_native2_read(self, datapath, setup_path):
             d1 = store["detector"]
             assert isinstance(d1, DataFrame)
 
-    @td.xfail_non_writeable
     def test_legacy_table_fixed_format_read_py2(self, datapath, setup_path):
         # GH 24510
         # legacy table with fixed format written in Python 2
@@ -4356,7 +4337,6 @@ def test_unicode_longer_encoded(self, setup_path):
             result = store.get("df")
             tm.assert_frame_equal(result, df)
 
-    @td.xfail_non_writeable
     def test_store_datetime_mixed(self, setup_path):
 
         df = DataFrame({"a": [1, 2, 3], "b": [1.0, 2.0, 3.0], "c": ["a", "b", "c"]})

pandas/util/_test_decorators.py

@@ -75,21 +75,6 @@ def safe_import(mod_name: str, min_version: Optional[str] = None):
     return False
 
 
-# TODO:
-# remove when gh-24839 is fixed.
-# this affects numpy 1.16 and pytables 3.4.4
-tables = safe_import("tables")
-xfail_non_writeable = pytest.mark.xfail(
-    tables
-    and LooseVersion(np.__version__) >= LooseVersion("1.16")
-    and LooseVersion(tables.__version__) < LooseVersion("3.5.1"),
-    reason=(
-        "gh-25511, gh-24839. pytables needs a "
-        "release beyond 3.4.4 to support numpy 1.16.x"
-    ),
-)
-
-
 def _skip_if_no_mpl():
     mod = safe_import("matplotlib")
     if mod:

requirements-dev.txt

@@ -67,7 +67,7 @@ fastparquet>=0.3.2
 pyarrow>=0.15.0
 python-snappy
 pyqt5>=5.9.2
-tables>=3.4.4
+tables>=3.5.1
 s3fs>=0.4.0
 fsspec>=0.7.4
 gcsfs>=0.6.0