From e3a2930649a28e05a8ae4b824df73eb5cc8cca85 Mon Sep 17 00:00:00 2001 From: Radek Benes Date: Tue, 19 Mar 2019 22:43:48 +0100 Subject: [PATCH 1/7] Unit test simulating GH 25766 --- pandas/tests/io/test_pytables.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index 42641d2c35c39..87c9a51028748 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -1223,6 +1223,16 @@ def test_append_all_nans(self): reloaded = read_hdf(path, 'df_with_missing') tm.assert_frame_equal(df_with_missing, reloaded) + def test_read_missing_key_close_store(self): + # GH 25766 + with ensure_clean_path(self.path) as path: + df = pd.DataFrame({'a': range(2), 'b': range(2)}) + df.to_hdf(path, 'k1') + + pytest.raises(KeyError, pd.read_hdf, path, 'k2') + + df.to_hdf(path, 'k2') + def test_append_frame_column_oriented(self): with ensure_clean_store(self.path) as store: From 4f765f7daf911c64138c367248448480140bafb3 Mon Sep 17 00:00:00 2001 From: Radek Benes Date: Tue, 19 Mar 2019 22:44:39 +0100 Subject: [PATCH 2/7] Catch KeyError and close file properly --- pandas/io/pytables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index b0c0727c638be..563cf3f89467f 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -376,7 +376,7 @@ def read_hdf(path_or_buf, key=None, mode='r', **kwargs): 'contains multiple datasets.') key = candidate_only_group._v_pathname return store.select(key, auto_close=auto_close, **kwargs) - except (ValueError, TypeError): + except (ValueError, TypeError, KeyError): # if there is an error, close the store try: store.close() From ded7dd034c6fc51f67ef4992070c32488fc942af Mon Sep 17 00:00:00 2001 From: Radek Benes Date: Sun, 24 Mar 2019 22:09:36 +0100 Subject: [PATCH 3/7] Whats new added --- doc/source/whatsnew/v0.25.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index ccf5c43280765..f7771cf9267dc 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -284,6 +284,7 @@ I/O - Bug in :meth:`DataFrame.to_string` and :meth:`DataFrame.to_latex` that would lead to incorrect output when the ``header`` keyword is used (:issue:`16718`) - Bug in :func:`read_csv` not properly interpreting the UTF8 encoded filenames on Windows on Python 3.6+ (:issue:`15086`) - Improved performance in :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` when converting columns that have missing values (:issue:`25772`) +- Bug in :func:`read_hdf` not properly close store when key is not found (:issue:`25766`) Plotting From 244cab4ecb1d27564580a5702a93f58235075173 Mon Sep 17 00:00:00 2001 From: Radek Benes Date: Mon, 25 Mar 2019 07:44:32 +0100 Subject: [PATCH 4/7] Correct pytest raises --- pandas/tests/io/test_pytables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index 87c9a51028748..9955d4fe8bacc 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -1229,7 +1229,8 @@ def test_read_missing_key_close_store(self): df = pd.DataFrame({'a': range(2), 'b': range(2)}) df.to_hdf(path, 'k1') - pytest.raises(KeyError, pd.read_hdf, path, 'k2') + with pytest.raises(KeyError): + pd.read_hdf(path, 'k2') df.to_hdf(path, 'k2') From a53b0396504a4c448a5c779dfb5a871cab4ae080 Mon Sep 17 00:00:00 2001 From: Radek Benes Date: Mon, 25 Mar 2019 07:51:52 +0100 Subject: [PATCH 5/7] Smoke test note --- pandas/tests/io/test_pytables.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/test_pytables.py b/pandas/tests/io/test_pytables.py index 9955d4fe8bacc..b947995611c38 100644 --- a/pandas/tests/io/test_pytables.py +++ b/pandas/tests/io/test_pytables.py @@ -1232,6 +1232,8 @@ def test_read_missing_key_close_store(self): with pytest.raises(KeyError): pd.read_hdf(path, 'k2') + # smoke test to test that file is properly closed after + # read with KeyError before another write df.to_hdf(path, 'k2') def test_append_frame_column_oriented(self): From c90b04718832fc36ac84fc309bbc34c9cc4b8908 Mon Sep 17 00:00:00 2001 From: William Ayd Date: Sat, 30 Mar 2019 14:24:22 +0100 Subject: [PATCH 6/7] Update doc/source/whatsnew/v0.25.0.rst Co-Authored-By: rbenes --- doc/source/whatsnew/v0.25.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index 725bd1cb25575..ba3de384a8054 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -329,7 +329,7 @@ I/O - Bug in :meth:`DataFrame.to_string` and :meth:`DataFrame.to_latex` that would lead to incorrect output when the ``header`` keyword is used (:issue:`16718`) - Bug in :func:`read_csv` not properly interpreting the UTF8 encoded filenames on Windows on Python 3.6+ (:issue:`15086`) - Improved performance in :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` when converting columns that have missing values (:issue:`25772`) -- Bug in :func:`read_hdf` not properly close store when key is not found (:issue:`25766`) +- Bug in :func:`read_hdf` not properly closing store after a KeyError is raised (:issue:`25766`) Plotting From 341b6fb3a251f601be6eaf44ee6f236ee5ccb92f Mon Sep 17 00:00:00 2001 From: Radek Benes Date: Sat, 30 Mar 2019 20:57:45 +0100 Subject: [PATCH 7/7] Whats new updated --- doc/source/whatsnew/v0.25.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index ba3de384a8054..08547801147f6 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -329,7 +329,7 @@ I/O - Bug in :meth:`DataFrame.to_string` and :meth:`DataFrame.to_latex` that would lead to incorrect output when the ``header`` keyword is used (:issue:`16718`) - Bug in :func:`read_csv` not properly interpreting the UTF8 encoded filenames on Windows on Python 3.6+ (:issue:`15086`) - Improved performance in :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` when converting columns that have missing values (:issue:`25772`) -- Bug in :func:`read_hdf` not properly closing store after a KeyError is raised (:issue:`25766`) +- Bug in :func:`read_hdf` not properly closing store after a ``KeyError`` is raised (:issue:`25766`) Plotting