From 5c8491276f22fb6293a6ff2388104367c0da7e09 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 26 Sep 2019 13:06:17 -0700 Subject: [PATCH 01/13] CLN: Exception in pickle loading --- pandas/compat/pickle_compat.py | 23 +++-------------------- pandas/io/pickle.py | 11 +++++------ 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py index b3c7b8a7c8b9f..53ceb8c8081a9 100644 --- a/pandas/compat/pickle_compat.py +++ b/pandas/compat/pickle_compat.py @@ -4,7 +4,6 @@ import copy import pickle as pkl -import sys from typing import TYPE_CHECKING import warnings @@ -25,14 +24,14 @@ def load_reduce(self): try: stack[-1] = func(*args) return - except Exception as e: + except TypeError as err: # If we have a deprecated function, # try to replace and try again. msg = "_reconstruct: First argument must be a sub-type of ndarray" - if msg in str(e): + if msg in str(err): try: cls = args[0] stack[-1] = object.__new__(cls) @@ -40,23 +39,7 @@ def load_reduce(self): except TypeError: pass - # try to re-encode the arguments - if getattr(self, "encoding", None) is not None: - args = tuple( - arg.encode(self.encoding) if isinstance(arg, str) else arg - for arg in args - ) - try: - stack[-1] = func(*args) - return - except TypeError: - pass - - # unknown exception, re-raise - if getattr(self, "is_verbose", None): - print(sys.exc_info()) - print(func, args) - raise + raise err _sparse_msg = """\ diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py index 4b9a52a1fb8f3..32c6c3580aba9 100644 --- a/pandas/io/pickle.py +++ b/pandas/io/pickle.py @@ -146,18 +146,17 @@ def read_pickle(path, compression="infer"): # 1) try standard libary Pickle # 2) try pickle_compat (older pandas version) to handle subclass changes - # 3) try pickle_compat with latin1 encoding try: with warnings.catch_warnings(record=True): # We want to silence any warnings about, e.g. moved modules. warnings.simplefilter("ignore", Warning) return pickle.load(f) - except Exception: - try: - return pc.load(f, encoding=None) - except Exception: - return pc.load(f, encoding="latin1") + except (ModuleNotFoundError, AttributeError): + # e.g. + # "No module named 'pandas.core.sparse.series'" + # "Can't get attribute '__nat_unpickle' on Date: Thu, 26 Sep 2019 13:54:05 -0700 Subject: [PATCH 02/13] PY35 compat --- pandas/io/pickle.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py index 32c6c3580aba9..a168840e59e2d 100644 --- a/pandas/io/pickle.py +++ b/pandas/io/pickle.py @@ -5,7 +5,7 @@ from numpy.lib.format import read_array -from pandas.compat import pickle_compat as pc +from pandas.compat import PY36, pickle_compat as pc from pandas.io.common import _get_handle, _stringify_path @@ -147,12 +147,16 @@ def read_pickle(path, compression="infer"): # 1) try standard libary Pickle # 2) try pickle_compat (older pandas version) to handle subclass changes + excs_to_catch = (AttributeError, ImportError) + if PY36: + excs_to_catch += (ModuleNotFoundError,) + try: with warnings.catch_warnings(record=True): # We want to silence any warnings about, e.g. moved modules. warnings.simplefilter("ignore", Warning) return pickle.load(f) - except (ModuleNotFoundError, AttributeError): + except excs_to_catch: # e.g. # "No module named 'pandas.core.sparse.series'" # "Can't get attribute '__nat_unpickle' on Date: Tue, 1 Oct 2019 10:25:54 -0500 Subject: [PATCH 03/13] Catch TypeError to fix pytables pickle error on 0.8.0 whatsnew --- pandas/io/pickle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py index a168840e59e2d..c68eb4198a3cd 100644 --- a/pandas/io/pickle.py +++ b/pandas/io/pickle.py @@ -147,7 +147,7 @@ def read_pickle(path, compression="infer"): # 1) try standard libary Pickle # 2) try pickle_compat (older pandas version) to handle subclass changes - excs_to_catch = (AttributeError, ImportError) + excs_to_catch = (TypeError, AttributeError, ImportError) if PY36: excs_to_catch += (ModuleNotFoundError,) @@ -160,6 +160,8 @@ def read_pickle(path, compression="infer"): # e.g. # "No module named 'pandas.core.sparse.series'" # "Can't get attribute '__nat_unpickle' on appears specific to pytables in 0.8.0 whatsnew, see GH#28645 return pc.load(f, encoding=None) finally: f.close() From cc30c50d90730e45a18eb34c00c8cbfe5afb8bfa Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 1 Oct 2019 15:41:39 -0500 Subject: [PATCH 04/13] restore encoding load --- pandas/io/pickle.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py index c68eb4198a3cd..490395dc171c1 100644 --- a/pandas/io/pickle.py +++ b/pandas/io/pickle.py @@ -162,7 +162,11 @@ def read_pickle(path, compression="infer"): # "Can't get attribute '__nat_unpickle' on appears specific to pytables in 0.8.0 whatsnew, see GH#28645 - return pc.load(f, encoding=None) + try: + return pc.load(f, encoding=None) + except UnicodeDecodeError: + # specific to 0.8.0 whatsnew, see GH#28645 + return pc.load(f, encoding="latin1") finally: f.close() for _f in fh: From b8e6f812f36b05229e0d8adc6bbd1aafc005adb3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 8 Oct 2019 09:04:24 -0700 Subject: [PATCH 05/13] revert change needed for 0.8.0 whatsnew --- doc/source/whatsnew/v0.8.0.rst | 4 ++-- pandas/io/pickle.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index 664325ac063c0..04493df7e9169 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -156,7 +156,7 @@ Other new features New plotting methods ~~~~~~~~~~~~~~~~~~~~ -.. ipython:: python +.. code-block:: python :suppress: import pandas as pd @@ -165,7 +165,7 @@ New plotting methods ``Series.plot`` now supports a ``secondary_y`` option: -.. ipython:: python +.. code-block:: python plt.figure() diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py index 490395dc171c1..c68eb4198a3cd 100644 --- a/pandas/io/pickle.py +++ b/pandas/io/pickle.py @@ -162,11 +162,7 @@ def read_pickle(path, compression="infer"): # "Can't get attribute '__nat_unpickle' on appears specific to pytables in 0.8.0 whatsnew, see GH#28645 - try: - return pc.load(f, encoding=None) - except UnicodeDecodeError: - # specific to 0.8.0 whatsnew, see GH#28645 - return pc.load(f, encoding="latin1") + return pc.load(f, encoding=None) finally: f.close() for _f in fh: From fc50b2f170dd7fa5b38137061a12a3172d2b8e71 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 8 Oct 2019 09:05:00 -0700 Subject: [PATCH 06/13] revert TypeError needed for 0.8.0 whatnsew --- pandas/io/pickle.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py index c68eb4198a3cd..a168840e59e2d 100644 --- a/pandas/io/pickle.py +++ b/pandas/io/pickle.py @@ -147,7 +147,7 @@ def read_pickle(path, compression="infer"): # 1) try standard libary Pickle # 2) try pickle_compat (older pandas version) to handle subclass changes - excs_to_catch = (TypeError, AttributeError, ImportError) + excs_to_catch = (AttributeError, ImportError) if PY36: excs_to_catch += (ModuleNotFoundError,) @@ -160,8 +160,6 @@ def read_pickle(path, compression="infer"): # e.g. # "No module named 'pandas.core.sparse.series'" # "Can't get attribute '__nat_unpickle' on appears specific to pytables in 0.8.0 whatsnew, see GH#28645 return pc.load(f, encoding=None) finally: f.close() From 9ab2a50ee7f26ab210d5d2abd7dd494a629baedf Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 8 Oct 2019 11:53:13 -0700 Subject: [PATCH 07/13] remove unrecognized option --- doc/source/whatsnew/v0.8.0.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index 04493df7e9169..072d1bae2a2b9 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -157,7 +157,6 @@ New plotting methods ~~~~~~~~~~~~~~~~~~~~ .. code-block:: python - :suppress: import pandas as pd fx = pd.read_pickle('data/fx_prices') From 0f75c0dad419226a67e1422f2b61e7939ee93ab2 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 25 Oct 2019 14:44:01 -0700 Subject: [PATCH 08/13] TST: add test for file written in py27 --- pandas/io/pickle.py | 3 +++ pandas/tests/io/data/test_py27.pkl | Bin 0 -> 943 bytes pandas/tests/io/test_pickle.py | 11 +++++++++++ 3 files changed, 14 insertions(+) create mode 100644 pandas/tests/io/data/test_py27.pkl diff --git a/pandas/io/pickle.py b/pandas/io/pickle.py index f62f2cc739305..df1996aa0dee0 100644 --- a/pandas/io/pickle.py +++ b/pandas/io/pickle.py @@ -157,6 +157,9 @@ def read_pickle(path, compression="infer"): # "No module named 'pandas.core.sparse.series'" # "Can't get attribute '__nat_unpickle' on CWuvZ0(MF=b>%5pv9W@ zqaV>jg4zh`Ps&0>7A7M}Yy{F9N-fH==p9jrf2!;3Mr_pmak%%K@0|OcbG~Mq5Ja*U zsfkAaXy`D>WDBw$2!;|326VsXLyCez6sL$ny{u}AE@%|aNuVe)3p0vy zSxW?3`9n2$D$QD1dn5_)lD0((PlLAVwXB7;62NqtwL@!@+wHFUNseh)pz-YX@7L9L%U(PnRRE#`)2FkSS z)8{NTOM#_Hm#_4C_?z;2i8b)WB}vF?4e{N$0# zAhsbtTJre2F?_YEzqi`b2V+x*kH5HUoIllB%{={R+-XAbXnP;n7r*Z}P#g>oI(Odw zI%>IL9bm={EKPb)H<%F&8z}OXA&q(_iIJd^zxh+0mTs+v4 zUH;$=c$sbMJ^i>J#P_e Date: Sat, 26 Oct 2019 08:17:03 -0700 Subject: [PATCH 09/13] revert --- doc/source/whatsnew/v0.8.0.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index 072d1bae2a2b9..664325ac063c0 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -156,7 +156,8 @@ Other new features New plotting methods ~~~~~~~~~~~~~~~~~~~~ -.. code-block:: python +.. ipython:: python + :suppress: import pandas as pd fx = pd.read_pickle('data/fx_prices') @@ -164,7 +165,7 @@ New plotting methods ``Series.plot`` now supports a ``secondary_y`` option: -.. code-block:: python +.. ipython:: python plt.figure() From 2e88e0cd195b480aa63595045c7ceadd64e20da7 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 28 Oct 2019 10:43:19 -0700 Subject: [PATCH 10/13] troubleshoot doc build --- pandas/compat/pickle_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py index f9e8b67e5d7bb..91efdf66feb89 100644 --- a/pandas/compat/pickle_compat.py +++ b/pandas/compat/pickle_compat.py @@ -39,7 +39,7 @@ def load_reduce(self): except TypeError: pass - raise err + raise TypeErrr(msg, args) _sparse_msg = """\ From 6007db9d3f05b3a9836b181fa050a520bba6097a Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 28 Oct 2019 12:09:52 -0700 Subject: [PATCH 11/13] troubleshoot docbuild --- pandas/compat/pickle_compat.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py index 91efdf66feb89..0d19402c07d49 100644 --- a/pandas/compat/pickle_compat.py +++ b/pandas/compat/pickle_compat.py @@ -39,7 +39,20 @@ def load_reduce(self): except TypeError: pass - raise TypeErrr(msg, args) + # try to re-encode the arguments + if getattr(self, "encoding", None) is not None: + # This is needed for a pytables pickle in 0.8.0 release notes, see + # GH#28645 + args = tuple( + arg.encode(self.encoding) if isinstance(arg, str) else arg + for arg in args + ) + try: + stack[-1] = func(*args) + return + except TypeError: + pass + raise _sparse_msg = """\ From 625f60ef71c1c7027cfea09d6321e0342d484075 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 29 Oct 2019 08:08:04 -0700 Subject: [PATCH 12/13] re-disable --- doc/source/whatsnew/v0.8.0.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.8.0.rst b/doc/source/whatsnew/v0.8.0.rst index 664325ac063c0..072d1bae2a2b9 100644 --- a/doc/source/whatsnew/v0.8.0.rst +++ b/doc/source/whatsnew/v0.8.0.rst @@ -156,8 +156,7 @@ Other new features New plotting methods ~~~~~~~~~~~~~~~~~~~~ -.. ipython:: python - :suppress: +.. code-block:: python import pandas as pd fx = pd.read_pickle('data/fx_prices') @@ -165,7 +164,7 @@ New plotting methods ``Series.plot`` now supports a ``secondary_y`` option: -.. ipython:: python +.. code-block:: python plt.figure() From 97bcd3a2319a03103a67d18b308d6bf1fd46560f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 29 Oct 2019 09:25:47 -0700 Subject: [PATCH 13/13] re-remove --- pandas/compat/pickle_compat.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py index 0d19402c07d49..458c0c07c7602 100644 --- a/pandas/compat/pickle_compat.py +++ b/pandas/compat/pickle_compat.py @@ -39,19 +39,6 @@ def load_reduce(self): except TypeError: pass - # try to re-encode the arguments - if getattr(self, "encoding", None) is not None: - # This is needed for a pytables pickle in 0.8.0 release notes, see - # GH#28645 - args = tuple( - arg.encode(self.encoding) if isinstance(arg, str) else arg - for arg in args - ) - try: - stack[-1] = func(*args) - return - except TypeError: - pass raise