From 5d1f3b7ed044204304366bb57d7029bc73c3b8ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Mon, 18 Jan 2021 10:36:10 -0500 Subject: [PATCH 1/3] REGR: codecs.open() is always opened in text mode --- pandas/io/common.py | 12 ++++++++---- pandas/tests/io/test_common.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index 47811d47e7238..998e8b63f8336 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -2,6 +2,7 @@ from __future__ import annotations import bz2 +import codecs from collections import abc import dataclasses import gzip @@ -857,9 +858,12 @@ def file_exists(filepath_or_buffer: FilePathOrBuffer) -> bool: def _is_binary_mode(handle: FilePathOrBuffer, mode: str) -> bool: """Whether the handle is opened in binary mode""" + # classes that expect string but have 'b' in mode + text_classes = (codecs.StreamReaderWriter,) + if isinstance(handle, text_classes): + return False + # classes that expect bytes - binary_classes = [BufferedIOBase, RawIOBase] + binary_classes = (BufferedIOBase, RawIOBase) - return isinstance(handle, tuple(binary_classes)) or "b" in getattr( - handle, "mode", mode - ) + return isinstance(handle, binary_classes) or "b" in getattr(handle, "mode", mode) diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index d31bee9aca135..8edfd3f16efdf 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -1,6 +1,7 @@ """ Tests for the pandas.io.common functionalities """ +import codecs from io import StringIO import mmap import os @@ -429,3 +430,19 @@ def test_default_errors(): file = Path(path) file.write_bytes(b"\xe4\na\n1") tm.assert_frame_equal(pd.read_csv(file, skiprows=[0]), pd.DataFrame({"a": [1]})) + + +@pytest.mark.parametrize("encoding", [None, "utf-8"]) +@pytest.mark.parametrize("format", ["csv", "json"]) +def test_codecs_encoding(encoding, format): + # GH39247 + expected = tm.makeDataFrame() + with tm.ensure_clean() as path: + with codecs.open(path, mode="w", encoding=encoding) as handle: + getattr(expected, f"to_{format}")(handle) + with codecs.open(path, mode="r", encoding=encoding) as handle: + if format == "csv": + df = pd.read_csv(handle, index_col=0) + else: + df = pd.read_json(handle) + tm.assert_frame_equal(expected, df) From 476e12a2704a13e8f2befe7eb718ae202b1506e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Mon, 18 Jan 2021 15:14:37 -0500 Subject: [PATCH 2/3] whatsnew --- doc/source/whatsnew/v1.2.1.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.2.1.rst b/doc/source/whatsnew/v1.2.1.rst index 24ba9be4383eb..d6f660e0fc085 100644 --- a/doc/source/whatsnew/v1.2.1.rst +++ b/doc/source/whatsnew/v1.2.1.rst @@ -34,6 +34,7 @@ Fixed regressions - Fixed regression in repr of float-like strings of an ``object`` dtype having trailing 0's truncated after the decimal (:issue:`38708`) - Fixed regression that raised ``AttributeError`` with PyArrow versions [0.16.0, 1.0.0) (:issue:`38801`) - Fixed regression in :func:`pandas.testing.assert_frame_equal` raising ``TypeError`` with ``check_like=True`` when :class:`Index` or columns have mixed dtype (:issue:`39168`) +- Fixed regression in :meth:`~DataFrame.to_csv` opening `codecs.StreamReaderWriter` in binary mode instead of in text mode (:issue:`39247`) We have reverted a commit that resulted in several plotting related regressions in pandas 1.2.0 (:issue:`38969`, :issue:`38736`, :issue:`38865`, :issue:`38947` and :issue:`39126`). As a result, bugs reported as fixed in pandas 1.2.0 related to inconsistent tick labeling in bar plots are again present (:issue:`26186` and :issue:`11465`) From 5f34396aedd53762270c5959619019752c5fbb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Mon, 18 Jan 2021 18:06:04 -0500 Subject: [PATCH 3/3] re-order --- doc/source/whatsnew/v1.2.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.1.rst b/doc/source/whatsnew/v1.2.1.rst index d6f660e0fc085..4e583ce4d3af9 100644 --- a/doc/source/whatsnew/v1.2.1.rst +++ b/doc/source/whatsnew/v1.2.1.rst @@ -15,6 +15,7 @@ including other versions of pandas. Fixed regressions ~~~~~~~~~~~~~~~~~ - Fixed regression in :meth:`~DataFrame.to_csv` that created corrupted zip files when there were more rows than ``chunksize`` (:issue:`38714`) +- Fixed regression in :meth:`~DataFrame.to_csv` opening ``codecs.StreamReaderWriter`` in binary mode instead of in text mode (:issue:`39247`) - Fixed regression in :meth:`read_csv` and other read functions were the encoding error policy (``errors``) did not default to ``"replace"`` when no encoding was specified (:issue:`38989`) - Fixed regression in :func:`read_excel` with non-rawbyte file handles (:issue:`38788`) - Fixed regression in :meth:`DataFrame.to_stata` not removing the created file when an error occured (:issue:`39202`) @@ -34,7 +35,6 @@ Fixed regressions - Fixed regression in repr of float-like strings of an ``object`` dtype having trailing 0's truncated after the decimal (:issue:`38708`) - Fixed regression that raised ``AttributeError`` with PyArrow versions [0.16.0, 1.0.0) (:issue:`38801`) - Fixed regression in :func:`pandas.testing.assert_frame_equal` raising ``TypeError`` with ``check_like=True`` when :class:`Index` or columns have mixed dtype (:issue:`39168`) -- Fixed regression in :meth:`~DataFrame.to_csv` opening `codecs.StreamReaderWriter` in binary mode instead of in text mode (:issue:`39247`) We have reverted a commit that resulted in several plotting related regressions in pandas 1.2.0 (:issue:`38969`, :issue:`38736`, :issue:`38865`, :issue:`38947` and :issue:`39126`). As a result, bugs reported as fixed in pandas 1.2.0 related to inconsistent tick labeling in bar plots are again present (:issue:`26186` and :issue:`11465`)