From 84442085418f37883b31e37cb5ffb83fc42f2122 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Sun, 6 Aug 2023 13:36:04 -0700 Subject: [PATCH 01/14] CI: Run tests in development mode --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 54e41ea449848..48ef21686a26f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 7daf9a2f2fdc7df9bb2fa7dda8c6531cdd2493a9 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Sun, 6 Aug 2023 19:23:56 -0700 Subject: [PATCH 02/14] Use finalizer --- pandas/io/common.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index 35e93d287f31a..39408eda95bdc 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -47,6 +47,7 @@ uses_relative, ) import warnings +import weakref import zipfile from pandas._typing import ( @@ -939,6 +940,10 @@ class _BufferedWriter(BytesIO, ABC): # type: ignore[misc] def write_to_buffer(self) -> None: ... + @abstractmethod + def close_buffer(self) -> None: + ... + def close(self) -> None: if self.closed: # already closed @@ -950,8 +955,7 @@ def close(self) -> None: with self.buffer: # type: ignore[attr-defined] self.write_to_buffer() else: - # error: "_BufferedWriter" has no attribute "buffer" - self.buffer.close() # type: ignore[attr-defined] + self.close_buffer() super().close() @@ -976,6 +980,10 @@ def __init__( fileobj=fileobj, # type: ignore[arg-type] **kwargs, ) + self._finalizer = weakref.finalize(self, lambda x: x.close(), self.buffer) + + def close_buffer(self) -> None: + self._finalizer() def extend_mode(self, mode: str) -> str: mode = mode.replace("b", "") @@ -1027,6 +1035,10 @@ def __init__( # Union[str, PathLike[str]], ReadBuffer[bytes], WriteBuffer[bytes]]"; # expected "Union[Union[str, PathLike[str]], IO[bytes]]" self.buffer = zipfile.ZipFile(file, mode, **kwargs) # type: ignore[arg-type] + self._finalizer = weakref.finalize(self, lambda x: x.close(), self.buffer) + + def close_buffer(self) -> None: + self._finalizer() def infer_filename(self) -> str | None: """ From fc9b2cf4084e40d1a423037b3cf8e7c3408cc8a4 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Sun, 6 Aug 2023 19:32:15 -0700 Subject: [PATCH 03/14] Specify more encoding --- pandas/_testing/contexts.py | 2 +- pandas/tests/io/parser/common/test_common_basic.py | 2 +- pandas/tests/io/parser/test_encoding.py | 2 +- pandas/tests/io/parser/test_read_fwf.py | 2 +- pandas/tests/io/parser/test_unsupported.py | 2 +- pandas/tests/io/test_sql.py | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pandas/_testing/contexts.py b/pandas/_testing/contexts.py index ac1a6a2450444..30818c14fc298 100644 --- a/pandas/_testing/contexts.py +++ b/pandas/_testing/contexts.py @@ -125,7 +125,7 @@ def ensure_clean( path.touch() handle_or_str: str | IO = str(path) - encoding = kwargs.pop("encoding", None) + encoding = kwargs.pop("encoding", "utf-8") if return_filelike: kwargs.setdefault("mode", "w+b") handle_or_str = open(path, encoding=encoding, **kwargs) diff --git a/pandas/tests/io/parser/common/test_common_basic.py b/pandas/tests/io/parser/common/test_common_basic.py index 9083d725887f1..00a26a755756f 100644 --- a/pandas/tests/io/parser/common/test_common_basic.py +++ b/pandas/tests/io/parser/common/test_common_basic.py @@ -856,7 +856,7 @@ def test_read_seek(all_parsers): prefix = "### DATA\n" content = "nkey,value\ntables,rectangular\n" with tm.ensure_clean() as path: - Path(path).write_text(prefix + content) + Path(path).write_text(prefix + content, encoding="utf-8") with open(path, encoding="utf-8") as file: file.readline() actual = parser.read_csv(file) diff --git a/pandas/tests/io/parser/test_encoding.py b/pandas/tests/io/parser/test_encoding.py index f6dbb24f36f18..3873bf31c1ed4 100644 --- a/pandas/tests/io/parser/test_encoding.py +++ b/pandas/tests/io/parser/test_encoding.py @@ -310,7 +310,7 @@ def test_not_readable(all_parsers, mode): content = b"abcd" if "t" in mode: content = "abcd" - with tempfile.SpooledTemporaryFile(mode=mode) as handle: + with tempfile.SpooledTemporaryFile(mode=mode, encoding="utf-8") as handle: handle.write(content) handle.seek(0) df = parser.read_csv(handle) diff --git a/pandas/tests/io/parser/test_read_fwf.py b/pandas/tests/io/parser/test_read_fwf.py index c19f8d2792a35..34cae289c0f22 100644 --- a/pandas/tests/io/parser/test_read_fwf.py +++ b/pandas/tests/io/parser/test_read_fwf.py @@ -702,7 +702,7 @@ def test_binary_mode(): [["bba", "bab", "b a"]], columns=["aaa", "aaa.1", "aaa.2"], index=[0] ) with tm.ensure_clean() as path: - Path(path).write_text(data) + Path(path).write_text(data, encoding="utf-8") with open(path, "rb") as file: df = read_fwf(file) file.seek(0) diff --git a/pandas/tests/io/parser/test_unsupported.py b/pandas/tests/io/parser/test_unsupported.py index 1a9d99b0b5c1f..f5a0bcd2c00fd 100644 --- a/pandas/tests/io/parser/test_unsupported.py +++ b/pandas/tests/io/parser/test_unsupported.py @@ -182,7 +182,7 @@ def test_close_file_handle_on_invalid_usecols(all_parsers): pytest.skip("GH#45547 causing timeouts on windows/mac builds 2022-01-22") with tm.ensure_clean("test.csv") as fname: - Path(fname).write_text("col1,col2\na,b\n1,2") + Path(fname).write_text("col1,col2\na,b\n1,2", encoding="utf-8") with tm.assert_produces_warning(False): with pytest.raises(error, match="col3"): parser.read_csv(fname, usecols=["col1", "col2", "col3"]) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index e2d985bdf1386..6aa8e31a52ab0 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -140,7 +140,7 @@ def create_and_load_iris_sqlite3(conn: sqlite3.Connection, iris_file: Path): "Name" TEXT )""" cur.execute(stmt) - with iris_file.open(newline=None) as csvfile: + with iris_file.open(newline=None, encoding="utf-8") as csvfile: reader = csv.reader(csvfile) next(reader) stmt = "INSERT INTO iris VALUES(?, ?, ?, ?, ?)" @@ -153,7 +153,7 @@ def create_and_load_iris(conn, iris_file: Path, dialect: str): iris = iris_table_metadata(dialect) - with iris_file.open(newline=None) as csvfile: + with iris_file.open(newline=None, encoding="utf-8") as csvfile: reader = csv.reader(csvfile) header = next(reader) params = [dict(zip(header, row)) for row in reader] From aeb2bf7832a10468137301dac0e9f1549d13461d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 7 Aug 2023 11:42:35 -0700 Subject: [PATCH 04/14] Set encoding if not binary --- pandas/_testing/contexts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/_testing/contexts.py b/pandas/_testing/contexts.py index 30818c14fc298..b2bb8e71fdf5c 100644 --- a/pandas/_testing/contexts.py +++ b/pandas/_testing/contexts.py @@ -125,9 +125,11 @@ def ensure_clean( path.touch() handle_or_str: str | IO = str(path) - encoding = kwargs.pop("encoding", "utf-8") + encoding = kwargs.pop("encoding", None) if return_filelike: kwargs.setdefault("mode", "w+b") + if encoding is None and "b" not in kwargs["mode"]: + encoding = "utf-8" handle_or_str = open(path, encoding=encoding, **kwargs) try: From ff9a6e7927bb865bbdff64278604b63a0fa52f1c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:37:10 -0700 Subject: [PATCH 05/14] address more tests --- pandas/io/common.py | 5 +++-- pandas/tests/io/formats/test_format.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index 0570efb0b8dfd..64abf6aeadeda 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -949,8 +949,9 @@ def close(self) -> None: # error: "_BufferedWriter" has no attribute "buffer" with self.buffer: # type: ignore[attr-defined] self.write_to_buffer() - elif hasattr(self.buffer, "close"): # type: ignore[attr-defined] - self.buffer.close() + else: + # error: "_BufferedWriter" has no attribute "buffer" + self.buffer.close() # type: ignore[attr-defined] super().close() diff --git a/pandas/tests/io/formats/test_format.py b/pandas/tests/io/formats/test_format.py index 592b8d206fa30..3b0dac21ef10c 100644 --- a/pandas/tests/io/formats/test_format.py +++ b/pandas/tests/io/formats/test_format.py @@ -95,6 +95,8 @@ def assert_filepath_or_buffer_equals( """ Assertion helper for checking filepath_or_buffer. """ + if encoding is None: + encoding = "utf-8" def _assert_filepath_or_buffer_equals(expected): if filepath_or_buffer_id == "string": From 7c73e082879141fd043e262d967ed73922762ced Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 7 Aug 2023 17:03:31 -0700 Subject: [PATCH 06/14] Make buffer property --- pandas/io/common.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index 64abf6aeadeda..aa3158860caaa 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -939,6 +939,11 @@ class _BufferedWriter(BytesIO, ABC): # type: ignore[misc] def write_to_buffer(self) -> None: ... + @property + @abstractmethod + def buffer(self) -> Any: + ... + def close(self) -> None: if self.closed: # already closed @@ -946,12 +951,10 @@ def close(self) -> None: if self.getvalue(): # write to buffer self.seek(0) - # error: "_BufferedWriter" has no attribute "buffer" - with self.buffer: # type: ignore[attr-defined] + with self.buffer: self.write_to_buffer() else: - # error: "_BufferedWriter" has no attribute "buffer" - self.buffer.close() # type: ignore[attr-defined] + self.buffer.close() super().close() @@ -970,13 +973,17 @@ def __init__( # error: Argument "fileobj" to "open" of "TarFile" has incompatible # type "Union[ReadBuffer[bytes], WriteBuffer[bytes], None]"; expected # "Optional[IO[bytes]]" - self.buffer = tarfile.TarFile.open( + self._buffer = tarfile.TarFile.open( name=name, mode=self.extend_mode(mode), fileobj=fileobj, # type: ignore[arg-type] **kwargs, ) + @property + def buffer(self) -> tarfile.TarFile: + return self._buffer + def extend_mode(self, mode: str) -> str: mode = mode.replace("b", "") if mode != "w": @@ -1026,10 +1033,14 @@ def __init__( # error: No overload variant of "ZipFile" matches argument types "str | # PathLike[str] | ReadBuffer[bytes] | WriteBuffer[bytes]", "str", "dict[str, # Any]" - self.buffer = zipfile.ZipFile( # type: ignore[call-overload] + self._buffer = zipfile.ZipFile( # type: ignore[call-overload] file, mode, **kwargs ) + @property + def buffer(self) -> zipfile.ZipFile: + return self._buffer + def infer_filename(self) -> str | None: """ If an explicit archive_name is not given, we still want the file inside the zip From 22002dcd16cb8e85f9c14d54e68bb578b2b2c237 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:04:34 -0700 Subject: [PATCH 07/14] Try this --- pandas/io/common.py | 52 ++++++++++++++++++++++----------------------- pyproject.toml | 1 + 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index aa3158860caaa..b8ea86ba1a136 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -939,24 +939,6 @@ class _BufferedWriter(BytesIO, ABC): # type: ignore[misc] def write_to_buffer(self) -> None: ... - @property - @abstractmethod - def buffer(self) -> Any: - ... - - def close(self) -> None: - if self.closed: - # already closed - return - if self.getvalue(): - # write to buffer - self.seek(0) - with self.buffer: - self.write_to_buffer() - else: - self.buffer.close() - super().close() - class _BytesTarFile(_BufferedWriter): def __init__( @@ -973,16 +955,25 @@ def __init__( # error: Argument "fileobj" to "open" of "TarFile" has incompatible # type "Union[ReadBuffer[bytes], WriteBuffer[bytes], None]"; expected # "Optional[IO[bytes]]" - self._buffer = tarfile.TarFile.open( + self.buffer = tarfile.TarFile.open( name=name, mode=self.extend_mode(mode), fileobj=fileobj, # type: ignore[arg-type] **kwargs, ) - @property - def buffer(self) -> tarfile.TarFile: - return self._buffer + def close(self) -> None: + if self.closed: + # already closed + return + if self.getbuffer().nbytes: + # write to buffer + self.seek(0) + with self.buffer: + self.write_to_buffer() + else: + self.buffer.close() + super().close() def extend_mode(self, mode: str) -> str: mode = mode.replace("b", "") @@ -1033,13 +1024,22 @@ def __init__( # error: No overload variant of "ZipFile" matches argument types "str | # PathLike[str] | ReadBuffer[bytes] | WriteBuffer[bytes]", "str", "dict[str, # Any]" - self._buffer = zipfile.ZipFile( # type: ignore[call-overload] + self.buffer = zipfile.ZipFile( # type: ignore[call-overload] file, mode, **kwargs ) - @property - def buffer(self) -> zipfile.ZipFile: - return self._buffer + def close(self) -> None: + if self.closed: + # already closed + return + if self.getbuffer().nbytes: + # write to buffer + self.seek(0) + with self.buffer: + self.write_to_buffer() + else: + self.buffer.close() + super().close() def infer_filename(self) -> str | None: """ diff --git a/pyproject.toml b/pyproject.toml index ae658329f42ee..b6d815cd8b7f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -502,6 +502,7 @@ filterwarnings = [ "ignore:distutils Version classes are deprecated:DeprecationWarning:numexpr", "ignore:distutils Version classes are deprecated:DeprecationWarning:fastparquet", "ignore:distutils Version classes are deprecated:DeprecationWarning:fsspec", + "ignore::EncodingWarning:parso", ] junit_family = "xunit2" markers = [ From 7614db884b7c6ee0e0fd3d6fe8d91fef312b944c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:26:39 -0700 Subject: [PATCH 08/14] Just ignore via error message and package, move super --- pandas/io/common.py | 45 +++++++++++++++++---------------------------- pyproject.toml | 2 +- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index b8ea86ba1a136..b28368a566062 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -939,6 +939,21 @@ class _BufferedWriter(BytesIO, ABC): # type: ignore[misc] def write_to_buffer(self) -> None: ... + def close(self) -> None: + if self.closed: + # already closed + return + if self.getvalue(): + # write to buffer + self.seek(0) + # error: "_BufferedWriter" has no attribute "buffer" + with self.buffer: # type: ignore[attr-defined] + self.write_to_buffer() + else: + # error: "_BufferedWriter" has no attribute "buffer" + self.buffer.close() # type: ignore[attr-defined] + super().close() + class _BytesTarFile(_BufferedWriter): def __init__( @@ -949,7 +964,6 @@ def __init__( archive_name: str | None = None, **kwargs, ) -> None: - super().__init__() self.archive_name = archive_name self.name = name # error: Argument "fileobj" to "open" of "TarFile" has incompatible @@ -961,19 +975,7 @@ def __init__( fileobj=fileobj, # type: ignore[arg-type] **kwargs, ) - - def close(self) -> None: - if self.closed: - # already closed - return - if self.getbuffer().nbytes: - # write to buffer - self.seek(0) - with self.buffer: - self.write_to_buffer() - else: - self.buffer.close() - super().close() + super().__init__() def extend_mode(self, mode: str) -> str: mode = mode.replace("b", "") @@ -1016,7 +1018,6 @@ def __init__( archive_name: str | None = None, **kwargs, ) -> None: - super().__init__() mode = mode.replace("b", "") self.archive_name = archive_name @@ -1027,19 +1028,7 @@ def __init__( self.buffer = zipfile.ZipFile( # type: ignore[call-overload] file, mode, **kwargs ) - - def close(self) -> None: - if self.closed: - # already closed - return - if self.getbuffer().nbytes: - # write to buffer - self.seek(0) - with self.buffer: - self.write_to_buffer() - else: - self.buffer.close() - super().close() + super().__init__() def infer_filename(self) -> str | None: """ diff --git a/pyproject.toml b/pyproject.toml index b6d815cd8b7f1..94916f3659e9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -502,7 +502,7 @@ filterwarnings = [ "ignore:distutils Version classes are deprecated:DeprecationWarning:numexpr", "ignore:distutils Version classes are deprecated:DeprecationWarning:fastparquet", "ignore:distutils Version classes are deprecated:DeprecationWarning:fsspec", - "ignore::EncodingWarning:parso", + "ignore:.*argument not specified::parso", ] junit_family = "xunit2" markers = [ From 3f82bddc0c3e1219d81e013f27808d255e0a8109 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Aug 2023 14:44:31 -0700 Subject: [PATCH 09/14] use try except and close --- pandas/io/common.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index b28368a566062..a74b9fb4511b5 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -964,18 +964,23 @@ def __init__( archive_name: str | None = None, **kwargs, ) -> None: + super().__init__() self.archive_name = archive_name self.name = name # error: Argument "fileobj" to "open" of "TarFile" has incompatible # type "Union[ReadBuffer[bytes], WriteBuffer[bytes], None]"; expected # "Optional[IO[bytes]]" - self.buffer = tarfile.TarFile.open( - name=name, - mode=self.extend_mode(mode), - fileobj=fileobj, # type: ignore[arg-type] - **kwargs, - ) - super().__init__() + try: + self.buffer = tarfile.TarFile.open( + name=name, + mode=self.extend_mode(mode), + fileobj=fileobj, # type: ignore[arg-type] + **kwargs, + ) + except Exception: + # Don't let close try to close non-existent self.buffer + BytesIO.close() + raise def extend_mode(self, mode: str) -> str: mode = mode.replace("b", "") @@ -1018,6 +1023,7 @@ def __init__( archive_name: str | None = None, **kwargs, ) -> None: + super().__init__() mode = mode.replace("b", "") self.archive_name = archive_name @@ -1025,10 +1031,14 @@ def __init__( # error: No overload variant of "ZipFile" matches argument types "str | # PathLike[str] | ReadBuffer[bytes] | WriteBuffer[bytes]", "str", "dict[str, # Any]" - self.buffer = zipfile.ZipFile( # type: ignore[call-overload] - file, mode, **kwargs - ) - super().__init__() + try: + self.buffer = zipfile.ZipFile( # type: ignore[call-overload] + file, mode, **kwargs + ) + except Exception: + # Don't let close try to close non-existent self.buffer + BytesIO.close() + raise def infer_filename(self) -> str | None: """ From 8672abcb4f100ee8932ad4416d081241721ed10b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Aug 2023 16:13:35 -0700 Subject: [PATCH 10/14] Give fake object --- pandas/io/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index a74b9fb4511b5..acb06f7214b5b 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -943,7 +943,7 @@ def close(self) -> None: if self.closed: # already closed return - if self.getvalue(): + if self.getbuffer().nbytes: # write to buffer self.seek(0) # error: "_BufferedWriter" has no attribute "buffer" @@ -978,8 +978,8 @@ def __init__( **kwargs, ) except Exception: - # Don't let close try to close non-existent self.buffer - BytesIO.close() + # Give super().close something to close on destruction + self.buffer = BytesIO() raise def extend_mode(self, mode: str) -> str: @@ -1036,8 +1036,8 @@ def __init__( file, mode, **kwargs ) except Exception: - # Don't let close try to close non-existent self.buffer - BytesIO.close() + # Give super().close something to close on destruction + self.buffer = BytesIO() raise def infer_filename(self) -> str | None: From f90a8c5e497e621bdbebe036c0f900d5281a3bdf Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Aug 2023 19:16:04 -0700 Subject: [PATCH 11/14] typing, ignore at test --- pandas/io/common.py | 46 +++++++------------ .../tests/arrays/categorical/test_warnings.py | 1 + pandas/tests/frame/test_api.py | 1 + pyproject.toml | 1 - 4 files changed, 19 insertions(+), 30 deletions(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index acb06f7214b5b..6be6f3f4300e4 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -935,6 +935,8 @@ class _BufferedWriter(BytesIO, ABC): # type: ignore[misc] This wrapper writes to the underlying buffer on close. """ + buffer = BytesIO() + @abstractmethod def write_to_buffer(self) -> None: ... @@ -946,12 +948,10 @@ def close(self) -> None: if self.getbuffer().nbytes: # write to buffer self.seek(0) - # error: "_BufferedWriter" has no attribute "buffer" - with self.buffer: # type: ignore[attr-defined] + with self.buffer: self.write_to_buffer() else: - # error: "_BufferedWriter" has no attribute "buffer" - self.buffer.close() # type: ignore[attr-defined] + self.buffer.close() super().close() @@ -967,20 +967,14 @@ def __init__( super().__init__() self.archive_name = archive_name self.name = name - # error: Argument "fileobj" to "open" of "TarFile" has incompatible - # type "Union[ReadBuffer[bytes], WriteBuffer[bytes], None]"; expected - # "Optional[IO[bytes]]" - try: - self.buffer = tarfile.TarFile.open( - name=name, - mode=self.extend_mode(mode), - fileobj=fileobj, # type: ignore[arg-type] - **kwargs, - ) - except Exception: - # Give super().close something to close on destruction - self.buffer = BytesIO() - raise + # error: Incompatible types in assignment (expression has type "TarFile", + # base class "_BufferedWriter" defined the type as "BytesIO") + self.buffer: tarfile.TarFile = tarfile.TarFile.open( # type: ignore[assignment] + name=name, + mode=self.extend_mode(mode), + fileobj=fileobj, + **kwargs, + ) def extend_mode(self, mode: str) -> str: mode = mode.replace("b", "") @@ -1028,17 +1022,11 @@ def __init__( self.archive_name = archive_name kwargs.setdefault("compression", zipfile.ZIP_DEFLATED) - # error: No overload variant of "ZipFile" matches argument types "str | - # PathLike[str] | ReadBuffer[bytes] | WriteBuffer[bytes]", "str", "dict[str, - # Any]" - try: - self.buffer = zipfile.ZipFile( # type: ignore[call-overload] - file, mode, **kwargs - ) - except Exception: - # Give super().close something to close on destruction - self.buffer = BytesIO() - raise + # error: Incompatible types in assignment (expression has type "ZipFile", + # base class "_BufferedWriter" defined the type as "BytesIO") + self.buffer: zipfile.ZipFile = zipfile.ZipFile( # type: ignore[assignment] + file, mode, **kwargs + ) def infer_filename(self) -> str | None: """ diff --git a/pandas/tests/arrays/categorical/test_warnings.py b/pandas/tests/arrays/categorical/test_warnings.py index 656da89c70196..3bf9ed060521b 100644 --- a/pandas/tests/arrays/categorical/test_warnings.py +++ b/pandas/tests/arrays/categorical/test_warnings.py @@ -7,6 +7,7 @@ class TestCategoricalWarnings: @async_mark() + @pytest.mark.filterwarnings("ignore:'encoding' argument not specified") async def test_tab_complete_warning(self, ip): # https://github.com/pandas-dev/pandas/issues/16409 pytest.importorskip("IPython", minversion="6.0.0") diff --git a/pandas/tests/frame/test_api.py b/pandas/tests/frame/test_api.py index 7ca5df0451d19..618b1fac046e1 100644 --- a/pandas/tests/frame/test_api.py +++ b/pandas/tests/frame/test_api.py @@ -289,6 +289,7 @@ def _check_f(base, f): _check_f(d.copy(), f) @async_mark() + @pytest.mark.filterwarnings("ignore:'encoding' argument not specified") async def test_tab_complete_warning(self, ip, frame_or_series): # GH 16409 pytest.importorskip("IPython", minversion="6.0.0") diff --git a/pyproject.toml b/pyproject.toml index 94916f3659e9f..ae658329f42ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -502,7 +502,6 @@ filterwarnings = [ "ignore:distutils Version classes are deprecated:DeprecationWarning:numexpr", "ignore:distutils Version classes are deprecated:DeprecationWarning:fastparquet", "ignore:distutils Version classes are deprecated:DeprecationWarning:fsspec", - "ignore:.*argument not specified::parso", ] junit_family = "xunit2" markers = [ From 82cd2260dd384a27dc32d72033356e261a93d66d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 8 Aug 2023 20:27:02 -0700 Subject: [PATCH 12/14] Don't raise on extra warnings --- pandas/tests/arrays/categorical/test_warnings.py | 3 +-- pandas/tests/frame/test_api.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/tests/arrays/categorical/test_warnings.py b/pandas/tests/arrays/categorical/test_warnings.py index 3bf9ed060521b..c0623faccd325 100644 --- a/pandas/tests/arrays/categorical/test_warnings.py +++ b/pandas/tests/arrays/categorical/test_warnings.py @@ -7,7 +7,6 @@ class TestCategoricalWarnings: @async_mark() - @pytest.mark.filterwarnings("ignore:'encoding' argument not specified") async def test_tab_complete_warning(self, ip): # https://github.com/pandas-dev/pandas/issues/16409 pytest.importorskip("IPython", minversion="6.0.0") @@ -18,6 +17,6 @@ async def test_tab_complete_warning(self, ip): # GH 31324 newer jedi version raises Deprecation warning; # appears resolved 2021-02-02 - with tm.assert_produces_warning(None): + with tm.assert_produces_warning(None, raise_on_extra_warnings=False): with provisionalcompleter("ignore"): list(ip.Completer.completions("c.", 1)) diff --git a/pandas/tests/frame/test_api.py b/pandas/tests/frame/test_api.py index 618b1fac046e1..8fc78629beb0a 100644 --- a/pandas/tests/frame/test_api.py +++ b/pandas/tests/frame/test_api.py @@ -289,7 +289,6 @@ def _check_f(base, f): _check_f(d.copy(), f) @async_mark() - @pytest.mark.filterwarnings("ignore:'encoding' argument not specified") async def test_tab_complete_warning(self, ip, frame_or_series): # GH 16409 pytest.importorskip("IPython", minversion="6.0.0") @@ -304,7 +303,7 @@ async def test_tab_complete_warning(self, ip, frame_or_series): # GH 31324 newer jedi version raises Deprecation warning; # appears resolved 2021-02-02 - with tm.assert_produces_warning(None): + with tm.assert_produces_warning(None, raise_on_extra_warnings=False): with provisionalcompleter("ignore"): list(ip.Completer.completions("obj.", 1)) From 296e4258e421e9cfdc7586c11610322569f45a95 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 9 Aug 2023 09:39:31 -0700 Subject: [PATCH 13/14] ignore on another test --- pandas/tests/resample/test_resampler_grouper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/resample/test_resampler_grouper.py b/pandas/tests/resample/test_resampler_grouper.py index 7144468dfc44c..62b0bc2012af1 100644 --- a/pandas/tests/resample/test_resampler_grouper.py +++ b/pandas/tests/resample/test_resampler_grouper.py @@ -41,7 +41,7 @@ async def test_tab_complete_ipython6_warning(ip): # GH 31324 newer jedi version raises Deprecation warning; # appears resolved 2021-02-02 - with tm.assert_produces_warning(None): + with tm.assert_produces_warning(None, raise_on_extra_warnings=False): with provisionalcompleter("ignore"): list(ip.Completer.completions("rs.", 1)) From 868e30a8d72745a6f9a83209f883db3b1cd4354a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 9 Aug 2023 11:00:20 -0700 Subject: [PATCH 14/14] another raise on extra --- pandas/tests/indexes/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 19711ba6329d4..fe6593186b5a8 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -1226,7 +1226,7 @@ async def test_tab_complete_warning(self, ip): # GH 31324 newer jedi version raises Deprecation warning; # appears resolved 2021-02-02 - with tm.assert_produces_warning(None): + with tm.assert_produces_warning(None, raise_on_extra_warnings=False): with provisionalcompleter("ignore"): list(ip.Completer.completions("idx.", 4))