From fd45ab23ef77c55c80744e440ac17440d0ea631e Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Sun, 6 Feb 2022 21:31:06 -0400 Subject: [PATCH 01/12] Fixing documentation format in read_csv add a mising ` in on_bad_lines option description in " New in version 1.3.0: " section , that was causing to make couple of lines red --- pandas/io/parsers/readers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py index 5f93eef4fd977..c5b84dd18ec13 100644 --- a/pandas/io/parsers/readers.py +++ b/pandas/io/parsers/readers.py @@ -374,7 +374,7 @@ - callable, function with signature ``(bad_line: list[str]) -> list[str] | None`` that will process a single bad line. ``bad_line`` is a list of strings split by the ``sep``. - If the function returns ``None`, the bad line will be ignored. + If the function returns ``None``, the bad line will be ignored. If the function returns a new list of strings with more elements than expected, a ``ParserWarning`` will be emitted while dropping extra elements. Only supported when ``engine="python"`` From 346eb6a7a86ee7616e08cf3b2c6dd6ebd8692498 Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Wed, 23 Mar 2022 21:43:37 -0400 Subject: [PATCH 02/12] remove store_option from docstring --- pandas/io/formats/style.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 54568d5cd18bb..5eb398dfa7419 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -545,8 +545,7 @@ def set_tooltips( @doc( NDFrame.to_excel, - klass="Styler", - storage_options=_shared_docs["storage_options"], + klass="Styler" ) def to_excel( self, From f97b9040d4977ed26a77e1b20211d6fc01cfb059 Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Wed, 23 Mar 2022 22:02:07 -0400 Subject: [PATCH 03/12] remove unused refrenced module --- pandas/io/formats/style.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 5eb398dfa7419..3d558845ee7c8 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -46,7 +46,6 @@ Series, ) from pandas.core.generic import NDFrame -from pandas.core.shared_docs import _shared_docs from pandas.io.formats.format import save_to_buffer @@ -543,10 +542,7 @@ def set_tooltips( return self - @doc( - NDFrame.to_excel, - klass="Styler" - ) + @doc(NDFrame.to_excel, klass="Styler") def to_excel( self, excel_writer, From eb2000f2d291f5ec1827c278878b87d80fbb0df6 Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Wed, 23 Mar 2022 22:37:13 -0400 Subject: [PATCH 04/12] add storage_option to the parameters of to_excel --- pandas/io/formats/style.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 3d558845ee7c8..edad3e82c6eed 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -26,6 +26,7 @@ IndexLabel, Level, Scalar, + StorageOptions, WriteBuffer, ) from pandas.compat._optional import import_optional_dependency @@ -46,6 +47,7 @@ Series, ) from pandas.core.generic import NDFrame +from pandas.core.shared_docs import _shared_docs from pandas.io.formats.format import save_to_buffer @@ -542,7 +544,11 @@ def set_tooltips( return self - @doc(NDFrame.to_excel, klass="Styler") + @doc( + NDFrame.to_excel, + klass="Styler", + storage_options=_shared_docs["storage_options"], + ) def to_excel( self, excel_writer, @@ -561,6 +567,7 @@ def to_excel( inf_rep: str = "inf", verbose: bool = True, freeze_panes: tuple[int, int] | None = None, + storage_options: StorageOptions = None, ) -> None: from pandas.io.formats.excel import ExcelFormatter @@ -583,6 +590,7 @@ def to_excel( startcol=startcol, freeze_panes=freeze_panes, engine=engine, + storage_options=storage_options, ) def to_latex( From 08237ad6e9cefaa66e5dd80a4e484c4e56a2e9cf Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Mon, 28 Mar 2022 21:45:06 -0400 Subject: [PATCH 05/12] add versionadded 1.5.0 blac hook --- pandas/io/formats/style.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index edad3e82c6eed..7e5f95192bfa6 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -569,6 +569,11 @@ def to_excel( freeze_panes: tuple[int, int] | None = None, storage_options: StorageOptions = None, ) -> None: + """ + {storage_options} + + .. versionadded:: 1.5.0 + """ from pandas.io.formats.excel import ExcelFormatter From 1c2a4e6ab5dc29b12ade2e2fe72697b400d2df0c Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Sat, 16 Apr 2022 21:39:10 -0400 Subject: [PATCH 06/12] add test unit test_styler_to_s3 --- pandas/tests/io/excel/test_style.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index c31e8ec022dcd..5f8026238d76b 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -205,3 +205,21 @@ def custom_converter(css): with contextlib.closing(openpyxl.load_workbook(path)) as wb: assert wb["custom"].cell(2, 2).font.color.value == "00111222" + + +def test_styler_to_s3(self, s3_resource, s3so): + import time + + mock_bucket_name, target_file = "pandas-test", "test.xlsx" + df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]}) + styler = df.style.set_sticky(axis="index") + styler.to_excel(f"s3://{mock_bucket_name}/{target_file}", storage_options=s3so) + timeout = 5 + while True: + if target_file in ( + obj.key for obj in s3_resource.Bucket("pandas-test").objects.all() + ): + break + time.sleep(0.1) + timeout -= 0.1 + assert timeout > 0, "Timed out waiting for file to appear on moto" From 27ff3acab3d20e05e4123c8da6177c9ddb89ee89 Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Mon, 18 Apr 2022 23:03:26 -0400 Subject: [PATCH 07/12] move import;add bug # to the testa --- pandas/tests/io/excel/test_style.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index 5f8026238d76b..b436e9e186d5a 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -1,4 +1,5 @@ import contextlib +import time import numpy as np import pytest @@ -208,7 +209,7 @@ def custom_converter(css): def test_styler_to_s3(self, s3_resource, s3so): - import time + # GH#46381 mock_bucket_name, target_file = "pandas-test", "test.xlsx" df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]}) From bf9446d6f04bd2a1d3d4e8e6a15f0e17eced8591 Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Wed, 25 May 2022 16:01:25 -0400 Subject: [PATCH 08/12] add reading excel from s3 to test the whole process --- pandas/tests/io/excel/test_style.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index b436e9e186d5a..02feeb60195d9 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -4,7 +4,10 @@ import numpy as np import pytest -from pandas import DataFrame +from pandas import ( + DataFrame, + read_excel, +) import pandas._testing as tm from pandas.io.excel import ExcelWriter @@ -224,3 +227,7 @@ def test_styler_to_s3(self, s3_resource, s3so): time.sleep(0.1) timeout -= 0.1 assert timeout > 0, "Timed out waiting for file to appear on moto" + result = read_excel( + f"s3://{mock_bucket_name}/{target_file}", index_col=0, storage_options=s3so + ) + tm.assert_frame_equal(result, df) From 6e9a212ea6445a65a45d3c53e0af965e52ffb03e Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Sun, 12 Jun 2022 22:54:53 -0400 Subject: [PATCH 09/12] parameterize the storage options versionadded description in documents --- pandas/core/generic.py | 8 ++++++-- pandas/io/formats/style.py | 6 +----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 1f524d07bfd3a..8d7c9856e8bc0 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -2080,7 +2080,11 @@ def _repr_data_resource_(self): # I/O Methods @final - @doc(klass="object", storage_options=_shared_docs["storage_options"]) + @doc( + klass="object", + storage_options=_shared_docs["storage_options"], + storage_options_versionadded="1.2.0", + ) def to_excel( self, excel_writer, @@ -2166,7 +2170,7 @@ def to_excel( is to be frozen. {storage_options} - .. versionadded:: 1.2.0 + .. versionadded:: {storage_options_versionadded} See Also -------- diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index 93d5f666ebb1d..29791cca7b291 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -548,6 +548,7 @@ def set_tooltips( NDFrame.to_excel, klass="Styler", storage_options=_shared_docs["storage_options"], + storage_options_versionadded="1.5.0", ) def to_excel( self, @@ -569,11 +570,6 @@ def to_excel( freeze_panes: tuple[int, int] | None = None, storage_options: StorageOptions = None, ) -> None: - """ - {storage_options} - - .. versionadded:: 1.5.0 - """ from pandas.io.formats.excel import ExcelFormatter From 73cb37d83de347c5ef911c34ed5b18b5a58f1415 Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Thu, 16 Jun 2022 22:11:18 -0400 Subject: [PATCH 10/12] remove self in test test_styler_to_s3 --- pandas/tests/io/excel/test_style.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index 02feeb60195d9..b417960f944b8 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -211,7 +211,7 @@ def custom_converter(css): assert wb["custom"].cell(2, 2).font.color.value == "00111222" -def test_styler_to_s3(self, s3_resource, s3so): +def test_styler_to_s3(s3_resource, s3so): # GH#46381 mock_bucket_name, target_file = "pandas-test", "test.xlsx" From 66e0d956467e66fe47b67d08ba6aee30b8e2e23d Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Sun, 19 Jun 2022 13:13:25 -0400 Subject: [PATCH 11/12] add necessary decorators for the test unit --- pandas/tests/io/excel/test_style.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index b417960f944b8..8652376a14c0c 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -1,6 +1,7 @@ import contextlib import time +from Cryptodome.SelfTest.Cipher.test_AES import td import numpy as np import pytest @@ -211,6 +212,8 @@ def custom_converter(css): assert wb["custom"].cell(2, 2).font.color.value == "00111222" +@pytest.mark.single_cpu +@td.skip_if_not_us_locale def test_styler_to_s3(s3_resource, s3so): # GH#46381 From 588871309ca4f820032c38291c7464c34b11e943 Mon Sep 17 00:00:00 2001 From: Ehsan Shirvanian Date: Sun, 19 Jun 2022 13:26:03 -0400 Subject: [PATCH 12/12] fix the import module in test file --- pandas/tests/io/excel/test_style.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/excel/test_style.py b/pandas/tests/io/excel/test_style.py index 8652376a14c0c..e25404e73f813 100644 --- a/pandas/tests/io/excel/test_style.py +++ b/pandas/tests/io/excel/test_style.py @@ -1,10 +1,11 @@ import contextlib import time -from Cryptodome.SelfTest.Cipher.test_AES import td import numpy as np import pytest +import pandas.util._test_decorators as td + from pandas import ( DataFrame, read_excel,