diff --git a/pandas/_testing/__init__.py b/pandas/_testing/__init__.py index 7908c9df60df8..de3dd58d3b716 100644 --- a/pandas/_testing/__init__.py +++ b/pandas/_testing/__init__.py @@ -97,7 +97,6 @@ from pandas._testing.contexts import ( decompress_file, ensure_clean, - ensure_safe_environment_variables, raises_chained_assignment_error, set_timezone, use_numexpr, @@ -1104,7 +1103,6 @@ def shares_memory(left, right) -> bool: "EMPTY_STRING_PATTERN", "ENDIAN", "ensure_clean", - "ensure_safe_environment_variables", "equalContents", "external_error_raised", "FLOAT_EA_DTYPES", diff --git a/pandas/_testing/contexts.py b/pandas/_testing/contexts.py index ba2c8c219dc41..f939bd42add93 100644 --- a/pandas/_testing/contexts.py +++ b/pandas/_testing/contexts.py @@ -138,22 +138,6 @@ def ensure_clean( path.unlink() -@contextmanager -def ensure_safe_environment_variables() -> Generator[None, None, None]: - """ - Get a context manager to safely set environment variables - - All changes will be undone on close, hence environment variables set - within this contextmanager will neither persist nor change global state. - """ - saved_environ = dict(os.environ) - try: - yield - finally: - os.environ.clear() - os.environ.update(saved_environ) - - @contextmanager def with_csv_dialect(name: str, **kwargs) -> Generator[None, None, None]: """ diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 68365c125a951..1fc867f95de53 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,4 +1,3 @@ -import os import shlex import subprocess import time @@ -13,8 +12,6 @@ ) import pandas.util._test_decorators as td -import pandas._testing as tm - import pandas.io.common as icom from pandas.io.parsers import read_csv @@ -58,7 +55,13 @@ def s3so(worker_id): @pytest.fixture(scope="session") -def s3_base(worker_id): +def monkeysession(): + with pytest.MonkeyPatch.context() as mp: + yield mp + + +@pytest.fixture(scope="session") +def s3_base(worker_id, monkeysession): """ Fixture for mocking S3 interaction. @@ -68,56 +71,55 @@ def s3_base(worker_id): pytest.importorskip("s3fs") pytest.importorskip("boto3") - with tm.ensure_safe_environment_variables(): - # temporary workaround as moto fails for botocore >= 1.11 otherwise, - # see https://github.com/spulec/moto/issues/1924 & 1952 - os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - if is_ci_environment(): - if is_platform_arm() or is_platform_mac() or is_platform_windows(): - # NOT RUN on Windows/macOS/ARM, only Ubuntu - # - subprocess in CI can cause timeouts - # - GitHub Actions do not support - # container services for the above OSs - # - CircleCI will probably hit the Docker rate pull limit - pytest.skip( - "S3 tests do not have a corresponding service in " - "Windows, macOS or ARM platforms" - ) - else: - yield "http://localhost:5000" + # temporary workaround as moto fails for botocore >= 1.11 otherwise, + # see https://github.com/spulec/moto/issues/1924 & 1952 + monkeysession.setenv("AWS_ACCESS_KEY_ID", "foobar_key") + monkeysession.setenv("AWS_SECRET_ACCESS_KEY", "foobar_secret") + if is_ci_environment(): + if is_platform_arm() or is_platform_mac() or is_platform_windows(): + # NOT RUN on Windows/macOS/ARM, only Ubuntu + # - subprocess in CI can cause timeouts + # - GitHub Actions do not support + # container services for the above OSs + # - CircleCI will probably hit the Docker rate pull limit + pytest.skip( + "S3 tests do not have a corresponding service in " + "Windows, macOS or ARM platforms" + ) else: - requests = pytest.importorskip("requests") - pytest.importorskip("moto", minversion="1.3.14") - pytest.importorskip("flask") # server mode needs flask too - - # Launching moto in server mode, i.e., as a separate process - # with an S3 endpoint on localhost - - worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") - endpoint_port = f"555{worker_id}" - endpoint_uri = f"http://127.0.0.1:{endpoint_port}/" - - # pipe to null to avoid logging in terminal - with subprocess.Popen( - shlex.split(f"moto_server s3 -p {endpoint_port}"), - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) as proc: - timeout = 5 - while timeout > 0: - try: - # OK to go once server is accepting connections - r = requests.get(endpoint_uri) - if r.ok: - break - except Exception: - pass - timeout -= 0.1 - time.sleep(0.1) - yield endpoint_uri - - proc.terminate() + yield "http://localhost:5000" + else: + requests = pytest.importorskip("requests") + pytest.importorskip("moto", minversion="1.3.14") + pytest.importorskip("flask") # server mode needs flask too + + # Launching moto in server mode, i.e., as a separate process + # with an S3 endpoint on localhost + + worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") + endpoint_port = f"555{worker_id}" + endpoint_uri = f"http://127.0.0.1:{endpoint_port}/" + + # pipe to null to avoid logging in terminal + with subprocess.Popen( + shlex.split(f"moto_server s3 -p {endpoint_port}"), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) as proc: + timeout = 5 + while timeout > 0: + try: + # OK to go once server is accepting connections + r = requests.get(endpoint_uri) + if r.ok: + break + except Exception: + pass + timeout -= 0.1 + time.sleep(0.1) + yield endpoint_uri + + proc.terminate() @pytest.fixture diff --git a/pandas/tests/io/test_s3.py b/pandas/tests/io/test_s3.py index 6702d58c139af..5171ec04b0bcf 100644 --- a/pandas/tests/io/test_s3.py +++ b/pandas/tests/io/test_s3.py @@ -1,5 +1,4 @@ from io import BytesIO -import os import pytest @@ -34,17 +33,16 @@ def test_read_without_creds_from_pub_bucket(): @td.skip_if_no("s3fs") @pytest.mark.network @tm.network -def test_read_with_creds_from_pub_bucket(): +def test_read_with_creds_from_pub_bucket(monkeypatch): # Ensure we can read from a public bucket with credentials # GH 34626 # Use Amazon Open Data Registry - https://registry.opendata.aws/gdelt - with tm.ensure_safe_environment_variables(): - # temporary workaround as moto fails for botocore >= 1.11 otherwise, - # see https://github.com/spulec/moto/issues/1924 & 1952 - os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - df = read_csv( - "s3://gdelt-open-data/events/1981.csv", nrows=5, sep="\t", header=None - ) - assert len(df) == 5 + # temporary workaround as moto fails for botocore >= 1.11 otherwise, + # see https://github.com/spulec/moto/issues/1924 & 1952 + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "foobar_key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "foobar_secret") + df = read_csv( + "s3://gdelt-open-data/events/1981.csv", nrows=5, sep="\t", header=None + ) + assert len(df) == 5