Skip to content

Commit 9c35ac9

Browse files
mroeschkemroeschke
authored
TST: Use monkeypatch instead of custom func for env variables (#53563)
* TST: Use monkeypatch instead of custom func for env variables * Make scopes match
1 parent 6e3c37c commit 9c35ac9

File tree

4 files changed

+64
-82
lines changed

4 files changed

+64
-82
lines changed

‎pandas/_testing/__init__.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,6 @@
9797
from pandas._testing.contexts import (
9898
decompress_file,
9999
ensure_clean,
100-
ensure_safe_environment_variables,
101100
raises_chained_assignment_error,
102101
set_timezone,
103102
use_numexpr,
@@ -1104,7 +1103,6 @@ def shares_memory(left, right) -> bool:
11041103
"EMPTY_STRING_PATTERN",
11051104
"ENDIAN",
11061105
"ensure_clean",
1107-
"ensure_safe_environment_variables",
11081106
"equalContents",
11091107
"external_error_raised",
11101108
"FLOAT_EA_DTYPES",

‎pandas/_testing/contexts.py

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -138,22 +138,6 @@ def ensure_clean(
138138
path.unlink()
139139

140140

141-
@contextmanager
142-
def ensure_safe_environment_variables() -> Generator[None, None, None]:
143-
"""
144-
Get a context manager to safely set environment variables
145-
146-
All changes will be undone on close, hence environment variables set
147-
within this contextmanager will neither persist nor change global state.
148-
"""
149-
saved_environ = dict(os.environ)
150-
try:
151-
yield
152-
finally:
153-
os.environ.clear()
154-
os.environ.update(saved_environ)
155-
156-
157141
@contextmanager
158142
def with_csv_dialect(name: str, **kwargs) -> Generator[None, None, None]:
159143
"""

‎pandas/tests/io/conftest.py

Lines changed: 55 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import os
21
import shlex
32
import subprocess
43
import time
@@ -13,8 +12,6 @@
1312
)
1413
import pandas.util._test_decorators as td
1514

16-
import pandas._testing as tm
17-
1815
import pandas.io.common as icom
1916
from pandas.io.parsers import read_csv
2017

@@ -58,7 +55,13 @@ def s3so(worker_id):
5855

5956

6057
@pytest.fixture(scope="session")
61-
def s3_base(worker_id):
58+
def monkeysession():
59+
with pytest.MonkeyPatch.context() as mp:
60+
yield mp
61+
62+
63+
@pytest.fixture(scope="session")
64+
def s3_base(worker_id, monkeysession):
6265
"""
6366
Fixture for mocking S3 interaction.
6467
@@ -68,56 +71,55 @@ def s3_base(worker_id):
6871
pytest.importorskip("s3fs")
6972
pytest.importorskip("boto3")
7073

71-
with tm.ensure_safe_environment_variables():
72-
# temporary workaround as moto fails for botocore >= 1.11 otherwise,
73-
# see https://github.com/spulec/moto/issues/1924 & 1952
74-
os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
75-
os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
76-
if is_ci_environment():
77-
if is_platform_arm() or is_platform_mac() or is_platform_windows():
78-
# NOT RUN on Windows/macOS/ARM, only Ubuntu
79-
# - subprocess in CI can cause timeouts
80-
# - GitHub Actions do not support
81-
# container services for the above OSs
82-
# - CircleCI will probably hit the Docker rate pull limit
83-
pytest.skip(
84-
"S3 tests do not have a corresponding service in "
85-
"Windows, macOS or ARM platforms"
86-
)
87-
else:
88-
yield "http://localhost:5000"
74+
# temporary workaround as moto fails for botocore >= 1.11 otherwise,
75+
# see https://github.com/spulec/moto/issues/1924 & 1952
76+
monkeysession.setenv("AWS_ACCESS_KEY_ID", "foobar_key")
77+
monkeysession.setenv("AWS_SECRET_ACCESS_KEY", "foobar_secret")
78+
if is_ci_environment():
79+
if is_platform_arm() or is_platform_mac() or is_platform_windows():
80+
# NOT RUN on Windows/macOS/ARM, only Ubuntu
81+
# - subprocess in CI can cause timeouts
82+
# - GitHub Actions do not support
83+
# container services for the above OSs
84+
# - CircleCI will probably hit the Docker rate pull limit
85+
pytest.skip(
86+
"S3 tests do not have a corresponding service in "
87+
"Windows, macOS or ARM platforms"
88+
)
8989
else:
90-
requests = pytest.importorskip("requests")
91-
pytest.importorskip("moto", minversion="1.3.14")
92-
pytest.importorskip("flask") # server mode needs flask too
93-
94-
# Launching moto in server mode, i.e., as a separate process
95-
# with an S3 endpoint on localhost
96-
97-
worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw")
98-
endpoint_port = f"555{worker_id}"
99-
endpoint_uri = f"http://127.0.0.1:{endpoint_port}/"
100-
101-
# pipe to null to avoid logging in terminal
102-
with subprocess.Popen(
103-
shlex.split(f"moto_server s3 -p {endpoint_port}"),
104-
stdout=subprocess.DEVNULL,
105-
stderr=subprocess.DEVNULL,
106-
) as proc:
107-
timeout = 5
108-
while timeout > 0:
109-
try:
110-
# OK to go once server is accepting connections
111-
r = requests.get(endpoint_uri)
112-
if r.ok:
113-
break
114-
except Exception:
115-
pass
116-
timeout -= 0.1
117-
time.sleep(0.1)
118-
yield endpoint_uri
119-
120-
proc.terminate()
90+
yield "http://localhost:5000"
91+
else:
92+
requests = pytest.importorskip("requests")
93+
pytest.importorskip("moto", minversion="1.3.14")
94+
pytest.importorskip("flask") # server mode needs flask too
95+
96+
# Launching moto in server mode, i.e., as a separate process
97+
# with an S3 endpoint on localhost
98+
99+
worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw")
100+
endpoint_port = f"555{worker_id}"
101+
endpoint_uri = f"http://127.0.0.1:{endpoint_port}/"
102+
103+
# pipe to null to avoid logging in terminal
104+
with subprocess.Popen(
105+
shlex.split(f"moto_server s3 -p {endpoint_port}"),
106+
stdout=subprocess.DEVNULL,
107+
stderr=subprocess.DEVNULL,
108+
) as proc:
109+
timeout = 5
110+
while timeout > 0:
111+
try:
112+
# OK to go once server is accepting connections
113+
r = requests.get(endpoint_uri)
114+
if r.ok:
115+
break
116+
except Exception:
117+
pass
118+
timeout -= 0.1
119+
time.sleep(0.1)
120+
yield endpoint_uri
121+
122+
proc.terminate()
121123

122124

123125
@pytest.fixture

‎pandas/tests/io/test_s3.py

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
from io import BytesIO
2-
import os
32

43
import pytest
54

@@ -34,17 +33,16 @@ def test_read_without_creds_from_pub_bucket():
3433
@td.skip_if_no("s3fs")
3534
@pytest.mark.network
3635
@tm.network
37-
def test_read_with_creds_from_pub_bucket():
36+
def test_read_with_creds_from_pub_bucket(monkeypatch):
3837
# Ensure we can read from a public bucket with credentials
3938
# GH 34626
4039
# Use Amazon Open Data Registry - https://registry.opendata.aws/gdelt
4140

42-
with tm.ensure_safe_environment_variables():
43-
# temporary workaround as moto fails for botocore >= 1.11 otherwise,
44-
# see https://github.com/spulec/moto/issues/1924 & 1952
45-
os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
46-
os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
47-
df = read_csv(
48-
"s3://gdelt-open-data/events/1981.csv", nrows=5, sep="\t", header=None
49-
)
50-
assert len(df) == 5
41+
# temporary workaround as moto fails for botocore >= 1.11 otherwise,
42+
# see https://github.com/spulec/moto/issues/1924 & 1952
43+
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "foobar_key")
44+
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "foobar_secret")
45+
df = read_csv(
46+
"s3://gdelt-open-data/events/1981.csv", nrows=5, sep="\t", header=None
47+
)
48+
assert len(df) == 5

0 commit comments

Comments
 (0)