diff --git a/.travis.yml b/.travis.yml index 03026647d6bb8..e0ab770ac46ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,6 +90,12 @@ before_install: - uname -a - git --version - git tag + # Because travis runs on Google Cloud and has a /etc/boto.cfg, + # it breaks moto import, see: + # https://github.com/spulec/moto/issues/1771 + # https://github.com/boto/boto/issues/3741 + # This overrides travis and tells it to look nowhere. + - export BOTO_CONFIG=/dev/null install: - echo "install start" diff --git a/ci/deps/travis-36.yaml b/ci/deps/travis-36.yaml index bfd69652730ed..37cea281d48a6 100644 --- a/ci/deps/travis-36.yaml +++ b/ci/deps/travis-36.yaml @@ -4,6 +4,7 @@ channels: - conda-forge dependencies: - beautifulsoup4 + - botocore>=1.11 - cython>=0.28.2 - dask - fastparquet @@ -35,10 +36,10 @@ dependencies: - pytest - pytest-xdist - pytest-cov - - moto - hypothesis>=3.58.0 - pip: - brotlipy - coverage + - moto - pandas-datareader - python-dateutil diff --git a/ci/deps/travis-37.yaml b/ci/deps/travis-37.yaml index a297786f6b14d..c503124d8cd26 100644 --- a/ci/deps/travis-37.yaml +++ b/ci/deps/travis-37.yaml @@ -5,6 +5,7 @@ channels: - c3i_test dependencies: - python=3.7 + - botocore>=1.11 - cython>=0.28.2 - numpy - python-dateutil @@ -14,3 +15,6 @@ dependencies: - pytest - pytest-xdist - hypothesis>=3.58.0 + - s3fs + - pip: + - moto diff --git a/environment.yml b/environment.yml index e31511e5b8afe..370001dffe683 100644 --- a/environment.yml +++ b/environment.yml @@ -26,6 +26,8 @@ dependencies: # optional - beautifulsoup4>=4.2.1 - blosc + - botocore>=1.11 + - boto3 - bottleneck>=1.2.0 - fastparquet>=0.1.2 - html5lib @@ -41,6 +43,7 @@ dependencies: - pytables>=3.4.2 - pytest-cov - pytest-xdist + - s3fs - scipy>=1.1 - seaborn - sqlalchemy diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 928519d39aed3..af6f7ac4ef528 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,4 +1,10 @@ +from distutils.version import LooseVersion +import os + import pytest + +import pandas.util.testing as tm + from pandas.io.parsers import read_csv @@ -37,43 +43,48 @@ def s3_resource(tips_file, jsonl_file): """ pytest.importorskip('s3fs') boto3 = pytest.importorskip('boto3') - # GH-24092. See if boto.plugin skips the test or fails. - try: - pytest.importorskip("boto.plugin") - except AttributeError: - raise pytest.skip("moto/moto error") - moto = pytest.importorskip('moto') - - test_s3_files = [ - ('tips.csv', tips_file), - ('tips.csv.gz', tips_file + '.gz'), - ('tips.csv.bz2', tips_file + '.bz2'), - ('items.jsonl', jsonl_file), - ] - - def add_tips_files(bucket_name): - for s3_key, file_name in test_s3_files: - with open(file_name, 'rb') as f: - conn.Bucket(bucket_name).put_object( - Key=s3_key, - Body=f) - - try: - - s3 = moto.mock_s3() - s3.start() - - # see gh-16135 - bucket = 'pandas-test' - conn = boto3.resource("s3", region_name="us-east-1") - - conn.create_bucket(Bucket=bucket) - add_tips_files(bucket) - - conn.create_bucket(Bucket='cant_get_it', ACL='private') - add_tips_files('cant_get_it') - yield conn - except: # noqa: flake8 - pytest.skip("failure to use s3 resource") - finally: - s3.stop() + botocore = pytest.importorskip('botocore') + + if LooseVersion(botocore.__version__) < LooseVersion("1.11.0"): + # botocore leaks an uncatchable ResourceWarning before 1.11.0; + # see GH 23731 and https://github.com/boto/botocore/issues/1464 + pytest.skip("botocore is leaking resources before 1.11.0") + + with tm.ensure_safe_environment_variables(): + # temporary workaround as moto fails for botocore >= 1.11 otherwise, + # see https://github.com/spulec/moto/issues/1924 & 1952 + os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") + os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") + + moto = pytest.importorskip('moto') + + test_s3_files = [ + ('tips.csv', tips_file), + ('tips.csv.gz', tips_file + '.gz'), + ('tips.csv.bz2', tips_file + '.bz2'), + ('items.jsonl', jsonl_file), + ] + + def add_tips_files(bucket_name): + for s3_key, file_name in test_s3_files: + with open(file_name, 'rb') as f: + conn.Bucket(bucket_name).put_object( + Key=s3_key, + Body=f) + + try: + s3 = moto.mock_s3() + s3.start() + + # see gh-16135 + bucket = 'pandas-test' + conn = boto3.resource("s3", region_name="us-east-1") + + conn.create_bucket(Bucket=bucket) + add_tips_files(bucket) + + conn.create_bucket(Bucket='cant_get_it', ACL='private') + add_tips_files('cant_get_it') + yield conn + finally: + s3.stop() diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 56a3cda1ba89f..8a408f5613a01 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -782,6 +782,22 @@ def ensure_clean_dir(): pass +@contextmanager +def ensure_safe_environment_variables(): + """ + Get a context manager to safely set environment variables + + All changes will be undone on close, hence environment variables set + within this contextmanager will neither persist nor change global state. + """ + saved_environ = dict(os.environ) + try: + yield + finally: + os.environ.clear() + os.environ.update(saved_environ) + + # ----------------------------------------------------------------------------- # Comparators diff --git a/requirements-dev.txt b/requirements-dev.txt index facadf384f770..ae32c82aacd58 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,6 +15,8 @@ sphinx sphinxcontrib-spelling beautifulsoup4>=4.2.1 blosc +botocore>=1.11 +boto3 bottleneck>=1.2.0 fastparquet>=0.1.2 html5lib @@ -30,6 +32,7 @@ pyarrow>=0.7.0 tables>=3.4.2 pytest-cov pytest-xdist +s3fs scipy>=1.1 seaborn sqlalchemy