diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbce7c02a9b0f..e0e898600ba95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,6 +162,14 @@ jobs: data_manager: name: Test experimental data manager runs-on: ubuntu-latest + services: + moto: + image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret + ports: + - 5000:5000 strategy: matrix: pattern: ["not slow and not network and not clipboard", "slow"] diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index c1255df8f4b20..3fdca951ae058 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -63,6 +63,14 @@ jobs: ports: - 5432:5432 + moto: + image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret + ports: + - 5000:5000 + steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index a7ee4cdad1145..19b96c866923f 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -49,6 +49,14 @@ jobs: # https://github.community/t/concurrecy-not-work-for-push/183068/7 group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }} cancel-in-progress: true + services: + moto: + image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret + ports: + - 5000:5000 steps: - name: Checkout diff --git a/ci/deps/actions-38-db-min.yaml b/ci/deps/actions-38-db-min.yaml index 41d80c89f6f6d..f2b5a49ebfa97 100644 --- a/ci/deps/actions-38-db-min.yaml +++ b/ci/deps/actions-38-db-min.yaml @@ -38,8 +38,6 @@ dependencies: - xlrd - xlsxwriter - xlwt - - moto - - flask # sql - psycopg2=2.8.4 diff --git a/ci/deps/actions-38-db.yaml b/ci/deps/actions-38-db.yaml index 7b73f43b7ba03..1a4e5d12f70df 100644 --- a/ci/deps/actions-38-db.yaml +++ b/ci/deps/actions-38-db.yaml @@ -14,6 +14,7 @@ dependencies: # pandas dependencies - aiobotocore<2.0.0 - beautifulsoup4 + - boto3 - botocore>=1.11 - dask - fastparquet>=0.4.0 @@ -22,8 +23,6 @@ dependencies: - geopandas - html5lib - matplotlib - - moto>=1.3.14 - - flask - nomkl - numexpr - numpy=1.18 diff --git a/ci/deps/actions-38-locale.yaml b/ci/deps/actions-38-locale.yaml index b7043735d9457..d401aa6434dd1 100644 --- a/ci/deps/actions-38-locale.yaml +++ b/ci/deps/actions-38-locale.yaml @@ -14,14 +14,12 @@ dependencies: # pandas dependencies - beautifulsoup4 - - flask - html5lib - ipython - jinja2 - jedi - lxml - matplotlib<3.3.0 - - moto - nomkl - numexpr - numpy<1.20 # GH#39541 compat with pyarrow<3 @@ -34,7 +32,6 @@ dependencies: - xlrd - xlsxwriter - xlwt - - moto - pyarrow=1.0.1 - pip - pip: diff --git a/ci/deps/actions-38-slow.yaml b/ci/deps/actions-38-slow.yaml index 903bd25655bd2..cfafcd679e9b9 100644 --- a/ci/deps/actions-38-slow.yaml +++ b/ci/deps/actions-38-slow.yaml @@ -13,6 +13,7 @@ dependencies: # pandas dependencies - beautifulsoup4 + - boto3 - fsspec>=0.7.4 - html5lib - lxml @@ -27,12 +28,9 @@ dependencies: - python-dateutil - pytz - s3fs>=0.4.0 - - moto>=1.3.14 - scipy - sqlalchemy - xlrd - xlsxwriter - xlwt - - moto - - flask - numba diff --git a/ci/deps/actions-39-slow.yaml b/ci/deps/actions-39-slow.yaml index 2d723354935d2..cb54210e81f23 100644 --- a/ci/deps/actions-39-slow.yaml +++ b/ci/deps/actions-39-slow.yaml @@ -15,14 +15,13 @@ dependencies: # pandas dependencies - beautifulsoup4 - bottleneck + - boto3 - fsspec>=0.8.0 - gcsfs - html5lib - jinja2 - lxml - matplotlib - - moto>=1.3.14 - - flask - numba - numexpr - numpy diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 8751651ece115..dc897453114f1 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -14,14 +14,13 @@ dependencies: # pandas dependencies - beautifulsoup4 - bottleneck + - boto3 - fsspec>=0.8.0 - gcsfs - html5lib - jinja2 - lxml - matplotlib - - moto>=1.3.14 - - flask - numba - numexpr - numpy diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index d4e2c482d1c1c..9b68eb6f4c55b 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -16,10 +16,8 @@ dependencies: - blosc - bottleneck - fastparquet>=0.4.0 - - flask - fsspec>=0.8.0 - matplotlib=3.3.2 - - moto>=1.3.14 - numba - numexpr - numpy=1.18 diff --git a/ci/deps/azure-windows-39.yaml b/ci/deps/azure-windows-39.yaml index 0e352a80a6d34..a0dde78c37261 100644 --- a/ci/deps/azure-windows-39.yaml +++ b/ci/deps/azure-windows-39.yaml @@ -21,8 +21,6 @@ dependencies: - jinja2 - lxml - matplotlib - - moto>=1.3.14 - - flask - numba - numexpr - numpy diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 48e8bfe461764..ba9f9aa3f6a49 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,4 +1,3 @@ -import logging import os import shlex import subprocess @@ -6,6 +5,12 @@ import pytest +from pandas.compat import ( + is_platform_arm, + is_platform_mac, + is_platform_windows, +) + import pandas._testing as tm from pandas.io.parsers import read_csv @@ -35,9 +40,8 @@ def feather_file(datapath): @pytest.fixture -def s3so(worker_id): - worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") - return {"client_kwargs": {"endpoint_url": f"http://127.0.0.1:555{worker_id}/"}} +def s3so(): + return {"client_kwargs": {"endpoint_url": "http://localhost:5000/"}} @pytest.fixture(scope="session") @@ -45,50 +49,63 @@ def s3_base(worker_id): """ Fixture for mocking S3 interaction. - Sets up moto server in separate process + Sets up moto server in separate process locally + Return url for motoserver/moto CI service """ pytest.importorskip("s3fs") pytest.importorskip("boto3") - requests = pytest.importorskip("requests") - logging.getLogger("requests").disabled = True with tm.ensure_safe_environment_variables(): # temporary workaround as moto fails for botocore >= 1.11 otherwise, # see https://github.com/spulec/moto/issues/1924 & 1952 os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - - pytest.importorskip("moto", minversion="1.3.14") - pytest.importorskip("flask") # server mode needs flask too - - # Launching moto in server mode, i.e., as a separate process - # with an S3 endpoint on localhost - - worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") - endpoint_port = f"555{worker_id}" - endpoint_uri = f"http://127.0.0.1:{endpoint_port}/" - - # pipe to null to avoid logging in terminal - with subprocess.Popen( - shlex.split(f"moto_server s3 -p {endpoint_port}"), - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) as proc: - - timeout = 5 - while timeout > 0: - try: - # OK to go once server is accepting connections - r = requests.get(endpoint_uri) - if r.ok: - break - except Exception: - pass - timeout -= 0.1 - time.sleep(0.1) - yield endpoint_uri - - proc.terminate() + if os.environ.get("PANDAS_CI", "0") == "1": + if is_platform_arm() or is_platform_mac() or is_platform_windows(): + # NOT RUN on Windows/MacOS/ARM, only Ubuntu + # - subprocess in CI can cause timeouts + # - Azure pipelines/Github Actions do not support + # container services for the above OSs + # - CircleCI will probably hit the Docker rate pull limit + pytest.skip( + "S3 tests do not have a corresponding service in " + "Windows, MacOS or ARM platforms" + ) + else: + yield "http://localhost:5000" + else: + requests = pytest.importorskip("requests") + pytest.importorskip("moto", minversion="1.3.14") + pytest.importorskip("flask") # server mode needs flask too + + # Launching moto in server mode, i.e., as a separate process + # with an S3 endpoint on localhost + + worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") + endpoint_port = f"555{worker_id}" + endpoint_uri = f"http://127.0.0.1:{endpoint_port}/" + + # pipe to null to avoid logging in terminal + with subprocess.Popen( + shlex.split(f"moto_server s3 -p {endpoint_port}"), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) as proc: + + timeout = 5 + while timeout > 0: + try: + # OK to go once server is accepting connections + r = requests.get(endpoint_uri) + if r.ok: + break + except Exception: + pass + timeout -= 0.1 + time.sleep(0.1) + yield endpoint_uri + + proc.terminate() @pytest.fixture()