From d97aebe98c7a88c77eec7ce14f4fe73961cb40fc Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 24 Nov 2021 20:31:16 -0800 Subject: [PATCH 01/18] WIP: CI/TST: Use CI moto service instead of subprocess --- .github/workflows/posix.yml | 6 ++++ azure-pipelines.yml | 10 ++++++ pandas/tests/io/conftest.py | 61 ++++--------------------------------- 3 files changed, 22 insertions(+), 55 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index f37f31686ef69..f89500b28db01 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -48,6 +48,12 @@ jobs: concurrency: group: ${{ github.ref }}-${{ matrix.settings[0] }} cancel-in-progress: ${{github.event_name == 'pull_request'}} + # https://github.com/qld-gov-au/ckanext-s3filestore/blob/master/.github/workflows/test.yml#L39 + services: + moto: + image: motoserver/moto + ports: + - 5000:5000 steps: - name: Checkout diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6c685d09ab55a..f454ea551ee77 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,6 +19,16 @@ variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas +resources: + containers: + - container: moto + image: motoserver/moto + ports: + - 5000:5000 + +services: + moto: moto + jobs: # Mac and Linux use the same template - template: ci/azure/posix.yml diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 8c34ca8056a07..a9ec26d5f50bb 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,13 +1,7 @@ -import logging -import os -import shlex -import subprocess import time import pytest -import pandas._testing as tm - from pandas.io.parsers import read_csv @@ -35,61 +29,18 @@ def feather_file(datapath): @pytest.fixture -def s3so(worker_id): - worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") - return {"client_kwargs": {"endpoint_url": f"http://127.0.0.1:555{worker_id}/"}} +def s3so(): + return {"client_kwargs": {"endpoint_url": "http://localhost:5000/"}} -@pytest.fixture(scope="session") -def s3_base(worker_id): +@pytest.fixture +def s3_base(): """ Fixture for mocking S3 interaction. - Sets up moto server in separate process + Connects to motoserver/moto CI service """ - pytest.importorskip("s3fs") - pytest.importorskip("boto3") - requests = pytest.importorskip("requests") - logging.getLogger("requests").disabled = True - - with tm.ensure_safe_environment_variables(): - # temporary workaround as moto fails for botocore >= 1.11 otherwise, - # see https://github.com/spulec/moto/issues/1924 & 1952 - os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") - os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - - pytest.importorskip("moto", minversion="1.3.14") - pytest.importorskip("flask") # server mode needs flask too - - # Launching moto in server mode, i.e., as a separate process - # with an S3 endpoint on localhost - - worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") - endpoint_port = f"555{worker_id}" - endpoint_uri = f"http://127.0.0.1:{endpoint_port}/" - - # pipe to null to avoid logging in terminal - proc = subprocess.Popen( - shlex.split(f"moto_server s3 -p {endpoint_port}"), - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - timeout = 5 - while timeout > 0: - try: - # OK to go once server is accepting connections - r = requests.get(endpoint_uri) - if r.ok: - break - except Exception: - pass - timeout -= 0.1 - time.sleep(0.1) - yield endpoint_uri - - proc.terminate() - proc.wait() + return "http://localhost:5000" @pytest.fixture() From 1997621cad2fa5012a577ee270563d78618be300 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 24 Nov 2021 22:57:26 -0800 Subject: [PATCH 02/18] Use importorskip, hardcode CI service --- pandas/tests/io/conftest.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index a9ec26d5f50bb..3c84170375859 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -33,18 +33,8 @@ def s3so(): return {"client_kwargs": {"endpoint_url": "http://localhost:5000/"}} -@pytest.fixture -def s3_base(): - """ - Fixture for mocking S3 interaction. - - Connects to motoserver/moto CI service - """ - return "http://localhost:5000" - - @pytest.fixture() -def s3_resource(s3_base, tips_file, jsonl_file, feather_file): +def s3_resource(tips_file, jsonl_file, feather_file): """ Sets up S3 bucket with contents @@ -59,8 +49,11 @@ def s3_resource(s3_base, tips_file, jsonl_file, feather_file): A private bucket "cant_get_it" is also created. The boto3 s3 resource is yielded by the fixture. """ - import boto3 - import s3fs + boto3 = pytest.importorskip("boto3") + s3fs = pytest.importorskip("s3fs") + + # motoserver/moto CI service + s3_base = "http://localhost:5000" test_s3_files = [ ("tips#1.csv", tips_file), From f6556fe7f73cf22ea66b7349e3604b0793a3aa41 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 10:12:59 -0800 Subject: [PATCH 03/18] Add moto to database --- .github/workflows/database.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index 6da47c86026ed..5145ca2d008fc 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -62,6 +62,11 @@ jobs: ports: - 5432:5432 + moto: + image: motoserver/moto + ports: + - 5000:5000 + steps: - name: Checkout uses: actions/checkout@v2 From 463580ed53438a0121aeeb0926a5016ffdd8524b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 10:19:34 -0800 Subject: [PATCH 04/18] Maybe fix resource format for azure? --- azure-pipelines.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f454ea551ee77..30f78f550bd64 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,27 +19,35 @@ variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas -resources: - containers: - - container: moto - image: motoserver/moto - ports: - - 5000:5000 - -services: - moto: moto - jobs: # Mac and Linux use the same template - template: ci/azure/posix.yml parameters: name: macOS vmImage: macOS-10.15 + resources: + containers: + - container: moto + image: motoserver/moto + ports: + - 5000:5000 + + services: + moto: moto - template: ci/azure/windows.yml parameters: name: Windows vmImage: windows-2019 + resources: + containers: + - container: moto + image: motoserver/moto + ports: + - 5000:5000 + + services: + moto: moto - job: py38_32bit pool: From 29f702521538a4e387459aadd8fc1d6a9484b2f6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 19:32:41 -0800 Subject: [PATCH 05/18] Set environment variables still due to botocore? --- pandas/tests/io/conftest.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 3c84170375859..1875dd45e0e65 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,7 +1,10 @@ +import os import time import pytest +import pandas._testing as tm + from pandas.io.parsers import read_csv @@ -33,8 +36,27 @@ def s3so(): return {"client_kwargs": {"endpoint_url": "http://localhost:5000/"}} +@pytest.fixture(scope="session") +def s3_base(worker_id): + """ + Fixture for mocking S3 interaction. + + Return url for motoserver/moto CI service + """ + pytest.importorskip("boto3") + pytest.importorskip("s3fs") + + with tm.ensure_safe_environment_variables(): + # temporary workaround as moto fails for botocore >= 1.11 otherwise, + # see https://github.com/spulec/moto/issues/1924 & 1952 + os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") + os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") + + return "http://localhost:5000" + + @pytest.fixture() -def s3_resource(tips_file, jsonl_file, feather_file): +def s3_resource(s3_base, tips_file, jsonl_file, feather_file): """ Sets up S3 bucket with contents @@ -49,11 +71,8 @@ def s3_resource(tips_file, jsonl_file, feather_file): A private bucket "cant_get_it" is also created. The boto3 s3 resource is yielded by the fixture. """ - boto3 = pytest.importorskip("boto3") - s3fs = pytest.importorskip("s3fs") - - # motoserver/moto CI service - s3_base = "http://localhost:5000" + import boto3 + import s3fs test_s3_files = [ ("tips#1.csv", tips_file), From 26fdc3ad0611fa0feae0d231fcc0d9aff2e74371 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 19:38:15 -0800 Subject: [PATCH 06/18] Move resource to top level --- azure-pipelines.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 30f78f550bd64..bc95d5720b3dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,19 +19,19 @@ variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas +resources: + containers: + - container: moto + image: motoserver/moto + ports: + - 5000:5000 + jobs: # Mac and Linux use the same template - template: ci/azure/posix.yml parameters: name: macOS vmImage: macOS-10.15 - resources: - containers: - - container: moto - image: motoserver/moto - ports: - - 5000:5000 - services: moto: moto @@ -39,13 +39,6 @@ jobs: parameters: name: Windows vmImage: windows-2019 - resources: - containers: - - container: moto - image: motoserver/moto - ports: - - 5000:5000 - services: moto: moto From 8ce202eb3e835ed457001ca4b2c4d4f8334058ac Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 19:40:16 -0800 Subject: [PATCH 07/18] Reorder pytest.importerskip --- pandas/tests/io/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 1875dd45e0e65..f5bd2f7ab36ef 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -43,8 +43,8 @@ def s3_base(worker_id): Return url for motoserver/moto CI service """ - pytest.importorskip("boto3") pytest.importorskip("s3fs") + pytest.importorskip("boto3") with tm.ensure_safe_environment_variables(): # temporary workaround as moto fails for botocore >= 1.11 otherwise, From 6f170005b286407f960cf5af8dc932fbe96866d2 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 19:53:56 -0800 Subject: [PATCH 08/18] Define azure service in template --- azure-pipelines.yml | 11 ----------- ci/azure/posix.yml | 9 +++++++++ ci/azure/windows.yml | 9 +++++++++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bc95d5720b3dd..6c685d09ab55a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,28 +19,17 @@ variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas -resources: - containers: - - container: moto - image: motoserver/moto - ports: - - 5000:5000 - jobs: # Mac and Linux use the same template - template: ci/azure/posix.yml parameters: name: macOS vmImage: macOS-10.15 - services: - moto: moto - template: ci/azure/windows.yml parameters: name: Windows vmImage: windows-2019 - services: - moto: moto - job: py38_32bit pool: diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index eaac17d50c315..4bc19a6bd7171 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -2,10 +2,19 @@ parameters: name: '' vmImage: '' +resources: + containers: + - container: moto + image: motoserver/moto + ports: + - 5000:5000 + jobs: - job: ${{ parameters.name }} pool: vmImage: ${{ parameters.vmImage }} + services: + moto: moto strategy: matrix: ${{ if eq(parameters.name, 'macOS') }}: diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 3bd20b1399be2..02e5c8ca723cf 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -2,10 +2,19 @@ parameters: name: '' vmImage: '' +resources: + containers: + - container: moto + image: motoserver/moto + ports: + - 5000:5000 + jobs: - job: ${{ parameters.name }} pool: vmImage: ${{ parameters.vmImage }} + services: + moto: moto strategy: matrix: py38_np18_1: From edc88410521b365e7022b00dd836790c79ca5a14 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 20:51:16 -0800 Subject: [PATCH 09/18] Remove moto & flask from CI, add support back for local running --- .github/workflows/database.yml | 3 +++ .github/workflows/posix.yml | 3 +++ ci/deps/actions-38-db-min.yaml | 2 -- ci/deps/actions-38-db.yaml | 2 -- ci/deps/actions-38-locale.yaml | 3 --- ci/deps/actions-38-slow.yaml | 3 --- ci/deps/actions-39-slow.yaml | 2 -- ci/deps/actions-39.yaml | 2 -- ci/deps/azure-windows-38.yaml | 2 -- ci/deps/azure-windows-39.yaml | 2 -- pandas/tests/io/conftest.py | 40 ++++++++++++++++++++++++++++++++-- 11 files changed, 44 insertions(+), 20 deletions(-) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index 5145ca2d008fc..ba9cfa524c3ab 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -64,6 +64,9 @@ jobs: moto: image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret ports: - 5000:5000 diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index f89500b28db01..639af0e867736 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -52,6 +52,9 @@ jobs: services: moto: image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret ports: - 5000:5000 diff --git a/ci/deps/actions-38-db-min.yaml b/ci/deps/actions-38-db-min.yaml index f875f2ef88949..9d419e1ae677e 100644 --- a/ci/deps/actions-38-db-min.yaml +++ b/ci/deps/actions-38-db-min.yaml @@ -38,8 +38,6 @@ dependencies: - xlrd - xlsxwriter - xlwt - - moto - - flask # sql - psycopg2=2.8.4 diff --git a/ci/deps/actions-38-db.yaml b/ci/deps/actions-38-db.yaml index 7b73f43b7ba03..d9d69d11f2943 100644 --- a/ci/deps/actions-38-db.yaml +++ b/ci/deps/actions-38-db.yaml @@ -22,8 +22,6 @@ dependencies: - geopandas - html5lib - matplotlib - - moto>=1.3.14 - - flask - nomkl - numexpr - numpy=1.18 diff --git a/ci/deps/actions-38-locale.yaml b/ci/deps/actions-38-locale.yaml index b7043735d9457..d401aa6434dd1 100644 --- a/ci/deps/actions-38-locale.yaml +++ b/ci/deps/actions-38-locale.yaml @@ -14,14 +14,12 @@ dependencies: # pandas dependencies - beautifulsoup4 - - flask - html5lib - ipython - jinja2 - jedi - lxml - matplotlib<3.3.0 - - moto - nomkl - numexpr - numpy<1.20 # GH#39541 compat with pyarrow<3 @@ -34,7 +32,6 @@ dependencies: - xlrd - xlsxwriter - xlwt - - moto - pyarrow=1.0.1 - pip - pip: diff --git a/ci/deps/actions-38-slow.yaml b/ci/deps/actions-38-slow.yaml index 903bd25655bd2..62c536166354f 100644 --- a/ci/deps/actions-38-slow.yaml +++ b/ci/deps/actions-38-slow.yaml @@ -27,12 +27,9 @@ dependencies: - python-dateutil - pytz - s3fs>=0.4.0 - - moto>=1.3.14 - scipy - sqlalchemy - xlrd - xlsxwriter - xlwt - - moto - - flask - numba diff --git a/ci/deps/actions-39-slow.yaml b/ci/deps/actions-39-slow.yaml index 2d723354935d2..4521a42fe3280 100644 --- a/ci/deps/actions-39-slow.yaml +++ b/ci/deps/actions-39-slow.yaml @@ -21,8 +21,6 @@ dependencies: - jinja2 - lxml - matplotlib - - moto>=1.3.14 - - flask - numba - numexpr - numpy diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 8751651ece115..3663b24f67c76 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -20,8 +20,6 @@ dependencies: - jinja2 - lxml - matplotlib - - moto>=1.3.14 - - flask - numba - numexpr - numpy diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index d4e2c482d1c1c..9b68eb6f4c55b 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -16,10 +16,8 @@ dependencies: - blosc - bottleneck - fastparquet>=0.4.0 - - flask - fsspec>=0.8.0 - matplotlib=3.3.2 - - moto>=1.3.14 - numba - numexpr - numpy=1.18 diff --git a/ci/deps/azure-windows-39.yaml b/ci/deps/azure-windows-39.yaml index 0e352a80a6d34..a0dde78c37261 100644 --- a/ci/deps/azure-windows-39.yaml +++ b/ci/deps/azure-windows-39.yaml @@ -21,8 +21,6 @@ dependencies: - jinja2 - lxml - matplotlib - - moto>=1.3.14 - - flask - numba - numexpr - numpy diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index f5bd2f7ab36ef..5a966b1a95a78 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -1,4 +1,6 @@ import os +import shlex +import subprocess import time import pytest @@ -41,6 +43,7 @@ def s3_base(worker_id): """ Fixture for mocking S3 interaction. + Sets up moto server in separate process locally Return url for motoserver/moto CI service """ pytest.importorskip("s3fs") @@ -51,8 +54,41 @@ def s3_base(worker_id): # see https://github.com/spulec/moto/issues/1924 & 1952 os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - - return "http://localhost:5000" + if os.environ.get("PANDAS_CI", "0") == "1": + return "http://localhost:5000" + else: + requests = pytest.importorskip("requests") + pytest.importorskip("moto", minversion="1.3.14") + pytest.importorskip("flask") # server mode needs flask too + + # Launching moto in server mode, i.e., as a separate process + # with an S3 endpoint on localhost + + worker_id = "5" if worker_id == "master" else worker_id.lstrip("gw") + endpoint_port = f"555{worker_id}" + endpoint_uri = f"http://127.0.0.1:{endpoint_port}/" + + # pipe to null to avoid logging in terminal + with subprocess.Popen( + shlex.split(f"moto_server s3 -p {endpoint_port}"), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) as proc: + + timeout = 5 + while timeout > 0: + try: + # OK to go once server is accepting connections + r = requests.get(endpoint_uri) + if r.ok: + break + except Exception: + pass + timeout -= 0.1 + time.sleep(0.1) + yield endpoint_uri + + proc.terminate() @pytest.fixture() From fe97638639deb8c539c5e0ddbf5142719c05627c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 21:02:50 -0800 Subject: [PATCH 10/18] Apparently templates do not support resources --- azure-pipelines.yml | 7 +++++++ ci/azure/posix.yml | 7 ------- ci/azure/windows.yml | 7 ------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6c685d09ab55a..d983b28cce7fa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,6 +19,13 @@ variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas +resources: + containers: + - container: moto + image: motoserver/moto + ports: + - 5000:5000 + jobs: # Mac and Linux use the same template - template: ci/azure/posix.yml diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 4bc19a6bd7171..bf720c317dc96 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -2,13 +2,6 @@ parameters: name: '' vmImage: '' -resources: - containers: - - container: moto - image: motoserver/moto - ports: - - 5000:5000 - jobs: - job: ${{ parameters.name }} pool: diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 02e5c8ca723cf..6f0668cfc4555 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -2,13 +2,6 @@ parameters: name: '' vmImage: '' -resources: - containers: - - container: moto - image: motoserver/moto - ports: - - 5000:5000 - jobs: - job: ${{ parameters.name }} pool: From 08a20cca7566b1caa7c6699f7dc28e4c7a77adba Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 25 Nov 2021 21:14:03 -0800 Subject: [PATCH 11/18] Dont use service for macOS on azure --- ci/azure/posix.yml | 2 -- pandas/tests/io/conftest.py | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index bf720c317dc96..eaac17d50c315 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -6,8 +6,6 @@ jobs: - job: ${{ parameters.name }} pool: vmImage: ${{ parameters.vmImage }} - services: - moto: moto strategy: matrix: ${{ if eq(parameters.name, 'macOS') }}: diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 5a966b1a95a78..776ed6ca1480c 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -5,6 +5,8 @@ import pytest +from pandas.compat import is_platform_mac + import pandas._testing as tm from pandas.io.parsers import read_csv @@ -54,7 +56,9 @@ def s3_base(worker_id): # see https://github.com/spulec/moto/issues/1924 & 1952 os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - if os.environ.get("PANDAS_CI", "0") == "1": + if os.environ.get("PANDAS_CI", "0") == "1" and not is_platform_mac(): + # MacOS on Azure pipelines does not support services + # Github Actions does... return "http://localhost:5000" else: requests = pytest.importorskip("requests") From 6f8c2730438829d5a59c46734f55c2f0e0cec4d0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 26 Nov 2021 16:48:33 -0800 Subject: [PATCH 12/18] Revert for window and mac --- azure-pipelines.yml | 7 ------- ci/azure/windows.yml | 2 -- pandas/tests/io/conftest.py | 13 +++++++++---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d983b28cce7fa..6c685d09ab55a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,13 +19,6 @@ variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas -resources: - containers: - - container: moto - image: motoserver/moto - ports: - - 5000:5000 - jobs: # Mac and Linux use the same template - template: ci/azure/posix.yml diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 6f0668cfc4555..3bd20b1399be2 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -6,8 +6,6 @@ jobs: - job: ${{ parameters.name }} pool: vmImage: ${{ parameters.vmImage }} - services: - moto: moto strategy: matrix: py38_np18_1: diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 776ed6ca1480c..0e2d190fab2e9 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -5,7 +5,10 @@ import pytest -from pandas.compat import is_platform_mac +from pandas.compat import ( + is_platform_mac, + is_platform_windows, +) import pandas._testing as tm @@ -56,9 +59,11 @@ def s3_base(worker_id): # see https://github.com/spulec/moto/issues/1924 & 1952 os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - if os.environ.get("PANDAS_CI", "0") == "1" and not is_platform_mac(): - # MacOS on Azure pipelines does not support services - # Github Actions does... + if os.environ.get("PANDAS_CI", "0") == "1" and not ( + is_platform_mac() or is_platform_windows() + ): + # Windows/MacOS on Azure pipelines/Github Actions + # does not support a moto services return "http://localhost:5000" else: requests = pytest.importorskip("requests") From ef4b020488ea7be6bceb587f089a7fa1ad3b7b8f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 26 Nov 2021 17:29:52 -0800 Subject: [PATCH 13/18] Add back deps for windows, ignore for circleci --- ci/deps/azure-windows-38.yaml | 2 ++ ci/deps/azure-windows-39.yaml | 2 ++ pandas/tests/io/conftest.py | 6 ++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index 9b68eb6f4c55b..d4e2c482d1c1c 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -16,8 +16,10 @@ dependencies: - blosc - bottleneck - fastparquet>=0.4.0 + - flask - fsspec>=0.8.0 - matplotlib=3.3.2 + - moto>=1.3.14 - numba - numexpr - numpy=1.18 diff --git a/ci/deps/azure-windows-39.yaml b/ci/deps/azure-windows-39.yaml index a0dde78c37261..0e352a80a6d34 100644 --- a/ci/deps/azure-windows-39.yaml +++ b/ci/deps/azure-windows-39.yaml @@ -21,6 +21,8 @@ dependencies: - jinja2 - lxml - matplotlib + - moto>=1.3.14 + - flask - numba - numexpr - numpy diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 0e2d190fab2e9..4b1109e8ffc66 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -6,6 +6,7 @@ import pytest from pandas.compat import ( + is_platform_arm, is_platform_mac, is_platform_windows, ) @@ -60,10 +61,11 @@ def s3_base(worker_id): os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") if os.environ.get("PANDAS_CI", "0") == "1" and not ( - is_platform_mac() or is_platform_windows() + is_platform_mac() or is_platform_windows() or is_platform_arm() ): # Windows/MacOS on Azure pipelines/Github Actions - # does not support a moto services + # does not support a container service + # Service on CircleCI will probably hit the Docker rate pull limit return "http://localhost:5000" else: requests = pytest.importorskip("requests") From 9693baa1f73b94a6624b54d312d47ab30dddc03f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 26 Nov 2021 20:33:39 -0800 Subject: [PATCH 14/18] Add boto3 to run s3_resource tests --- ci/deps/actions-38-db.yaml | 1 + ci/deps/actions-38-slow.yaml | 1 + ci/deps/actions-39-slow.yaml | 1 + ci/deps/actions-39.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/ci/deps/actions-38-db.yaml b/ci/deps/actions-38-db.yaml index d9d69d11f2943..1a4e5d12f70df 100644 --- a/ci/deps/actions-38-db.yaml +++ b/ci/deps/actions-38-db.yaml @@ -14,6 +14,7 @@ dependencies: # pandas dependencies - aiobotocore<2.0.0 - beautifulsoup4 + - boto3 - botocore>=1.11 - dask - fastparquet>=0.4.0 diff --git a/ci/deps/actions-38-slow.yaml b/ci/deps/actions-38-slow.yaml index 62c536166354f..cfafcd679e9b9 100644 --- a/ci/deps/actions-38-slow.yaml +++ b/ci/deps/actions-38-slow.yaml @@ -13,6 +13,7 @@ dependencies: # pandas dependencies - beautifulsoup4 + - boto3 - fsspec>=0.7.4 - html5lib - lxml diff --git a/ci/deps/actions-39-slow.yaml b/ci/deps/actions-39-slow.yaml index 4521a42fe3280..cb54210e81f23 100644 --- a/ci/deps/actions-39-slow.yaml +++ b/ci/deps/actions-39-slow.yaml @@ -15,6 +15,7 @@ dependencies: # pandas dependencies - beautifulsoup4 - bottleneck + - boto3 - fsspec>=0.8.0 - gcsfs - html5lib diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 3663b24f67c76..dc897453114f1 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -14,6 +14,7 @@ dependencies: # pandas dependencies - beautifulsoup4 - bottleneck + - boto3 - fsspec>=0.8.0 - gcsfs - html5lib From 70a20d85e7bfef537166ce064a00fb2172877bfe Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 26 Nov 2021 21:26:31 -0800 Subject: [PATCH 15/18] Have fixture yield, don't run on windows --- ci/deps/azure-windows-38.yaml | 2 -- ci/deps/azure-windows-39.yaml | 2 -- pandas/tests/io/conftest.py | 12 ++---------- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index d4e2c482d1c1c..9b68eb6f4c55b 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -16,10 +16,8 @@ dependencies: - blosc - bottleneck - fastparquet>=0.4.0 - - flask - fsspec>=0.8.0 - matplotlib=3.3.2 - - moto>=1.3.14 - numba - numexpr - numpy=1.18 diff --git a/ci/deps/azure-windows-39.yaml b/ci/deps/azure-windows-39.yaml index 0e352a80a6d34..a0dde78c37261 100644 --- a/ci/deps/azure-windows-39.yaml +++ b/ci/deps/azure-windows-39.yaml @@ -21,8 +21,6 @@ dependencies: - jinja2 - lxml - matplotlib - - moto>=1.3.14 - - flask - numba - numexpr - numpy diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 4b1109e8ffc66..975a54d344105 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -5,12 +5,6 @@ import pytest -from pandas.compat import ( - is_platform_arm, - is_platform_mac, - is_platform_windows, -) - import pandas._testing as tm from pandas.io.parsers import read_csv @@ -60,13 +54,11 @@ def s3_base(worker_id): # see https://github.com/spulec/moto/issues/1924 & 1952 os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - if os.environ.get("PANDAS_CI", "0") == "1" and not ( - is_platform_mac() or is_platform_windows() or is_platform_arm() - ): + if os.environ.get("PANDAS_CI", "0") == "1": # Windows/MacOS on Azure pipelines/Github Actions # does not support a container service # Service on CircleCI will probably hit the Docker rate pull limit - return "http://localhost:5000" + yield "http://localhost:5000" else: requests = pytest.importorskip("requests") pytest.importorskip("moto", minversion="1.3.14") From 762a79b3a5ddb6b721bf158e01d08cefaf71ec96 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 26 Nov 2021 22:44:19 -0800 Subject: [PATCH 16/18] Add service to arraymanager test --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23e452f682b60..a14d8237ac5f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,14 @@ jobs: data_manager: name: Test experimental data manager runs-on: ubuntu-latest + services: + moto: + image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret + ports: + - 5000:5000 strategy: matrix: pattern: ["not slow and not network and not clipboard", "slow"] From 09481d546c003e72333d07ddd640200919391859 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Sun, 28 Nov 2021 12:20:43 -0800 Subject: [PATCH 17/18] Add comments and remove url --- .github/workflows/posix.yml | 1 - pandas/tests/io/conftest.py | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 639af0e867736..55f95ed41b12b 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -48,7 +48,6 @@ jobs: concurrency: group: ${{ github.ref }}-${{ matrix.settings[0] }} cancel-in-progress: ${{github.event_name == 'pull_request'}} - # https://github.com/qld-gov-au/ckanext-s3filestore/blob/master/.github/workflows/test.yml#L39 services: moto: image: motoserver/moto diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index 975a54d344105..fb90e9c3c8f93 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -55,9 +55,11 @@ def s3_base(worker_id): os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") if os.environ.get("PANDAS_CI", "0") == "1": - # Windows/MacOS on Azure pipelines/Github Actions - # does not support a container service - # Service on CircleCI will probably hit the Docker rate pull limit + # NOT RUN on Windows/MacOS/ARM, only Ubuntu + # - subprocess in CI can cause timeouts + # - Azure pipelines/Github Actions do not support + # container services for the above OSs + # - CircleCI will probably hit the Docker rate pull limit yield "http://localhost:5000" else: requests = pytest.importorskip("requests") From c2eed39de19d5a38cb51cb18d9d5ee31f3ff86d0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Sun, 28 Nov 2021 17:18:59 -0800 Subject: [PATCH 18/18] Be very explicit just in case the dependencies get added to the build --- pandas/tests/io/conftest.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/conftest.py b/pandas/tests/io/conftest.py index fb90e9c3c8f93..ba9f9aa3f6a49 100644 --- a/pandas/tests/io/conftest.py +++ b/pandas/tests/io/conftest.py @@ -5,6 +5,12 @@ import pytest +from pandas.compat import ( + is_platform_arm, + is_platform_mac, + is_platform_windows, +) + import pandas._testing as tm from pandas.io.parsers import read_csv @@ -55,12 +61,18 @@ def s3_base(worker_id): os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") if os.environ.get("PANDAS_CI", "0") == "1": - # NOT RUN on Windows/MacOS/ARM, only Ubuntu - # - subprocess in CI can cause timeouts - # - Azure pipelines/Github Actions do not support - # container services for the above OSs - # - CircleCI will probably hit the Docker rate pull limit - yield "http://localhost:5000" + if is_platform_arm() or is_platform_mac() or is_platform_windows(): + # NOT RUN on Windows/MacOS/ARM, only Ubuntu + # - subprocess in CI can cause timeouts + # - Azure pipelines/Github Actions do not support + # container services for the above OSs + # - CircleCI will probably hit the Docker rate pull limit + pytest.skip( + "S3 tests do not have a corresponding service in " + "Windows, MacOS or ARM platforms" + ) + else: + yield "http://localhost:5000" else: requests = pytest.importorskip("requests") pytest.importorskip("moto", minversion="1.3.14")