From 953b88ac28a4363638358fefb5165ef01567692b Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 2 Jan 2021 12:01:34 -0600 Subject: [PATCH 1/9] CI: test 2 workers on arm64 #36719 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8ede978074a9c..584c8f8191c4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,13 +37,13 @@ matrix: include: - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=2 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" allow_failures: # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719 - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=2 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" before_install: From 351c379e87b629b810a00356fc129910b0b525e3 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 2 Jan 2021 13:36:18 -0600 Subject: [PATCH 2/9] CI: test 4 workers on arm64 #36719 --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 584c8f8191c4b..93a4a3d5255e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,13 +37,13 @@ matrix: include: - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=2 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=auto ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" allow_failures: # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719 - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=2 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=auto ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" before_install: @@ -60,9 +60,9 @@ before_install: install: - echo "install start" - - ci/prep_cython_cache.sh + # - ci/prep_cython_cache.sh - ci/setup_env.sh - - ci/submit_cython_cache.sh + # - ci/submit_cython_cache.sh - echo "install done" script: From 8dbd8c0537e935b648ca2d18dcb70bc8367804b4 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 2 Jan 2021 14:23:26 -0600 Subject: [PATCH 3/9] CI: test 4 workers on arm64 #36719 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93a4a3d5255e7..0eae5f3e2a1b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,13 +37,13 @@ matrix: include: - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=auto ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=4 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" allow_failures: # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719 - arch: arm64 env: - - JOB="3.7, arm64" PYTEST_WORKERS=auto ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" + - JOB="3.7, arm64" PYTEST_WORKERS=4 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" before_install: From 89c720709bcf6ec7c3e2748816bde9efa492a5f5 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 2 Jan 2021 22:23:58 -0600 Subject: [PATCH 4/9] update miniforge version --- ci/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index c36422884f2ec..6ad4b85aba2a6 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -37,7 +37,7 @@ else fi if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh" + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.9.2-5/Miniforge3-4.9.2-5-Linux-aarch64.sh" else CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" fi From 630b023cf303d81483edb90e198858ed9304af44 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 2 Jan 2021 23:00:47 -0600 Subject: [PATCH 5/9] try dist=no --- .travis.yml | 4 ++-- ci/run_tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0eae5f3e2a1b3..5b6f92b94f226 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,9 +60,9 @@ before_install: install: - echo "install start" - # - ci/prep_cython_cache.sh + - ci/prep_cython_cache.sh - ci/setup_env.sh - # - ci/submit_cython_cache.sh + - ci/submit_cython_cache.sh - echo "install done" script: diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 593939431d5eb..ea820bdd8138d 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -20,7 +20,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=no -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then # GH#37455 windows py38 build appears to be running out of memory From 8000531e4ba59bca175be9ad7760b327801887f2 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 3 Jan 2021 10:34:43 -0600 Subject: [PATCH 6/9] xfail tests --- .travis.yml | 9 +-------- pandas/compat/__init__.py | 1 + pandas/tests/indexes/interval/test_astype.py | 3 +++ pandas/tests/tools/test_to_numeric.py | 4 +++- pandas/tests/window/test_rolling.py | 2 ++ 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b6f92b94f226..e97994262e825 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ env: git: depth: false -matrix: +jobs: fast_finish: true include: @@ -39,13 +39,6 @@ matrix: env: - JOB="3.7, arm64" PYTEST_WORKERS=4 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" - allow_failures: - # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719 - - arch: arm64 - env: - - JOB="3.7, arm64" PYTEST_WORKERS=4 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)" - - before_install: - echo "before_install" # Use blocking IO on travis. Ref: https://github.com/travis-ci/travis-ci/issues/8920#issuecomment-352661024 diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index 2ac9b9e2c875c..1a429b65b52db 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -17,6 +17,7 @@ PY39 = sys.version_info >= (3, 9) PYPY = platform.python_implementation() == "PyPy" IS64 = sys.maxsize > 2 ** 32 +ARM64 = platform.machine() == "arm64" def set_function_name(f: F, name: str, cls) -> F: diff --git a/pandas/tests/indexes/interval/test_astype.py b/pandas/tests/indexes/interval/test_astype.py index 840e3f641e8ba..8d306e73a6545 100644 --- a/pandas/tests/indexes/interval/test_astype.py +++ b/pandas/tests/indexes/interval/test_astype.py @@ -3,6 +3,8 @@ import numpy as np import pytest +from pandas.compat import ARM64 + from pandas.core.dtypes.dtypes import CategoricalDtype, IntervalDtype from pandas import ( @@ -165,6 +167,7 @@ def test_subtype_integer_with_non_integer_borders(self, subtype): ) tm.assert_index_equal(result, expected) + @pytest.mark.xfail(ARM64, reason="GH 38923") def test_subtype_integer_errors(self): # float64 -> uint64 fails with negative values index = interval_range(-10.0, 10.0) diff --git a/pandas/tests/tools/test_to_numeric.py b/pandas/tests/tools/test_to_numeric.py index 80446e464985c..0e5612a191a02 100644 --- a/pandas/tests/tools/test_to_numeric.py +++ b/pandas/tests/tools/test_to_numeric.py @@ -4,6 +4,8 @@ from numpy import iinfo import pytest +from pandas.compat import ARM64 + import pandas as pd from pandas import DataFrame, Index, Series, to_numeric import pandas._testing as tm @@ -747,7 +749,7 @@ def test_to_numeric_from_nullable_string(values, expected): "UInt64", "signed", "UInt64", - marks=pytest.mark.xfail(reason="GH38798"), + marks=pytest.mark.xfail(not ARM64, reason="GH38798"), ), ([1, 1], "Int64", "unsigned", "UInt8"), ([1.0, 1.0], "Float32", "unsigned", "UInt8"), diff --git a/pandas/tests/window/test_rolling.py b/pandas/tests/window/test_rolling.py index 84056299093cf..a1bd672a3c42e 100644 --- a/pandas/tests/window/test_rolling.py +++ b/pandas/tests/window/test_rolling.py @@ -3,6 +3,7 @@ import numpy as np import pytest +from pandas.compat import ARM64 from pandas.errors import UnsupportedFunctionCall from pandas import ( @@ -891,6 +892,7 @@ def test_rolling_sem(frame_or_series): tm.assert_series_equal(result, expected) +@pytest.mark.xfail(ARM64, reason="GH 38921") @pytest.mark.parametrize( ("func", "third_value", "values"), [ From 57fe2e58fbb908b4c0ffc4e2360241a0a2222d77 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 3 Jan 2021 11:45:44 -0600 Subject: [PATCH 7/9] add comment --- ci/run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index ea820bdd8138d..b50b1c104b4af 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -20,6 +20,10 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi +# With --dist=no, pytest distributs one test at a time. +# If using other options, running the test suite would be extremely slow on arm64 machine. +# See https://pypi.org/project/pytest-xdist/ for details. +# GH 36719 PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=no -s --strict-markers --durations=30 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then From 83ca3fed54f322ae36036d7edce4a1b5f6b8febd Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 3 Jan 2021 14:13:26 -0600 Subject: [PATCH 8/9] check platform --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e97994262e825..01336f59e0b3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,6 +62,7 @@ script: - echo "script start" - echo "$JOB" - source activate pandas-dev + - python -c "import platform; print(platform.machine());" - ci/run_tests.sh after_script: From 7941a15ed65920598444a71016a8b166e73a0be8 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 3 Jan 2021 15:29:49 -0600 Subject: [PATCH 9/9] fix arch check --- .travis.yml | 1 - pandas/compat/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01336f59e0b3a..e97994262e825 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,7 +62,6 @@ script: - echo "script start" - echo "$JOB" - source activate pandas-dev - - python -c "import platform; print(platform.machine());" - ci/run_tests.sh after_script: diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index 1a429b65b52db..9081bc29f5271 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -17,7 +17,7 @@ PY39 = sys.version_info >= (3, 9) PYPY = platform.python_implementation() == "PyPy" IS64 = sys.maxsize > 2 ** 32 -ARM64 = platform.machine() == "arm64" +ARM64 = platform.machine() == "arm64" or platform.machine() == "aarch64" def set_function_name(f: F, name: str, cls) -> F: