From 38a67725d2c13948a97280f8bf47dfb54eb9478d Mon Sep 17 00:00:00 2001 From: ossdev07 Date: Fri, 3 Jan 2020 10:50:48 +0000 Subject: [PATCH 1/4] added travis-ci support for arm64 Signed-off-by: ossdev07 --- .travis.yml | 9 +++- ci/deps/travis-37-aarch64.yaml | 22 +++++++++ ci/run_tests.sh | 33 ++++++++++--- ci/setup_env.sh | 86 ++++++++++++++++++++++++++++------ 4 files changed, 127 insertions(+), 23 deletions(-) create mode 100644 ci/deps/travis-37-aarch64.yaml diff --git a/.travis.yml b/.travis.yml index 98826a2d9e115..3958912db4daf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,13 @@ matrix: - env: - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)" + - dist: trusty + os: linux + sudo: true + arch: arm64 + env: + - JOB="3.7" ENV_FILE="ci/deps/travis-37-aarch64.yaml" PATTERN="(not slow and not network)" + - env: - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1" services: @@ -98,6 +105,6 @@ script: after_script: - echo "after_script start" - - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + - source activate pandas-dev && pushd /tmp && if [ `uname -m` = 'aarch64' ];then VAR="python3.7" ; else VAR="python";fi && $VAR -c "import pandas; pandas.show_versions();" && popd - ci/print_skipped.py - echo "after_script done" diff --git a/ci/deps/travis-37-aarch64.yaml b/ci/deps/travis-37-aarch64.yaml new file mode 100644 index 0000000000000..779d7600b1ca2 --- /dev/null +++ b/ci/deps/travis-37-aarch64.yaml @@ -0,0 +1,22 @@ +name: pandas-dev +channels: + - defaults + - conda-forge + - c3i_test +dependencies: + - python=3.7.* + + # tools + - cython>=0.29.13 + - pytest>=5.0.1 + - pytest-xdist>=1.21 + - hypothesis>=3.58.0 + + # pandas dependencies + - botocore>=1.11 + - numpy + - python-dateutil + - pytz + - pip + - pip: + - moto diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 0cb1f4aabf352..ef89403788abf 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -5,6 +5,17 @@ # https://github.com/pytest-dev/pytest/issues/1075 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') +if [ -n "$LOCALE_OVERRIDE" ]; then + export LC_ALL="$LOCALE_OVERRIDE" + export LANG="$LOCALE_OVERRIDE" + PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'` + if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then + echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE" + # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed + # exit 1 + fi +fi + if [[ "not network" == *"$PATTERN"* ]]; then export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; fi @@ -14,19 +25,27 @@ if [ "$COVERAGE" ]; then COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME" fi -# If no X server is found, we use xvfb to emulate it -if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then - export DISPLAY=":0" - XVFB="xvfb-run " +# Travis does not have have an X server +if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + DISPLAY=DISPLAY=:99.0 + if [ `uname -m` = 'aarch64' ]; then + PYTEST_CMD="xvfb-run -e /dev/stdout pytest -m \"$PATTERN\" -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" + else + PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" + fi fi PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" echo $PYTEST_CMD -sh -c "$PYTEST_CMD" +if [ `uname -m` = 'aarch64' ]; then + sudo sh -c "$PYTEST_CMD" +else + sh -c "$PYTEST_CMD" +fi if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then echo "uploading coverage" - echo "bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME" - bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME + echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME" + bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME fi diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 76f47b7ed3634..1156ec5b7cc94 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -17,8 +17,13 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then echo fi -MINICONDA_DIR="$HOME/miniconda3" +if [ `uname -m` = 'aarch64' ]; then + MINICONDA_DIR="$HOME/archiconda3" + IS_SUDO="sudo" +else + MINICONDA_DIR="$HOME/miniconda3" +fi if [ -d "$MINICONDA_DIR" ]; then echo @@ -41,9 +46,24 @@ else exit 1 fi -wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh -chmod +x miniconda.sh -./miniconda.sh -b +if [ `uname -m` = 'aarch64' ]; then + wget -q "https://github.com/Archiconda/build-tools/releases/download/0.2.3/Archiconda3-0.2.3-Linux-aarch64.sh" -O archiconda.sh + chmod +x archiconda.sh + $IS_SUDO apt-get install python-dev + $IS_SUDO apt-get install python3-pip + $IS_SUDO apt-get install lib$ARCHICONDA_PYTHON-dev + $IS_SUDO apt-get install xvfb + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib:/usr/local/lib:/usr/local/bin/python + ./archiconda.sh -b + echo "chmod MINICONDA_DIR" + $IS_SUDO chmod -R 777 $MINICONDA_DIR + $IS_SUDO cp $MINICONDA_DIR/bin/* /usr/bin/ + $IS_SUDO rm /usr/bin/lsb_release +else + wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh + chmod +x miniconda.sh + ./miniconda.sh -b +fi export PATH=$MINICONDA_DIR/bin:$PATH @@ -55,8 +75,8 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -conda install pip conda # create conda to create a historical artifact for pip & setuptools -conda update -n base conda +$IS_SUDO conda install pip # create conda to create a historical artifact for pip & setuptools +$IS_SUDO conda update -n base conda echo "conda info -a" conda info -a @@ -96,8 +116,18 @@ conda list conda remove --all -q -y -n pandas-dev echo +if [ `uname -m` = 'aarch64' ]; then + $IS_SUDO chmod -R 777 $MINICONDA_DIR + $IS_SUDO conda install botocore + $IS_SUDO conda install numpy + $IS_SUDO conda install python-dateutil=2.8.0 + $IS_SUDO conda install hypothesis + $IS_SUDO conda install pytz + $IS_SUDO chmod -R 777 $MINICONDA_DIR +fi + echo "conda env create -q --file=${ENV_FILE}" -time conda env create -q --file="${ENV_FILE}" +time $IS_SUDO conda env create -q --file="${ENV_FILE}" if [[ "$BITS32" == "yes" ]]; then @@ -111,13 +141,17 @@ source activate pandas-dev echo echo "remove any installed pandas package" echo "w/o removing anything else" -conda remove pandas -y --force || true -pip uninstall -y pandas || true +$IS_SUDO conda remove pandas -y --force || true +if [ `uname -m` = 'aarch64' ]; then + $IS_SUDO $ARCHICONDA_PYTHON -m pip uninstall -y pandas || true +else + pip uninstall -y pandas || true +fi echo echo "remove postgres if has been installed with conda" echo "we use the one from the CI" -conda remove postgresql -y --force || true +$IS_SUDO conda remove postgresql -y --force || true echo echo "remove qt" @@ -131,7 +165,10 @@ conda list pandas # Make sure any error below is reported as such echo "[Build extensions]" -python setup.py build_ext -q -i -j2 +if [ `uname -m` = 'aarch64' ]; then + sudo chmod -R 777 /home/travis/.ccache +fi +python setup.py build_ext -q -i # TODO: Some of our environments end up with old versions of pip (10.x) # Adding a new enough version of pip to the requirements explodes the @@ -140,21 +177,40 @@ python setup.py build_ext -q -i -j2 # - py35_compat # - py36_32bit echo "[Updating pip]" -python -m pip install --no-deps -U pip wheel setuptools +if [ `uname -m` = 'aarch64' ]; then + sudo chmod -R 777 /home/travis/archiconda3/envs/pandas-dev/lib/$ARCHICONDA_PYTHON/site-packages + $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-forked + $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-xdist + $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-deps -U pip wheel setuptools + sudo chmod -R 777 $MINICONDA_DIR +else + python -m pip install --no-deps -U pip wheel setuptools +fi echo "[Install pandas]" -python -m pip install --no-build-isolation -e . +if [ `uname -m` = 'aarch64' ]; then + $IS_SUDO chmod -R 777 $MINICONDA_DIR + $IS_SUDO $ARCHICONDA_PYTHON -m pip install numpy + $IS_SUDO $ARCHICONDA_PYTHON -m pip install hypothesis + $IS_SUDO chmod -R 777 /home/travis/.cache/ + $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-build-isolation -e . +else + python -m pip install --no-build-isolation -e . +fi echo echo "conda list" conda list # Install DB for Linux - if [[ -n ${SQL:0} ]]; then echo "installing dbs" + if [ `uname -m` = 'aarch64' ]; then + sudo systemctl start mysql + else + psql -c 'create database pandas_nosetest;' -U postgres + fi mysql -e 'create database pandas_nosetest;' - psql -c 'create database pandas_nosetest;' -U postgres else echo "not using dbs on non-linux Travis builds or Azure Pipelines" fi From 854b7f8b6e56ca14943f7e0b4d93589f759c83fa Mon Sep 17 00:00:00 2001 From: ossdev07 <39188636+ossdev07@users.noreply.github.com> Date: Mon, 6 Jan 2020 12:38:22 +0530 Subject: [PATCH 2/4] Update run_tests.sh --- ci/run_tests.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index ef89403788abf..584a5b2eefe6f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -5,17 +5,6 @@ # https://github.com/pytest-dev/pytest/issues/1075 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') -if [ -n "$LOCALE_OVERRIDE" ]; then - export LC_ALL="$LOCALE_OVERRIDE" - export LANG="$LOCALE_OVERRIDE" - PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'` - if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then - echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE" - # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed - # exit 1 - fi -fi - if [[ "not network" == *"$PATTERN"* ]]; then export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; fi @@ -47,5 +36,5 @@ fi if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then echo "uploading coverage" echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME" - bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME + bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME fi From 900937d7c7cef67125be12c3578d0fc329a898dc Mon Sep 17 00:00:00 2001 From: ossdev07 Date: Tue, 3 Mar 2020 13:04:05 +0000 Subject: [PATCH 3/4] Switched from Archiconda to Miniforge Signed-off-by: ossdev07 --- .travis.yml | 10 ++---- ci/run_tests.sh | 26 ++++++--------- ci/setup_env.sh | 89 +++++++++++-------------------------------------- 3 files changed, 34 insertions(+), 91 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3958912db4daf..4fdcae485c61f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ cache: directories: - $HOME/.cache # cython cache - $HOME/.ccache # compiler cache - env: global: # create a github personal access token @@ -38,12 +37,9 @@ matrix: - env: - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)" - - dist: trusty - os: linux - sudo: true - arch: arm64 + - arch: arm64 env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37-aarch64.yaml" PATTERN="(not slow and not network)" + - JOB="3.7" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)" - env: - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1" @@ -105,6 +101,6 @@ script: after_script: - echo "after_script start" - - source activate pandas-dev && pushd /tmp && if [ `uname -m` = 'aarch64' ];then VAR="python3.7" ; else VAR="python";fi && $VAR -c "import pandas; pandas.show_versions();" && popd + - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - ci/print_skipped.py - echo "after_script done" diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 584a5b2eefe6f..a9294bc52a4c5 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -14,27 +14,23 @@ if [ "$COVERAGE" ]; then COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME" fi -# Travis does not have have an X server -if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - DISPLAY=DISPLAY=:99.0 - if [ `uname -m` = 'aarch64' ]; then - PYTEST_CMD="xvfb-run -e /dev/stdout pytest -m \"$PATTERN\" -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" - else - PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD" - fi +# If no X server is found, we use xvfb to emulate it +if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then + export DISPLAY=":0" + XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" - -echo $PYTEST_CMD -if [ `uname -m` = 'aarch64' ]; then - sudo sh -c "$PYTEST_CMD" +if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then + PYTEST_CMD="pytest -m \"$PATTERN\" --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"; else - sh -c "$PYTEST_CMD" + PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" fi +echo $PYTEST_CMD +sh -c "$PYTEST_CMD" + if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then echo "uploading coverage" - echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME" + echo "bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME" bash <(curl -s https://codecov.io/bash) -Z -c -f $COVERAGE_FNAME fi diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 1156ec5b7cc94..6c264fe8b1de2 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -17,13 +17,8 @@ if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then echo fi +MINICONDA_DIR="$HOME/miniconda3" -if [ `uname -m` = 'aarch64' ]; then - MINICONDA_DIR="$HOME/archiconda3" - IS_SUDO="sudo" -else - MINICONDA_DIR="$HOME/miniconda3" -fi if [ -d "$MINICONDA_DIR" ]; then echo @@ -46,24 +41,16 @@ else exit 1 fi -if [ `uname -m` = 'aarch64' ]; then - wget -q "https://github.com/Archiconda/build-tools/releases/download/0.2.3/Archiconda3-0.2.3-Linux-aarch64.sh" -O archiconda.sh - chmod +x archiconda.sh - $IS_SUDO apt-get install python-dev - $IS_SUDO apt-get install python3-pip - $IS_SUDO apt-get install lib$ARCHICONDA_PYTHON-dev - $IS_SUDO apt-get install xvfb - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib:/usr/local/lib:/usr/local/bin/python - ./archiconda.sh -b - echo "chmod MINICONDA_DIR" - $IS_SUDO chmod -R 777 $MINICONDA_DIR - $IS_SUDO cp $MINICONDA_DIR/bin/* /usr/bin/ - $IS_SUDO rm /usr/bin/lsb_release +if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh"; else - wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh - chmod +x miniconda.sh - ./miniconda.sh -b + CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"; fi +wget -q $CONDA_URL -O miniconda.sh; +chmod +x miniconda.sh + +# Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3. +./miniconda.sh -b -p $MINICONDA_DIR export PATH=$MINICONDA_DIR/bin:$PATH @@ -75,8 +62,8 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -$IS_SUDO conda install pip # create conda to create a historical artifact for pip & setuptools -$IS_SUDO conda update -n base conda +conda install pip conda # create conda to create a historical artifact for pip & setuptools +conda update -n base conda echo "conda info -a" conda info -a @@ -116,18 +103,8 @@ conda list conda remove --all -q -y -n pandas-dev echo -if [ `uname -m` = 'aarch64' ]; then - $IS_SUDO chmod -R 777 $MINICONDA_DIR - $IS_SUDO conda install botocore - $IS_SUDO conda install numpy - $IS_SUDO conda install python-dateutil=2.8.0 - $IS_SUDO conda install hypothesis - $IS_SUDO conda install pytz - $IS_SUDO chmod -R 777 $MINICONDA_DIR -fi - echo "conda env create -q --file=${ENV_FILE}" -time $IS_SUDO conda env create -q --file="${ENV_FILE}" +time conda env create -q --file="${ENV_FILE}" if [[ "$BITS32" == "yes" ]]; then @@ -141,17 +118,13 @@ source activate pandas-dev echo echo "remove any installed pandas package" echo "w/o removing anything else" -$IS_SUDO conda remove pandas -y --force || true -if [ `uname -m` = 'aarch64' ]; then - $IS_SUDO $ARCHICONDA_PYTHON -m pip uninstall -y pandas || true -else - pip uninstall -y pandas || true -fi +conda remove pandas -y --force || true +pip uninstall -y pandas || true echo echo "remove postgres if has been installed with conda" echo "we use the one from the CI" -$IS_SUDO conda remove postgresql -y --force || true +conda remove postgresql -y --force || true echo echo "remove qt" @@ -165,10 +138,7 @@ conda list pandas # Make sure any error below is reported as such echo "[Build extensions]" -if [ `uname -m` = 'aarch64' ]; then - sudo chmod -R 777 /home/travis/.ccache -fi -python setup.py build_ext -q -i +python setup.py build_ext -q -i -j2 # TODO: Some of our environments end up with old versions of pip (10.x) # Adding a new enough version of pip to the requirements explodes the @@ -177,40 +147,21 @@ python setup.py build_ext -q -i # - py35_compat # - py36_32bit echo "[Updating pip]" -if [ `uname -m` = 'aarch64' ]; then - sudo chmod -R 777 /home/travis/archiconda3/envs/pandas-dev/lib/$ARCHICONDA_PYTHON/site-packages - $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-forked - $IS_SUDO $ARCHICONDA_PYTHON -m pip install pytest-xdist - $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-deps -U pip wheel setuptools - sudo chmod -R 777 $MINICONDA_DIR -else - python -m pip install --no-deps -U pip wheel setuptools -fi +python -m pip install --no-deps -U pip wheel setuptools echo "[Install pandas]" -if [ `uname -m` = 'aarch64' ]; then - $IS_SUDO chmod -R 777 $MINICONDA_DIR - $IS_SUDO $ARCHICONDA_PYTHON -m pip install numpy - $IS_SUDO $ARCHICONDA_PYTHON -m pip install hypothesis - $IS_SUDO chmod -R 777 /home/travis/.cache/ - $IS_SUDO $ARCHICONDA_PYTHON -m pip install --no-build-isolation -e . -else - python -m pip install --no-build-isolation -e . -fi +python -m pip install --no-build-isolation -e . echo echo "conda list" conda list # Install DB for Linux + if [[ -n ${SQL:0} ]]; then echo "installing dbs" - if [ `uname -m` = 'aarch64' ]; then - sudo systemctl start mysql - else - psql -c 'create database pandas_nosetest;' -U postgres - fi mysql -e 'create database pandas_nosetest;' + psql -c 'create database pandas_nosetest;' -U postgres else echo "not using dbs on non-linux Travis builds or Azure Pipelines" fi From 92d0c12780eb6d5b64e9723b33dfbacacb3e3e13 Mon Sep 17 00:00:00 2001 From: ossdev07 Date: Wed, 11 Mar 2020 18:57:15 +0530 Subject: [PATCH 4/4] Switched from Archiconda to Miniforge --- .travis.yml | 14 ++++++++++---- azure-pipelines.yml | 3 +++ ...travis-37-aarch64.yaml => travis-37-arm64.yaml} | 1 - ci/run_tests.sh | 6 +----- ci/setup_env.sh | 7 ++++--- pandas/_libs/src/ujson/lib/ultrajson.h | 2 +- 6 files changed, 19 insertions(+), 14 deletions(-) rename ci/deps/{travis-37-aarch64.yaml => travis-37-arm64.yaml} (95%) diff --git a/.travis.yml b/.travis.yml index 4fdcae485c61f..7943ca370af1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,11 @@ cache: directories: - $HOME/.cache # cython cache - $HOME/.ccache # compiler cache + env: global: + # Variable for test workers + - PYTEST_WORKERS="auto" # create a github personal access token # cd pandas-dev/pandas # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas @@ -39,7 +42,7 @@ matrix: - arch: arm64 env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)" + - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)" - env: - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1" @@ -62,9 +65,12 @@ matrix: - mysql - postgresql allow_failures: - - dist: bionic - python: 3.9-dev - env: + - arch: arm64 + env: + - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)" + - dist: bionic + python: 3.9-dev + env: - JOB="3.9-dev" PATTERN="(not slow and not network)" before_install: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d042bda77d4e8..e45cafc02cb61 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,6 +5,9 @@ trigger: pr: - master +variables: + PYTEST_WORKERS: auto + jobs: # Mac and Linux use the same template - template: ci/azure/posix.yml diff --git a/ci/deps/travis-37-aarch64.yaml b/ci/deps/travis-37-arm64.yaml similarity index 95% rename from ci/deps/travis-37-aarch64.yaml rename to ci/deps/travis-37-arm64.yaml index 779d7600b1ca2..5cb53489be225 100644 --- a/ci/deps/travis-37-aarch64.yaml +++ b/ci/deps/travis-37-arm64.yaml @@ -2,7 +2,6 @@ name: pandas-dev channels: - defaults - conda-forge - - c3i_test dependencies: - python=3.7.* diff --git a/ci/run_tests.sh b/ci/run_tests.sh index a9294bc52a4c5..b198b39e277b9 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -20,11 +20,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then - PYTEST_CMD="pytest -m \"$PATTERN\" --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"; -else - PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" -fi +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas" echo $PYTEST_CMD sh -c "$PYTEST_CMD" diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 6c264fe8b1de2..4d551294dbb21 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -42,11 +42,12 @@ else fi if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then - CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh"; + sudo apt-get -y install xvfb + CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh" else - CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"; + CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" fi -wget -q $CONDA_URL -O miniconda.sh; +wget -q $CONDA_URL -O miniconda.sh chmod +x miniconda.sh # Installation path is required for ARM64 platform as miniforge script installs in path $HOME/miniforge3. diff --git a/pandas/_libs/src/ujson/lib/ultrajson.h b/pandas/_libs/src/ujson/lib/ultrajson.h index b40ac9856d6a6..acb66b668e8dc 100644 --- a/pandas/_libs/src/ujson/lib/ultrajson.h +++ b/pandas/_libs/src/ujson/lib/ultrajson.h @@ -108,7 +108,7 @@ typedef uint32_t JSUINT32; #define FASTCALL_MSVC -#if !defined __x86_64__ +#if !defined __x86_64__ && !defined __aarch64__ #define FASTCALL_ATTR __attribute__((fastcall)) #else #define FASTCALL_ATTR