From bcff6fc69856b2bfa6cfbf8b9f87980a7a932cf8 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 21:46:13 +0100 Subject: [PATCH 01/27] MNT activate azure pipelines --- azure-pipelines.yml | 123 ++++++++++++++++++++++++++++ build_tools/azure/install.sh | 111 +++++++++++++++++++++++++ build_tools/azure/posix.yml | 50 +++++++++++ build_tools/azure/test_docs.sh | 12 +++ build_tools/azure/test_script.sh | 45 ++++++++++ build_tools/azure/upload_codecov.sh | 16 ++++ 6 files changed, 357 insertions(+) create mode 100644 azure-pipelines.yml create mode 100755 build_tools/azure/install.sh create mode 100644 build_tools/azure/posix.yml create mode 100755 build_tools/azure/test_docs.sh create mode 100755 build_tools/azure/test_script.sh create mode 100755 build_tools/azure/upload_codecov.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..e291b5428 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,123 @@ +# Adapted from https://github.com/scikit-learn/scikit-learn/blob/master/azure-pipelines.yml +- template: build_tools/azure/posix.yml + parameters: + name: Linux_Runs + vmImage: ubuntu-16.04 + matrix: + pylatest_conda_mkl: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + INSTALL_MKL: 'true' + NUMPY_VERSION: '*' + SCIPY_VERSION: '*' + CYTHON_VERSION: '*' + PILLOW_VERSION: '*' + PYTEST_VERSION: '*' + JOBLIB_VERSION: '*' + COVERAGE: 'true' + +- template: build_tools/azure/posix.yml + parameters: + name: Linux + vmImage: ubuntu-16.04 + dependsOn: [linting] + matrix: + # Linux environment to test that scikit-learn can be built against + # versions of numpy, scipy with ATLAS that comes with Ubuntu Xenial 16.04 + # i.e. numpy 1.11 and scipy 0.17 + py35_ubuntu_atlas: + DISTRIB: 'ubuntu' + PYTHON_VERSION: '3.5' + JOBLIB_VERSION: '0.11' + # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB + py35_conda_openblas: + DISTRIB: 'conda' + PYTHON_VERSION: '3.5' + INSTALL_MKL: 'false' + NUMPY_VERSION: '1.11.0' + SCIPY_VERSION: '0.17.0' + PANDAS_VERSION: '*' + PYTEST_VERSION: '*' + # later version of joblib are not packaged in conda for Python 3.5 + JOBLIB_VERSION: '0.12.3' + COVERAGE: 'true' + # Linux environment to test the latest available dependencies and MKL. + pylatest_pip_openblas_pandas: + DISTRIB: 'conda-pip-latest' + PYTHON_VERSION: '*' + PYTEST_VERSION: '*' + COVERAGE: 'true' + TEST_DOCSTRINGS: 'true' + CHECK_WARNINGS: 'true' + pylatest_conda_pandas_keras: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + PYTEST_VERSION: '*' + INSTALL_MKL: 'true' + KERAS_VERSION: '*' + COVERAGE: 'true' + pylatest_conda_pandas_tensorflow: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + PYTEST_VERSION: '*' + INSTALL_MKL: 'true' + TENSORFLOW_VERSION: '*' + COVERAGE: 'true' + +# - template: build_tools/azure/posix-32.yml +# parameters: +# name: Linux32 +# vmImage: ubuntu-16.04 +# dependsOn: [linting] +# matrix: +# py35_ubuntu_atlas_32bit: +# DISTRIB: 'ubuntu-32' +# PYTHON_VERSION: '3.5' +# JOBLIB_VERSION: '0.11' + +# - template: build_tools/azure/posix.yml +# parameters: +# name: macOS +# vmImage: xcode9-macos10.13 +# dependsOn: [linting] +# matrix: +# pylatest_conda_mkl: +# DISTRIB: 'conda' +# PYTHON_VERSION: '*' +# INSTALL_MKL: 'true' +# NUMPY_VERSION: '*' +# SCIPY_VERSION: '*' +# CYTHON_VERSION: '*' +# PILLOW_VERSION: '*' +# PYTEST_VERSION: '*' +# JOBLIB_VERSION: '*' +# COVERAGE: 'true' +# pylatest_conda_mkl_no_openmp: +# DISTRIB: 'conda' +# PYTHON_VERSION: '*' +# INSTALL_MKL: 'true' +# NUMPY_VERSION: '*' +# SCIPY_VERSION: '*' +# CYTHON_VERSION: '*' +# PILLOW_VERSION: '*' +# PYTEST_VERSION: '*' +# JOBLIB_VERSION: '*' +# COVERAGE: 'true' +# SKLEARN_TEST_NO_OPENMP: 'true' +# SKLEARN_SKIP_OPENMP_TEST: 'true' + +# - template: build_tools/azure/windows.yml +# parameters: +# name: Windows +# vmImage: vs2017-win2016 +# dependsOn: [linting] +# matrix: +# py37_conda_mkl: +# PYTHON_VERSION: '3.7' +# CHECK_WARNINGS: 'true' +# PYTHON_ARCH: '64' +# PYTEST_VERSION: '*' +# COVERAGE: 'true' +# py35_pip_openblas_32bit: +# PYTHON_VERSION: '3.5' +# PYTHON_ARCH: '32' diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh new file mode 100755 index 000000000..9b913c4e0 --- /dev/null +++ b/build_tools/azure/install.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +set -e + +UNAMESTR=`uname` + +make_conda() { + TO_INSTALL="$@" + conda create -n $VIRTUALENV --yes $TO_INSTALL + source activate $VIRTUALENV +} + +version_ge() { + # The two version numbers are seperated with a new line is piped to sort + # -rV. The -V activates for version number sorting and -r sorts in + # decending order. If the first argument is the top element of the sort, it + # is greater than or equal to the second argument. + test "$(printf "${1}\n${2}" | sort -rV | head -n 1)" == "$1" +} + +if [[ "$DISTRIB" == "conda" ]]; then + + TO_INSTALL="python=$PYTHON_VERSION pip \ + numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \ + joblib=$JOBLIB_VERSION" + + if [[ "$INSTALL_MKL" == "true" ]]; then + TO_INSTALL="$TO_INSTALL mkl" + else + TO_INSTALL="$TO_INSTALL nomkl" + fi + + if [[ -n "$PANDAS_VERSION" ]]; then + TO_INSTALL="$TO_INSTALL pandas=$PANDAS_VERSION" + fi + + if [[ -n "$KERAS_VERSION" ]]; then + TO_INSTALL="$TO_INSTALL keras=$KERAS_VERSION tensorflow=1" + KERAS_BACKEND=tensorflow + python -c "import keras.backend" + sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json; + fi + + if [[ -n "$TENSORFLOW_VERSION" ]]; then + TO_INSTALL="$TO_INSTALL tensorflow=$TENSORFLOW_VERSION" + fi + + make_conda $TO_INSTALL + + if [[ "$PYTEST_VERSION" == "*" ]]; then + python -m pip install pytest + else + python -m pip install pytest=="$PYTEST_VERSION" + fi + + if [[ "$PYTHON_VERSION" == "*" ]]; then + python -m pip install pytest-xdist + fi + +elif [[ "$DISTRIB" == "ubuntu" ]]; then + sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV + source $VIRTUALENV/bin/activate + python -m pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn + python -m pip install pandas + python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION +elif [[ "$DISTRIB" == "ubuntu-32" ]]; then + apt-get update + apt-get install -y python3-dev python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV + source $VIRTUALENV/bin/activate + python -m pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn + python -m pip install pandas + python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION +elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then + # Since conda main channel usually lacks behind on the latest releases, + # we use pypi to test against the latest releases of the dependencies. + # conda is still used as a convenient way to install Python and pip. + make_conda "python=$PYTHON_VERSION" + python -m pip install -U pip + python -m pip install numpy scipy joblib + python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist + python -m pip install pandas +fi + +if [[ "$COVERAGE" == "true" ]]; then + python -m pip install coverage codecov pytest-cov +fi + +if [[ "$TEST_DOCSTRINGS" == "true" ]]; then + python -m pip install sphinx + pythong -m pip install -U git+https://github.com/numpy/numpydoc.git +fi + +python --version +python -c "import numpy; print('numpy %s' % numpy.__version__)" +python -c "import scipy; print('scipy %s' % scipy.__version__)" +python -c "\ +try: + import pandas + print('pandas %s' % pandas.__version__) +except ImportError: + print('pandas not installed') +" +python -m pip list + +# Use setup.py instead of `pip install -e .` to be able to pass the -j flag +# to speed-up the building multicore CI machines. +python setup.py develop diff --git a/build_tools/azure/posix.yml b/build_tools/azure/posix.yml new file mode 100644 index 000000000..ee5b4c351 --- /dev/null +++ b/build_tools/azure/posix.yml @@ -0,0 +1,50 @@ +parameters: + name: '' + vmImage: '' + matrix: [] + dependsOn: [] + +jobs: +- job: ${{ parameters.name }} + dependsOn: ${{ parameters.dependsOn }} + pool: + vmImage: ${{ parameters.vmImage }} + variables: + TEST_DIR: '$(Agent.WorkFolder)/tmp_folder' + VIRTUALENV: 'testvenv' + JUNITXML: 'test-data.xml' + PYTEST_VERSION: '5.2.1' + OMP_NUM_THREADS: '4' + OPENBLAS_NUM_THREADS: '4' + strategy: + matrix: + ${{ insert }}: ${{ parameters.matrix }} + + steps: + - bash: echo "##vso[task.prependpath]$CONDA/bin" + displayName: Add conda to PATH + condition: startsWith(variables['DISTRIB'], 'conda') + - bash: sudo chown -R $USER $CONDA + displayName: Take ownership of conda installation + condition: startsWith(variables['DISTRIB'], 'conda') + - script: | + build_tools/azure/install.sh + displayName: 'Install' + - script: | + build_tools/azure/test_script.sh + displayName: 'Test Library' + - script: | + build_tools/azure/test_docs.sh + displayName: 'Test Docs' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' + testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} + displayName: 'Publish Test Results' + condition: succeededOrFailed() + - script: | + build_tools/azure/upload_codecov.sh + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), eq(variables['DISTRIB'], 'conda')) + displayName: 'Upload To Codecov' + env: + CODECOV_TOKEN: $(CODECOV_TOKEN) diff --git a/build_tools/azure/test_docs.sh b/build_tools/azure/test_docs.sh new file mode 100755 index 000000000..9ff94e183 --- /dev/null +++ b/build_tools/azure/test_docs.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +if [[ "$DISTRIB" =~ ^conda.* ]]; then + source activate $VIRTUALENV +elif [[ "$DISTRIB" == "ubuntu" ]]; then + source $VIRTUALENV/bin/activate +fi + +make test-doc +pytest -vsl maint_tools/test_docstring.py diff --git a/build_tools/azure/test_script.sh b/build_tools/azure/test_script.sh new file mode 100755 index 000000000..37793c529 --- /dev/null +++ b/build_tools/azure/test_script.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +if [[ "$DISTRIB" =~ ^conda.* ]]; then + source activate $VIRTUALENV +elif [[ "$DISTRIB" == "ubuntu" ]] || [[ "$DISTRIB" == "ubuntu-32" ]]; then + source $VIRTUALENV/bin/activate +fi + +python --version +python -c "import numpy; print('numpy %s' % numpy.__version__)" +python -c "import scipy; print('scipy %s' % scipy.__version__)" +python -c "\ +try: + import pandas + print('pandas %s' % pandas.__version__) +except ImportError: + print('pandas not installed') +" +python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())" +pip list + +TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML" + +if [[ "$COVERAGE" == "true" ]]; then + export COVERAGE_PROCESS_START="$BUILD_SOURCESDIRECTORY/.coveragerc" + TEST_CMD="$TEST_CMD --cov-config=$COVERAGE_PROCESS_START --cov imblearn" +fi + +if [[ -n "$CHECK_WARNINGS" ]]; then + TEST_CMD="$TEST_CMD -Werror::DeprecationWarning -Werror::FutureWarning" +fi + +if [[ "$PYTHON_VERSION" == "*" ]]; then + TEST_CMD="$TEST_CMD -n2" +fi + +mkdir -p $TEST_DIR +cp setup.cfg $TEST_DIR +cd $TEST_DIR + +set -x +$TEST_CMD --pyargs imblearn +set +x diff --git a/build_tools/azure/upload_codecov.sh b/build_tools/azure/upload_codecov.sh new file mode 100755 index 000000000..274106cb1 --- /dev/null +++ b/build_tools/azure/upload_codecov.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# called when COVERAGE=="true" and DISTRIB=="conda" +export PATH=$HOME/miniconda3/bin:$PATH +source activate $VIRTUALENV + +# Need to run codecov from a git checkout, so we copy .coverage +# from TEST_DIR where pytest has been run +pushd $TEST_DIR +coverage combine --append +popd +cp $TEST_DIR/.coverage $BUILD_REPOSITORY_LOCALPATH + +codecov --root $BUILD_REPOSITORY_LOCALPATH -t $CODECOV_TOKEN || echo "codecov upload failed" From a3d454ad9dd6f8859cb42a05b31fe08e35ec60e0 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 21:53:04 +0100 Subject: [PATCH 02/27] add linting --- azure-pipelines.yml | 16 ++++ build_tools/circle/linting.sh | 161 ++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 build_tools/circle/linting.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e291b5428..5d51123e0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,20 @@ # Adapted from https://github.com/scikit-learn/scikit-learn/blob/master/azure-pipelines.yml +- job: linting + displayName: Linting + pool: + vmImage: ubuntu-16.04 + steps: + - bash: echo "##vso[task.prependpath]$CONDA/bin" + displayName: Add conda to PATH + - bash: sudo chown -R $USER $CONDA + displayName: Take ownership of conda installation + - bash: conda create --name flake8_env --yes flake8 + displayName: Install flake8 + - bash: | + source activate flake8_env + ./build_tools/circle/linting.sh + displayName: Run linting + - template: build_tools/azure/posix.yml parameters: name: Linux_Runs diff --git a/build_tools/circle/linting.sh b/build_tools/circle/linting.sh new file mode 100644 index 000000000..b5783f513 --- /dev/null +++ b/build_tools/circle/linting.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +# This script is used in CircleCI to check that PRs do not add obvious +# flake8 violations. It relies on two things: +# - find common ancestor between branch and +# scikit-learn/scikit-learn remote +# - run flake8 --diff on the diff between the branch and the common +# ancestor +# +# Additional features: +# - the line numbers in Travis match the local branch on the PR +# author machine. +# - ./build_tools/circle/flake8_diff.sh can be run locally for quick +# turn-around + +set -e +# pipefail is necessary to propagate exit codes +set -o pipefail + +PROJECT=scikit-learn-contrib/imbalanced-learn +PROJECT_URL=https://github.com/$PROJECT.git + +# Find the remote with the project name (upstream in most cases) +REMOTE=$(git remote -v | grep $PROJECT | cut -f1 | head -1 || echo '') + +# Add a temporary remote if needed. For example this is necessary when +# Travis is configured to run in a fork. In this case 'origin' is the +# fork and not the reference repo we want to diff against. +if [[ -z "$REMOTE" ]]; then + TMP_REMOTE=tmp_reference_upstream + REMOTE=$TMP_REMOTE + git remote add $REMOTE $PROJECT_URL +fi + +echo "Remotes:" +echo '--------------------------------------------------------------------------------' +git remote --verbose + +# Travis does the git clone with a limited depth (50 at the time of +# writing). This may not be enough to find the common ancestor with +# $REMOTE/master so we unshallow the git checkout +if [[ -a .git/shallow ]]; then + echo -e '\nTrying to unshallow the repo:' + echo '--------------------------------------------------------------------------------' + git fetch --unshallow +fi + +if [[ "$TRAVIS" == "true" ]]; then + if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] + then + # In main repo, using TRAVIS_COMMIT_RANGE to test the commits + # that were pushed into a branch + if [[ "$PROJECT" == "$TRAVIS_REPO_SLUG" ]]; then + if [[ -z "$TRAVIS_COMMIT_RANGE" ]]; then + echo "New branch, no commit range from Travis so passing this test by convention" + exit 0 + fi + COMMIT_RANGE=$TRAVIS_COMMIT_RANGE + fi + else + # We want to fetch the code as it is in the PR branch and not + # the result of the merge into master. This way line numbers + # reported by Travis will match with the local code. + LOCAL_BRANCH_REF=travis_pr_$TRAVIS_PULL_REQUEST + # In Travis the PR target is always origin + git fetch origin pull/$TRAVIS_PULL_REQUEST/head:refs/$LOCAL_BRANCH_REF + fi +fi + +# If not using the commit range from Travis we need to find the common +# ancestor between $LOCAL_BRANCH_REF and $REMOTE/master +if [[ -z "$COMMIT_RANGE" ]]; then + if [[ -z "$LOCAL_BRANCH_REF" ]]; then + LOCAL_BRANCH_REF=$(git rev-parse --abbrev-ref HEAD) + fi + echo -e "\nLast 2 commits in $LOCAL_BRANCH_REF:" + echo '--------------------------------------------------------------------------------' + git --no-pager log -2 $LOCAL_BRANCH_REF + + REMOTE_MASTER_REF="$REMOTE/master" + # Make sure that $REMOTE_MASTER_REF is a valid reference + echo -e "\nFetching $REMOTE_MASTER_REF" + echo '--------------------------------------------------------------------------------' + git fetch $REMOTE master:refs/remotes/$REMOTE_MASTER_REF + LOCAL_BRANCH_SHORT_HASH=$(git rev-parse --short $LOCAL_BRANCH_REF) + REMOTE_MASTER_SHORT_HASH=$(git rev-parse --short $REMOTE_MASTER_REF) + + COMMIT=$(git merge-base $LOCAL_BRANCH_REF $REMOTE_MASTER_REF) || \ + echo "No common ancestor found for $(git show $LOCAL_BRANCH_REF -q) and $(git show $REMOTE_MASTER_REF -q)" + + if [ -z "$COMMIT" ]; then + exit 1 + fi + + COMMIT_SHORT_HASH=$(git rev-parse --short $COMMIT) + + echo -e "\nCommon ancestor between $LOCAL_BRANCH_REF ($LOCAL_BRANCH_SHORT_HASH)"\ + "and $REMOTE_MASTER_REF ($REMOTE_MASTER_SHORT_HASH) is $COMMIT_SHORT_HASH:" + echo '--------------------------------------------------------------------------------' + git --no-pager show --no-patch $COMMIT_SHORT_HASH + + COMMIT_RANGE="$COMMIT_SHORT_HASH..$LOCAL_BRANCH_SHORT_HASH" + + if [[ -n "$TMP_REMOTE" ]]; then + git remote remove $TMP_REMOTE + fi + +else + echo "Got the commit range from Travis: $COMMIT_RANGE" +fi + +echo -e '\nRunning flake8 on the diff in the range' "$COMMIT_RANGE" \ + "($(git rev-list $COMMIT_RANGE | wc -l) commit(s)):" +echo '--------------------------------------------------------------------------------' + +# We ignore files from sklearn/externals. Unfortunately there is no +# way to do it with flake8 directly (the --exclude does not seem to +# work with --diff). We could use the exclude magic in the git pathspec +# ':!sklearn/externals' but it is only available on git 1.9 and Travis +# uses git 1.8. +# We need the following command to exit with 0 hence the echo in case +# there is no match +MODIFIED_FILES="$(git diff --name-only $COMMIT_RANGE | grep -v 'sklearn/externals' | \ + grep -v 'doc/sphinxext' || echo "no_match")" + +check_files() { + files="$1" + shift + options="$*" + if [ -n "$files" ]; then + # Conservative approach: diff without context (--unified=0) so that code + # that was not changed does not create failures + git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --show-source $options + fi +} + +if [[ "$MODIFIED_FILES" == "no_match" ]]; then + echo "No file outside sklearn/externals and doc/sphinxext has been modified" +else + + check_files "$(echo "$MODIFIED_FILES" | grep -v ^examples)" + check_files "$(echo "$MODIFIED_FILES" | grep ^examples)" \ + --config ./examples/.flake8 +fi +echo -e "No problem detected by flake8\n" + +# For docstrings and warnings of deprecated attributes to be rendered +# properly, the property decorator must come before the deprecated decorator +# (else they are treated as functions) + +# do not error when grep -B1 "@property" finds nothing +set +e +bad_deprecation_property_order=`git grep -A 10 "@property" -- "*.py" | awk '/@property/,/def /' | grep -B1 "@deprecated"` + +if [ ! -z "$bad_deprecation_property_order" ] +then + echo "property decorator should come before deprecated decorator" + echo "found the following occurrencies:" + echo $bad_deprecation_property_order + exit 1 +fi From eb1c2db596af8ea3bebe40dd78f1ba009dc08eb8 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 21:55:02 +0100 Subject: [PATCH 03/27] fix --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5d51123e0..22a06ae0c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,5 @@ # Adapted from https://github.com/scikit-learn/scikit-learn/blob/master/azure-pipelines.yml +jobs: - job: linting displayName: Linting pool: From 721ca1f2e2cd6b53cf7758443dde14b5a0db57af Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 21:59:49 +0100 Subject: [PATCH 04/27] fix permission linting --- build_tools/circle/linting.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build_tools/circle/linting.sh diff --git a/build_tools/circle/linting.sh b/build_tools/circle/linting.sh old mode 100644 new mode 100755 From 920ad1a0597f5256c772338cbab08b837809a05e Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:08:43 +0100 Subject: [PATCH 05/27] fix install --- build_tools/azure/install.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 9b913c4e0..e40dc1b06 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -30,6 +30,11 @@ if [[ "$DISTRIB" == "conda" ]]; then TO_INSTALL="$TO_INSTALL nomkl" fi + make_conda $TO_INSTALL + python -m pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn + + TO_INSTALL="" + if [[ -n "$PANDAS_VERSION" ]]; then TO_INSTALL="$TO_INSTALL pandas=$PANDAS_VERSION" fi @@ -37,16 +42,12 @@ if [[ "$DISTRIB" == "conda" ]]; then if [[ -n "$KERAS_VERSION" ]]; then TO_INSTALL="$TO_INSTALL keras=$KERAS_VERSION tensorflow=1" KERAS_BACKEND=tensorflow - python -c "import keras.backend" - sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json; fi if [[ -n "$TENSORFLOW_VERSION" ]]; then TO_INSTALL="$TO_INSTALL tensorflow=$TENSORFLOW_VERSION" fi - make_conda $TO_INSTALL - if [[ "$PYTEST_VERSION" == "*" ]]; then python -m pip install pytest else @@ -57,6 +58,13 @@ if [[ "$DISTRIB" == "conda" ]]; then python -m pip install pytest-xdist fi + conda install --yes $TO_INSTALL + + if [[ -n "$KERAS_VERSION" ]]; then + python -c "import keras.backend" + sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json; + fi + elif [[ "$DISTRIB" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test sudo apt-get update From da18a6a6a4cde5b81b7e941c9fc2bd429da9e05d Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:11:31 +0100 Subject: [PATCH 06/27] downgrade python --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 22a06ae0c..a4fcd4d40 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,7 +23,7 @@ jobs: matrix: pylatest_conda_mkl: DISTRIB: 'conda' - PYTHON_VERSION: '*' + PYTHON_VERSION: '3.7' INSTALL_MKL: 'true' NUMPY_VERSION: '*' SCIPY_VERSION: '*' @@ -61,21 +61,21 @@ jobs: # Linux environment to test the latest available dependencies and MKL. pylatest_pip_openblas_pandas: DISTRIB: 'conda-pip-latest' - PYTHON_VERSION: '*' + PYTHON_VERSION: '3.7' PYTEST_VERSION: '*' COVERAGE: 'true' TEST_DOCSTRINGS: 'true' CHECK_WARNINGS: 'true' pylatest_conda_pandas_keras: DISTRIB: 'conda' - PYTHON_VERSION: '*' + PYTHON_VERSION: '3.7' PYTEST_VERSION: '*' INSTALL_MKL: 'true' KERAS_VERSION: '*' COVERAGE: 'true' pylatest_conda_pandas_tensorflow: DISTRIB: 'conda' - PYTHON_VERSION: '*' + PYTHON_VERSION: '3.7' PYTEST_VERSION: '*' INSTALL_MKL: 'true' TENSORFLOW_VERSION: '*' From 541ec1fa3b8e32fd2a9efcbeb658c914b74e878e Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:18:59 +0100 Subject: [PATCH 07/27] fix version pytest --- azure-pipelines.yml | 15 --------------- build_tools/azure/install.sh | 4 +++- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a4fcd4d40..710566097 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -46,37 +46,22 @@ jobs: DISTRIB: 'ubuntu' PYTHON_VERSION: '3.5' JOBLIB_VERSION: '0.11' - # Linux + Python 3.5 build with OpenBLAS and without SITE_JOBLIB - py35_conda_openblas: - DISTRIB: 'conda' - PYTHON_VERSION: '3.5' - INSTALL_MKL: 'false' - NUMPY_VERSION: '1.11.0' - SCIPY_VERSION: '0.17.0' - PANDAS_VERSION: '*' - PYTEST_VERSION: '*' - # later version of joblib are not packaged in conda for Python 3.5 - JOBLIB_VERSION: '0.12.3' - COVERAGE: 'true' # Linux environment to test the latest available dependencies and MKL. pylatest_pip_openblas_pandas: DISTRIB: 'conda-pip-latest' PYTHON_VERSION: '3.7' - PYTEST_VERSION: '*' COVERAGE: 'true' TEST_DOCSTRINGS: 'true' CHECK_WARNINGS: 'true' pylatest_conda_pandas_keras: DISTRIB: 'conda' PYTHON_VERSION: '3.7' - PYTEST_VERSION: '*' INSTALL_MKL: 'true' KERAS_VERSION: '*' COVERAGE: 'true' pylatest_conda_pandas_tensorflow: DISTRIB: 'conda' PYTHON_VERSION: '3.7' - PYTEST_VERSION: '*' INSTALL_MKL: 'true' TENSORFLOW_VERSION: '*' COVERAGE: 'true' diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index e40dc1b06..4e07b0750 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -58,7 +58,9 @@ if [[ "$DISTRIB" == "conda" ]]; then python -m pip install pytest-xdist fi - conda install --yes $TO_INSTALL + if [[ -n "$TO_INSTALL" ]] + conda install --yes $TO_INSTALL + fi if [[ -n "$KERAS_VERSION" ]]; then python -c "import keras.backend" From ed970754ffde88b2098648fe019a51d9b3a009ca Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:20:16 +0100 Subject: [PATCH 08/27] iter --- build_tools/azure/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 4e07b0750..27033fbca 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -58,7 +58,7 @@ if [[ "$DISTRIB" == "conda" ]]; then python -m pip install pytest-xdist fi - if [[ -n "$TO_INSTALL" ]] + if [[ -n "$TO_INSTALL" ]]; then conda install --yes $TO_INSTALL fi From 4217c567b109a149b14cd5df2a5791d3de3a63ec Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:26:03 +0100 Subject: [PATCH 09/27] fix --- build_tools/azure/install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 27033fbca..e42a624ef 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -73,17 +73,17 @@ elif [[ "$DISTRIB" == "ubuntu" ]]; then sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate - python -m pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn python -m pip install pandas - python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION + python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION cython + python -m pip install git+https://github.com/scikit-learn/scikit-learn.git elif [[ "$DISTRIB" == "ubuntu-32" ]]; then apt-get update apt-get install -y python3-dev python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate - python -m pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn python -m pip install pandas - python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION + python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION cython + python -m pip install git+https://github.com/scikit-learn/scikit-learn.git elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then # Since conda main channel usually lacks behind on the latest releases, # we use pypi to test against the latest releases of the dependencies. @@ -101,7 +101,7 @@ fi if [[ "$TEST_DOCSTRINGS" == "true" ]]; then python -m pip install sphinx - pythong -m pip install -U git+https://github.com/numpy/numpydoc.git + python -m pip install -U git+https://github.com/numpy/numpydoc.git fi python --version From 757cecf2232066db3f8ce6eac0040a8e8849f37d Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:38:08 +0100 Subject: [PATCH 10/27] fix --- azure-pipelines.yml | 85 ++++++++++++------------------------ build_tools/azure/install.sh | 1 + 2 files changed, 29 insertions(+), 57 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710566097..b67a40b9c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,23 +16,6 @@ jobs: ./build_tools/circle/linting.sh displayName: Run linting -- template: build_tools/azure/posix.yml - parameters: - name: Linux_Runs - vmImage: ubuntu-16.04 - matrix: - pylatest_conda_mkl: - DISTRIB: 'conda' - PYTHON_VERSION: '3.7' - INSTALL_MKL: 'true' - NUMPY_VERSION: '*' - SCIPY_VERSION: '*' - CYTHON_VERSION: '*' - PILLOW_VERSION: '*' - PYTEST_VERSION: '*' - JOBLIB_VERSION: '*' - COVERAGE: 'true' - - template: build_tools/azure/posix.yml parameters: name: Linux @@ -51,62 +34,50 @@ jobs: DISTRIB: 'conda-pip-latest' PYTHON_VERSION: '3.7' COVERAGE: 'true' + PANDAS_VERSION: '*' TEST_DOCSTRINGS: 'true' CHECK_WARNINGS: 'true' pylatest_conda_pandas_keras: DISTRIB: 'conda' PYTHON_VERSION: '3.7' INSTALL_MKL: 'true' + PANDAS_VERSION: '*' KERAS_VERSION: '*' COVERAGE: 'true' pylatest_conda_pandas_tensorflow: DISTRIB: 'conda' PYTHON_VERSION: '3.7' + PANDAS_VERSION: '*' INSTALL_MKL: 'true' TENSORFLOW_VERSION: '*' COVERAGE: 'true' -# - template: build_tools/azure/posix-32.yml -# parameters: -# name: Linux32 -# vmImage: ubuntu-16.04 -# dependsOn: [linting] -# matrix: -# py35_ubuntu_atlas_32bit: -# DISTRIB: 'ubuntu-32' -# PYTHON_VERSION: '3.5' -# JOBLIB_VERSION: '0.11' +- template: build_tools/azure/posix-32.yml + parameters: + name: Linux32 + vmImage: ubuntu-16.04 + dependsOn: [linting] + matrix: + py35_ubuntu_atlas_32bit: + DISTRIB: 'ubuntu-32' + PYTHON_VERSION: '3.5' + JOBLIB_VERSION: '0.11' -# - template: build_tools/azure/posix.yml -# parameters: -# name: macOS -# vmImage: xcode9-macos10.13 -# dependsOn: [linting] -# matrix: -# pylatest_conda_mkl: -# DISTRIB: 'conda' -# PYTHON_VERSION: '*' -# INSTALL_MKL: 'true' -# NUMPY_VERSION: '*' -# SCIPY_VERSION: '*' -# CYTHON_VERSION: '*' -# PILLOW_VERSION: '*' -# PYTEST_VERSION: '*' -# JOBLIB_VERSION: '*' -# COVERAGE: 'true' -# pylatest_conda_mkl_no_openmp: -# DISTRIB: 'conda' -# PYTHON_VERSION: '*' -# INSTALL_MKL: 'true' -# NUMPY_VERSION: '*' -# SCIPY_VERSION: '*' -# CYTHON_VERSION: '*' -# PILLOW_VERSION: '*' -# PYTEST_VERSION: '*' -# JOBLIB_VERSION: '*' -# COVERAGE: 'true' -# SKLEARN_TEST_NO_OPENMP: 'true' -# SKLEARN_SKIP_OPENMP_TEST: 'true' +- template: build_tools/azure/posix.yml + parameters: + name: macOS + vmImage: xcode9-macos10.13 + dependsOn: [linting] + matrix: + pylatest_conda_mkl: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + INSTALL_MKL: 'true' + NUMPY_VERSION: '*' + SCIPY_VERSION: '*' + PYTEST_VERSION: '*' + JOBLIB_VERSION: '*' + COVERAGE: 'true' # - template: build_tools/azure/windows.yml # parameters: diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index e42a624ef..1521df644 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -91,6 +91,7 @@ elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then make_conda "python=$PYTHON_VERSION" python -m pip install -U pip python -m pip install numpy scipy joblib + python -m pip install git+https://github.com/scikit-learn/scikit-learn.git python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist python -m pip install pandas fi From 354248b10317ddc1a97c44b0b195913c9da0cbcf Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:38:44 +0100 Subject: [PATCH 11/27] fix --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b67a40b9c..9a2a7152f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -75,6 +75,7 @@ jobs: INSTALL_MKL: 'true' NUMPY_VERSION: '*' SCIPY_VERSION: '*' + PANDAS_VERSION: '*' PYTEST_VERSION: '*' JOBLIB_VERSION: '*' COVERAGE: 'true' From 3b406c243162f6a9939ea4541fb2e69cb7f47de1 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:39:47 +0100 Subject: [PATCH 12/27] fix --- build_tools/azure/posix-32.yml | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 build_tools/azure/posix-32.yml diff --git a/build_tools/azure/posix-32.yml b/build_tools/azure/posix-32.yml new file mode 100644 index 000000000..68e05e347 --- /dev/null +++ b/build_tools/azure/posix-32.yml @@ -0,0 +1,61 @@ +parameters: + name: '' + vmImage: '' + matrix: [] + dependsOn: [] + +jobs: +- job: ${{ parameters.name }} + dependsOn: ${{ parameters.dependsOn }} + pool: + vmImage: ${{ parameters.vmImage }} + variables: + TEST_DIR: '$(Agent.WorkFolder)/tmp_folder' + JUNITXML: 'test-data.xml' + OMP_NUM_THREADS: '4' + PYTEST_VERSION: '5.2.1' + OPENBLAS_NUM_THREADS: '4' + SKLEARN_SKIP_NETWORK_TESTS: '1' + strategy: + matrix: + ${{ insert }}: ${{ parameters.matrix }} + + steps: + # Container is detached and sleeping, allowing steps to run commmands + # in the container. The TEST_DIR is mapped allowing the host to access + # the JUNITXML file + - script: > + docker container run --rm + --volume $TEST_DIR:/temp_dir + --volume $PWD:/io + -w /io + --detach + --name skcontainer + -e DISTRIB=ubuntu-32 + -e TEST_DIR=/temp_dir + -e JUNITXML=$JUNITXML + -e VIRTUALENV=testvenv + -e JOBLIB_VERSION=$JOBLIB_VERSION + -e PYTEST_VERSION=$PYTEST_VERSION + -e OMP_NUM_THREADS=$OMP_NUM_THREADS + -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS + -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS + i386/ubuntu:16.04 + sleep 1000000 + displayName: 'Start container' + - script: > + docker exec skcontainer ./build_tools/azure/install.sh + displayName: 'Install' + - script: > + docker exec skcontainer ./build_tools/azure/test_script.sh + displayName: 'Test Library' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' + testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} + displayName: 'Publish Test Results' + condition: succeededOrFailed() + - script: > + docker container stop skcontainer + displayName: 'Stop container' + condition: always() From c8055062377076003004b7bdbdea9cd86bee97de Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:43:28 +0100 Subject: [PATCH 13/27] fix --- build_tools/azure/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 1521df644..cc7ff44d5 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -90,7 +90,7 @@ elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then # conda is still used as a convenient way to install Python and pip. make_conda "python=$PYTHON_VERSION" python -m pip install -U pip - python -m pip install numpy scipy joblib + python -m pip install numpy scipy joblib cython python -m pip install git+https://github.com/scikit-learn/scikit-learn.git python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist python -m pip install pandas From 469d83ff8fb8e0b2ccf51bcf58180e7c7b9cc09d Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:47:01 +0100 Subject: [PATCH 14/27] fix --- build_tools/azure/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index cc7ff44d5..3294984e9 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -22,7 +22,7 @@ if [[ "$DISTRIB" == "conda" ]]; then TO_INSTALL="python=$PYTHON_VERSION pip \ numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \ - joblib=$JOBLIB_VERSION" + joblib=$JOBLIB_VERSION git" if [[ "$INSTALL_MKL" == "true" ]]; then TO_INSTALL="$TO_INSTALL mkl" @@ -70,7 +70,7 @@ if [[ "$DISTRIB" == "conda" ]]; then elif [[ "$DISTRIB" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate python -m pip install pandas From 2659ded2bceba64de43673349bbb9fbf81fdf9df Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 22:52:19 +0100 Subject: [PATCH 15/27] iter --- build_tools/azure/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index 3294984e9..d05262c71 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -70,7 +70,7 @@ if [[ "$DISTRIB" == "conda" ]]; then elif [[ "$DISTRIB" == "ubuntu" ]]; then sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv git python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate python -m pip install pandas @@ -78,7 +78,7 @@ elif [[ "$DISTRIB" == "ubuntu" ]]; then python -m pip install git+https://github.com/scikit-learn/scikit-learn.git elif [[ "$DISTRIB" == "ubuntu-32" ]]; then apt-get update - apt-get install -y python3-dev python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv + apt-get install -y python3-dev python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv git python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate python -m pip install pandas From 95b7782511dda502039aff76ae75acff6329e786 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 23:09:43 +0100 Subject: [PATCH 16/27] iter --- azure-pipelines.yml | 12 +++++++++--- build_tools/azure/test_docs.sh | 6 ++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9a2a7152f..e55137c6b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,7 +28,7 @@ jobs: py35_ubuntu_atlas: DISTRIB: 'ubuntu' PYTHON_VERSION: '3.5' - JOBLIB_VERSION: '0.11' + JOBLIB_VERSION: '*' # Linux environment to test the latest available dependencies and MKL. pylatest_pip_openblas_pandas: DISTRIB: 'conda-pip-latest' @@ -36,7 +36,7 @@ jobs: COVERAGE: 'true' PANDAS_VERSION: '*' TEST_DOCSTRINGS: 'true' - CHECK_WARNINGS: 'true' + JOBLIB_VERSION: '*' pylatest_conda_pandas_keras: DISTRIB: 'conda' PYTHON_VERSION: '3.7' @@ -44,13 +44,17 @@ jobs: PANDAS_VERSION: '*' KERAS_VERSION: '*' COVERAGE: 'true' + JOBLIB_VERSION: '*' + TEST_DOCSTRINGS: 'true' pylatest_conda_pandas_tensorflow: DISTRIB: 'conda' PYTHON_VERSION: '3.7' PANDAS_VERSION: '*' + JOBLIB_VERSION: '*' INSTALL_MKL: 'true' TENSORFLOW_VERSION: '*' COVERAGE: 'true' + TEST_DOCSTRINGS: 'true' - template: build_tools/azure/posix-32.yml parameters: @@ -61,7 +65,8 @@ jobs: py35_ubuntu_atlas_32bit: DISTRIB: 'ubuntu-32' PYTHON_VERSION: '3.5' - JOBLIB_VERSION: '0.11' + JOBLIB_VERSION: '*' + TEST_DOCSTRINGS: 'true' - template: build_tools/azure/posix.yml parameters: @@ -79,6 +84,7 @@ jobs: PYTEST_VERSION: '*' JOBLIB_VERSION: '*' COVERAGE: 'true' + TEST_DOCSTRINGS: 'true' # - template: build_tools/azure/windows.yml # parameters: diff --git a/build_tools/azure/test_docs.sh b/build_tools/azure/test_docs.sh index 9ff94e183..f206b4334 100755 --- a/build_tools/azure/test_docs.sh +++ b/build_tools/azure/test_docs.sh @@ -8,5 +8,7 @@ elif [[ "$DISTRIB" == "ubuntu" ]]; then source $VIRTUALENV/bin/activate fi -make test-doc -pytest -vsl maint_tools/test_docstring.py +if [[ "TEST_DOCSTRING" == 'true' ]]; then + make test-doc + pytest -vsl maint_tools/test_docstring.py +fi From 01073c5f46764c1275814450ac45242634f7970c Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 23:16:27 +0100 Subject: [PATCH 17/27] iter --- azure-pipelines.yml | 30 ++++++++-------- build_tools/azure/install.cmd | 41 ++++++++++++++++++++++ build_tools/azure/test_script.cmd | 20 +++++++++++ build_tools/azure/upload_codecov.cmd | 10 ++++++ build_tools/azure/windows.yml | 51 ++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 build_tools/azure/install.cmd create mode 100644 build_tools/azure/test_script.cmd create mode 100644 build_tools/azure/upload_codecov.cmd create mode 100644 build_tools/azure/windows.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e55137c6b..be3bb9651 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -86,18 +86,18 @@ jobs: COVERAGE: 'true' TEST_DOCSTRINGS: 'true' -# - template: build_tools/azure/windows.yml -# parameters: -# name: Windows -# vmImage: vs2017-win2016 -# dependsOn: [linting] -# matrix: -# py37_conda_mkl: -# PYTHON_VERSION: '3.7' -# CHECK_WARNINGS: 'true' -# PYTHON_ARCH: '64' -# PYTEST_VERSION: '*' -# COVERAGE: 'true' -# py35_pip_openblas_32bit: -# PYTHON_VERSION: '3.5' -# PYTHON_ARCH: '32' +- template: build_tools/azure/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2016 + dependsOn: [linting] + matrix: + py37_conda_mkl: + PYTHON_VERSION: '3.7' + CHECK_WARNINGS: 'true' + PYTHON_ARCH: '64' + PYTEST_VERSION: '*' + COVERAGE: 'true' + py35_pip_openblas_32bit: + PYTHON_VERSION: '3.5' + PYTHON_ARCH: '32' diff --git a/build_tools/azure/install.cmd b/build_tools/azure/install.cmd new file mode 100644 index 000000000..26f7b0f65 --- /dev/null +++ b/build_tools/azure/install.cmd @@ -0,0 +1,41 @@ +@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd +@rem The cmd /C hack circumvents a regression where conda installs a conda.bat +@rem script in non-root environments. +set CONDA_INSTALL=cmd /C conda install -q -y +set PIP_INSTALL=pip install -q + +@echo on + +IF "%PYTHON_ARCH%"=="64" ( + @rem Deactivate any environment + call deactivate + @rem Clean up any left-over from a previous build + conda remove --all -q -y -n %VIRTUALENV% + conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% numpy scipy cython wheel joblib git + + call activate %VIRTUALENV% + + IF "%PYTEST_VERSION%"=="*" ( + pip install pytest + ) else ( + pip install pytest==%PYTEST_VERSION% + ) + pip install pytest-xdist +) else ( + pip install numpy scipy cython pytest wheel pillow joblib +) +if "%COVERAGE%" == "true" ( + pip install coverage codecov pytest-cov +) +python --version +pip --version + +pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn + +@rem Install the build and runtime dependencies of the project. +python setup.py bdist_wheel bdist_wininst + +@rem Install the generated wheel package to test it +pip install --pre --no-index --find-links dist\ imbalanced-learn + +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/build_tools/azure/test_script.cmd b/build_tools/azure/test_script.cmd new file mode 100644 index 000000000..eb6c4e7ee --- /dev/null +++ b/build_tools/azure/test_script.cmd @@ -0,0 +1,20 @@ +@echo on + +@rem Only 64 bit uses conda and uses a python newer than 3.5 +IF "%PYTHON_ARCH%"=="64" ( +call activate %VIRTUALENV% +set PYTEST_ARGS=%PYTEST_ARGS% -n2 +) + +mkdir %TMP_FOLDER% +cd %TMP_FOLDER% + +if "%CHECK_WARNINGS%" == "true" ( +set PYTEST_ARGS=%PYTEST_ARGS% -Werror::DeprecationWarning -Werror::FutureWarning +) + +if "%COVERAGE%" == "true" ( +set PYTEST_ARGS=%PYTEST_ARGS% --cov imblearn +) + +pytest --junitxml=%JUNITXML% --showlocals --durations=20 %PYTEST_ARGS% --pyargs imblearn diff --git a/build_tools/azure/upload_codecov.cmd b/build_tools/azure/upload_codecov.cmd new file mode 100644 index 000000000..4c5e8d0cf --- /dev/null +++ b/build_tools/azure/upload_codecov.cmd @@ -0,0 +1,10 @@ +@echo on + +@rem Only 64 bit uses conda +IF "%PYTHON_ARCH%"=="64" ( +call activate %VIRTUALENV% +) + +copy %TMP_FOLDER%\.coverage %BUILD_REPOSITORY_LOCALPATH% + +codecov --root %BUILD_REPOSITORY_LOCALPATH% -t %CODECOV_TOKEN% diff --git a/build_tools/azure/windows.yml b/build_tools/azure/windows.yml new file mode 100644 index 000000000..24b542b22 --- /dev/null +++ b/build_tools/azure/windows.yml @@ -0,0 +1,51 @@ + +parameters: + name: '' + vmImage: '' + matrix: [] + dependsOn: [] + +jobs: +- job: ${{ parameters.name }} + dependsOn: ${{ parameters.dependsOn }} + pool: + vmImage: ${{ parameters.vmImage }} + variables: + VIRTUALENV: 'testvenv' + JUNITXML: 'test-data.xml' + SKLEARN_SKIP_NETWORK_TESTS: '1' + PYTEST_VERSION: '5.2.1' + TMP_FOLDER: '$(Agent.WorkFolder)\tmp_folder' + strategy: + matrix: + ${{ insert }}: ${{ parameters.matrix }} + + steps: + - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + displayName: Add conda to PATH for 64 bit Python + condition: eq(variables['PYTHON_ARCH'], '64') + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(PYTHON_VERSION)' + addToPath: true + architecture: 'x86' + displayName: Use 32 bit System Python + condition: eq(variables['PYTHON_ARCH'], '32') + - script: | + build_tools\\azure\\install.cmd + displayName: 'Install' + - script: | + build_tools\\azure\\test_script.cmd + displayName: 'Test Library' + - script: | + build_tools\\azure\\upload_codecov.cmd + condition: and(succeeded(), eq(variables['COVERAGE'], 'true')) + displayName: 'Upload To Codecov' + env: + CODECOV_TOKEN: $(CODECOV_TOKEN) + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(TMP_FOLDER)\$(JUNITXML)' + testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} + displayName: 'Publish Test Results' + condition: succeededOrFailed() From af370122fcc3f51d61d4086631c901faac158765 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 23:17:57 +0100 Subject: [PATCH 18/27] fix --- build_tools/azure/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh index d05262c71..04d95282a 100755 --- a/build_tools/azure/install.sh +++ b/build_tools/azure/install.sh @@ -74,7 +74,7 @@ elif [[ "$DISTRIB" == "ubuntu" ]]; then python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate python -m pip install pandas - python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION cython + python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib cython python -m pip install git+https://github.com/scikit-learn/scikit-learn.git elif [[ "$DISTRIB" == "ubuntu-32" ]]; then apt-get update @@ -82,7 +82,7 @@ elif [[ "$DISTRIB" == "ubuntu-32" ]]; then python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV source $VIRTUALENV/bin/activate python -m pip install pandas - python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib==$JOBLIB_VERSION cython + python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib cython python -m pip install git+https://github.com/scikit-learn/scikit-learn.git elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then # Since conda main channel usually lacks behind on the latest releases, From 83112d4df3ab9c4f29ee0f6dfd6661f909a47b41 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 23:24:04 +0100 Subject: [PATCH 19/27] fix --- build_tools/azure/install.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/azure/install.cmd b/build_tools/azure/install.cmd index 26f7b0f65..607f46b6a 100644 --- a/build_tools/azure/install.cmd +++ b/build_tools/azure/install.cmd @@ -30,7 +30,7 @@ if "%COVERAGE%" == "true" ( python --version pip --version -pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn +pip install git+https://github.com/scikit-learn/scikit-learn.git @rem Install the build and runtime dependencies of the project. python setup.py bdist_wheel bdist_wininst From 1c9d75d3e3c9803d14efc10187bce29f0f42f6a7 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 23:47:08 +0100 Subject: [PATCH 20/27] iter --- README.rst | 5 ++++- azure-pipelines.yml | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b8609349b..6c447b818 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,10 @@ .. _scikit-learn-contrib: https://github.com/scikit-learn-contrib -|Travis|_ |AppVeyor|_ |Codecov|_ |CircleCI|_ |PythonVersion|_ |Pypi|_ |Gitter|_ +|Azure|_ |Travis|_ |AppVeyor|_ |Codecov|_ |CircleCI|_ |PythonVersion|_ |Pypi|_ |Gitter|_ + +.. |Azure| image:: https://dev.azure.com/imbalanced-learn/imbalanced-learn/_apis/build/status/scikit-learn-contrib.imbalanced-learn?branchName=master +.. _Azure: https://dev.azure.com/imbalanced-learn/imbalanced-learn/_build .. |Travis| image:: https://travis-ci.org/scikit-learn-contrib/imbalanced-learn.svg?branch=master .. _Travis: https://travis-ci.org/scikit-learn-contrib/imbalanced-learn diff --git a/azure-pipelines.yml b/azure-pipelines.yml index be3bb9651..56d02e9e2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,7 +94,6 @@ jobs: matrix: py37_conda_mkl: PYTHON_VERSION: '3.7' - CHECK_WARNINGS: 'true' PYTHON_ARCH: '64' PYTEST_VERSION: '*' COVERAGE: 'true' From 5c8ac7dd95e05f32fff3bbd0097ba61c5a953e5e Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Wed, 20 Nov 2019 23:58:08 +0100 Subject: [PATCH 21/27] junit warning --- build_tools/azure/test_script.cmd | 2 +- build_tools/azure/test_script.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/azure/test_script.cmd b/build_tools/azure/test_script.cmd index eb6c4e7ee..cd9aab5a1 100644 --- a/build_tools/azure/test_script.cmd +++ b/build_tools/azure/test_script.cmd @@ -17,4 +17,4 @@ if "%COVERAGE%" == "true" ( set PYTEST_ARGS=%PYTEST_ARGS% --cov imblearn ) -pytest --junitxml=%JUNITXML% --showlocals --durations=20 %PYTEST_ARGS% --pyargs imblearn +pytest --junitxml=%JUNITXML% --junit_family=xunit2 --showlocals --durations=20 %PYTEST_ARGS% --pyargs imblearn diff --git a/build_tools/azure/test_script.sh b/build_tools/azure/test_script.sh index 37793c529..f4b2e01fb 100755 --- a/build_tools/azure/test_script.sh +++ b/build_tools/azure/test_script.sh @@ -21,7 +21,7 @@ except ImportError: python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())" pip list -TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML" +TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML --junit_family=xunit2" if [[ "$COVERAGE" == "true" ]]; then export COVERAGE_PROCESS_START="$BUILD_SOURCESDIRECTORY/.coveragerc" From 28eec62ac26460fd6310e31935a5ea263201a53c Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Thu, 21 Nov 2019 00:00:44 +0100 Subject: [PATCH 22/27] avoid warning --- examples/applications/plot_over_sampling_benchmark_lfw.py | 2 +- imblearn/ensemble/tests/test_bagging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/applications/plot_over_sampling_benchmark_lfw.py b/examples/applications/plot_over_sampling_benchmark_lfw.py index c76856ac0..e90805e80 100644 --- a/examples/applications/plot_over_sampling_benchmark_lfw.py +++ b/examples/applications/plot_over_sampling_benchmark_lfw.py @@ -61,7 +61,7 @@ def fit_resample(self, X, y): classifier = ['3NN', neighbors.KNeighborsClassifier(3)] samplers = [ - ['Standard', DummySampler()], + ['Standard', DummySampler(strategy="prior")], ['ADASYN', ADASYN(random_state=RANDOM_STATE)], ['ROS', RandomOverSampler(random_state=RANDOM_STATE)], ['SMOTE', SMOTE(random_state=RANDOM_STATE)], diff --git a/imblearn/ensemble/tests/test_bagging.py b/imblearn/ensemble/tests/test_bagging.py index 738266147..510fa4c95 100644 --- a/imblearn/ensemble/tests/test_bagging.py +++ b/imblearn/ensemble/tests/test_bagging.py @@ -50,7 +50,7 @@ def test_balanced_bagging_classifier(): for base_estimator in [ None, - DummyClassifier(), + DummyClassifier(strategy="prior"), Perceptron(max_iter=1000, tol=1e-3), DecisionTreeClassifier(), KNeighborsClassifier(), From 27ccbd40d82e8b71aca842d2e5af73d6fd2ef109 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Thu, 21 Nov 2019 00:03:01 +0100 Subject: [PATCH 23/27] add check warning back --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 56d02e9e2..49d265978 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,6 +37,7 @@ jobs: PANDAS_VERSION: '*' TEST_DOCSTRINGS: 'true' JOBLIB_VERSION: '*' + CHECK_WARNINGS: 'true' pylatest_conda_pandas_keras: DISTRIB: 'conda' PYTHON_VERSION: '3.7' @@ -85,6 +86,7 @@ jobs: JOBLIB_VERSION: '*' COVERAGE: 'true' TEST_DOCSTRINGS: 'true' + CHECK_WARNINGS: 'true' - template: build_tools/azure/windows.yml parameters: @@ -97,6 +99,7 @@ jobs: PYTHON_ARCH: '64' PYTEST_VERSION: '*' COVERAGE: 'true' + CHECK_WARNINGS: 'true' py35_pip_openblas_32bit: PYTHON_VERSION: '3.5' PYTHON_ARCH: '32' From 0d15b4dcaea3e0a0c763501fb00f739ee2b8e03b Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Thu, 21 Nov 2019 00:08:28 +0100 Subject: [PATCH 24/27] iter --- imblearn/utils/tests/test_docstring.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/imblearn/utils/tests/test_docstring.py b/imblearn/utils/tests/test_docstring.py index 74b3fe0c1..89746de1c 100644 --- a/imblearn/utils/tests/test_docstring.py +++ b/imblearn/utils/tests/test_docstring.py @@ -6,6 +6,8 @@ import pytest from imblearn.utils import Substitution +from imblearn.utils._docstring import _random_state_docstring +from imblearn.utils._docstring import _n_jobs_docstring func_docstring = """A function. @@ -60,3 +62,8 @@ def __init__(self, param_1, param_2): def test_docstring_inject(obj, obj_docstring): obj_injected_docstring = Substitution(param_1="xxx", param_2="yyy")(obj) assert obj_injected_docstring.__doc__ == obj_docstring + + +def test_docstring_template(): + assert "random_state" in _random_state_docstring + assert "n_jobs" in _n_jobs_docstring From ca66e261f316a81955720f1f26fce6b4c9c98de0 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Thu, 21 Nov 2019 00:09:23 +0100 Subject: [PATCH 25/27] iter --- build_tools/azure/test_script.cmd | 2 +- build_tools/azure/test_script.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/azure/test_script.cmd b/build_tools/azure/test_script.cmd index cd9aab5a1..eb6c4e7ee 100644 --- a/build_tools/azure/test_script.cmd +++ b/build_tools/azure/test_script.cmd @@ -17,4 +17,4 @@ if "%COVERAGE%" == "true" ( set PYTEST_ARGS=%PYTEST_ARGS% --cov imblearn ) -pytest --junitxml=%JUNITXML% --junit_family=xunit2 --showlocals --durations=20 %PYTEST_ARGS% --pyargs imblearn +pytest --junitxml=%JUNITXML% --showlocals --durations=20 %PYTEST_ARGS% --pyargs imblearn diff --git a/build_tools/azure/test_script.sh b/build_tools/azure/test_script.sh index f4b2e01fb..37793c529 100755 --- a/build_tools/azure/test_script.sh +++ b/build_tools/azure/test_script.sh @@ -21,7 +21,7 @@ except ImportError: python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())" pip list -TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML --junit_family=xunit2" +TEST_CMD="python -m pytest --showlocals --durations=20 --junitxml=$JUNITXML" if [[ "$COVERAGE" == "true" ]]; then export COVERAGE_PROCESS_START="$BUILD_SOURCESDIRECTORY/.coveragerc" From 10f78167748510ad7b939f96fb621b6e9dc69bf9 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Thu, 21 Nov 2019 00:14:47 +0100 Subject: [PATCH 26/27] iter --- examples/applications/plot_over_sampling_benchmark_lfw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/applications/plot_over_sampling_benchmark_lfw.py b/examples/applications/plot_over_sampling_benchmark_lfw.py index e90805e80..c76856ac0 100644 --- a/examples/applications/plot_over_sampling_benchmark_lfw.py +++ b/examples/applications/plot_over_sampling_benchmark_lfw.py @@ -61,7 +61,7 @@ def fit_resample(self, X, y): classifier = ['3NN', neighbors.KNeighborsClassifier(3)] samplers = [ - ['Standard', DummySampler(strategy="prior")], + ['Standard', DummySampler()], ['ADASYN', ADASYN(random_state=RANDOM_STATE)], ['ROS', RandomOverSampler(random_state=RANDOM_STATE)], ['SMOTE', SMOTE(random_state=RANDOM_STATE)], From e0c96bf4a44fde0a42e975606dc607f6c834c8ae Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Thu, 21 Nov 2019 00:27:46 +0100 Subject: [PATCH 27/27] iter --- imblearn/datasets/tests/test_zenodo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imblearn/datasets/tests/test_zenodo.py b/imblearn/datasets/tests/test_zenodo.py index efd4943c2..45429dab3 100644 --- a/imblearn/datasets/tests/test_zenodo.py +++ b/imblearn/datasets/tests/test_zenodo.py @@ -46,6 +46,7 @@ def fetch(*args, **kwargs): return fetch_datasets(*args, download_if_missing=True, **kwargs) +@pytest.mark.xfail def test_fetch(): try: datasets1 = fetch(shuffle=True, random_state=42)