MNT activate azure pipelines

README.rst

@@ -4,7 +4,10 @@
 
 .. _scikit-learn-contrib: https://github.com/scikit-learn-contrib
 
-|Travis|_ |AppVeyor|_ |Codecov|_ |CircleCI|_ |PythonVersion|_ |Pypi|_ |Gitter|_
+|Azure|_ |Travis|_ |AppVeyor|_ |Codecov|_ |CircleCI|_ |PythonVersion|_ |Pypi|_ |Gitter|_
+
+.. |Azure| image:: https://dev.azure.com/imbalanced-learn/imbalanced-learn/_apis/build/status/scikit-learn-contrib.imbalanced-learn?branchName=master
+.. _Azure: https://dev.azure.com/imbalanced-learn/imbalanced-learn/_build
 
 .. |Travis| image:: https://travis-ci.org/scikit-learn-contrib/imbalanced-learn.svg?branch=master
 .. _Travis: https://travis-ci.org/scikit-learn-contrib/imbalanced-learn

azure-pipelines.yml

@@ -1,36 +1,105 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+# Adapted from https://github.com/scikit-learn/scikit-learn/blob/master/azure-pipelines.yml
+jobs:
+- job: linting
+  displayName: Linting
+  pool:
+    vmImage: ubuntu-16.04
+  steps:
+    - bash: echo "##vso[task.prependpath]$CONDA/bin"
+      displayName: Add conda to PATH
+    - bash: sudo chown -R $USER $CONDA
+      displayName: Take ownership of conda installation
+    - bash: conda create --name flake8_env --yes flake8
+      displayName: Install flake8
+    - bash: |
+        source activate flake8_env
+        ./build_tools/circle/linting.sh
+      displayName: Run linting
 
-trigger:
-- master
+- template: build_tools/azure/posix.yml
+  parameters:
+    name: Linux
+    vmImage: ubuntu-16.04
+    dependsOn: [linting]
+    matrix:
+      # Linux environment to test that scikit-learn can be built against
+      # versions of numpy, scipy with ATLAS that comes with Ubuntu Xenial 16.04
+      # i.e. numpy 1.11 and scipy 0.17
+      py35_ubuntu_atlas:
+        DISTRIB: 'ubuntu'
+        PYTHON_VERSION: '3.5'
+        JOBLIB_VERSION: '*'
+      # Linux environment to test the latest available dependencies and MKL.
+      pylatest_pip_openblas_pandas:
+        DISTRIB: 'conda-pip-latest'
+        PYTHON_VERSION: '3.7'
+        COVERAGE: 'true'
+        PANDAS_VERSION: '*'
+        TEST_DOCSTRINGS: 'true'
+        JOBLIB_VERSION: '*'
+        CHECK_WARNINGS: 'true'
+      pylatest_conda_pandas_keras:
+        DISTRIB: 'conda'
+        PYTHON_VERSION: '3.7'
+        INSTALL_MKL: 'true'
+        PANDAS_VERSION: '*'
+        KERAS_VERSION: '*'
+        COVERAGE: 'true'
+        JOBLIB_VERSION: '*'
+        TEST_DOCSTRINGS: 'true'
+      pylatest_conda_pandas_tensorflow:
+        DISTRIB: 'conda'
+        PYTHON_VERSION: '3.7'
+        PANDAS_VERSION: '*'
+        JOBLIB_VERSION: '*'
+        INSTALL_MKL: 'true'
+        TENSORFLOW_VERSION: '*'
+        COVERAGE: 'true'
+        TEST_DOCSTRINGS: 'true'
 
-pool:
-  vmImage: 'ubuntu-latest'
-strategy:
-  matrix:
-    Python27:
-      python.version: '2.7'
-    Python35:
-      python.version: '3.5'
-    Python36:
-      python.version: '3.6'
-    Python37:
-      python.version: '3.7'
+- template: build_tools/azure/posix-32.yml
+  parameters:
+    name: Linux32
+    vmImage: ubuntu-16.04
+    dependsOn: [linting]
+    matrix:
+      py35_ubuntu_atlas_32bit:
+        DISTRIB: 'ubuntu-32'
+        PYTHON_VERSION: '3.5'
+        JOBLIB_VERSION: '*'
+        TEST_DOCSTRINGS: 'true'
 
-steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-  displayName: 'Use Python $(python.version)'
+- template: build_tools/azure/posix.yml
+  parameters:
+    name: macOS
+    vmImage: xcode9-macos10.13
+    dependsOn: [linting]
+    matrix:
+      pylatest_conda_mkl:
+        DISTRIB: 'conda'
+        PYTHON_VERSION: '*'
+        INSTALL_MKL: 'true'
+        NUMPY_VERSION: '*'
+        SCIPY_VERSION: '*'
+        PANDAS_VERSION: '*'
+        PYTEST_VERSION: '*'
+        JOBLIB_VERSION: '*'
+        COVERAGE: 'true'
+        TEST_DOCSTRINGS: 'true'
+        CHECK_WARNINGS: 'true'
 
-- script: |
-    python -m pip install --upgrade pip
-    pip install -r requirements.txt
-  displayName: 'Install dependencies'
-
-- script: |
-    pip install pytest pytest-azurepipelines
-    pytest
-  displayName: 'pytest'
+- template: build_tools/azure/windows.yml
+  parameters:
+    name: Windows
+    vmImage: vs2017-win2016
+    dependsOn: [linting]
+    matrix:
+      py37_conda_mkl:
+        PYTHON_VERSION: '3.7'
+        PYTHON_ARCH: '64'
+        PYTEST_VERSION: '*'
+        COVERAGE: 'true'
+        CHECK_WARNINGS: 'true'
+      py35_pip_openblas_32bit:
+        PYTHON_VERSION: '3.5'
+        PYTHON_ARCH: '32'

build_tools/azure/install.cmd

@@ -0,0 +1,41 @@
+@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd
+@rem The cmd /C hack circumvents a regression where conda installs a conda.bat
+@rem script in non-root environments.
+set CONDA_INSTALL=cmd /C conda install -q -y
+set PIP_INSTALL=pip install -q
+
+@echo on
+
+IF "%PYTHON_ARCH%"=="64" (
+    @rem Deactivate any environment
+    call deactivate
+    @rem Clean up any left-over from a previous build
+    conda remove --all -q -y -n %VIRTUALENV%
+    conda create -n %VIRTUALENV% -q -y python=%PYTHON_VERSION% numpy scipy cython wheel joblib git
+
+    call activate %VIRTUALENV%
+
+    IF "%PYTEST_VERSION%"=="*" (
+        pip install pytest
+    ) else (
+        pip install pytest==%PYTEST_VERSION%
+    )
+    pip install pytest-xdist
+) else (
+    pip install numpy scipy cython pytest wheel pillow joblib
+)
+if "%COVERAGE%" == "true" (
+    pip install coverage codecov pytest-cov
+)
+python --version
+pip --version
+
+pip install git+https://github.com/scikit-learn/scikit-learn.git
+
+@rem Install the build and runtime dependencies of the project.
+python setup.py bdist_wheel bdist_wininst
+
+@rem Install the generated wheel package to test it
+pip install --pre --no-index --find-links dist\ imbalanced-learn
+
+if %errorlevel% neq 0 exit /b %errorlevel%

build_tools/azure/install.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+set -e
+
+UNAMESTR=`uname`
+
+make_conda() {
+    TO_INSTALL="$@"
+    conda create -n $VIRTUALENV --yes $TO_INSTALL
+    source activate $VIRTUALENV
+}
+
+version_ge() {
+    # The two version numbers are seperated with a new line is piped to sort
+    # -rV. The -V activates for version number sorting and -r sorts in
+    # decending order. If the first argument is the top element of the sort, it
+    # is greater than or equal to the second argument.
+    test "$(printf "${1}\n${2}" | sort -rV | head -n 1)" == "$1"
+}
+
+if [[ "$DISTRIB" == "conda" ]]; then
+
+    TO_INSTALL="python=$PYTHON_VERSION pip \
+                numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION \
+                joblib=$JOBLIB_VERSION git"
+
+    if [[ "$INSTALL_MKL" == "true" ]]; then
+        TO_INSTALL="$TO_INSTALL mkl"
+    else
+        TO_INSTALL="$TO_INSTALL nomkl"
+    fi
+
+    make_conda $TO_INSTALL
+    python -m pip install --pre -f https://sklearn-nightly.scdn8.secure.raxcdn.com scikit-learn
+
+    TO_INSTALL=""
+
+    if [[ -n "$PANDAS_VERSION" ]]; then
+        TO_INSTALL="$TO_INSTALL pandas=$PANDAS_VERSION"
+    fi
+
+    if [[ -n "$KERAS_VERSION" ]]; then
+        TO_INSTALL="$TO_INSTALL keras=$KERAS_VERSION tensorflow=1"
+        KERAS_BACKEND=tensorflow
+    fi
+
+    if [[ -n "$TENSORFLOW_VERSION" ]]; then
+        TO_INSTALL="$TO_INSTALL tensorflow=$TENSORFLOW_VERSION"
+    fi
+
+    if [[ "$PYTEST_VERSION" == "*" ]]; then
+        python -m pip install pytest
+    else
+        python -m pip install pytest=="$PYTEST_VERSION"
+    fi
+
+    if [[ "$PYTHON_VERSION" == "*" ]]; then
+        python -m pip install pytest-xdist
+    fi
+
+    if [[ -n "$TO_INSTALL" ]]; then
+       conda install --yes $TO_INSTALL
+    fi
+
+    if [[ -n "$KERAS_VERSION" ]]; then
+        python -c "import keras.backend"
+        sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json;
+    fi
+
+elif [[ "$DISTRIB" == "ubuntu" ]]; then
+    sudo add-apt-repository --remove ppa:ubuntu-toolchain-r/test
+    sudo apt-get update
+    sudo apt-get install python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv git
+    python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
+    source $VIRTUALENV/bin/activate
+    python -m pip install pandas
+    python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib cython
+    python -m pip install git+https://github.com/scikit-learn/scikit-learn.git
+elif [[ "$DISTRIB" == "ubuntu-32" ]]; then
+    apt-get update
+    apt-get install -y python3-dev python3-scipy libatlas3-base libatlas-base-dev libatlas-dev python3-virtualenv git
+    python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
+    source $VIRTUALENV/bin/activate
+    python -m pip install pandas
+    python -m pip install pytest==$PYTEST_VERSION pytest-cov joblib cython
+    python -m pip install git+https://github.com/scikit-learn/scikit-learn.git
+elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then
+    # Since conda main channel usually lacks behind on the latest releases,
+    # we use pypi to test against the latest releases of the dependencies.
+    # conda is still used as a convenient way to install Python and pip.
+    make_conda "python=$PYTHON_VERSION"
+    python -m pip install -U pip
+    python -m pip install numpy scipy joblib cython
+    python -m pip install git+https://github.com/scikit-learn/scikit-learn.git
+    python -m pip install pytest==$PYTEST_VERSION pytest-cov pytest-xdist
+    python -m pip install pandas
+fi
+
+if [[ "$COVERAGE" == "true" ]]; then
+    python -m pip install coverage codecov pytest-cov
+fi
+
+if [[ "$TEST_DOCSTRINGS" == "true" ]]; then
+    python -m pip install sphinx
+    python -m pip install -U git+https://github.com/numpy/numpydoc.git
+fi
+
+python --version
+python -c "import numpy; print('numpy %s' % numpy.__version__)"
+python -c "import scipy; print('scipy %s' % scipy.__version__)"
+python -c "\
+try:
+    import pandas
+    print('pandas %s' % pandas.__version__)
+except ImportError:
+    print('pandas not installed')
+"
+python -m pip list
+
+# Use setup.py instead of `pip install -e .` to be able to pass the -j flag
+# to speed-up the building multicore CI machines.
+python setup.py develop

build_tools/azure/posix-32.yml

@@ -0,0 +1,61 @@
+parameters:
+  name: ''
+  vmImage: ''
+  matrix: []
+  dependsOn: []
+
+jobs:
+- job: ${{ parameters.name }}
+  dependsOn: ${{ parameters.dependsOn }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  variables:
+    TEST_DIR: '$(Agent.WorkFolder)/tmp_folder'
+    JUNITXML: 'test-data.xml'
+    OMP_NUM_THREADS: '4'
+    PYTEST_VERSION: '5.2.1'
+    OPENBLAS_NUM_THREADS: '4'
+    SKLEARN_SKIP_NETWORK_TESTS: '1'
+  strategy:
+    matrix:
+      ${{ insert }}: ${{ parameters.matrix }}
+
+  steps:
+    # Container is detached and sleeping, allowing steps to run commmands
+    # in the container. The TEST_DIR is mapped allowing the host to access
+    # the JUNITXML file
+    - script: >
+        docker container run --rm
+        --volume $TEST_DIR:/temp_dir
+        --volume $PWD:/io
+        -w /io
+        --detach
+        --name skcontainer
+        -e DISTRIB=ubuntu-32
+        -e TEST_DIR=/temp_dir
+        -e JUNITXML=$JUNITXML
+        -e VIRTUALENV=testvenv
+        -e JOBLIB_VERSION=$JOBLIB_VERSION
+        -e PYTEST_VERSION=$PYTEST_VERSION
+        -e OMP_NUM_THREADS=$OMP_NUM_THREADS
+        -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS
+        -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS
+        i386/ubuntu:16.04
+        sleep 1000000
+      displayName: 'Start container'
+    - script: >
+        docker exec skcontainer ./build_tools/azure/install.sh
+      displayName: 'Install'
+    - script: >
+        docker exec skcontainer ./build_tools/azure/test_script.sh
+      displayName: 'Test Library'
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFiles: '$(TEST_DIR)/$(JUNITXML)'
+        testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }}
+      displayName: 'Publish Test Results'
+      condition: succeededOrFailed()
+    - script: >
+        docker container stop skcontainer
+      displayName: 'Stop container'
+      condition: always()