From 1979f292f1714b5ace7d67d096e2efaa4d02c04d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 7 Aug 2021 09:38:50 -0700 Subject: [PATCH 01/11] Update windows.yml --- azure/windows.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/azure/windows.yml b/azure/windows.yml index 9334f50..680641b 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -66,14 +66,18 @@ jobs: set -e pip install --timeout=60 numpy==$NP_BUILD_DEP pip install --timeout=60 $TEST_DEPENDS Cython==$CYTHON_BUILD_DEP - pip install twine wheel + pip install twine wheel delvewheel pushd pandas cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/$PYTHON_ARCH/Microsoft.VC142.CRT/msvcp140.dll" pandas/_libs/window - cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/$PYTHON_ARCH/Microsoft.VC142.CRT/concrt140.dll" pandas/_libs/window + cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/$PYTHON_ARCH/Microsoft.VC142.CRT/vcruntime140_1.dll" pandas/_libs/window python setup.py build python setup.py bdist_wheel ls dist twine check dist/* + # Delvewheel doesn't support using the directory path :( + # We are just using delvewheel right now to list dll dependencies not copy them + WHEEL_PATH = $(python -c "import os; print(os.listdir('.')[0])") + delvewheel show WHEEL_PATH popd displayName: Build wheel condition: eq(variables['SKIP_BUILD'], 'false') @@ -125,3 +129,6 @@ jobs: echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple" displayName: Upload to anaconda.org (only if secret token is retrieved) condition: ne(variables['TOKEN'], '') + + - publish: pandas/dist/pandas-*.whl + artifact: Wheel From c094de314d5f7f52312e6033018b7b877932bcf8 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 7 Aug 2021 09:44:05 -0700 Subject: [PATCH 02/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index 680641b..6d3101d 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -69,7 +69,7 @@ jobs: pip install twine wheel delvewheel pushd pandas cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/$PYTHON_ARCH/Microsoft.VC142.CRT/msvcp140.dll" pandas/_libs/window - cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/$PYTHON_ARCH/Microsoft.VC142.CRT/vcruntime140_1.dll" pandas/_libs/window + cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/x64/Microsoft.VC142.CRT/vcruntime140_1.dll" pandas/_libs/window python setup.py build python setup.py bdist_wheel ls dist From 9ba46c904ab476e80809dc35019827f362c6f320 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 7 Aug 2021 10:09:27 -0700 Subject: [PATCH 03/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index 6d3101d..bee0797 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -76,7 +76,7 @@ jobs: twine check dist/* # Delvewheel doesn't support using the directory path :( # We are just using delvewheel right now to list dll dependencies not copy them - WHEEL_PATH = $(python -c "import os; print(os.listdir('.')[0])") + export WHEEL_PATH = "dist/$(ls dist)" delvewheel show WHEEL_PATH popd displayName: Build wheel From 17b708425439165d425c0d4d92fa89612364c1b4 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 7 Aug 2021 10:44:35 -0700 Subject: [PATCH 04/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index bee0797..c539014 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -76,7 +76,7 @@ jobs: twine check dist/* # Delvewheel doesn't support using the directory path :( # We are just using delvewheel right now to list dll dependencies not copy them - export WHEEL_PATH = "dist/$(ls dist)" + export WHEEL_PATH="dist/$(ls dist)" delvewheel show WHEEL_PATH popd displayName: Build wheel From aaf8226a517902aba6229bd3cec2011424ab839b Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 7 Aug 2021 11:14:18 -0700 Subject: [PATCH 05/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index c539014..fd8a1c5 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -77,7 +77,7 @@ jobs: # Delvewheel doesn't support using the directory path :( # We are just using delvewheel right now to list dll dependencies not copy them export WHEEL_PATH="dist/$(ls dist)" - delvewheel show WHEEL_PATH + delvewheel show $WHEEL_PATH popd displayName: Build wheel condition: eq(variables['SKIP_BUILD'], 'false') From c8b562ad24bb1b85372de7adbc50ffc31c6fe2f7 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 8 Aug 2021 08:33:16 -0700 Subject: [PATCH 06/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index fd8a1c5..e8ed363 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -130,5 +130,5 @@ jobs: displayName: Upload to anaconda.org (only if secret token is retrieved) condition: ne(variables['TOKEN'], '') - - publish: pandas/dist/pandas-*.whl + - publish: pandas/dist artifact: Wheel From 6034d06a6af69a2c1906e214e2edba27a54e54fc Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 8 Aug 2021 10:03:22 -0700 Subject: [PATCH 07/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index e8ed363..ed67223 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -131,4 +131,4 @@ jobs: condition: ne(variables['TOKEN'], '') - publish: pandas/dist - artifact: Wheel + artifact: ${{parameters.name}} From e7289b04b17420d368bdd0ccfca041fd8066b76f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:11:11 -0700 Subject: [PATCH 08/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index ed67223..dcb9681 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -131,4 +131,4 @@ jobs: condition: ne(variables['TOKEN'], '') - publish: pandas/dist - artifact: ${{parameters.name}} + artifact: ${{Agent.JobName}} From 45b9c31ab6babcfb0be299ee80e5a7a464f5b4d5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:19:48 -0700 Subject: [PATCH 09/11] Update windows.yml --- azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/windows.yml b/azure/windows.yml index be7b3be..30e0e3f 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -131,4 +131,4 @@ jobs: condition: ne(variables['TOKEN'], '') - publish: pandas/dist - artifact: ${{Agent.JobName}} + artifact: $(Agent.JobName) From 1d66714fb3f1b242e2c2f8bd3dc693eeb856013e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 16 Aug 2021 17:39:18 -0700 Subject: [PATCH 10/11] Update azure-pipelines.yml --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3f35941..4c0475f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,7 +17,7 @@ jobs: - template: azure/windows.yml parameters: name: windows - vmImage: vs2017-win2016 + vmImage: windows-2019 matrix: py_3.7_32: PYTHON_VERSION: "3.7" @@ -97,7 +97,7 @@ jobs: - template: azure/posix.yml parameters: name: macOS - vmImage: macOS-10.14 + vmImage: macOS-10.15 matrix: py_3.7_64: MB_PYTHON_VERSION: "3.7" From c0639a552d030c6af08f4ddd186e72f2d9e38434 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 17 Aug 2021 14:24:08 -0700 Subject: [PATCH 11/11] Update windows.yml --- azure/windows.yml | 222 ++++++++++++++++++++++++---------------------- 1 file changed, 115 insertions(+), 107 deletions(-) diff --git a/azure/windows.yml b/azure/windows.yml index 30e0e3f..98b1413 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -19,116 +19,124 @@ jobs: strategy: matrix: ${{ insert }}: ${{ parameters.matrix }} - steps: - - checkout: self - submodules: true + stages: + - stage build: + steps: + - checkout: self + submodules: true - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - architecture: $(PYTHON_ARCH) - displayName: Set python version + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PYTHON_VERSION) + architecture: $(PYTHON_ARCH) + displayName: Set python version - - bash: | - set -e - echo PYTHON $PYTHON_VERSION $PYTHON_ARCH - echo Build Reason: $BUILD_REASON - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - pip --version - displayName: Check that we have the expected version and architecture for Python - - bash: | - set -e - SKIP_BUILD="false" - if [ "$BUILD_REASON" == "Schedule" ]; then - BUILD_COMMIT=$NIGHTLY_BUILD_COMMIT - if [ "$NIGHTLY_BUILD" != "true" ]; then - SKIP_BUILD="true" + - bash: | + set -e + echo PYTHON $PYTHON_VERSION $PYTHON_ARCH + echo Build Reason: $BUILD_REASON + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + pip --version + displayName: Check that we have the expected version and architecture for Python + - bash: | + set -e + SKIP_BUILD="false" + if [ "$BUILD_REASON" == "Schedule" ]; then + BUILD_COMMIT=$NIGHTLY_BUILD_COMMIT + if [ "$NIGHTLY_BUILD" != "true" ]; then + SKIP_BUILD="true" + fi fi - fi - echo "Building pandas@$BUILD_COMMIT" - echo "##vso[task.setvariable variable=BUILD_COMMIT]$BUILD_COMMIT" - echo "##vso[task.setvariable variable=SKIP_BUILD]$SKIP_BUILD" - # Store original Python path to be able to create test_venv pointing - # to same Python version. - PYTHON_EXE=`which python` - echo "##vso[task.setvariable variable=PYTHON_EXE]$PYTHON_EXE" - displayName: Define build env variables - - bash: | - set -e - cd pandas - git checkout $BUILD_COMMIT - git clean -fxd - git reset --hard - displayName: Checkout pandas commit - condition: eq(variables['SKIP_BUILD'], 'false') - - bash: | - set -e - pip install --timeout=60 numpy==$NP_BUILD_DEP - pip install --timeout=60 $TEST_DEPENDS Cython==$CYTHON_BUILD_DEP - pip install twine wheel delvewheel - pushd pandas - cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/$PYTHON_ARCH/Microsoft.VC142.CRT/msvcp140.dll" pandas/_libs/window - cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/x64/Microsoft.VC142.CRT/vcruntime140_1.dll" pandas/_libs/window - python setup.py build - python setup.py bdist_wheel - ls dist - twine check dist/* - # Delvewheel doesn't support using the directory path :( - # We are just using delvewheel right now to list dll dependencies not copy them - export WHEEL_PATH="dist/$(ls dist)" - delvewheel show $WHEEL_PATH - popd - displayName: Build wheel - condition: eq(variables['SKIP_BUILD'], 'false') - - bash: | - set -e - source extra_functions.sh - source config.sh - setup_test_venv - pip install pandas/dist/pandas-*.whl - run_tests - teardown_test_venv - displayName: Install wheel and test - condition: eq(variables['SKIP_BUILD'], 'false') + echo "Building pandas@$BUILD_COMMIT" + echo "##vso[task.setvariable variable=BUILD_COMMIT]$BUILD_COMMIT" + echo "##vso[task.setvariable variable=SKIP_BUILD]$SKIP_BUILD" + # Store original Python path to be able to create test_venv pointing + # to same Python version. + PYTHON_EXE=`which python` + echo "##vso[task.setvariable variable=PYTHON_EXE]$PYTHON_EXE" + displayName: Define build env variables + - bash: | + set -e + cd pandas + git checkout $BUILD_COMMIT + git clean -fxd + git reset --hard + displayName: Checkout pandas commit + condition: eq(variables['SKIP_BUILD'], 'false') + - bash: | + set -e + pip install --timeout=60 numpy==$NP_BUILD_DEP + pip install --timeout=60 $TEST_DEPENDS Cython==$CYTHON_BUILD_DEP + pip install twine wheel delvewheel + pushd pandas + cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/$PYTHON_ARCH/Microsoft.VC142.CRT/msvcp140.dll" pandas/_libs/window + cp "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Redist/MSVC/14.29.30036/x64/Microsoft.VC142.CRT/vcruntime140_1.dll" pandas/_libs/window + python setup.py build + python setup.py bdist_wheel + ls dist + twine check dist/* + # Delvewheel doesn't support using the directory path :( + # We are just using delvewheel right now to list dll dependencies not copy them + export WHEEL_PATH="dist/$(ls dist)" + delvewheel show $WHEEL_PATH + popd + displayName: Build wheel + condition: eq(variables['SKIP_BUILD'], 'false') + - bash: | + set -e + source extra_functions.sh + source config.sh + setup_test_venv + pip install pandas/dist/pandas-*.whl + run_tests + teardown_test_venv + displayName: Install wheel and test + condition: eq(variables['SKIP_BUILD'], 'false') - - bash: echo "##vso[task.prependpath]$CONDA/Scripts" - displayName: Add conda to PATH - condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest')) + - bash: echo "##vso[task.prependpath]$CONDA/Scripts" + displayName: Add conda to PATH + condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest')) - - bash: conda install -q -y anaconda-client - displayName: Install anaconda-client - condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest')) + - bash: conda install -q -y anaconda-client + displayName: Install anaconda-client + condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest')) - - bash: | - set -e - if [ "$BUILD_REASON" == "Schedule" ]; then - ANACONDA_ORG="scipy-wheels-nightly" - TOKEN="$PANDAS_NIGHTLY_UPLOAD_TOKEN" - else - ANACONDA_ORG="multibuild-wheels-staging" - TOKEN="$PANDAS_STAGING_UPLOAD_TOKEN" - fi - if [ "$TOKEN" == "" ]; then - echo "##[warning] Could not find anaconda.org upload token in secret variables" - fi - echo "##vso[task.setvariable variable=TOKEN]$TOKEN" - echo "##vso[task.setvariable variable=ANACONDA_ORG]$ANACONDA_ORG" - displayName: Retrieve secret upload token - condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest')) - env: - # Secret variables need to mapped to env variables explicitly: - PANDAS_NIGHTLY_UPLOAD_TOKEN: $(PANDAS_NIGHTLY_UPLOAD_TOKEN) - PANDAS_STAGING_UPLOAD_TOKEN: $(PANDAS_STAGING_UPLOAD_TOKEN) - - bash: | - set -e - # The --force option forces a replacement if the remote file already - # exists. - ls pandas/dist/pandas-*.whl - anaconda -t $TOKEN upload --force -u $ANACONDA_ORG pandas/dist/pandas-*.whl - echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple" - displayName: Upload to anaconda.org (only if secret token is retrieved) - condition: ne(variables['TOKEN'], '') - - - publish: pandas/dist - artifact: $(Agent.JobName) + - bash: | + set -e + if [ "$BUILD_REASON" == "Schedule" ]; then + ANACONDA_ORG="scipy-wheels-nightly" + TOKEN="$PANDAS_NIGHTLY_UPLOAD_TOKEN" + else + ANACONDA_ORG="multibuild-wheels-staging" + TOKEN="$PANDAS_STAGING_UPLOAD_TOKEN" + fi + if [ "$TOKEN" == "" ]; then + echo "##[warning] Could not find anaconda.org upload token in secret variables" + fi + echo "##vso[task.setvariable variable=TOKEN]$TOKEN" + echo "##vso[task.setvariable variable=ANACONDA_ORG]$ANACONDA_ORG" + displayName: Retrieve secret upload token + condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest')) + env: + # Secret variables need to mapped to env variables explicitly: + PANDAS_NIGHTLY_UPLOAD_TOKEN: $(PANDAS_NIGHTLY_UPLOAD_TOKEN) + PANDAS_STAGING_UPLOAD_TOKEN: $(PANDAS_STAGING_UPLOAD_TOKEN) + - bash: | + set -e + # The --force option forces a replacement if the remote file already + # exists. + ls pandas/dist/pandas-*.whl + anaconda -t $TOKEN upload --force -u $ANACONDA_ORG pandas/dist/pandas-*.whl + echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple" + displayName: Upload to anaconda.org (only if secret token is retrieved) + condition: ne(variables['TOKEN'], '') + + - publish: pandas/dist + artifact: $(Agent.JobName) + + - stage verify: + dependsOn: build + steps: + - download: current + artifact: $(Agent.JobName)