From 6b7e1e71e2d5ccaf3bfe068507ac329da39b7065 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 21 Jul 2020 20:05:30 -0700 Subject: [PATCH 1/7] PYTHON-1631 Automate release process git_tag_only: false # TODO: change to true once POC is done --- .evergreen/build-mac.sh | 15 +++++++++++ .evergreen/build-manylinux.sh | 21 +++++++++++++++ .evergreen/build-wheels.sh | 25 ++++++++++++++++++ .evergreen/build-windows.sh | 20 +++++++++++++++ .evergreen/config.yml | 48 +++++++++++++++++++++++++++++++++++ .evergreen/release.sh | 9 +++++++ 6 files changed, 138 insertions(+) create mode 100755 .evergreen/build-mac.sh create mode 100755 .evergreen/build-manylinux.sh create mode 100755 .evergreen/build-wheels.sh create mode 100755 .evergreen/build-windows.sh create mode 100755 .evergreen/release.sh diff --git a/.evergreen/build-mac.sh b/.evergreen/build-mac.sh new file mode 100755 index 0000000000..a1e76636e2 --- /dev/null +++ b/.evergreen/build-mac.sh @@ -0,0 +1,15 @@ +#!/bin/bash -ex + + +#python setup.py sdist + +for VERSION in 2.7 3.4 3.5 3.6 3.7 3.8; do + if [[ $VERSION == "2.7" ]]; then + rm -rf build + python$VERSION setup.py bdist_egg + fi + rm -rf build + python$VERSION setup.py bdist_wheel +done + +ls dist diff --git a/.evergreen/build-manylinux.sh b/.evergreen/build-manylinux.sh new file mode 100755 index 0000000000..8807cde728 --- /dev/null +++ b/.evergreen/build-manylinux.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex + +docker version + +docker pull quay.io/pypa/manylinux1_x86_64 +docker pull quay.io/pypa/manylinux1_i686 +docker pull quay.io/pypa/manylinux2014_x86_64 +docker pull quay.io/pypa/manylinux2014_i686 +docker pull quay.io/pypa/manylinux2014_aarch64 +docker pull quay.io/pypa/manylinux2014_ppc64le +docker pull quay.io/pypa/manylinux2014_s390x + +docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux1_x86_64 /pymongo/.evergreen/build-wheels.sh +docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux1_i686 linux32 /pymongo/.evergreen/build-wheels.sh +docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_x86_64 /pymongo/.evergreen/build-wheels.sh +docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_i686 /pymongo/.evergreen/build-wheels.sh +docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh +docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_ppc64le /pymongo/.evergreen/build-wheels.sh +docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_s390x /pymongo/.evergreen/build-wheels.sh + +ls dist diff --git a/.evergreen/build-wheels.sh b/.evergreen/build-wheels.sh new file mode 100755 index 0000000000..c810700a29 --- /dev/null +++ b/.evergreen/build-wheels.sh @@ -0,0 +1,25 @@ +#!/bin/bash -ex +cd /pymongo + +# Compile wheels +for PYBIN in /opt/python/*/bin; do + # Skip Python 3.3 and 3.9. + if [[ "$PYBIN" == *"cp33"* || "$PYBIN" == *"cp39"* ]]; then + continue + fi + # https://github.com/pypa/manylinux/issues/49 + rm -rf build + ${PYBIN}/python setup.py bdist_wheel +done + +# https://github.com/pypa/manylinux/issues/49 +rm -rf build + +# Audit wheels and write multilinux1 tag +for whl in dist/*.whl; do + # Skip already built manylinux1 wheels. + if [[ "$whl" != *"manylinux"* ]]; then + auditwheel repair $whl -w dist + rm $whl + fi +done diff --git a/.evergreen/build-windows.sh b/.evergreen/build-windows.sh new file mode 100755 index 0000000000..d5df1f356e --- /dev/null +++ b/.evergreen/build-windows.sh @@ -0,0 +1,20 @@ +#!/bin/bash -ex + +versions=(27 34 35 36 37 38) + +for VERSION in 27 34 35 36 37 38; do + PYTHON=C:/Python/Python${VERSION}/python.exe + PYTHON32=C:/Python/32/Python${VERSION}/python.exe + if [[ $VERSION == "2.7" ]]; then + rm -rf build + $PYTHON setup.py bdist_egg + rm -rf build + $PYTHON32 setup.py bdist_egg + fi + rm -rf build + $PYTHON setup.py bdist_wheel + rm -rf build + $PYTHON32 setup.py bdist_wheel +done + +ls dist diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ac5e4147f9..bdadd4a1f6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -756,6 +756,21 @@ functions: -v \ --fault revoked +# https://github.com/mongodb/mongodb-atlas-service-broker/blob/e69ae3523da5c0a35f9f134119d332dba9602ab5/.evergreen.yaml + "setup_docker": + - command: shell.exec + params: + script: | + # sudo pacman --refresh --noconfirm -S community/docker + systemctl start docker.service + + "teardown_docker": + - command: shell.exec + params: + script: | + # Remove all Docker images + docker rmi -f $(docker images -a -q) &> /dev/null || true + pre: - func: "fetch source" - func: "prepare resources" @@ -773,6 +788,7 @@ post: - func: "upload test results" - func: "stop mongo-orchestration" - func: "cleanup" + - func: "teardown_docker" tasks: @@ -806,6 +822,20 @@ tasks: genhtml --version || true valgrind --version || true + + - name: "release" + tags: ["release"] + git_tag_only: false # TODO: change to true once POC is done + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + set -o xtrace + ${PREPARE_SHELL} + .evergreen/release.sh + # Standard test tasks {{{ - name: "mockupdb" @@ -1912,6 +1942,17 @@ axes: variables: SETDEFAULTENCODING: "cp1251" +task_groups: + - name: release_group + setup_group: + - func: "fetch source" + - func: "setup_docker" + - func: "build_docker" + tasks: + - release + teardown_group: + - func: "teardown_docker" + buildvariants: - matrix_name: "tests-all" matrix_spec: @@ -2519,6 +2560,13 @@ buildvariants: - name: "aws-auth-test-4.4" - name: "aws-auth-test-latest" +- matrix_name: "Release" + matrix_spec: + platform: [ubuntu-16.04, windows-64-vsMulti-small] + display_name: "Release ${platform}" + tasks: + - name: "release" + # Platform notes # i386 builds of OpenSSL or Cyrus SASL are not available # Ubuntu16.04 ppc64le is only supported by MongoDB 3.4+ diff --git a/.evergreen/release.sh b/.evergreen/release.sh new file mode 100755 index 0000000000..759786b934 --- /dev/null +++ b/.evergreen/release.sh @@ -0,0 +1,9 @@ +#!/bin/bash -ex + +if [ $(uname -s) = "Darwin" ]; then + .evergreen/build-mac.sh +elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + .evergreen/build-windows.sh +else + .evergreen/build-manylinux.sh +fi From d2406bed59af11618440cd1a6e2f2c783284f5f3 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Wed, 22 Jul 2020 17:56:13 -0700 Subject: [PATCH 2/7] Skip manylinux2014_aarch64/ppc64le/s390x for now [2020/07/23 00:24:00.482] + docker run --rm -v /data/mci/cd100cec6341abda533450fb3f2fab99/src:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh [2020/07/23 00:24:01.186] standard_init_linux.go:211: exec user process caused "exec format error" --- .evergreen/build-manylinux.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.evergreen/build-manylinux.sh b/.evergreen/build-manylinux.sh index 8807cde728..90c9fd539d 100755 --- a/.evergreen/build-manylinux.sh +++ b/.evergreen/build-manylinux.sh @@ -6,16 +6,22 @@ docker pull quay.io/pypa/manylinux1_x86_64 docker pull quay.io/pypa/manylinux1_i686 docker pull quay.io/pypa/manylinux2014_x86_64 docker pull quay.io/pypa/manylinux2014_i686 -docker pull quay.io/pypa/manylinux2014_aarch64 -docker pull quay.io/pypa/manylinux2014_ppc64le -docker pull quay.io/pypa/manylinux2014_s390x +# This works on macOS locally but not on linux in evergreen. +# [2020/07/23 00:24:00.482] + docker run --rm -v /data/mci/cd100cec6341abda533450fb3f2fab99/src:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh +# [2020/07/23 00:24:01.186] standard_init_linux.go:211: exec user process caused "exec format error" +# +# Could be related to: +# https://github.com/pypa/manylinux/issues/410 +#docker pull quay.io/pypa/manylinux2014_aarch64 +#docker pull quay.io/pypa/manylinux2014_ppc64le +#docker pull quay.io/pypa/manylinux2014_s390x docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux1_x86_64 /pymongo/.evergreen/build-wheels.sh docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux1_i686 linux32 /pymongo/.evergreen/build-wheels.sh docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_x86_64 /pymongo/.evergreen/build-wheels.sh docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_i686 /pymongo/.evergreen/build-wheels.sh -docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh -docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_ppc64le /pymongo/.evergreen/build-wheels.sh -docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_s390x /pymongo/.evergreen/build-wheels.sh +#docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh +#docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_ppc64le /pymongo/.evergreen/build-wheels.sh +#docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_s390x /pymongo/.evergreen/build-wheels.sh ls dist From db3c3ea41c55b280c3561f4cfb58958fef8a0dbf Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 23 Jul 2020 10:55:22 -0700 Subject: [PATCH 3/7] PYTHON-1631 Add uploading and use 2020-03-20-2fda31c for 3.4 support --- .evergreen/build-mac.sh | 3 --- .evergreen/build-manylinux.sh | 43 +++++++++++++++++++++++------------ .evergreen/build-windows.sh | 2 -- .evergreen/config.yml | 16 +++++++++++++ 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/.evergreen/build-mac.sh b/.evergreen/build-mac.sh index a1e76636e2..95be59c340 100755 --- a/.evergreen/build-mac.sh +++ b/.evergreen/build-mac.sh @@ -1,8 +1,5 @@ #!/bin/bash -ex - -#python setup.py sdist - for VERSION in 2.7 3.4 3.5 3.6 3.7 3.8; do if [[ $VERSION == "2.7" ]]; then rm -rf build diff --git a/.evergreen/build-manylinux.sh b/.evergreen/build-manylinux.sh index 90c9fd539d..524f58ea6d 100755 --- a/.evergreen/build-manylinux.sh +++ b/.evergreen/build-manylinux.sh @@ -2,26 +2,39 @@ docker version -docker pull quay.io/pypa/manylinux1_x86_64 -docker pull quay.io/pypa/manylinux1_i686 -docker pull quay.io/pypa/manylinux2014_x86_64 -docker pull quay.io/pypa/manylinux2014_i686 -# This works on macOS locally but not on linux in evergreen. +# 2020-03-20-2fda31c Was the last release to include Python 3.4. +images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \ + quay.io/pypa/manylinux1_i686:2020-03-20-2fda31c \ + quay.io/pypa/manylinux1_x86_64 \ + quay.io/pypa/manylinux1_i686 \ + quay.io/pypa/manylinux2014_x86_64 \ + quay.io/pypa/manylinux2014_i686) +# aarch64/ppc64le/s390x work on macOS locally but not on linux in evergreen: # [2020/07/23 00:24:00.482] + docker run --rm -v /data/mci/cd100cec6341abda533450fb3f2fab99/src:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh # [2020/07/23 00:24:01.186] standard_init_linux.go:211: exec user process caused "exec format error" # # Could be related to: # https://github.com/pypa/manylinux/issues/410 -#docker pull quay.io/pypa/manylinux2014_aarch64 -#docker pull quay.io/pypa/manylinux2014_ppc64le -#docker pull quay.io/pypa/manylinux2014_s390x +# quay.io/pypa/manylinux2014_aarch64 \ +# quay.io/pypa/manylinux2014_ppc64le \ +# quay.io/pypa/manylinux2014_s390x) -docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux1_x86_64 /pymongo/.evergreen/build-wheels.sh -docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux1_i686 linux32 /pymongo/.evergreen/build-wheels.sh -docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_x86_64 /pymongo/.evergreen/build-wheels.sh -docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_i686 /pymongo/.evergreen/build-wheels.sh -#docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh -#docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_ppc64le /pymongo/.evergreen/build-wheels.sh -#docker run --rm -v `pwd`:/pymongo quay.io/pypa/manylinux2014_s390x /pymongo/.evergreen/build-wheels.sh +for image in "${images[@]}"; do + docker pull $image + docker run --rm -v `pwd`:/pymongo $image /pymongo/.evergreen/build-wheels.sh +done ls dist + +# Check for any unexpected files. +unexpected=$(find dist \! \( -iname dist -or \ + -iname '*cp27*' -or \ + -iname '*cp34*' -or \ + -iname '*cp35*' -or \ + -iname '*cp36*' -or \ + -iname '*cp37*' -or \ + -iname '*cp38*' \)) +if [ -n "$unexpected" ]; then + echo "Unexpected files:" $unexpected + exit 1 +fi \ No newline at end of file diff --git a/.evergreen/build-windows.sh b/.evergreen/build-windows.sh index d5df1f356e..a34506f551 100755 --- a/.evergreen/build-windows.sh +++ b/.evergreen/build-windows.sh @@ -1,7 +1,5 @@ #!/bin/bash -ex -versions=(27 34 35 36 37 38) - for VERSION in 27 34 35 36 37 38; do PYTHON=C:/Python/Python${VERSION}/python.exe PYTHON32=C:/Python/32/Python${VERSION}/python.exe diff --git a/.evergreen/config.yml b/.evergreen/config.yml index bdadd4a1f6..becf8c1e22 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -835,6 +835,22 @@ tasks: set -o xtrace ${PREPARE_SHELL} .evergreen/release.sh + - command: archive.targz_pack + params: + target: "release-files.tgz" + source_dir: "src/dist" + include: + - "*" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: release-files.tgz + remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/release/${task_id}-${execution}-release-files.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Release files # Standard test tasks {{{ From 69a82b7434d569b4ad7391bf7481f05750dde677 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 27 Jul 2020 12:53:30 -0700 Subject: [PATCH 4/7] Remove unused code --- .evergreen/build-manylinux.sh | 2 +- .evergreen/config.yml | 21 --------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/.evergreen/build-manylinux.sh b/.evergreen/build-manylinux.sh index 524f58ea6d..34f9b04e48 100755 --- a/.evergreen/build-manylinux.sh +++ b/.evergreen/build-manylinux.sh @@ -37,4 +37,4 @@ unexpected=$(find dist \! \( -iname dist -or \ if [ -n "$unexpected" ]; then echo "Unexpected files:" $unexpected exit 1 -fi \ No newline at end of file +fi diff --git a/.evergreen/config.yml b/.evergreen/config.yml index becf8c1e22..f8a0bbf3b2 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -756,14 +756,6 @@ functions: -v \ --fault revoked -# https://github.com/mongodb/mongodb-atlas-service-broker/blob/e69ae3523da5c0a35f9f134119d332dba9602ab5/.evergreen.yaml - "setup_docker": - - command: shell.exec - params: - script: | - # sudo pacman --refresh --noconfirm -S community/docker - systemctl start docker.service - "teardown_docker": - command: shell.exec params: @@ -1958,17 +1950,6 @@ axes: variables: SETDEFAULTENCODING: "cp1251" -task_groups: - - name: release_group - setup_group: - - func: "fetch source" - - func: "setup_docker" - - func: "build_docker" - tasks: - - release - teardown_group: - - func: "teardown_docker" - buildvariants: - matrix_name: "tests-all" matrix_spec: @@ -2122,8 +2103,6 @@ buildvariants: - ".latest" - ".4.4" - ".4.2" - variables: - set_xtrace_on: on - matrix_name: "tests-python-version-rhel62-test-ssl" matrix_spec: From cac08d4b1e70921840565c7c34f0b05ddfe6391f Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 27 Jul 2020 14:33:50 -0700 Subject: [PATCH 5/7] Use Ubuntu 18 not 16 --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f8a0bbf3b2..cc18000274 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2557,7 +2557,7 @@ buildvariants: - matrix_name: "Release" matrix_spec: - platform: [ubuntu-16.04, windows-64-vsMulti-small] + platform: [ubuntu-18.04, windows-64-vsMulti-small] display_name: "Release ${platform}" tasks: - name: "release" From 06c0ff13ea74c211ad0861be3c9bab674867914a Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 27 Jul 2020 17:15:43 -0700 Subject: [PATCH 6/7] Rename docker script --- .evergreen/{build-wheels.sh => build-manylinux-internal.sh} | 0 .evergreen/build-manylinux.sh | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename .evergreen/{build-wheels.sh => build-manylinux-internal.sh} (100%) diff --git a/.evergreen/build-wheels.sh b/.evergreen/build-manylinux-internal.sh similarity index 100% rename from .evergreen/build-wheels.sh rename to .evergreen/build-manylinux-internal.sh diff --git a/.evergreen/build-manylinux.sh b/.evergreen/build-manylinux.sh index 34f9b04e48..16b05dbea4 100755 --- a/.evergreen/build-manylinux.sh +++ b/.evergreen/build-manylinux.sh @@ -10,7 +10,7 @@ images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \ quay.io/pypa/manylinux2014_x86_64 \ quay.io/pypa/manylinux2014_i686) # aarch64/ppc64le/s390x work on macOS locally but not on linux in evergreen: -# [2020/07/23 00:24:00.482] + docker run --rm -v /data/mci/cd100cec6341abda533450fb3f2fab99/src:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-wheels.sh +# [2020/07/23 00:24:00.482] + docker run --rm -v /data/mci/cd100cec6341abda533450fb3f2fab99/src:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-manylinux-internal.sh # [2020/07/23 00:24:01.186] standard_init_linux.go:211: exec user process caused "exec format error" # # Could be related to: @@ -21,7 +21,7 @@ images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \ for image in "${images[@]}"; do docker pull $image - docker run --rm -v `pwd`:/pymongo $image /pymongo/.evergreen/build-wheels.sh + docker run --rm -v `pwd`:/pymongo $image /pymongo/.evergreen/build-manylinux-internal.sh done ls dist From 27e425c689370a08afbffceb58e92517b89b3094 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 27 Jul 2020 17:19:55 -0700 Subject: [PATCH 7/7] Enable git_tag_only: true --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index cc18000274..24cdd26238 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -817,7 +817,7 @@ tasks: - name: "release" tags: ["release"] - git_tag_only: false # TODO: change to true once POC is done + git_tag_only: true commands: - command: shell.exec type: test