From 31cfea136694deb36f94f66933bcc067e2d2ae9f Mon Sep 17 00:00:00 2001 From: Allan Renucci Date: Sat, 9 Sep 2017 13:31:46 +0200 Subject: [PATCH] Revert "Simplify CI builds (#3084)" This reverts commit c53bcbcb60dea3223c2ee86e0b9c734052a03bbf. --- .drone.yml | 89 +++++++++++------------------- project/scripts/genDocs | 109 ++++++++++++++++++++----------------- project/scripts/sbt | 35 +++++++----- project/scripts/sbtPublish | 38 ++++++++----- 4 files changed, 136 insertions(+), 135 deletions(-) diff --git a/.drone.yml b/.drone.yml index 6a114f7029ed..519f56566323 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,61 +1,17 @@ pipeline: - # TESTS: - # We run tests in parallel. Tests run in a copy of the working directory to avoid conflict - test_legacy: - group: test - image: lampepfl/dotty:2017-09-08 - commands: - - cp -R . /tmp/0/ && cd /tmp/0/ - - ./project/scripts/sbt legacyTests - test: - group: test - image: lampepfl/dotty:2017-09-08 - commands: - - cp -R . /tmp/1/ && cd /tmp/1/ - - ./project/scripts/sbt test - - test_bootstrapped: - group: test - image: lampepfl/dotty:2017-09-08 - commands: - - cp -R . /tmp/2/ && cd /tmp/2/ - - ./project/scripts/sbt dotty-bootstrapped/test - - test_optimised: - group: test - image: lampepfl/dotty:2017-09-08 - commands: - - cp -R . /tmp/3/ && cd /tmp/3/ - - ./project/scripts/sbt ";set bootstrapOptimised in ThisBuild := true ;dotty-bootstrapped/test" - - test_sbt: - group: test - image: lampepfl/dotty:2017-09-08 - commands: - - cp -R . /tmp/4/ && cd /tmp/4/ - - ./project/scripts/sbt sbt-dotty/scripted - when: - # sbt scripted tests are slow and don't run on PRs - event: [ push, tag, deployment ] - - # DOCUMENTATION: - documentation: - image: lampepfl/dotty:2017-09-08 + image: lampepfl/dotty:2017-09-05 commands: - - ./project/scripts/genDocs - secrets: [ bot_pass ] - when: - event: push + - ./project/scripts/sbt "${CI_TEST}" - # PUBLISHING: - # Publishing expect NIGHTLYBUILD or RELEASEBUILD to be set. See dottyVersion in Build.scala publish_nightly: - image: lampepfl/dotty:2017-09-08 + image: lampepfl/dotty:2017-09-05 environment: - NIGHTLYBUILD=yes commands: - - ./project/scripts/sbtPublish ";dotty-bootstrapped/publishSigned ;sonatypeRelease" + - ./project/scripts/sbt ";clean ;publishLocal" "${CI_PUBLISH}" + - ./project/scripts/sbt "sbt-dotty/scripted source-dependencies/*" "${CI_PUBLISH}" + - ./project/scripts/sbtPublish "${CI_PUBLISH}" "$SONATYPE_USER" "$SONATYPE_PW" "$PGP_PW" ";dotty-bootstrapped/publishSigned ;sonatypeRelease" volumes: - /home/drone/keys:/keys secrets: [ sonatype_user, sonatype_pw, pgp_pw ] @@ -64,11 +20,13 @@ pipeline: environment: nightly publish_release: - image: lampepfl/dotty:2017-09-08 + image: lampepfl/dotty:2017-09-05 environment: - RELEASEBUILD=yes commands: - - ./project/scripts/sbtPublish ";dotty-bootstrapped/publishSigned ;sonatypeRelease" + - ./project/scripts/sbt ";clean ;publishLocal" "${CI_PUBLISH}" + - ./project/scripts/sbt "sbt-dotty/scripted source-dependencies/*" "${CI_PUBLISH}" + - ./project/scripts/sbtPublish "${CI_PUBLISH}" "$SONATYPE_USER" "$SONATYPE_PW" "$PGP_PW" ";dotty-bootstrapped/publishSigned ;sonatypeRelease" volumes: - /home/drone/keys:/keys secrets: [ sonatype_user, sonatype_pw, pgp_pw ] @@ -77,11 +35,11 @@ pipeline: environment: release publish_sbt_release: - image: lampepfl/dotty:2017-09-08 + image: lampepfl/dotty:2017-09-05 environment: - RELEASEBUILD=yes commands: - - ./project/scripts/sbtPublish ";sbt-dotty/publishSigned ;sonatypeRelease" + - ./project/scripts/sbtPublish "${CI_PUBLISH}" "$SONATYPE_USER" "$SONATYPE_PW" "$PGP_PW" ";sbt-dotty/publishSigned ;sonatypeRelease" volumes: - /home/drone/keys:/keys secrets: [ sonatype_user, sonatype_pw, pgp_pw ] @@ -89,14 +47,31 @@ pipeline: event: deployment environment: sbt_release - # NOTIFICATIONS: + documentation: + image: lampepfl/dotty:2017-09-05 + commands: + - ./project/scripts/genDocs "${CI_PUBLISH}" "$BOT_PASS" + secrets: [ bot_pass ] + when: + event: push + slack: image: plugins/slack channel: dotty secrets: [ slack_webhook ] when: status: [ success, failure ] - event: [ push, tag, deployment ] + event: [ push, deployment ] -# CI runs only for the master branch (including PRs) branches: master + +matrix: + include: + - CI_TEST: legacyTests + CI_PUBLISH: true + - CI_TEST: ;test;sbt-dotty/scripted compilerReporter/*;sbt-dotty/scripted discovery/*;sbt-dotty/scripted sbt-dotty/* + CI_PUBLISH: false + - CI_TEST: dotty-bootstrapped/test + CI_PUBLISH: false + - CI_TEST: ;set bootstrapOptimised in ThisBuild := true ;dotty-bootstrapped/test + CI_PUBLISH: false diff --git a/project/scripts/genDocs b/project/scripts/genDocs index 0e9ec8302fc6..464c1f21f286 100755 --- a/project/scripts/genDocs +++ b/project/scripts/genDocs @@ -1,57 +1,68 @@ #!/usr/bin/env bash -# Usage: -# BOT_PASS= ./genDocs +# Usage: ./genDocs set -e - # set extended glob, needed for rm everything but x shopt -s extglob -# make sure that BOT_PASS is set -if [ -z "$BOT_PASS" ]; then - echo "Error: BOT_PASS env unset, unable to push without password" 1>&2 - exit 1 +if [ $1 = true ]; then + + # make sure that BOT_PASS is set + if [ -z ${2+x} ]; then + echo "BOT_PASS unset, unable to push without password" 1>&2 + exit 1 + else + BOT_PASS=$2 + fi + + echo "Working directory: $PWD" + + # this command will generate docs in $PWD/docs/_site + sbt -J-Xmx4096m \ + -J-XX:ReservedCodeCacheSize=512m \ + -J-XX:MaxMetaspaceSize=1024m \ + -Ddotty.drone.mem=4096m \ + -ivy /var/cache/drone/ivy2 \ + "genDocs" + + # make sure that the previous command actually succeeded + if [ ! -d "$PWD/docs/_site" ]; then + echo "Output directory did not exist: $PWD/docs/_site" 1>&2 + exit 1 + fi + + # save current head for commit message in gh-pages + GIT_HEAD=$(git rev-parse HEAD) + + # check out correct branch + git fetch origin gh-pages:gh-pages + git checkout gh-pages + + # move newly generated _site dir to $PWD + mv $PWD/docs/_site . + + # remove everything BUT _site dir + rm -rf !(_site) + + # copy new contents to $PWD + mv _site/* . + + # remove now empty _site dir + rm -rf _site + + # set github credentials + git config user.name "dotty-bot" + git config user.email "felix.mulder@epfl.ch" + + # add all contents of $PWD to commit + git add -A + git commit -m "Update gh-pages site for $GIT_HEAD" || echo "nothing new to commit" + + # push using dotty-bot to origin + git push https://dotty-bot:$BOT_PASS@github.com/lampepfl/dotty.git || echo "couldn't push, since nothing was added" +else + # wrong parameter passed, should only generate docs if argument is "test" + # to avoid multiple site gens + echo "Not generating docs for $1" fi - -echo "Working directory: $PWD" - -# this command will generate docs in $PWD/docs/_site -SBT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/sbt" -eval "$SBT" "genDocs" - -# make sure that the previous command actually succeeded -if [ ! -d "$PWD/docs/_site" ]; then - echo "Output directory did not exist: $PWD/docs/_site" 1>&2 - exit 1 -fi - -# save current head for commit message in gh-pages -GIT_HEAD=$(git rev-parse HEAD) - -# check out correct branch -git fetch origin gh-pages:gh-pages -git checkout gh-pages - -# move newly generated _site dir to $PWD -mv $PWD/docs/_site . - -# remove everything BUT _site dir -rm -rf !(_site) - -# copy new contents to $PWD -mv _site/* . - -# remove now empty _site dir -rm -rf _site - -# set github credentials -git config user.name "dotty-bot" -git config user.email "felix.mulder@epfl.ch" - -# add all contents of $PWD to commit -git add -A -git commit -m "Update gh-pages site for $GIT_HEAD" || echo "nothing new to commit" - -# push using dotty-bot to origin -git push https://dotty-bot:$BOT_PASS@github.com/lampepfl/dotty.git || echo "couldn't push, since nothing was added" diff --git a/project/scripts/sbt b/project/scripts/sbt index a86e35e04774..d9c12d004f87 100755 --- a/project/scripts/sbt +++ b/project/scripts/sbt @@ -1,19 +1,26 @@ #!/usr/bin/env bash -# Usage: -# ./sbt +CI_TEST=$1 -CMD="$1" - -if [ -z "$CMD" ]; then - echo "Error: missing sbt command" - exit 1 +if [ -n "$2" ]; then + if [ $2 = true ]; then + SHOULD_RUN=true + else + SHOULD_RUN=false + fi +else + SHOULD_RUN=true fi -# run sbt with the supplied arg -sbt -J-Xmx4096m \ - -J-XX:ReservedCodeCacheSize=512m \ - -J-XX:MaxMetaspaceSize=1024m \ - -Ddotty.drone.mem=4096m \ - -ivy /var/cache/drone/ivy2 \ - "$CMD" +if [ $SHOULD_RUN = true ]; then + # get the ivy2 cache: + ln -s /var/cache/drone/ivy2 "$HOME/.ivy2" || true + + # run sbt with the supplied arg: + sbt -J-Xmx4096m \ + -J-XX:ReservedCodeCacheSize=512m \ + -J-XX:MaxMetaspaceSize=1024m \ + -Ddotty.drone.mem=4096m "$CI_TEST" +else + echo "Not running, \$2=$2, SHOULD_RUN=$SHOULD_RUN" +fi diff --git a/project/scripts/sbtPublish b/project/scripts/sbtPublish index b29233df10e1..d47401070010 100755 --- a/project/scripts/sbtPublish +++ b/project/scripts/sbtPublish @@ -1,26 +1,32 @@ #!/usr/bin/env bash -# Usage: -# SONATYPE_USER= SONATYPE_PW= PGP_PW= ./sbtPublish +# If the script should publish to maven: +SHOULD_RUN=$1 -set -e +# Sonatype credentials: +SONATYPE_USER=$2 +SONATYPE_PW=$3 + +# PGP Credentials: +PGP_PW=$4 # Release command: -RELEASE_CMD="$1" +RELEASE_CMD=$5 -if [ -z "$SONATYPE_USER" ] || [ -z "$SONATYPE_PW" ] || [ -z "$PGP_PW" ]; then - echo "Error: SONATYPE_USER, SONATYPE_PW or PGP_PW env unset" - exit 1 +if [ $SHOULD_RUN = false ]; then + exit 0 fi -if [ ! "$NIGHTLYBUILD" = "yes" ] && [ ! "$RELEASEBUILD" = "yes" ]; then - echo "Neither NIGHTLYBUILD nor RELEASEBUILD env var set to \"yes\"" +if [ -z "$2" -o -z "$3" -o -z "$4" ]; then + echo "invalid args: this script takes 4 args, usage:" + echo "" + echo " ./project/scripts/sbt SHOULD_RUN SONATYPE_USER SONATYPE_PW PGP_PW" exit 1 fi -if [ -z "$RELEASE_CMD" ]; then - echo "Error: missing publish command" - exit 1 +if [ ! "$NIGHTLYBUILD" = "yes" ] && [ ! "$RELEASEBUILD" = "yes" ]; then + echo "Neither NIGHTLYBUILD or RELEASEBUILD env var set to \"yes\"" + exit 1 fi CMD=" ;set credentials in ThisBuild := Seq(Credentials(\"Sonatype Nexus Repository Manager\", \"oss.sonatype.org\", \"$2\", \"$3\"))" @@ -29,6 +35,8 @@ CMD="$CMD ;set pgpSecretRing := file(\"/keys/secring.asc\")" CMD="$CMD ;set pgpPublicRing := file(\"/keys/pubring.asc\")" CMD="$CMD $RELEASE_CMD" -# run sbt with the supplied arg -SBT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/sbt" -eval "$SBT" "$CMD" +sbt -J-Xmx4096m \ + -J-XX:ReservedCodeCacheSize=512m \ + -J-XX:MaxMetaspaceSize=1024m \ + -Ddotty.drone.mem=4096m \ + "$CMD"