Switch to Travis CI for building releases

adriaanm · adriaanm · commit 41e376a26539 · 2018-03-07T11:19:32.000+01:00
Both nightly and full releases migrate to Travis.
PR validation remains on our Jenkins cluster for now.

Main difference is that we don't use our artifactory as
a cache, nor do we need to wipe stuff, since Travis
gives us a fresh machine. We do cache the ivy/sbt cache.

TODO: decide whether to publish a "mergely" (a nightly
for each merge, or skip this script on merge and use
a scheduled job for publishing the nightly.)

Note: we don't use `travis encrypt-file` because it
nukes the iv/key variables on each invocation.. too much magic

Instead, I did:

```
cat /dev/urandom | head -c 10000 | openssl sha1 &gt; ./secret
openssl aes-256-cbc -pass "file:./secret" -in gpg_subkey -out admin/files/gpg_subkey.enc
travis encrypt "GPG_SUBKEY_SECRET=$(cat ./secret)"
```
diff --git a/.travis.yml b/.travis.yml
@@ -1,19 +1,37 @@
-# opt-in to Travis's newer/faster container-based infrastructure
-sudo: false
+sudo: required # GCE VMs have better performance (will be upgrading to premium VMs soon)
 
 # this builds the spec using jekyll
 # based on http://www.paperplanes.de/2013/8/13/deploying-your-jekyll-blog-to-s3-with-travis-ci.html
-language: ruby
+
+language: scala
+jdk: openjdk8
+
+# the spec is built with jekyll
 rvm:
   - 2.2
-script: bundle exec jekyll build -s spec/ -d build/spec
+
+cache:
+  directories:
+    - $HOME/.ivy2/cache
+    - $HOME/.sbt
+
+script:
+  - (cd admin && ./init.sh)
+  - scripts/jobs/integrate/bootstrap
+  - bundle exec jekyll build -s spec/ -d build/spec
+
 install: bundle install
 
 # cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret
 # openssl aes-256-cbc -pass "file:./secret" -in id_dsa_spec212_b4096 -out spec/id_dsa_travis.enc -a
 # travis encrypt "PRIV_KEY_SECRET=`cat ./secret`"
 env:
-  - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY="
+  global:
+    - secure: "TuJOUtALynPd+MV1AuMeIpVb8BUBHr7Ul7FS48XhS2PyuTRpEBkSWybYcNg3AXyzmWDAuOjUxbaNMQBvP8vvehTbIYls5H5wTGKvj0D0TNVaPIXjF8bA8KyNat9xGNzhnWm2/2BMaWpKBJWRF7Jb+zHhijMYCJEbkMtoiE5R/mY="
+    - secure: "T1fxtvLTxioyXJYiC/zVYdNYsBOt+0Piw+xE04rB1pzeKahm9+G2mISdcAyqv6/vze9eIJt6jNHHpKX32/Z3Cs1/Ruha4m3k+jblj3S0SbxV6ht2ieJXLT5WoUPFRrU68KXI8wqUadXpjxeJJV53qF2FC4lhfMUsw1IwwMhdaE8=" # PRIVATE_REPO_PASS
+    - secure: "feE5A8mYNpkNQKVwCj3aXrwjVrJWh/4ENpRfFlr2HOD9ORk1GORD5Yq907WZd+dTkYK54Lh1gA+qHOCIDgJHbi9ZLU+kjzEjtYKF6lQy6Wb0LI8smTOnAA6IWVVYifiXw8d66MI2MKZb2jjGeIzy8Q00SZjLhEGjLyTeCIB88Ws=" # SONA_USER
+    - secure: "ek3As5q2tL8UBXcxSBbv4v5YgsoPD41SCzPOSu72kzfbngyxgQxrcziU5pIM+Lib9KaWex7hVVWNL38tMyDbu+0OpDv8bPjMujzlDx5I2pJUfuOJo7QRYsJE1nsXcY4cA72cCLfbRcLEkvtDAhcdLSaUOqlyQe5BY4X4fY5eoPA=" # SONA_PASS
+    - secure: "dbAvl6KEuLwZ0MVQPZihFsPzCdiLbX0EFk3so+hcfEbksrmLQ1tn4X5ZM7Wy1UDR8uN9lxngEwHch7a7lKqpugzmXMew9Wnikr9WBWbJT77Z+XJ/jHI6YuiCRpRo+nvxXGp9Ry80tSIgx5eju0J83IaJL41BWlBkvyAd7YAHORI=" # GPG_SUBKEY_SECRET
 
 # ^^^ set PRIV_KEY_SECRET to password used to encrypt spec/id_dsa_travis.enc
 
@@ -23,3 +41,8 @@ after_success:
 
 # using S3 would be simpler, but we want to upload to scala-lang.org
 # after_success: bundle exec s3_website push --headless
+
+before_cache:
+  # Cleanup the cached directories to avoid unnecessary cache updates
+  - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
+  - find $HOME/.sbt        -name "*.lock"               -print -delete
diff --git a/admin/files/credentials-private-repo b/admin/files/credentials-private-repo
@@ -0,0 +1,4 @@
+realm=Artifactory Realm
+host=scala-ci.typesafe.com
+user=scala-ci
+password=${PRIVATE_REPO_PASS}
diff --git a/admin/files/credentials-sonatype b/admin/files/credentials-sonatype
@@ -0,0 +1,4 @@
+realm=Sonatype Nexus Repository Manager
+host=oss.sonatype.org
+user=${SONA_USER}
+password=${SONA_PASS}
diff --git a/admin/files/gpg.sbt b/admin/files/gpg.sbt
@@ -0,0 +1 @@
+addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")
diff --git a/admin/files/gpg_subkey.enc b/admin/files/gpg_subkey.enc
diff --git a/admin/files/m2-settings.xml b/admin/files/m2-settings.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<settings>
+  <servers>
+    <server>
+      <id>sonatype-nexus</id>
+      <username>${SONA_USER}</username>
+      <password>${SONA_PASS}</password>
+    </server>
+    <server>
+      <id>private-repo</id>
+      <username>scala-ci</username>
+      <password>${PRIVATE_REPO_PASS}</password>
+    </server>
+  </servers>
+
+  <mirrors>
+    <!--
+      codehaus doesn't exist anymore. it doesn't 404, but serves up
+      bad POMs, so we disable it globally by overriding with a mirror
+      with an invalid URL.
+      this entry could be removed after the scala-ide/scala-refactoring
+      repo removes their reference to codehaus-snapshots from their pom.xml
+    -->
+    <mirror>
+      <id>codehaus-snapshots-mirror</id>
+      <name>Maven Codehaus snapshot repository</name>
+      <url>file:///codehaus-does-not-exist-anymore</url>
+      <mirrorOf>codehaus-snapshots</mirrorOf>
+    </mirror>
+  </mirrors>
+</settings>
diff --git a/admin/files/sonatype-curl b/admin/files/sonatype-curl
@@ -0,0 +1 @@
+user = ${SONA_USER}:${SONA_PASS}
diff --git a/admin/init.sh b/admin/init.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+
+sensitive() {
+  perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-private-repo > ~/.credentials-private-repo
+  perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/credentials-sonatype     > ~/.credentials-sonatype
+  perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/sonatype-curl            > ~/.sonatype-curl
+  # perl -p -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' < files/m2-settings.xml          > ~/.m2/settings.xml  -- not needed anymore (used for ide integration?)
+
+  openssl aes-256-cbc -d -pass "pass:$GPG_SUBKEY_SECRET" -in files/gpg_subkey.enc | gpg --import
+}
+
+# directories needed by sensitive part
+# mkdir -p ~/.m2 -- not needed anymore (used for ide integration?)
+mkdir -p ~/.ssh
+
+# don't let anything escape from the sensitive part (e.g. leak environment var by echoing to log on failure)
+sensitive >/dev/null 2>&1
+
+# pgp signing doesn't work without public key??
+gpg --keyserver pgp.mit.edu --recv-keys 0xa9052b1b6d92e560
+
+# just to verify
+gpg --list-keys
+gpg --list-secret-keys
+
+mkdir -p ~/.sbt/0.13/plugins
+cp files/gpg.sbt ~/.sbt/0.13/plugins/
+
+export SBT_CMD=$(which sbt)
diff --git a/project/ScriptCommands.scala b/project/ScriptCommands.scala
@@ -6,6 +6,8 @@ import BuildSettings.autoImport._
 
 /** Custom commands for use by the Jenkins scripts. This keeps the surface area and call syntax small. */
 object ScriptCommands {
+  def env(key: String) = Option(System.getenv(key)).getOrElse("")
+
   def all = Seq(
     setupPublishCore,
     setupValidateTest,
@@ -80,7 +82,7 @@ object ScriptCommands {
       baseVersionSuffix in Global := "SPLIT",
       resolvers in Global += "scala-pr" at url,
       publishTo in Global := Some("sonatype-releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2"),
-      credentials in Global += Credentials(Path.userHome / ".credentials-sonatype"),
+      credentials in Global += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", env("SONA_USER"), env("SONA_PASS")),
       pgpPassphrase in Global := Some(Array.empty)
     ) ++ enableOptimizer
   }
@@ -114,7 +116,11 @@ object ScriptCommands {
   private[this] def publishTarget(url: String) = {
     // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088):
     val url2 = if(url.startsWith("file:")) url else url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis
-    Seq(publishTo in Global := Some("scala-pr-publish" at url2))
+
+    Seq(
+      publishTo in Global := Some("scala-pr-publish" at url2),
+      credentials in Global += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", env("PRIVATE_REPO_PASS"))
+    )
   }
 
   /** Like `Def.sequential` but accumulate all results */
diff --git a/scripts/common b/scripts/common
@@ -159,19 +159,36 @@ EOF
 # Takes a variable number of additional repositories as argument.
 # See http://www.scala-sbt.org/0.13/docs/Proxy-Repositories.html
 function generateRepositoriesConfig() {
-  jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
   sbtRepositoryConfig="$scriptsDir/sbt-repositories-config"
   echo > "$sbtRepositoryConfig" '[repositories]'
   if [[ $# -gt 0 ]]; then
     for i in $(seq 1 $#); do
       echo >> "$sbtRepositoryConfig" "  script-repo-$i: ${!i}"
     done
   fi
+
+  if [ "${TRAVIS}" != "true" ]; then
+    jcenterCacheUrl=${jcenterCacheUrl-"https://scala-ci.typesafe.com/artifactory/jcenter/"}
+    echo "jcenter-cache: $jcenterCacheUrl" >> "$sbtRepositoryConfig"
+  fi
+
   cat >> "$sbtRepositoryConfig" << EOF
-  jcenter-cache: $jcenterCacheUrl
-  typesafe-ivy-releases: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
-  sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
-  maven-central
   local
+  maven-central
+  typesafe-ivy-releases-boot: https://repo.lightbend.com/typesafe/ivy-releases/, [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
+  typesafe-ivy-releases: https://dl.bintray.com/typesafe/ivy-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
+  sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
 EOF
 }
+
+
+# https://github.com/travis-ci/docs-travis-ci-com/issues/949
+travis_fold_start() {
+  echo ""
+  echo -e "travis_fold:start:$1\033[33;1m$2\033[0m"
+}
+
+travis_fold_end() {
+  echo -e "\ntravis_fold:end:$1\r"
+  echo ""
+}
diff --git a/scripts/jobs/integrate/bootstrap b/scripts/jobs/integrate/bootstrap
@@ -77,13 +77,14 @@ publishSonatypeTaskCore=${publishSonatypeTaskCore-"publishSigned"}
 publishSonatypeTaskModules=${publishSonatypeTaskModules-"publishSigned"}
 
 forceRebuild=${forceRebuild-no}
-
 sbtBuildTask=${sbtBuildTask-"testAll"} # TESTING leave empty to avoid the sanity check
 testStability=${testStability-yes}
 
 clean="clean" # TESTING leave empty to speed up testing
 
-baseDir=${WORKSPACE-`pwd`}
+WORKSPACE=${WORKSPACE-`pwd`}
+baseDir=${WORKSPACE}
+
 scriptsDir="$baseDir/scripts"
 . $scriptsDir/common
 
@@ -99,7 +100,9 @@ mkdir -p $baseDir/resolutionScratch_
 # repo to publish builds
 integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"}
 
-generateRepositoriesConfig $integrationRepoUrl
+if [ "${TRAVIS}" != "true" ]; then
+  generateRepositoriesConfig $integrationRepoUrl
+fi
 
 # ARGH trying to get this to work on multiple versions of sbt-extras...
 # the old version (on jenkins, and I don't want to upgrade for risk of breaking other builds) honors -sbt-dir
@@ -152,19 +155,22 @@ function st_stagingRepoClose() {
 #### sbt tools
 
 sbtBuild() {
-  echo "### sbtBuild: "$SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
-  $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@" >> $baseDir/logs/builds 2>&1
+  travis_fold_start build "Building $(basename $PWD) with $@"
+  $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" "${publishTasks[@]}" "$@"
+  travis_fold_end build
 }
 
 sbtResolve() {
   cd $baseDir/resolutionScratch_
   touch build.sbt
   # Can be set to `full` if a module requires cross-versioning against the full Scala version, like the continuations plugin used to.
   cross=${4-binary}
-  echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
+  # echo "### sbtResolve: $SBT_CMD -no-colors $sbtArgs " "${scalaVersionTasks[@]}" "\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
+  travis_fold_start resolve "Resolving \"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross"
   $SBT_CMD -no-colors $sbtArgs "${scalaVersionTasks[@]}" \
     "set libraryDependencies := Seq(\"$1\" % \"$2\" % \"$3\" cross CrossVersion.$cross)" \
-      'show update' >> $baseDir/logs/resolution 2>&1
+      'show update'
+  travis_fold_end resolve
 }
 
 # Oh boy... can't use scaladoc to document scala-xml if scaladoc depends on the same version of scala-xml.
@@ -229,15 +235,15 @@ buildScalaCheck(){
 
 # build modules, using ${buildTasks[@]} (except for ScalaCheck, which is hard-coded to publish to artifactory)
 buildModules() {
-  publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-private-repo")' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")")
+  publishTasks=('set credentials += Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", System.getenv("PRIVATE_REPO_PASS"))' "set every publishTo := Some(\"publish-repo\" at \"$integrationRepoUrl\")")
   buildTasks=($publishPrivateTask)
   buildXML
   # buildScalaCheck
   buildPartest
 }
 
 buildPublishedModules() {
-  publishTasks=('set credentials += Credentials(Path.userHome / ".credentials-sonatype")' "set pgpPassphrase := Some(Array.empty)")
+  publishTasks=('set credentials += Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", System.getenv("SONA_USER"), System.getenv("SONA_PASS"))' "set pgpPassphrase := Some(Array.empty)")
   buildTasks=($publishSonatypeTaskModules)
   buildXML
   buildPartest
@@ -283,7 +289,9 @@ determineScalaVersion() {
   if [ -z "$SCALA_VER_BASE" ]; then
     echo "No SCALA_VER_BASE specified."
 
+    travis_fold_start determineScalaVersion "Determining Scala version"
     $SBT_CMD $sbtArgs 'set baseVersionSuffix in Global := "SHA"' generateBuildCharacterPropertiesFile
+    travis_fold_end determineScalaVersion
     parseScalaProperties "buildcharacter.properties"
     SCALA_VER_BASE="$maven_version_base"
     SCALA_VER_SUFFIX="$maven_version_suffix"
@@ -376,8 +384,6 @@ bootstrap() {
 
   #### (Optional) STARR.
   if [ ! -z "$STARR_REF" ]; then
-      echo "### Building STARR"
-
       STARR_DIR=./scala-starr
       STARR_VER_SUFFIX="-$(git rev-parse --short $STARR_REF)-starr"
       STARR_VER=$SCALA_VER_BASE$STARR_VER_SUFFIX
@@ -386,21 +392,24 @@ bootstrap() {
         git clone --reference $WORKSPACE/.git $WORKSPACE/.git $STARR_DIR
         cd $STARR_DIR
         git co $STARR_REF
-        $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish >> $baseDir/logs/builds 2>&1
+        travis_fold_start starr "Building starr"
+        $SBT_CMD -no-colors $sbtArgs --warn "setupBootstrapStarr $integrationRepoUrl $STARR_VER" $clean publish
+        travis_fold_end starr
       )
   fi
 
   #### LOCKER
 
-  echo "### Building locker"
-
   # for bootstrapping, publish core (or at least smallest subset we can get away with)
   # so that we can build modules with this version of Scala and publish them locally
   # must publish under $SCALA_VER so that the modules will depend on this (binary) version of Scala
   # publish more than just core: partest needs scalap
   # in sabbus lingo, the resulting Scala build will be used as starr to build the released Scala compiler
   if [ ! -z "$STARR_VER" ]; then SET_STARR=-Dstarr.version=$STARR_VER; fi
-  $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish >> $baseDir/logs/builds 2>&1
+
+  travis_fold_start locker "Building locker"
+  $SBT_CMD -no-colors $sbtArgs $SET_STARR --warn "setupBootstrapLocker $integrationRepoUrl $SCALA_VER" $clean publish
+  travis_fold_end locker
 
   echo "### Building modules using locker"
 
@@ -425,6 +434,7 @@ bootstrap() {
   cd $baseDir
   rm -rf build/
 
+  travis_fold_start quick "Building bootstrapped"
   $SBT_CMD $sbtArgs \
       --warn \
       -Dstarr.version=$SCALA_VER \
@@ -434,6 +444,7 @@ bootstrap() {
       $sbtBuildTask \
       dist/mkQuick \
       publish
+  travis_fold_end quick
 
   # clear ivy cache (and to be sure, local as well), so the next round of sbt builds sees the fresh scala
   rm -rf $baseDir/ivy2
@@ -443,8 +454,7 @@ bootstrap() {
 }
 
 testStability() {
-  echo "### Testing stability"
-
+  travis_fold_start stab "Testing stability"
   cd $baseDir
 
   # Run stability tests using the just built version as "quick" and a new version as "strap"
@@ -460,6 +470,8 @@ testStability() {
   mv build/quick build/strap
   mv quick1 build/quick
   $scriptsDir/stability-test.sh
+
+  travis_fold_end stab
 }
 
 # assumes we just bootstrapped, and current directory is $baseDir
@@ -469,15 +481,16 @@ testStability() {
 publishSonatype() {
   # stage to sonatype, along with all modules -Dmaven.version.suffix/-Dbuild.release not necessary,
   # since we're just publishing an existing build
-  echo "### Publishing core to sonatype"
+  travis_fold_start sona "Publishing core to sonatype"
   $SBT_CMD $sbtArgs \
       --warn \
       -Dstarr.version=$SCALA_VER \
       ${updatedModuleVersions[@]} \
       "setupBootstrapPublish $integrationRepoUrl $SCALA_VER" \
       $publishSonatypeTaskCore
+  travis_fold_end sona
 
-  echo "### Publishing modules to sonatype"
+  # echo "### Publishing modules to sonatype"
   # build/test/publish scala core modules to sonatype (this will start a new staging repo)
   # (was hoping we could make everything go to the same staging repo, but it's not timing that causes two staging repos to be opened)
   # NOTE: only publish those for which versions are set

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")`