travis: Make more network requests retryable

.travis.yml

@@ -45,7 +45,7 @@ matrix:
       os: osx
       osx_image: xcode8.2
       install: &osx_install_sccache >
-        curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
+        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/sccache
     - env: >
         RUST_CHECK_TARGET=check
@@ -63,7 +63,7 @@ matrix:
       os: osx
       osx_image: xcode8.2
       install: >
-        curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
+        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/sccache &&
           brew uninstall --ignore-dependencies openssl &&
           brew install openssl --universal --without-test

appveyor.yml

@@ -103,7 +103,7 @@ install:
   # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
   # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
   # file exists in there (which it doesn't by default).
-  - if defined MINGW_URL appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
+  - if defined MINGW_URL appveyor-retry appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
   - if defined MINGW_URL 7z x -y %MINGW_ARCHIVE% > nul
   - if defined MINGW_URL set PATH=%CD%\%MINGW_DIR%\bin;C:\msys64\usr\bin;%PATH%
 
@@ -115,12 +115,12 @@ install:
   - set PATH=C:\Python27;%PATH%
 
   # Download and install sccache
-  - appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc
+  - appveyor-retry appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc
   - mv 2017-02-24-sccache-x86_64-pc-windows-msvc sccache
   - set PATH=%PATH%;%CD%
 
   # Install InnoSetup to get `iscc` used to produce installers
-  - choco install -y InnoSetup
+  - appveyor-retry choco install -y InnoSetup
   - set PATH="C:\Program Files (x86)\Inno Setup 5";%PATH%
 
   # Help debug some handle issues on AppVeyor
@@ -131,7 +131,7 @@ install:
   - handle.exe -accepteula -help
 
 test_script:
-  - git submodule update --init
+  - appveyor-retry sh -c 'git submodule deinit -f . && git submodule update --init'
   - set SRC=.
   - set NO_CCACHE=1
   - sh src/ci/run.sh

src/bootstrap/mk/Makefile.in

@@ -71,6 +71,8 @@ install:
 	$(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS)
 tidy:
 	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS)
+prepare:
+	$(Q)$(BOOTSTRAP) build nonexistent/path/to/trigger/cargo/metadata
 
 check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu:
 	$(Q)$(BOOTSTRAP) test --target arm-linux-androideabi

src/ci/docker/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -19,7 +19,9 @@ ci_dir="`dirname $docker_dir`"
 src_dir="`dirname $ci_dir`"
 root_dir="`dirname $src_dir`"
 
-docker \
+source "$ci_dir/shared.sh"
+
+retry docker \
   build \
   --rm \
   -t rust-ci \

src/ci/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -20,6 +20,9 @@ if [ "$NO_CHANGE_USER" = "" ]; then
   fi
 fi
 
+ci_dir=`cd $(dirname $0) && pwd`
+source "$ci_dir/shared.sh"
+
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
@@ -54,16 +57,17 @@ else
   fi
 fi
 
-set -ex
-
 $SRC/configure $RUST_CONFIGURE_ARGS
+retry make prepare
 
 if [ "$TRAVIS_OS_NAME" = "osx" ]; then
     ncpus=$(sysctl -n hw.ncpu)
 else
     ncpus=$(grep processor /proc/cpuinfo | wc -l)
 fi
 
+set -x
+
 if [ ! -z "$SCRIPT" ]; then
   sh -x -c "$SCRIPT"
 else

src/ci/shared.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# See http://unix.stackexchange.com/questions/82598
+function retry {
+  local n=1
+  local max=5
+  local delay=15
+  while true; do
+    echo "Attempting:" "$@"
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+      else
+        echo "The command has failed after $n attempts."
+        exit 1
+      fi
+    }
+  done
+}