diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 99b8ddf7db1f0..5c0b43c1d24e1 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -368,6 +368,7 @@ impl<'a> Builder<'a> { check::Rustdoc ), Kind::Test => describe!( + crate::toolstate::ToolStateCheck, test::Tidy, test::Ui, test::CompileFail, diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 7ea2bb126a641..1f4a4f923e048 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -169,7 +169,6 @@ mod job { pub use crate::config::Config; use crate::flags::Subcommand; use crate::cache::{Interned, INTERNER}; -use crate::toolstate::ToolState; const LLVM_TOOLS: &[&str] = &[ "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility @@ -1074,32 +1073,6 @@ impl Build { } } - /// Updates the actual toolstate of a tool. - /// - /// The toolstates are saved to the file specified by the key - /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be - /// done. The file is updated immediately after this function completes. - pub fn save_toolstate(&self, tool: &str, state: ToolState) { - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = t!(fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(path)); - - let mut current_toolstates: HashMap, ToolState> = - serde_json::from_reader(&mut file).unwrap_or_default(); - current_toolstates.insert(tool.into(), state); - t!(file.seek(SeekFrom::Start(0))); - t!(file.set_len(0)); - t!(serde_json::to_writer(file, ¤t_toolstates)); - } - } - fn in_tree_crates(&self, root: &str) -> Vec<&Crate> { let mut ret = Vec::new(); let mut list = vec![INTERNER.intern_str(root)]; diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs index e86209be91fe1..a6d9ef38e57b1 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/toolstate.rs @@ -1,4 +1,28 @@ use serde::{Deserialize, Serialize}; +use build_helper::t; +use std::time; +use std::fs; +use std::io::{Seek, SeekFrom}; +use std::collections::HashMap; +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use std::fmt; +use std::process::Command; +use std::path::PathBuf; +use std::env; + +// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. +const BETA_WEEK_START: u64 = 35; + +#[cfg(linux)] +const OS: Option<&str> = Some("linux"); + +#[cfg(windows)] +const OS: Option<&str> = Some("windows"); + +#[cfg(all(not(linux), not(windows)))] +const OS: Option<&str> = None; + +type ToolstateData = HashMap, ToolState>; #[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] @@ -12,9 +36,392 @@ pub enum ToolState { BuildFail = 0, } +impl fmt::Display for ToolState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", match self { + ToolState::TestFail => "test-fail", + ToolState::TestPass => "test-pass", + ToolState::BuildFail => "build-fail", + }) + } +} + impl Default for ToolState { fn default() -> Self { // err on the safe side ToolState::BuildFail } } + +/// Number of days after the last promotion of beta. +/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. +/// The Wednesday after this has value 0. +/// We track this value to prevent regressing tools in the last week of the 6-week cycle. +fn days_since_beta_promotion() -> u64 { + let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); + (since_epoch.as_secs() / 86400 - 20) % 42 +} + +// These tools must test-pass on the beta/stable channels. +// +// On the nightly channel, their build step must be attempted, but they may not +// be able to build successfully. +static STABLE_TOOLS: &[(&str, &str)] = &[ + ("book", "src/doc/book"), + ("nomicon", "src/doc/nomicon"), + ("reference", "src/doc/reference"), + ("rust-by-example", "src/doc/rust-by-example"), + ("edition-guide", "src/doc/edition-guide"), + ("rls", "src/tools/rls"), + ("rustfmt", "src/tools/rustfmt"), + ("clippy-driver", "src/tools/clippy"), +]; + +// These tools are permitted to not build on the beta/stable channels. +// +// We do require that we checked whether they build or not on the tools builder, +// though, as otherwise we will be unable to file an issue if they start +// failing. +static NIGHTLY_TOOLS: &[(&str, &str)] = &[ + ("miri", "src/tools/miri"), + ("embedded-book", "src/doc/embedded-book"), + ("rustc-guide", "src/doc/rustc-guide"), +]; + +fn print_error(tool: &str, submodule: &str) { + eprintln!(""); + eprintln!("We detected that this PR updated '{}', but its tests failed.", tool); + eprintln!(""); + eprintln!("If you do intend to update '{}', please check the error messages above and", tool); + eprintln!("commit another update."); + eprintln!(""); + eprintln!("If you do NOT intend to update '{}', please ensure you did not accidentally", tool); + eprintln!("change the submodule at '{}'. You may ask your reviewer for the", submodule); + eprintln!("proper steps."); + std::process::exit(3); +} + +fn check_changed_files(toolstates: &HashMap, ToolState>) { + // Changed files + let output = std::process::Command::new("git") + .arg("diff") + .arg("--name-status") + .arg("HEAD") + .arg("HEAD^") + .output(); + let output = match output { + Ok(o) => o, + Err(e) => { + eprintln!("Failed to get changed files: {:?}", e); + std::process::exit(1); + } + }; + + let output = t!(String::from_utf8(output.stdout)); + + for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + let changed = output.lines().any(|l| { + l.starts_with("M") && l.ends_with(submodule) + }); + eprintln!("Verifying status of {}...", tool); + if !changed { + continue; + } + + eprintln!("This PR updated '{}', verifying if status is 'test-pass'...", submodule); + if toolstates[*tool] != ToolState::TestPass { + print_error(tool, submodule); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ToolStateCheck; + +impl Step for ToolStateCheck { + type Output = (); + + /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will verify the validity of all our links in the + /// documentation to ensure we don't have a bunch of dead ones. + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run { + return; + } + + let days_since_beta_promotion = days_since_beta_promotion(); + let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; + let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); + let toolstates = builder.toolstates(); + + let mut did_error = false; + + for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + if !toolstates.contains_key(*tool) { + did_error = true; + eprintln!("error: Tool `{}` was not recorded in tool state.", tool); + } + } + + if did_error { + std::process::exit(1); + } + + check_changed_files(&toolstates); + + for (tool, _) in STABLE_TOOLS.iter() { + let state = toolstates[*tool]; + + if state != ToolState::TestPass { + if !is_nightly { + did_error = true; + eprintln!("error: Tool `{}` should be test-pass but is {}", tool, state); + } else if in_beta_week { + did_error = true; + eprintln!("error: Tool `{}` should be test-pass but is {} during beta week.", + tool, state); + } + } + } + + if did_error { + std::process::exit(1); + } + + if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { + commit_toolstate_change(&toolstates, in_beta_week); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("check-tools") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ToolStateCheck); + } +} + +impl Builder<'_> { + fn toolstates(&self) -> HashMap, ToolState> { + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = t!(fs::OpenOptions::new() + .create(true) + .write(true) + .read(true) + .open(path)); + + serde_json::from_reader(&mut file).unwrap_or_default() + } else { + Default::default() + } + } + + /// Updates the actual toolstate of a tool. + /// + /// The toolstates are saved to the file specified by the key + /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be + /// done. The file is updated immediately after this function completes. + pub fn save_toolstate(&self, tool: &str, state: ToolState) { + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = t!(fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(path)); + + let mut current_toolstates: HashMap, ToolState> = + serde_json::from_reader(&mut file).unwrap_or_default(); + current_toolstates.insert(tool.into(), state); + t!(file.seek(SeekFrom::Start(0))); + t!(file.set_len(0)); + t!(serde_json::to_writer(file, ¤t_toolstates)); + } + } +} + +/// This function `commit_toolstate_change` provides functionality for pushing a change +/// to the `rust-toolstate` repository. +/// +/// The function relies on a GitHub bot user, which should have a Personal access +/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for +/// some reason you need to change the token, please update the Azure Pipelines +/// variable group. +/// +/// 1. Generate a new Personal access token: +/// +/// * Login to the bot account, and go to Settings -> Developer settings -> +/// Personal access tokens +/// * Click "Generate new token" +/// * Enable the "public_repo" permission, then click "Generate token" +/// * Copy the generated token (should be a 40-digit hexadecimal number). +/// Save it somewhere secure, as the token would be gone once you leave +/// the page. +/// +/// 2. Update the variable group in Azure Pipelines +/// +/// * Ping a member of the infrastructure team to do this. +/// +/// 4. Replace the email address below if the bot account identity is changed +/// +/// * See +/// if a private email by GitHub is wanted. +fn commit_toolstate_change( + current_toolstate: &ToolstateData, + in_beta_week: bool, +) { + fn git_config(key: &str, value: &str) { + let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git config key={} value={} successful (status: {:?})", key, value, status); + } + } + + // If changing anything here, then please check that src/ci/publish_toolstate.sh is up to date + // as well. + git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); + git_config("user.name", "Rust Toolstate Update"); + git_config("credential.helper", "store"); + + let credential = format!( + "https://{}:x-oauth-basic@github.com\n", + t!(env::var("TOOLSTATE_REPO_ACCESS_TOKEN")), + ); + let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); + t!(fs::write(&git_credential_path, credential)); + + let status = Command::new("git").arg("clone") + .arg("--depth=1") + .arg(t!(env::var("TOOLSTATE_REPO"))) + .status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git clone successful (status: {:?})", status); + } + + let old_toolstate = t!(fs::read("rust-toolstate/_data/latest.json")); + let old_toolstate: Vec = t!(serde_json::from_slice(&old_toolstate)); + + let message = format!("({} CI update)", OS.expect("linux/windows only")); + let mut success = false; + for _ in 1..=5 { + // Update the toolstate results (the new commit-to-toolstate mapping) in the toolstate repo. + change_toolstate(¤t_toolstate, &old_toolstate, in_beta_week); + + // `git commit` failing means nothing to commit. + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("commit") + .arg("-a") + .arg("-m") + .arg(&message) + .status()); + if !status.success() { + success = true; + break; + } + + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("push") + .arg("origin") + .arg("master") + .status()); + // If we successfully push, exit. + if status.success() { + success = true; + break; + } + eprintln!("Sleeping for 3 seconds before retrying push"); + std::thread::sleep(std::time::Duration::from_secs(3)); + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("fetch") + .arg("origin") + .arg("master") + .status()); + assert!(status.success()); + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("reset") + .arg("--hard") + .arg("origin/master") + .status()); + assert!(status.success()); + } + + if !success { + panic!("Failed to update toolstate repository with new data"); + } +} + +fn change_toolstate( + current_toolstate: &ToolstateData, + old_toolstate: &[RepoState], + in_beta_week: bool, +) { + let mut regressed = false; + for repo_state in old_toolstate { + let tool = &repo_state.tool; + let state = if cfg!(linux) { + &repo_state.linux + } else if cfg!(windows) { + &repo_state.windows + } else { + unimplemented!() + }; + let new_state = current_toolstate[tool.as_str()]; + + if new_state != *state { + eprintln!("The state of `{}` has changed from `{}` to `{}`", tool, state, new_state); + if (new_state as u8) < (*state as u8) { + if !["rustc-guide", "miri", "embedded-book"].contains(&tool.as_str()) { + regressed = true; + } + } + } + } + + if regressed && in_beta_week { + std::process::exit(1); + } + + let commit = t!(std::process::Command::new("git") + .arg("rev-parse") + .arg("HEAD") + .output()); + let commit = t!(String::from_utf8(commit.stdout)); + + let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); + + let history_path = format!("rust-toolstate/history/{}.tsv", OS.expect("linux/windows only")); + let mut file = t!(fs::read_to_string(&history_path)); + let end_of_first_line = file.find('\n').unwrap(); + file.insert_str(end_of_first_line, &format!("{}\t{}\n", commit, toolstate_serialized)); + t!(fs::write(&history_path, file)); +} + +#[derive(Debug, Serialize, Deserialize)] +struct RepoState { + tool: String, + windows: ToolState, + linux: ToolState, + commit: String, + datetime: String, +} diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 70d6bad297dce..5248e41725836 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -163,7 +163,7 @@ jobs: NO_LLVM_ASSERTIONS: 1 # MSVC tools tests x86_64-msvc-tools: - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json # 32/64-bit MinGW builds. diff --git a/src/ci/azure-pipelines/master.yml b/src/ci/azure-pipelines/master.yml index e2baa923d99f7..9c5a15a3cf4ed 100644 --- a/src/ci/azure-pipelines/master.yml +++ b/src/ci/azure-pipelines/master.yml @@ -16,10 +16,7 @@ steps: - checkout: self fetchDepth: 2 -- script: | - export MESSAGE_FILE=$(mktemp -t msg.XXXXXX) - . src/ci/docker/x86_64-gnu-tools/repo.sh - commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN" +- script: src/ci/publish_toolstate.sh displayName: Publish toolstate env: TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) diff --git a/src/ci/docker/x86_64-gnu-tools/Dockerfile b/src/ci/docker/x86_64-gnu-tools/Dockerfile index 7687a6ca23e18..51193402ee842 100644 --- a/src/ci/docker/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/x86_64-gnu-tools/Dockerfile @@ -17,9 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -COPY x86_64-gnu-tools/checkregression.py /tmp/ COPY x86_64-gnu-tools/checktools.sh /tmp/ -COPY x86_64-gnu-tools/repo.sh /tmp/ # Run rustbook with `linkcheck` feature enabled ENV CHECK_LINKS 1 @@ -27,4 +25,4 @@ ENV CHECK_LINKS 1 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --save-toolstates=/tmp/toolstate/toolstates.json -ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstate/toolstates.json linux +ENV SCRIPT /tmp/checktools.sh ../x.py diff --git a/src/ci/docker/x86_64-gnu-tools/checkregression.py b/src/ci/docker/x86_64-gnu-tools/checkregression.py deleted file mode 100755 index 4fbb8c4d20349..0000000000000 --- a/src/ci/docker/x86_64-gnu-tools/checkregression.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -## This script has two purposes: detect any tool that *regressed*, which is used -## during the week before the beta branches to reject PRs; and detect any tool -## that *changed* to see if we need to update the toolstate repo. - -import sys -import json - -# Regressions for these tools during the beta cutoff week do not cause failure. -# See `status_check` in `checktools.sh` for tools that have to pass on the -# beta/stable branches. -REGRESSION_OK = ["rustc-guide", "miri", "embedded-book"] - -if __name__ == '__main__': - os_name = sys.argv[1] - toolstate_file = sys.argv[2] - current_state = sys.argv[3] - verb = sys.argv[4] # 'regressed' or 'changed' - - with open(toolstate_file, 'r') as f: - toolstate = json.load(f) - with open(current_state, 'r') as f: - current = json.load(f) - - regressed = False - for cur in current: - tool = cur['tool'] - state = cur[os_name] - new_state = toolstate.get(tool, '') - if verb == 'regressed': - updated = new_state < state - elif verb == 'changed': - updated = new_state != state - else: - print('Unknown verb {}'.format(updated)) - sys.exit(2) - if updated: - print( - 'The state of "{}" has {} from "{}" to "{}"' - .format(tool, verb, state, new_state) - ) - if not (verb == 'regressed' and tool in REGRESSION_OK): - regressed = True - - if regressed: - sys.exit(1) diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh index ebb8c0bda53ee..e57fe2271398f 100755 --- a/src/ci/docker/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh @@ -3,18 +3,6 @@ set -eu X_PY="$1" -TOOLSTATE_FILE="$(realpath -m $2)" -OS="$3" -COMMIT="$(git rev-parse HEAD)" -CHANGED_FILES="$(git diff --name-status HEAD HEAD^)" -SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))" -# ^ Number of days after the last promotion of beta. -# Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. -# The Wednesday after this has value 0. -# We track this value to prevent regressing tools in the last week of the 6-week cycle. - -mkdir -p "$(dirname $TOOLSTATE_FILE)" -touch "$TOOLSTATE_FILE" # Try to test all the tools and store the build/test success in the TOOLSTATE_FILE @@ -34,106 +22,4 @@ python2.7 "$X_PY" test --no-fail-fast \ set -e -cat "$TOOLSTATE_FILE" -echo - -# This function checks if a particular tool is *not* in status "test-pass". -check_tool_failed() { - grep -vq '"'"$1"'":"test-pass"' "$TOOLSTATE_FILE" -} - -# This function checks that if a tool's submodule changed, the tool's state must improve -verify_submodule_changed() { - echo "Verifying status of $1..." - if echo "$CHANGED_FILES" | grep -q "^M[[:blank:]]$2$"; then - echo "This PR updated '$2', verifying if status is 'test-pass'..." - if check_tool_failed "$1"; then - echo - echo "⚠️ We detected that this PR updated '$1', but its tests failed." - echo - echo "If you do intend to update '$1', please check the error messages above and" - echo "commit another update." - echo - echo "If you do NOT intend to update '$1', please ensure you did not accidentally" - echo "change the submodule at '$2'. You may ask your reviewer for the" - echo "proper steps." - exit 3 - fi - fi -} - -# deduplicates the submodule check and the assertion that on beta some tools MUST be passing. -# $1 should be "submodule_changed" to only check tools that got changed by this PR, -# or "beta_required" to check all tools that have $2 set to "beta". -check_dispatch() { - if [ "$1" = submodule_changed ]; then - # ignore $2 (branch id) - verify_submodule_changed $3 $4 - elif [ "$2" = beta ]; then - echo "Requiring test passing for $3..." - if check_tool_failed "$3"; then - exit 4 - fi - fi -} - -# List all tools here. -# This function gets called with "submodule_changed" for each PR that changed a submodule, -# and with "beta_required" for each PR that lands on beta/stable. -# The purpose of this function is to *reject* PRs if a tool is not "test-pass" and -# (a) the tool's submodule has been updated, or (b) we landed on beta/stable and the -# tool has to "test-pass" on that branch. -status_check() { - check_dispatch $1 beta book src/doc/book - check_dispatch $1 beta nomicon src/doc/nomicon - check_dispatch $1 beta reference src/doc/reference - check_dispatch $1 beta rust-by-example src/doc/rust-by-example - check_dispatch $1 beta edition-guide src/doc/edition-guide - check_dispatch $1 beta rls src/tools/rls - check_dispatch $1 beta rustfmt src/tools/rustfmt - check_dispatch $1 beta clippy-driver src/tools/clippy - # These tools are not required on the beta/stable branches, but they *do* cause - # PRs to fail if a submodule update does not fix them. - # They will still cause failure during the beta cutoff week, unless `checkregression.py` - # exempts them from that. - check_dispatch $1 nightly miri src/tools/miri - check_dispatch $1 nightly embedded-book src/doc/embedded-book - check_dispatch $1 nightly rustc-guide src/doc/rustc-guide -} - -# If this PR is intended to update one of these tools, do not let the build pass -# when they do not test-pass. - -status_check "submodule_changed" - -CHECK_NOT="$(readlink -f "$(dirname $0)/checkregression.py")" -# This callback is called by `commit_toolstate_change`, see `repo.sh`. -change_toolstate() { - # only update the history - if python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" changed; then - echo 'Toolstate is not changed. Not updating.' - else - if [ $SIX_WEEK_CYCLE -ge 35 ]; then - # Reject any regressions during the week before beta cutoff. - python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" regressed - fi - sed -i "1 a\\ -$COMMIT\t$(cat "$TOOLSTATE_FILE") -" "history/$OS.tsv" - fi -} - -if [ "$RUST_RELEASE_CHANNEL" = nightly ]; then - if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then - . "$(dirname $0)/repo.sh" - MESSAGE_FILE=$(mktemp -t msg.XXXXXX) - echo "($OS CI update)" > "$MESSAGE_FILE" - commit_toolstate_change "$MESSAGE_FILE" change_toolstate - rm -f "$MESSAGE_FILE" - fi - exit 0 -fi - -# abort compilation if an important tool doesn't build -# (this code is reachable if not on the nightly channel) -status_check "beta_required" +python2.7 "$X_PY" test check-tools diff --git a/src/ci/docker/x86_64-gnu-tools/repo.sh b/src/ci/docker/x86_64-gnu-tools/repo.sh deleted file mode 100644 index 82700a00fb6af..0000000000000 --- a/src/ci/docker/x86_64-gnu-tools/repo.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh - -# This file provides the function `commit_toolstate_change` for pushing a change -# to the `rust-toolstate` repository. -# -# The function relies on a GitHub bot user, which should have a Personal access -# token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for -# some reason you need to change the token, please update the Azure Pipelines -# variable group. -# -# 1. Generate a new Personal access token: -# -# * Login to the bot account, and go to Settings -> Developer settings -> -# Personal access tokens -# * Click "Generate new token" -# * Enable the "public_repo" permission, then click "Generate token" -# * Copy the generated token (should be a 40-digit hexadecimal number). -# Save it somewhere secure, as the token would be gone once you leave -# the page. -# -# 2. Update the variable group in Azure Pipelines -# -# * Ping a member of the infrastructure team to do this. -# -# 4. Replace the email address below if the bot account identity is changed -# -# * See -# if a private email by GitHub is wanted. - -commit_toolstate_change() { - OLDFLAGS="$-" - set -eu - - git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com' - git config --global user.name 'Rust Toolstate Update' - git config --global credential.helper store - printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \ - > "$HOME/.git-credentials" - git clone --depth=1 $TOOLSTATE_REPO - - cd rust-toolstate - FAILURE=1 - MESSAGE_FILE="$1" - shift - for RETRY_COUNT in 1 2 3 4 5; do - # Call the callback. - # - If we are in the `auto` branch (pre-landing), this is called from `checktools.sh` and - # the callback is `change_toolstate` in that file. The purpose of this is to publish the - # test results (the new commit-to-toolstate mapping) in the toolstate repo. - # - If we are in the `master` branch (post-landing), this is called by the CI pipeline - # and the callback is `src/tools/publish_toolstate.py`. The purpose is to publish - # the new "current" toolstate in the toolstate repo. - "$@" - # `git commit` failing means nothing to commit. - FAILURE=0 - git commit -a -F "$MESSAGE_FILE" || break - # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering. - git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1) - FAILURE=1 - git fetch origin master - git reset --hard origin/master - done - cd .. - - set +eu - set "-$OLDFLAGS" - return $FAILURE -} diff --git a/src/ci/publish_toolstate.sh b/src/ci/publish_toolstate.sh new file mode 100644 index 0000000000000..d8ff74078220e --- /dev/null +++ b/src/ci/publish_toolstate.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +set -eu + +# The following lines are also found in src/bootstrap/toolstate.rs, +# so if updating here, please also update that file. + +export MESSAGE_FILE=$(mktemp -t msg.XXXXXX) + +git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com' +git config --global user.name 'Rust Toolstate Update' +git config --global credential.helper store +printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \ + > "$HOME/.git-credentials" +git clone --depth=1 $TOOLSTATE_REPO + +cd rust-toolstate +FAILURE=1 +for RETRY_COUNT in 1 2 3 4 5; do + # The purpose is to publish the new "current" toolstate in the toolstate repo. + "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \ + "$(git log --format=%s -n1 HEAD)" \ + "$MESSAGE_FILE" \ + "$TOOLSTATE_REPO_ACCESS_TOKEN" + # `git commit` failing means nothing to commit. + FAILURE=0 + git commit -a -F "$MESSAGE_FILE" || break + # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering. + git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1) + FAILURE=1 + git fetch origin master + git reset --hard origin/master +done