Allow configuring where artifacts are downloaded from

src/bootstrap/bootstrap.py

@@ -1043,7 +1043,7 @@ def bootstrap(help_triggered):
     build.checksums_sha256 = data["checksums_sha256"]
     build.stage0_compiler = Stage0Toolchain(data["compiler"])
 
-    build.set_dist_environment(data["dist_server"])
+    build.set_dist_environment(data["config"]["dist_server"])
 
     build.build = args.build or build.build_triple()
 

src/bootstrap/builder.rs

@@ -870,20 +870,23 @@ impl<'a> Builder<'a> {
         self.try_run(patchelf.arg(fname));
     }
 
-    pub(crate) fn download_component(
-        &self,
-        base: &str,
-        url: &str,
-        dest_path: &Path,
-        help_on_error: &str,
-    ) {
+    pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
-        self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
+        // While bootstrap itself only supports http and https downloads, downstream forks might
+        // need to download components from other protocols. The match allows them adding more
+        // protocols without worrying about merge conficts if we change the HTTP implementation.
+        match url.split_once("://").map(|(proto, _)| proto) {
+            Some("http") | Some("https") => {
+                self.download_http_with_retries(&tempfile, url, help_on_error)
+            }
+            Some(other) => panic!("unsupported protocol {other} in {url}"),
+            None => panic!("no protocol in {url}"),
+        }
         t!(std::fs::rename(&tempfile, dest_path));
     }
 
-    fn download_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
+    fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
         println!("downloading {}", url);
         // Try curl. If that fails and we are on windows, fallback to PowerShell.
         let mut curl = Command::new("curl");

src/bootstrap/config.rs

@@ -20,7 +20,6 @@ use crate::channel::GitInfo;
 pub use crate::flags::Subcommand;
 use crate::flags::{Color, Flags};
 use crate::util::{exe, output, program_out_of_date, t};
-use crate::RustfmtMetadata;
 use once_cell::sync::OnceCell;
 use serde::{Deserialize, Deserializer};
 
@@ -73,6 +72,7 @@ pub struct Config {
     pub test_compare_mode: bool,
     pub color: Color,
     pub patch_binaries_for_nix: bool,
+    pub stage0_metadata: Stage0Metadata,
 
     pub on_fail: Option<String>,
     pub stage: u32,
@@ -212,6 +212,28 @@ pub struct Config {
     pub out: PathBuf,
 }
 
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Metadata {
+    pub config: Stage0Config,
+    pub checksums_sha256: HashMap<String, String>,
+    pub rustfmt: Option<RustfmtMetadata>,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Config {
+    pub dist_server: String,
+    pub artifacts_server: String,
+    pub artifacts_with_llvm_assertions_server: String,
+    pub git_merge_commit_email: String,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct RustfmtMetadata {
+    pub date: String,
+    pub version: String,
+}
+
 #[derive(Clone, Debug)]
 pub enum RustfmtState {
     SystemToolchain(PathBuf),
@@ -776,6 +798,9 @@ impl Config {
         config.llvm_profile_use = flags.llvm_profile_use;
         config.llvm_profile_generate = flags.llvm_profile_generate;
 
+        let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
+        config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
+
         #[cfg(test)]
         let get_toml = |_| TomlConfig::default();
         #[cfg(not(test))]
@@ -1103,8 +1128,11 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit =
-                download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
+            config.download_rustc_commit = download_ci_rustc_commit(
+                &config.stage0_metadata,
+                rust.download_rustc,
+                config.verbose > 0,
+            );
         } else {
             config.rust_profile_use = flags.rust_profile_use;
             config.rust_profile_generate = flags.rust_profile_generate;
@@ -1424,7 +1452,11 @@ fn threads_from_config(v: u32) -> u32 {
 }
 
 /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
-fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
+fn download_ci_rustc_commit(
+    stage0_metadata: &Stage0Metadata,
+    download_rustc: Option<StringOrBool>,
+    verbose: bool,
+) -> Option<String> {
     // If `download-rustc` is not set, default to rebuilding.
     let if_unchanged = match download_rustc {
         None | Some(StringOrBool::Bool(false)) => return None,
@@ -1443,13 +1475,12 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 
     // Look for a version to compare to based on the current commit.
     // Only commits merged by bors will have CI artifacts.
-    let merge_base = output(Command::new("git").args(&[
-        "rev-list",
-        "--author=bors@rust-lang.org",
-        "-n1",
-        "--first-parent",
-        "HEAD",
-    ]));
+    let merge_base = output(
+        Command::new("git")
+            .arg("rev-list")
+            .arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email))
+            .args(&["-n1", "--first-parent", "HEAD"]),
+    );
     let commit = merge_base.trim_end();
     if commit.is_empty() {
         println!("error: could not find commit hash for downloading rustc");
@@ -1484,7 +1515,7 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 }
 
 fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
-    let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?;
+    let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
     let channel = format!("{version}-{date}");
 
     let host = builder.config.build;
@@ -1568,13 +1599,13 @@ fn download_component(
     let tarball = cache_dir.join(&filename);
     let (base_url, url, should_verify) = match mode {
         DownloadSource::CI => (
-            "https://ci-artifacts.rust-lang.org/rustc-builds".to_string(),
+            builder.config.stage0_metadata.config.artifacts_server.clone(),
             format!("{key}/{filename}"),
             false,
         ),
         DownloadSource::Dist => {
             let dist_server = env::var("RUSTUP_DIST_SERVER")
-                .unwrap_or(builder.stage0_metadata.dist_server.to_string());
+                .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
             // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
             (dist_server, format!("dist/{key}/{filename}"), true)
         }
@@ -1590,7 +1621,7 @@ fn download_component(
             target at this time, see https://doc.rust-lang.org/nightly\
             /rustc/platform-support.html for more information."
         );
-        let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error);
+        let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
         if tarball.exists() {
             if builder.verify(&tarball, sha256) {
                 builder.unpack(&tarball, &bin_root, prefix);
@@ -1610,7 +1641,7 @@ fn download_component(
         None
     };
 
-    builder.download_component(&base_url, &url, &tarball, "");
+    builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
     if let Some(sha256) = checksum {
         if !builder.verify(&tarball, sha256) {
             panic!("failed to verify {}", tarball.display());

src/bootstrap/lib.rs

@@ -118,7 +118,6 @@ use std::os::windows::fs::symlink_file;
 
 use filetime::FileTime;
 use once_cell::sync::OnceCell;
-use serde::Deserialize;
 
 use crate::builder::Kind;
 use crate::config::{LlvmLibunwind, TargetSelection};
@@ -294,8 +293,6 @@ pub struct Build {
     hosts: Vec<TargetSelection>,
     targets: Vec<TargetSelection>,
 
-    // Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents
-    stage0_metadata: Stage0Metadata,
     initial_rustc: PathBuf,
     initial_cargo: PathBuf,
     initial_lld: PathBuf,
@@ -322,18 +319,6 @@ pub struct Build {
     metrics: metrics::BuildMetrics,
 }
 
-#[derive(Deserialize)]
-struct Stage0Metadata {
-    dist_server: String,
-    checksums_sha256: HashMap<String, String>,
-    rustfmt: Option<RustfmtMetadata>,
-}
-#[derive(Deserialize)]
-struct RustfmtMetadata {
-    date: String,
-    version: String,
-}
-
 #[derive(Debug)]
 struct Crate {
     name: Interned<String>,
@@ -482,11 +467,7 @@ impl Build {
             bootstrap_out
         };
 
-        let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json")));
-        let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
-
         let mut build = Build {
-            stage0_metadata,
             initial_rustc: config.initial_rustc.clone(),
             initial_cargo: config.initial_cargo.clone(),
             initial_lld,

src/bootstrap/native.rs

@@ -121,7 +121,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
     let mut rev_list = Command::new("git");
     rev_list.args(&[
         PathBuf::from("rev-list"),
-        "--author=bors@rust-lang.org".into(),
+        format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
         "-n1".into(),
         "--first-parent".into(),
         "HEAD".into(),
@@ -170,11 +170,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
     if !rustc_cache.exists() {
         t!(fs::create_dir_all(&rustc_cache));
     }
-    let base = "https://ci-artifacts.rust-lang.org";
-    let url = if llvm_assertions {
-        format!("rustc-builds-alt/{}", llvm_sha)
+    let base = if llvm_assertions {
+        &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
     } else {
-        format!("rustc-builds/{}", llvm_sha)
+        &builder.config.stage0_metadata.config.artifacts_server
     };
     let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
     let tarball = rustc_cache.join(&filename);
@@ -187,7 +186,11 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
 \ndownload-ci-llvm = false
 \n
 ";
-        builder.download_component(base, &format!("{}/{}", url, filename), &tarball, help_on_error);
+        builder.download_component(
+            &format!("{base}/{llvm_sha}/{filename}"),
+            &tarball,
+            help_on_error,
+        );
     }
     let llvm_root = builder.config.ci_llvm_root();
     builder.unpack(&tarball, &llvm_root, "rust-dev");

src/stage0.json

@@ -1,6 +1,20 @@
 {
-  "__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.",
-  "dist_server": "https://static.rust-lang.org",
+  "config": {
+    "dist_server": "https://static.rust-lang.org",
+    "artifacts_server": "https://ci-artifacts.rust-lang.org/rustc-builds",
+    "artifacts_with_llvm_assertions_server": "https://ci-artifacts.rust-lang.org/rustc-builds-alt",
+    "git_merge_commit_email": "bors@rust-lang.org"
+  },
+  "__comments": [
+    "The configuration above this comment is editable, and can be changed",
+    "by forks of the repository if they have alternate values.",
+    "",
+    "The section below is generated by `./x.py run src/tools/bump-stage0`,",
+    "run that command again to update the bootstrap compiler.",
+    "",
+    "All changes below this comment will be overridden the next time the",
+    "tool is executed."
+  ],
   "compiler": {
     "date": "2022-05-20",
     "version": "beta"