From 79e86e3ea067cf531d728fb15652e4fa76f54b63 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 1 Oct 2022 23:05:32 -0500 Subject: [PATCH 1/9] More build-manifest docs --- src/tools/build-manifest/README.md | 32 +++++++++++++++++----------- src/tools/build-manifest/src/main.rs | 6 +----- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index 44c96f31d0ba8..b5c6371e553a5 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -1,7 +1,16 @@ # build-manifest -This tool generates the manifests uploaded to static.rust-lang.org and used by -rustup. The tool is invoked by the bootstrap tool. +This tool generates the manifests uploaded to static.rust-lang.org and used by rustup. +You can see a full list of all manifests at . + +This gets called by `promote-release` via `x.py dist hash-and-sign`. + +## Adding a new component + +There are several steps involved here. +1. Add a new `Step` to `dist.rs`. This should usually be named after the filename of the uploaded tarball. See https://github.com/rust-lang/rust/pull/101799/files#diff-2c56335faa24486df09ba392d8900c57e2fac4633e1f7038469bcf9ed3feb871 for an example. + a. If appropriate, call `tarball.is_preview(true)` for the component. +3. Add a new `PkgType` to build-manifest. Fix all the compile errors as appropriate. ## Testing changes locally @@ -9,19 +18,16 @@ In order to test the changes locally you need to have a valid dist directory available locally. If you don't want to build all the compiler, you can easily create one from the nightly artifacts with: -``` -#!/bin/bash -for cmpn in rust rustc rust-std rust-docs cargo; do - wget https://static.rust-lang.org/dist/${cmpn}-nightly-x86_64-unknown-linux-gnu.tar.gz +```sh +for component in rust rustc rust-std rust-docs cargo; do + wget -P build/dist https://static.rust-lang.org/dist/${component}-nightly-x86_64-unknown-linux-gnu.tar.gz done ``` -Then, you can generate the manifest and all the packages from `path/to/dist` to -`path/to/output` with: +Then, you can generate the manifest and all the packages from `build/dist` to +`build/manifest` with: +```sh +mkdir -p build/manifest +cargo +nightly run -p build-manifest build/dist build/manifest 1970-01-01 http://example.com nightly ``` -$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com CHANNEL -``` - -Remember to replace `CHANNEL` with the channel you produced dist artifacts of -and `VERSION` with the current Rust version. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index b0006cb90bdd6..983f4ca713882 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -1,8 +1,4 @@ -//! Build a dist manifest, hash and sign everything. -//! This gets called by `promote-release` -//! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release) -//! via `x.py dist hash-and-sign`; the cmdline arguments are set up -//! by rustbuild (in `src/bootstrap/dist.rs`). +#![doc = include_str!("../README.md")] mod checksum; mod manifest; From 494cb47852ef7bb2cc14f1f440d77722bf0f759b Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 1 Oct 2022 23:42:21 -0500 Subject: [PATCH 2/9] build-manifest: Add a macro that makes it impossible to typo `-preview`, or have a mismatch between parsing and stringifying --- src/tools/build-manifest/src/versions.rs | 84 +++++++++++------------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 0186194a41f55..61e8825753c5d 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -8,58 +8,48 @@ use tar::Archive; const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; -#[derive(Debug, Hash, Eq, PartialEq, Clone)] -pub(crate) enum PkgType { - Rust, - RustSrc, - Rustc, - Cargo, - Rls, - RustAnalyzer, - Clippy, - Rustfmt, - LlvmTools, - Miri, - JsonDocs, - Other(String), -} - -impl PkgType { - pub(crate) fn from_component(component: &str) -> Self { - match component { - "rust" => PkgType::Rust, - "rust-src" => PkgType::RustSrc, - "rustc" => PkgType::Rustc, - "cargo" => PkgType::Cargo, - "rls" | "rls-preview" => PkgType::Rls, - "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer, - "clippy" | "clippy-preview" => PkgType::Clippy, - "rustfmt" | "rustfmt-preview" => PkgType::Rustfmt, - "llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools, - "miri" | "miri-preview" => PkgType::Miri, - "rust-docs-json" | "rust-docs-json-preview" => PkgType::JsonDocs, - other => PkgType::Other(other.into()), +macro_rules! pkg_type { + ( $($variant:ident = $component:literal $(; preview = true $(@$is_preview:tt)? )? ),+ $(,)? ) => { + #[derive(Debug, Hash, Eq, PartialEq, Clone)] + pub(crate) enum PkgType { + $($variant,)+ + Other(String), } - } - /// First part of the tarball name. - fn tarball_component_name(&self) -> &str { - match self { - PkgType::Rust => "rust", - PkgType::RustSrc => "rust-src", - PkgType::Rustc => "rustc", - PkgType::Cargo => "cargo", - PkgType::Rls => "rls", - PkgType::RustAnalyzer => "rust-analyzer", - PkgType::Clippy => "clippy", - PkgType::Rustfmt => "rustfmt", - PkgType::LlvmTools => "llvm-tools", - PkgType::Miri => "miri", - PkgType::JsonDocs => "rust-docs-json", - PkgType::Other(component) => component, + impl PkgType { + pub(crate) fn from_component(component: &str) -> Self { + match component { + $( $component $( | concat!($($is_preview)? $component, "-preview") )? => PkgType::$variant,)+ + _ => PkgType::Other(component.into()), + } + } + + /// First part of the tarball name. + fn tarball_component_name(&self) -> &str { + match self { + $( PkgType::$variant => $component,)+ + PkgType::Other(component) => component, + } + } } } +} +pkg_type! { + Rust = "rust", + RustSrc = "rust-src", + Rustc = "rustc", + Cargo = "cargo", + Rls = "rls"; preview = true, + RustAnalyzer = "rust-analyzer"; preview = true, + Clippy = "clippy"; preview = true, + Rustfmt = "rustfmt"; preview = true, + LlvmTools = "llvm-tools"; preview = true, + Miri = "miri"; preview = true, + JsonDocs = "rust-docs-json"; preview = true, +} + +impl PkgType { /// Whether this package has the same version as Rust itself, or has its own `version` and /// `git-commit-hash` files inside the tarball. fn should_use_rust_version(&self) -> bool { From 69a74955fd9dfd28647b8a4c18abc3dcecb5696d Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 2 Oct 2022 00:25:08 -0500 Subject: [PATCH 3/9] Use `PkgType` to determine which packages to add to the manifest Previously, these had to be hard-coded (i.e. specified in both `PkgType` and `fn package`). Now they only have to be specified in `PkgType`. --- src/tools/build-manifest/src/main.rs | 35 +++++------- src/tools/build-manifest/src/versions.rs | 70 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 983f4ca713882..726275988e095 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -283,28 +283,15 @@ impl Builder { } fn add_packages_to(&mut self, manifest: &mut Manifest) { - macro_rules! package { - ($name:expr, $targets:expr) => { - self.package($name, &mut manifest.pkg, $targets, &[]) - }; + for pkg in PkgType::all() { + let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] }; + self.package( + &pkg.manifest_component_name(), + &mut manifest.pkg, + pkg.targets(), + fallback, + ); } - package!("rustc", HOSTS); - package!("rustc-dev", HOSTS); - package!("reproducible-artifacts", HOSTS); - package!("rustc-docs", HOSTS); - package!("cargo", HOSTS); - package!("rust-mingw", MINGW); - package!("rust-std", TARGETS); - self.package("rust-docs", &mut manifest.pkg, HOSTS, DOCS_FALLBACK); - self.package("rust-docs-json-preview", &mut manifest.pkg, HOSTS, DOCS_FALLBACK); - package!("rust-src", &["*"]); - package!("rls-preview", HOSTS); - package!("rust-analyzer-preview", HOSTS); - package!("clippy-preview", HOSTS); - package!("miri-preview", HOSTS); - package!("rustfmt-preview", HOSTS); - package!("rust-analysis", TARGETS); - package!("llvm-tools-preview", TARGETS); } fn add_artifacts_to(&mut self, manifest: &mut Manifest) { @@ -500,6 +487,12 @@ impl Builder { targets: &[&str], fallback: &[(&str, &str)], ) { + if pkgname == "rust" { + // This is handled specially by `rust_package` later. + // Order is important, so don't call `rust_package` here. + return; + } + let version_info = self .versions .version(&PkgType::from_component(pkgname)) diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 61e8825753c5d..5aef232d7abf4 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -17,6 +17,13 @@ macro_rules! pkg_type { } impl PkgType { + fn is_preview(&self) -> bool { + match self { + $( $( $($is_preview)? PkgType::$variant => true, )? )+ + _ => false, + } + } + pub(crate) fn from_component(component: &str) -> Self { match component { $( $component $( | concat!($($is_preview)? $component, "-preview") )? => PkgType::$variant,)+ @@ -31,6 +38,11 @@ macro_rules! pkg_type { PkgType::Other(component) => component, } } + + /// Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate. + pub(crate) fn all() -> &'static [PkgType] { + &[ $(PkgType::$variant),+ ] + } } } } @@ -39,7 +51,14 @@ pkg_type! { Rust = "rust", RustSrc = "rust-src", Rustc = "rustc", + RustcDev = "rustc-dev", + RustcDocs = "rustc-docs", + ReproducibleArtifacts = "reproducible-artifacts", + RustMingw = "rust-mingw", + RustStd = "rust-std", Cargo = "cargo", + HtmlDocs = "rust-docs", + RustAnalysis = "rust-analysis", Rls = "rls"; preview = true, RustAnalyzer = "rust-analyzer"; preview = true, Clippy = "clippy"; preview = true, @@ -50,6 +69,15 @@ pkg_type! { } impl PkgType { + // / Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate. + pub(crate) fn manifest_component_name(&self) -> String { + if self.is_preview() { + format!("{}-preview", self.tarball_component_name()) + } else { + self.tarball_component_name().to_string() + } + } + /// Whether this package has the same version as Rust itself, or has its own `version` and /// `git-commit-hash` files inside the tarball. fn should_use_rust_version(&self) -> bool { @@ -63,17 +91,59 @@ impl PkgType { PkgType::Miri => false, PkgType::Rust => true, + PkgType::RustStd => true, PkgType::RustSrc => true, PkgType::Rustc => true, PkgType::JsonDocs => true, + PkgType::HtmlDocs => true, + PkgType::RustcDev => true, + PkgType::RustcDocs => true, + PkgType::ReproducibleArtifacts => true, + PkgType::RustMingw => true, + PkgType::RustAnalysis => true, PkgType::Other(_) => true, } } + pub(crate) fn targets(&self) -> &[&str] { + use crate::{HOSTS, MINGW, TARGETS}; + use PkgType::*; + + match self { + Rust => HOSTS, // doesn't matter in practice, but return something to avoid panicking + Rustc => HOSTS, + RustcDev => HOSTS, + ReproducibleArtifacts => HOSTS, + RustcDocs => HOSTS, + Cargo => HOSTS, + RustMingw => MINGW, + RustStd => TARGETS, + HtmlDocs => HOSTS, + JsonDocs => HOSTS, + RustSrc => &["*"], + Rls => HOSTS, + RustAnalyzer => HOSTS, + Clippy => HOSTS, + Miri => HOSTS, + Rustfmt => HOSTS, + RustAnalysis => TARGETS, + LlvmTools => TARGETS, + Other(pkg) => panic!("add {pkg} to the list of known `PkgType`s"), + } + } + /// Whether this package is target-independent or not. fn target_independent(&self) -> bool { *self == PkgType::RustSrc } + + /// Whether to package these target-specific docs for another similar target. + pub(crate) fn use_docs_fallback(&self) -> bool { + match self { + PkgType::JsonDocs | PkgType::HtmlDocs => true, + _ => false, + } + } } #[derive(Debug, Default, Clone)] From a3dd94e702f2d640f5f84f9f43f25fd79dfeae95 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 2 Oct 2022 00:28:25 -0500 Subject: [PATCH 4/9] Use `PkgType::is_preview` to determine whether to add a rename to the manifest This caught a missing preview rename for `llvm-tools`. --- src/tools/build-manifest/src/main.rs | 11 +++++------ src/tools/build-manifest/src/versions.rs | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 726275988e095..f09e1e5718ee3 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -365,12 +365,11 @@ impl Builder { let mut rename = |from: &str, to: &str| { manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() }) }; - rename("rls", "rls-preview"); - rename("rustfmt", "rustfmt-preview"); - rename("clippy", "clippy-preview"); - rename("miri", "miri-preview"); - rename("rust-docs-json", "rust-docs-json-preview"); - rename("rust-analyzer", "rust-analyzer-preview"); + for pkg in PkgType::all() { + if pkg.is_preview() { + rename(pkg.tarball_component_name(), &pkg.manifest_component_name()); + } + } } fn rust_package(&mut self, manifest: &Manifest) -> Package { diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 5aef232d7abf4..967433a86ad3e 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -17,7 +17,7 @@ macro_rules! pkg_type { } impl PkgType { - fn is_preview(&self) -> bool { + pub(crate) fn is_preview(&self) -> bool { match self { $( $( $($is_preview)? PkgType::$variant => true, )? )+ _ => false, @@ -32,7 +32,7 @@ macro_rules! pkg_type { } /// First part of the tarball name. - fn tarball_component_name(&self) -> &str { + pub(crate) fn tarball_component_name(&self) -> &str { match self { $( PkgType::$variant => $component,)+ PkgType::Other(component) => component, From 6cbf07959692c662126d0971f767ba14fd175ce5 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 2 Oct 2022 00:48:50 -0500 Subject: [PATCH 5/9] Use an exhaustive match in `target_host_combination`. This avoids bugs where components are added to one part of the manifest but not another. --- src/tools/build-manifest/src/main.rs | 83 ++++++++++++++---------- src/tools/build-manifest/src/versions.rs | 3 +- 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index f09e1e5718ee3..efe4412726a9a 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -401,42 +401,55 @@ impl Builder { let mut components = Vec::new(); let mut extensions = Vec::new(); - let host_component = |pkg| Component::from_str(pkg, host); - - // rustc/rust-std/cargo/docs are all required, - // and so is rust-mingw if it's available for the target. - components.extend(vec![ - host_component("rustc"), - host_component("rust-std"), - host_component("cargo"), - host_component("rust-docs"), - ]); - if host.contains("pc-windows-gnu") { - components.push(host_component("rust-mingw")); - } + let host_component = |pkg: &_| Component::from_str(pkg, host); - // Tools are always present in the manifest, - // but might be marked as unavailable if they weren't built. - extensions.extend(vec![ - host_component("clippy-preview"), - host_component("miri-preview"), - host_component("rls-preview"), - host_component("rust-analyzer-preview"), - host_component("rustfmt-preview"), - host_component("llvm-tools-preview"), - host_component("rust-analysis"), - host_component("rust-docs-json-preview"), - ]); - - extensions.extend( - TARGETS - .iter() - .filter(|&&target| target != host) - .map(|target| Component::from_str("rust-std", target)), - ); - extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target))); - extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target))); - extensions.push(Component::from_str("rust-src", "*")); + for pkg in PkgType::all() { + match pkg { + // rustc/rust-std/cargo/docs are all required + PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => { + components.push(host_component(&pkg.manifest_component_name())); + } + PkgType::RustStd => { + components.push(host_component(&pkg.manifest_component_name())); + extensions.extend( + TARGETS.iter().filter(|&&target| target != host).map(|target| { + Component::from_str(&pkg.manifest_component_name(), target) + }), + ); + } + // so is rust-mingw if it's available for the target + PkgType::RustMingw => { + if host.contains("pc-windows-gnu") { + components.push(host_component("rust-mingw")); + } + } + // Tools are always present in the manifest, + // but might be marked as unavailable if they weren't built. + PkgType::Clippy + | PkgType::Miri + | PkgType::Rls + | PkgType::RustAnalyzer + | PkgType::Rustfmt + | PkgType::LlvmTools + | PkgType::RustAnalysis + | PkgType::JsonDocs => { + extensions.push(host_component(&pkg.manifest_component_name())); + } + PkgType::RustcDev | PkgType::RustcDocs => { + extensions.extend( + HOSTS.iter().map(|target| { + Component::from_str(&pkg.manifest_component_name(), target) + }), + ); + } + PkgType::RustSrc => { + extensions.push(Component::from_str(&pkg.manifest_component_name(), "*")); + } + PkgType::Rust | PkgType::Other(_) => {} + // FIXME: is this correct? maybe we should add it so rustup knows about it ... + PkgType::ReproducibleArtifacts => {} + } + } // If the components/extensions don't actually exist for this // particular host/target combination then nix it entirely from our diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 967433a86ad3e..cea34905db6aa 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -39,7 +39,6 @@ macro_rules! pkg_type { } } - /// Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate. pub(crate) fn all() -> &'static [PkgType] { &[ $(PkgType::$variant),+ ] } @@ -69,7 +68,7 @@ pkg_type! { } impl PkgType { - // / Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate. + /// Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate. pub(crate) fn manifest_component_name(&self) -> String { if self.is_preview() { format!("{}-preview", self.tarball_component_name()) From 1e9d2187ed0addf59e5e73811454e46246b4fdf0 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 27 Oct 2022 12:07:34 -0500 Subject: [PATCH 6/9] Use PkgType in more places In particular, this avoids serializing and parsing the pkg to a string, which allows getting rid of `PkgType::Other` altogether --- src/tools/build-manifest/README.md | 1 + src/tools/build-manifest/src/main.rs | 57 ++++++++++-------------- src/tools/build-manifest/src/manifest.rs | 5 ++- src/tools/build-manifest/src/versions.rs | 11 ----- 4 files changed, 27 insertions(+), 47 deletions(-) diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index b5c6371e553a5..40dce38742a14 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -2,6 +2,7 @@ This tool generates the manifests uploaded to static.rust-lang.org and used by rustup. You can see a full list of all manifests at . +This listing is updated by every 7 days. This gets called by `promote-release` via `x.py dist hash-and-sign`. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index efe4412726a9a..ebc220cfce590 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -180,7 +180,7 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin" static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; -static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-docs-json-preview"]; +static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs]; macro_rules! t { ($e:expr) => { @@ -285,12 +285,7 @@ impl Builder { fn add_packages_to(&mut self, manifest: &mut Manifest) { for pkg in PkgType::all() { let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] }; - self.package( - &pkg.manifest_component_name(), - &mut manifest.pkg, - pkg.targets(), - fallback, - ); + self.package(pkg, &mut manifest.pkg, fallback); } } @@ -401,26 +396,27 @@ impl Builder { let mut components = Vec::new(); let mut extensions = Vec::new(); - let host_component = |pkg: &_| Component::from_str(pkg, host); + let host_component = |pkg: &_| Component::from_pkg(pkg, host); for pkg in PkgType::all() { match pkg { // rustc/rust-std/cargo/docs are all required PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => { - components.push(host_component(&pkg.manifest_component_name())); + components.push(host_component(pkg)); } PkgType::RustStd => { - components.push(host_component(&pkg.manifest_component_name())); + components.push(host_component(pkg)); extensions.extend( - TARGETS.iter().filter(|&&target| target != host).map(|target| { - Component::from_str(&pkg.manifest_component_name(), target) - }), + TARGETS + .iter() + .filter(|&&target| target != host) + .map(|target| Component::from_pkg(pkg, target)), ); } // so is rust-mingw if it's available for the target PkgType::RustMingw => { if host.contains("pc-windows-gnu") { - components.push(host_component("rust-mingw")); + components.push(host_component(pkg)); } } // Tools are always present in the manifest, @@ -433,20 +429,16 @@ impl Builder { | PkgType::LlvmTools | PkgType::RustAnalysis | PkgType::JsonDocs => { - extensions.push(host_component(&pkg.manifest_component_name())); + extensions.push(host_component(pkg)); } PkgType::RustcDev | PkgType::RustcDocs => { - extensions.extend( - HOSTS.iter().map(|target| { - Component::from_str(&pkg.manifest_component_name(), target) - }), - ); + extensions.extend(HOSTS.iter().map(|target| Component::from_pkg(pkg, target))); } PkgType::RustSrc => { - extensions.push(Component::from_str(&pkg.manifest_component_name(), "*")); + extensions.push(Component::from_pkg(pkg, "*")); } - PkgType::Rust | PkgType::Other(_) => {} - // FIXME: is this correct? maybe we should add it so rustup knows about it ... + PkgType::Rust => {} + // NOTE: this is intentional, these artifacts aren't intended to be used with rustup PkgType::ReproducibleArtifacts => {} } } @@ -494,31 +486,27 @@ impl Builder { fn package( &mut self, - pkgname: &str, + pkg: &PkgType, dst: &mut BTreeMap, - targets: &[&str], fallback: &[(&str, &str)], ) { - if pkgname == "rust" { + if *pkg == PkgType::Rust { // This is handled specially by `rust_package` later. // Order is important, so don't call `rust_package` here. return; } - let version_info = self - .versions - .version(&PkgType::from_component(pkgname)) - .expect("failed to load package version"); + let version_info = self.versions.version(&pkg).expect("failed to load package version"); let mut is_present = version_info.present; // Never ship nightly-only components for other trains. - if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { + if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkg) { is_present = false; // Pretend the component is entirely missing. } macro_rules! tarball_name { ($target_name:expr) => { - self.versions.tarball_name(&PkgType::from_component(pkgname), $target_name).unwrap() + self.versions.tarball_name(pkg, $target_name).unwrap() }; } let mut target_from_compressed_tar = |target_name| { @@ -547,7 +535,8 @@ impl Builder { Target::unavailable() }; - let targets = targets + let targets = pkg + .targets() .iter() .map(|name| { let target = if is_present { @@ -562,7 +551,7 @@ impl Builder { .collect(); dst.insert( - pkgname.to_string(), + pkg.manifest_component_name(), Package { version: version_info.version.unwrap_or_default(), git_commit_hash: version_info.git_commit, diff --git a/src/tools/build-manifest/src/manifest.rs b/src/tools/build-manifest/src/manifest.rs index 547c270d89ab7..a9f19d8e5653f 100644 --- a/src/tools/build-manifest/src/manifest.rs +++ b/src/tools/build-manifest/src/manifest.rs @@ -1,3 +1,4 @@ +use crate::versions::PkgType; use crate::Builder; use serde::{Serialize, Serializer}; use std::collections::BTreeMap; @@ -116,8 +117,8 @@ pub(crate) struct Component { } impl Component { - pub(crate) fn from_str(pkg: &str, target: &str) -> Self { - Self { pkg: pkg.to_string(), target: target.to_string() } + pub(crate) fn from_pkg(pkg: &PkgType, target: &str) -> Self { + Self { pkg: pkg.manifest_component_name(), target: target.to_string() } } } diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index cea34905db6aa..dde9745afb785 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -13,7 +13,6 @@ macro_rules! pkg_type { #[derive(Debug, Hash, Eq, PartialEq, Clone)] pub(crate) enum PkgType { $($variant,)+ - Other(String), } impl PkgType { @@ -24,18 +23,10 @@ macro_rules! pkg_type { } } - pub(crate) fn from_component(component: &str) -> Self { - match component { - $( $component $( | concat!($($is_preview)? $component, "-preview") )? => PkgType::$variant,)+ - _ => PkgType::Other(component.into()), - } - } - /// First part of the tarball name. pub(crate) fn tarball_component_name(&self) -> &str { match self { $( PkgType::$variant => $component,)+ - PkgType::Other(component) => component, } } @@ -100,7 +91,6 @@ impl PkgType { PkgType::ReproducibleArtifacts => true, PkgType::RustMingw => true, PkgType::RustAnalysis => true, - PkgType::Other(_) => true, } } @@ -127,7 +117,6 @@ impl PkgType { Rustfmt => HOSTS, RustAnalysis => TARGETS, LlvmTools => TARGETS, - Other(pkg) => panic!("add {pkg} to the list of known `PkgType`s"), } } From fde05ea0898bf0436f9a85d6f58064cba9c8336c Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 27 Oct 2022 12:23:27 -0500 Subject: [PATCH 7/9] Use PkgType for profiles, too This makes it easier to remove `is_preview` from components in the future if we choose to do so. --- src/tools/build-manifest/src/main.rs | 61 +++++++++++----------------- 1 file changed, 24 insertions(+), 37 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index ebc220cfce590..92be95c9420c5 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -311,44 +311,28 @@ impl Builder { } fn add_profiles_to(&mut self, manifest: &mut Manifest) { - let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs); - profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]); - profile( - "default", - &[ - "rustc", - "cargo", - "rust-std", - "rust-mingw", - "rust-docs", - "rustfmt-preview", - "clippy-preview", - ], - ); - profile( - "complete", - &[ - "rustc", - "cargo", - "rust-std", - "rust-mingw", - "rust-docs", - "rustfmt-preview", - "clippy-preview", - "rls-preview", - "rust-analyzer-preview", - "rust-src", - "llvm-tools-preview", - "rust-analysis", - "miri-preview", - ], - ); + use PkgType::*; + + let mut profile = |name, pkgs: &_| self.profile(name, &mut manifest.profiles, pkgs); + + // Use a Vec here to make sure we don't exclude any components in an earlier profile. + let minimal = vec![Rustc, Cargo, RustStd, RustMingw]; + profile("minimal", &minimal); + + let mut default = minimal; + default.extend([HtmlDocs, Rustfmt, Clippy]); + profile("default", &default); + + // NOTE: this profile is effectively deprecated; do not add new components to it. + let mut complete = default; + complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]); + profile("complete", &complete); // The compiler libraries are not stable for end users, and they're also huge, so we only // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible // for users to install the additional component manually, if needed. if self.versions.channel() == "nightly" { - self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]); + self.extend_profile("complete", &mut manifest.profiles, &[RustcDev]); // Do not include the rustc-docs component for now, as it causes // conflicts with the rust-docs component when installed. See // #75833. @@ -468,20 +452,23 @@ impl Builder { &mut self, profile_name: &str, dst: &mut BTreeMap>, - pkgs: &[&str], + pkgs: &[PkgType], ) { - dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect()); + dst.insert( + profile_name.to_owned(), + pkgs.iter().map(|s| s.manifest_component_name()).collect(), + ); } fn extend_profile( &mut self, profile_name: &str, dst: &mut BTreeMap>, - pkgs: &[&str], + pkgs: &[PkgType], ) { dst.get_mut(profile_name) .expect("existing profile") - .extend(pkgs.iter().map(|s| (*s).to_owned())); + .extend(pkgs.iter().map(|s| s.manifest_component_name())); } fn package( From 1aab5d570baca5d776b9e6c670791391fe39bd6d Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 1 Oct 2022 23:05:32 -0500 Subject: [PATCH 8/9] More build-manifest docs --- src/tools/build-manifest/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index 40dce38742a14..9d30c554186be 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -8,10 +8,9 @@ This gets called by `promote-release` Date: Sun, 30 Oct 2022 16:38:40 -0500 Subject: [PATCH 9/9] Remove unnecessary argument to `package` --- src/tools/build-manifest/src/main.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 92be95c9420c5..371e60f969e58 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -284,8 +284,7 @@ impl Builder { fn add_packages_to(&mut self, manifest: &mut Manifest) { for pkg in PkgType::all() { - let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] }; - self.package(pkg, &mut manifest.pkg, fallback); + self.package(pkg, &mut manifest.pkg); } } @@ -471,18 +470,14 @@ impl Builder { .extend(pkgs.iter().map(|s| s.manifest_component_name())); } - fn package( - &mut self, - pkg: &PkgType, - dst: &mut BTreeMap, - fallback: &[(&str, &str)], - ) { + fn package(&mut self, pkg: &PkgType, dst: &mut BTreeMap) { if *pkg == PkgType::Rust { // This is handled specially by `rust_package` later. // Order is important, so don't call `rust_package` here. return; } + let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] }; let version_info = self.versions.version(&pkg).expect("failed to load package version"); let mut is_present = version_info.present;