From 86d32ef92ef8bdfbb73dbf34a02ac7128b7ee0df Mon Sep 17 00:00:00 2001 From: Tshepang Mbambo Date: Tue, 5 Sep 2023 10:53:46 +0200 Subject: [PATCH 01/22] issue has since been fixed --- config.example.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config.example.toml b/config.example.toml index 5c4bee875532f..95f0c95bc5c46 100644 --- a/config.example.toml +++ b/config.example.toml @@ -455,8 +455,6 @@ changelog-seen = 2 # Sets the number of codegen units to build the standard library with, # regardless of what the codegen-unit setting for the rest of the compiler is. -# NOTE: building with anything other than 1 is known to occasionally have bugs. -# See https://github.com/rust-lang/rust/issues/83600. #codegen-units-std = codegen-units # Whether or not debug assertions are enabled for the compiler and standard library. From 7db123dfbfe8d50cc83aac693161de8232d37e6e Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 9 Sep 2023 20:20:48 -0400 Subject: [PATCH 02/22] Add myself to the mailmap --- .mailmap | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.mailmap b/.mailmap index c072f6282f19b..2cc8c5de43f25 100644 --- a/.mailmap +++ b/.mailmap @@ -545,6 +545,8 @@ Timothy Maloney Tomas Koutsky Torsten Weber Torsten Weber +Trevor Gross +Trevor Gross Trevor Spiteri Tshepang Mbambo Ty Overby From 074fb2c6b7c4c7fb1de7d303a37632c02dd10cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 18:02:45 +0200 Subject: [PATCH 03/22] Store target triple in environment --- src/tools/opt-dist/src/environment/linux.rs | 14 +++++++++++++- src/tools/opt-dist/src/environment/mod.rs | 10 ++++------ src/tools/opt-dist/src/environment/windows.rs | 9 +++++++-- src/tools/opt-dist/src/main.rs | 4 +++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs index 58b7e6d23067c..bd8dd037ae5a2 100644 --- a/src/tools/opt-dist/src/environment/linux.rs +++ b/src/tools/opt-dist/src/environment/linux.rs @@ -3,9 +3,21 @@ use crate::exec::cmd; use crate::utils::io::copy_directory; use camino::{Utf8Path, Utf8PathBuf}; -pub(super) struct LinuxEnvironment; +pub(super) struct LinuxEnvironment { + target_triple: String, +} + +impl LinuxEnvironment { + pub fn new(target_triple: String) -> Self { + Self { target_triple } + } +} impl Environment for LinuxEnvironment { + fn host_triple(&self) -> &str { + &self.target_triple + } + fn python_binary(&self) -> &'static str { "python3" } diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs index a8650fad011d9..9f7f31d4791df 100644 --- a/src/tools/opt-dist/src/environment/mod.rs +++ b/src/tools/opt-dist/src/environment/mod.rs @@ -6,9 +6,7 @@ mod linux; mod windows; pub trait Environment { - fn host_triple(&self) -> String { - std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing") - } + fn host_triple(&self) -> &str; fn python_binary(&self) -> &'static str; @@ -69,9 +67,9 @@ pub trait Environment { fn skipped_tests(&self) -> &'static [&'static str]; } -pub fn create_environment() -> Box { +pub fn create_environment(target_triple: String) -> Box { #[cfg(target_family = "unix")] - return Box::new(linux::LinuxEnvironment); + return Box::new(linux::LinuxEnvironment::new(target_triple)); #[cfg(target_family = "windows")] - return Box::new(windows::WindowsEnvironment::new()); + return Box::new(windows::WindowsEnvironment::new(target_triple)); } diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs index 79399391798cb..90ddc01e8935c 100644 --- a/src/tools/opt-dist/src/environment/windows.rs +++ b/src/tools/opt-dist/src/environment/windows.rs @@ -9,15 +9,20 @@ use zip::ZipArchive; pub(super) struct WindowsEnvironment { checkout_dir: Utf8PathBuf, + target_triple: String, } impl WindowsEnvironment { - pub fn new() -> Self { - Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap() } + pub fn new(target_triple: String) -> Self { + Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple } } } impl Environment for WindowsEnvironment { + fn host_triple(&self) -> &str { + &self.target_triple + } + fn python_binary(&self) -> &'static str { "python" } diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 8ab19674d05b7..484ca5b3b8640 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -171,6 +171,8 @@ fn main() -> anyhow::Result<()> { .parse_default_env() .init(); + let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + let mut build_args: Vec = std::env::args().skip(1).collect(); println!("Running optimized build pipeline with args `{}`", build_args.join(" ")); @@ -202,7 +204,7 @@ fn main() -> anyhow::Result<()> { } let mut timer = Timer::new(); - let env = create_environment(); + let env = create_environment(target_triple); let result = execute_pipeline(env.as_ref(), &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); From 95500f494118b74c965a94acce3e92a803ee8e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 18:07:31 +0200 Subject: [PATCH 04/22] Make executable extension platform, rather than environment dependent --- src/tools/opt-dist/src/environment/linux.rs | 4 ---- src/tools/opt-dist/src/environment/mod.rs | 20 +++++++++++++------ src/tools/opt-dist/src/environment/windows.rs | 4 ---- src/tools/opt-dist/src/tests.rs | 8 ++++---- src/tools/opt-dist/src/training.rs | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs index bd8dd037ae5a2..61f0513729850 100644 --- a/src/tools/opt-dist/src/environment/linux.rs +++ b/src/tools/opt-dist/src/environment/linux.rs @@ -57,10 +57,6 @@ impl Environment for LinuxEnvironment { true } - fn executable_extension(&self) -> &'static str { - "" - } - fn skipped_tests(&self) -> &'static [&'static str] { &[ // Fails because of linker errors, as of June 2023. diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs index 9f7f31d4791df..271b41316abef 100644 --- a/src/tools/opt-dist/src/environment/mod.rs +++ b/src/tools/opt-dist/src/environment/mod.rs @@ -31,21 +31,21 @@ pub trait Environment { self.build_artifacts() .join("stage0") .join("bin") - .join(format!("cargo{}", self.executable_extension())) + .join(format!("cargo{}", executable_extension())) } fn rustc_stage_0(&self) -> Utf8PathBuf { self.build_artifacts() .join("stage0") .join("bin") - .join(format!("rustc{}", self.executable_extension())) + .join(format!("rustc{}", executable_extension())) } fn rustc_stage_2(&self) -> Utf8PathBuf { self.build_artifacts() .join("stage2") .join("bin") - .join(format!("rustc{}", self.executable_extension())) + .join(format!("rustc{}", executable_extension())) } /// Path to the built rustc-perf benchmark suite. @@ -60,9 +60,6 @@ pub trait Environment { fn supports_shared_llvm(&self) -> bool; - /// What is the extension of binary executables in this environment? - fn executable_extension(&self) -> &'static str; - /// List of test paths that should be skipped when testing the optimized artifacts. fn skipped_tests(&self) -> &'static [&'static str]; } @@ -73,3 +70,14 @@ pub fn create_environment(target_triple: String) -> Box { #[cfg(target_family = "windows")] return Box::new(windows::WindowsEnvironment::new(target_triple)); } + +/// What is the extension of binary executables on this platform? +#[cfg(target_family = "unix")] +pub fn executable_extension() -> &'static str { + "" +} + +#[cfg(target_family = "windows")] +pub fn executable_extension() -> &'static str { + ".exe" +} diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs index 90ddc01e8935c..f705461ac81d3 100644 --- a/src/tools/opt-dist/src/environment/windows.rs +++ b/src/tools/opt-dist/src/environment/windows.rs @@ -84,10 +84,6 @@ impl Environment for WindowsEnvironment { false } - fn executable_extension(&self) -> &'static str { - ".exe" - } - fn skipped_tests(&self) -> &'static [&'static str] { &[ // Fails as of June 2023. diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 3dd1a3223f5a5..e6b8df9c86203 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -1,4 +1,4 @@ -use crate::environment::Environment; +use crate::environment::{executable_extension, Environment}; use crate::exec::cmd; use crate::utils::io::{copy_directory, find_file_in_dir, unpack_archive}; use anyhow::Context; @@ -45,9 +45,9 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { &rustc_dir.join("lib").join("rustlib").join("src"), )?; - let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", env.executable_extension())); + let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", executable_extension())); assert!(rustc_path.is_file()); - let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", env.executable_extension())); + let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", executable_extension())); assert!(cargo_path.is_file()); // Specify path to a LLVM config so that LLVM is not rebuilt. @@ -56,7 +56,7 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { .build_artifacts() .join("llvm") .join("bin") - .join(format!("llvm-config{}", env.executable_extension())); + .join(format!("llvm-config{}", executable_extension())); assert!(llvm_config.is_file()); let config_content = format!( diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 59c73fbd6953e..950dd1127a9ba 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -1,4 +1,4 @@ -use crate::environment::Environment; +use crate::environment::{executable_extension, Environment}; use crate::exec::{cmd, CmdBuilder}; use crate::utils::io::{count_files, delete_directory}; use crate::utils::with_log_group; @@ -86,7 +86,7 @@ fn merge_llvm_profiles( .build_artifacts() .join("llvm") .join("build") - .join(format!("bin/llvm-profdata{}", env.executable_extension())), + .join(format!("bin/llvm-profdata{}", executable_extension())), }; cmd(&[llvm_profdata.as_str(), "merge", "-o", merged_path.as_str(), profile_dir.as_str()]) From f17047bc9046b3e1218200c6b649ee59ac6d0a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 18:32:32 +0200 Subject: [PATCH 05/22] Refactor Environment --- .github/workflows/ci.yml | 2 +- Cargo.lock | 76 +++++++- .../host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- src/ci/github-actions/ci.yml | 2 +- src/tools/opt-dist/Cargo.toml | 2 + src/tools/opt-dist/src/environment.rs | 100 +++++++++++ src/tools/opt-dist/src/environment/linux.rs | 66 ------- src/tools/opt-dist/src/environment/mod.rs | 83 --------- src/tools/opt-dist/src/environment/windows.rs | 93 ---------- src/tools/opt-dist/src/exec.rs | 4 +- src/tools/opt-dist/src/main.rs | 162 ++++++++++++++++-- src/tools/opt-dist/src/tests.rs | 2 +- src/tools/opt-dist/src/training.rs | 16 +- src/tools/opt-dist/src/utils/mod.rs | 4 +- 14 files changed, 335 insertions(+), 279 deletions(-) create mode 100644 src/tools/opt-dist/src/environment.rs delete mode 100644 src/tools/opt-dist/src/environment/linux.rs delete mode 100644 src/tools/opt-dist/src/environment/mod.rs delete mode 100644 src/tools/opt-dist/src/environment/windows.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3680136d89ff7..0c20e4dc5351b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -393,7 +393,7 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-i686-msvc diff --git a/Cargo.lock b/Cargo.lock index 295a5b43a6f32..51fce77150c03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -858,14 +858,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.3", + "darling_macro 0.20.3", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", ] [[package]] @@ -882,13 +906,24 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ - "darling_core", + "darling_core 0.20.3", "quote", "syn 2.0.29", ] @@ -919,6 +954,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -938,7 +1004,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d" dependencies = [ - "darling", + "darling 0.20.3", "proc-macro2", "quote", "syn 2.0.29", @@ -2584,6 +2650,8 @@ dependencies = [ "anyhow", "build_helper", "camino", + "clap", + "derive_builder", "env_logger 0.10.0", "fs_extra", "glob", diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index f6954275ad440..6f1b2a6a638be 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.lto=thin ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ - ./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \ + ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ build-manifest bootstrap diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 89b82d59d3176..1712d65ece971 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -624,7 +624,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index 3f7dba81c3a9f..f1c3dd6aa9bb0 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" glob = "0.3" tempfile = "3.5" +derive_builder = "0.12" +clap = { version = "4", features = ["derive"] } diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs new file mode 100644 index 0000000000000..e730e61060431 --- /dev/null +++ b/src/tools/opt-dist/src/environment.rs @@ -0,0 +1,100 @@ +use camino::Utf8PathBuf; +use derive_builder::Builder; + +#[derive(Builder)] +pub struct Environment { + host_triple: String, + python_binary: String, + /// The rustc checkout, where the compiler source is located. + checkout_dir: Utf8PathBuf, + /// The main directory where the build occurs. + build_dir: Utf8PathBuf, + /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) + /// will be stored. + artifact_dir: Utf8PathBuf, + /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. + host_llvm_dir: Utf8PathBuf, + /// List of test paths that should be skipped when testing the optimized artifacts. + skipped_tests: Vec<&'static str>, + use_bolt: bool, + shared_llvm: bool, +} + +impl Environment { + pub fn host_triple(&self) -> &str { + &self.host_triple + } + + pub fn python_binary(&self) -> &str { + &self.python_binary + } + + pub fn checkout_path(&self) -> Utf8PathBuf { + self.checkout_dir.clone() + } + + pub fn build_root(&self) -> Utf8PathBuf { + self.build_dir.clone() + } + + pub fn build_artifacts(&self) -> Utf8PathBuf { + self.build_root().join("build").join(&self.host_triple) + } + + pub fn artifact_dir(&self) -> Utf8PathBuf { + self.artifact_dir.clone() + } + + pub fn cargo_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("cargo{}", executable_extension())) + } + + pub fn rustc_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + pub fn rustc_stage_2(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage2") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + /// Path to the built rustc-perf benchmark suite. + pub fn rustc_perf_dir(&self) -> Utf8PathBuf { + self.artifact_dir.join("rustc-perf") + } + + pub fn host_llvm_dir(&self) -> Utf8PathBuf { + self.host_llvm_dir.clone() + } + + pub fn use_bolt(&self) -> bool { + self.use_bolt + } + + pub fn supports_shared_llvm(&self) -> bool { + self.shared_llvm + } + + pub fn skipped_tests(&self) -> &[&'static str] { + &self.skipped_tests + } +} + +/// What is the extension of binary executables on this platform? +#[cfg(target_family = "unix")] +pub fn executable_extension() -> &'static str { + "" +} + +#[cfg(target_family = "windows")] +pub fn executable_extension() -> &'static str { + ".exe" +} diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs deleted file mode 100644 index 61f0513729850..0000000000000 --- a/src/tools/opt-dist/src/environment/linux.rs +++ /dev/null @@ -1,66 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::copy_directory; -use camino::{Utf8Path, Utf8PathBuf}; - -pub(super) struct LinuxEnvironment { - target_triple: String, -} - -impl LinuxEnvironment { - pub fn new(target_triple: String) -> Self { - Self { target_triple } - } -} - -impl Environment for LinuxEnvironment { - fn host_triple(&self) -> &str { - &self.target_triple - } - - fn python_binary(&self) -> &'static str { - "python3" - } - - fn checkout_path(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/checkout") - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/rustroot") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path().join("obj") - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // /tmp/rustc-perf comes from the x64 dist Dockerfile - copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?; - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - Ok(()) - } - - fn supports_bolt(&self) -> bool { - true - } - - fn supports_shared_llvm(&self) -> bool { - true - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs deleted file mode 100644 index 271b41316abef..0000000000000 --- a/src/tools/opt-dist/src/environment/mod.rs +++ /dev/null @@ -1,83 +0,0 @@ -use camino::Utf8PathBuf; - -#[cfg(target_family = "unix")] -mod linux; -#[cfg(target_family = "windows")] -mod windows; - -pub trait Environment { - fn host_triple(&self) -> &str; - - fn python_binary(&self) -> &'static str; - - /// The rustc checkout, where the compiler source is located. - fn checkout_path(&self) -> Utf8PathBuf; - - /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. - fn host_llvm_dir(&self) -> Utf8PathBuf; - - /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) - /// will be stored. - fn opt_artifacts(&self) -> Utf8PathBuf; - - /// The main directory where the build occurs. - fn build_root(&self) -> Utf8PathBuf; - - fn build_artifacts(&self) -> Utf8PathBuf { - self.build_root().join("build").join(self.host_triple()) - } - - fn cargo_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("cargo{}", executable_extension())) - } - - fn rustc_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("rustc{}", executable_extension())) - } - - fn rustc_stage_2(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage2") - .join("bin") - .join(format!("rustc{}", executable_extension())) - } - - /// Path to the built rustc-perf benchmark suite. - fn rustc_perf_dir(&self) -> Utf8PathBuf { - self.opt_artifacts().join("rustc-perf") - } - - /// Download and/or compile rustc-perf. - fn prepare_rustc_perf(&self) -> anyhow::Result<()>; - - fn supports_bolt(&self) -> bool; - - fn supports_shared_llvm(&self) -> bool; - - /// List of test paths that should be skipped when testing the optimized artifacts. - fn skipped_tests(&self) -> &'static [&'static str]; -} - -pub fn create_environment(target_triple: String) -> Box { - #[cfg(target_family = "unix")] - return Box::new(linux::LinuxEnvironment::new(target_triple)); - #[cfg(target_family = "windows")] - return Box::new(windows::WindowsEnvironment::new(target_triple)); -} - -/// What is the extension of binary executables on this platform? -#[cfg(target_family = "unix")] -pub fn executable_extension() -> &'static str { - "" -} - -#[cfg(target_family = "windows")] -pub fn executable_extension() -> &'static str { - ".exe" -} diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs deleted file mode 100644 index f705461ac81d3..0000000000000 --- a/src/tools/opt-dist/src/environment/windows.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::move_directory; -use crate::utils::retry_action; -use camino::Utf8PathBuf; -use std::io::Cursor; -use std::time::Duration; -use zip::ZipArchive; - -pub(super) struct WindowsEnvironment { - checkout_dir: Utf8PathBuf, - target_triple: String, -} - -impl WindowsEnvironment { - pub fn new(target_triple: String) -> Self { - Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple } - } -} - -impl Environment for WindowsEnvironment { - fn host_triple(&self) -> &str { - &self.target_triple - } - - fn python_binary(&self) -> &'static str { - "python" - } - - fn checkout_path(&self) -> Utf8PathBuf { - self.checkout_dir.clone() - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - self.checkout_path().join("citools").join("clang-rust") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - self.checkout_path().join("opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path() - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // FIXME: add some mechanism for synchronization of this commit SHA with - // Linux (which builds rustc-perf in a Dockerfile) - // rustc-perf version from 2023-05-30 - const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; - - let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(60 * 2)) - .connect_timeout(Duration::from_secs(60 * 2)) - .build()?; - let response = retry_action( - || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), - "Download rustc-perf archive", - 5, - )?; - - let mut archive = ZipArchive::new(Cursor::new(response))?; - archive.extract(self.rustc_perf_dir())?; - move_directory( - &self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), - &self.rustc_perf_dir(), - )?; - - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - - Ok(()) - } - - fn supports_bolt(&self) -> bool { - false - } - - fn supports_shared_llvm(&self) -> bool { - false - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails as of June 2023. - "tests\\codegen\\vec-shrink-panik.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs index 4765dceb5dddd..04e0184528a5f 100644 --- a/src/tools/opt-dist/src/exec.rs +++ b/src/tools/opt-dist/src/exec.rs @@ -96,7 +96,7 @@ pub struct Bootstrap { } impl Bootstrap { - pub fn build(env: &dyn Environment) -> Self { + pub fn build(env: &Environment) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&[ env.python_binary(), @@ -114,7 +114,7 @@ impl Bootstrap { Self { cmd, metrics_path } } - pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self { + pub fn dist(env: &Environment, dist_args: &[String]) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::>()) .env("RUST_BACKTRACE", "full"); diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 484ca5b3b8640..6a3bdd7cc75b4 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -1,17 +1,22 @@ use crate::bolt::{bolt_optimize, with_bolt_instrumented}; use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; +use clap::Parser; use log::LevelFilter; +use std::io::Cursor; +use std::time::Duration; use utils::io; +use zip::ZipArchive; -use crate::environment::{create_environment, Environment}; -use crate::exec::Bootstrap; +use crate::environment::{Environment, EnvironmentBuilder}; +use crate::exec::{cmd, Bootstrap}; use crate::tests::run_tests; use crate::timer::Timer; use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles}; -use crate::utils::io::reset_directory; +use crate::utils::io::{copy_directory, move_directory, reset_directory}; use crate::utils::{ clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space, - with_log_group, + retry_action, with_log_group, }; mod bolt; @@ -23,24 +28,104 @@ mod timer; mod training; mod utils; +#[derive(clap::Parser, Debug)] +struct Args { + #[clap(subcommand)] + env: EnvironmentCmd, +} + +#[derive(clap::Parser, Clone, Debug)] +struct SharedArgs { + // Arguments passed to `x` to perform the final (dist) build. + build_args: Vec, +} + +#[derive(clap::Parser, Clone, Debug)] +enum EnvironmentCmd { + LinuxCi { + #[clap(flatten)] + shared: SharedArgs, + }, + WindowsCi { + #[clap(flatten)] + shared: SharedArgs, + }, +} + fn is_try_build() -> bool { std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0" } +fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> { + let (env, args) = match args.env { + EnvironmentCmd::LinuxCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir = Utf8PathBuf::from("/checkout"); + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python3".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(Utf8PathBuf::from("/rustroot")) + .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) + .build_dir(checkout_dir.join("obj")) + .shared_llvm(true) + .use_bolt(true) + .skipped_tests(vec![ + // Fails because of linker errors, as of June 2023. + "tests/ui/process/nofile-limit.rs", + ]) + .build()?; + // /tmp/rustc-perf comes from the x64 dist Dockerfile + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?) + })?; + + (env, shared.build_args) + } + EnvironmentCmd::WindowsCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?; + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(checkout_dir.join("citools").join("clang-rust")) + .artifact_dir(checkout_dir.join("opt-artifacts")) + .build_dir(checkout_dir) + .shared_llvm(false) + .use_bolt(false) + .skipped_tests(vec![ + // Fails as of June 2023. + "tests\\codegen\\vec-shrink-panik.rs", + ]) + .build()?; + + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) + })?; + + (env, shared.build_args) + } + }; + Ok((env, args)) +} + fn execute_pipeline( - env: &dyn Environment, + env: &Environment, timer: &mut Timer, dist_args: Vec, ) -> anyhow::Result<()> { - reset_directory(&env.opt_artifacts())?; - - with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?; + reset_directory(&env.artifact_dir())?; // Stage 1: Build PGO instrumented rustc // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the // same time can cause issues, because the host and in-tree LLVM versions can diverge. let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| { - let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo"); + let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo"); stage.section("Build PGO instrumented rustc and LLVM", |section| { let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root); @@ -74,7 +159,7 @@ fn execute_pipeline( // Remove the previous, uninstrumented build of LLVM. clear_llvm_files(env)?; - let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo"); + let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo"); stage.section("Build PGO instrumented LLVM", |section| { Bootstrap::build(env) @@ -95,7 +180,7 @@ fn execute_pipeline( Ok(profile) })?; - let llvm_bolt_profile = if env.supports_bolt() { + let llvm_bolt_profile = if env.use_bolt() { // Stage 3: Build BOLT instrumented LLVM // We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles. // Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build. @@ -171,10 +256,9 @@ fn main() -> anyhow::Result<()> { .parse_default_env() .init(); - let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + let args = Args::parse(); - let mut build_args: Vec = std::env::args().skip(1).collect(); - println!("Running optimized build pipeline with args `{}`", build_args.join(" ")); + println!("Running optimized build pipeline with args `{:?}`", args); with_log_group("Environment values", || { println!("Environment values\n{}", format_env_variables()); @@ -186,6 +270,8 @@ fn main() -> anyhow::Result<()> { } }); + let (env, mut build_args) = create_environment(args).context("Cannot create environment")?; + // Skip components that are not needed for try builds to speed them up if is_try_build() { log::info!("Skipping building of unimportant components for a try build"); @@ -204,14 +290,56 @@ fn main() -> anyhow::Result<()> { } let mut timer = Timer::new(); - let env = create_environment(target_triple); - let result = execute_pipeline(env.as_ref(), &mut timer, build_args); + let result = execute_pipeline(&env, &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); print_free_disk_space()?; result.context("Optimized build pipeline has failed")?; - print_binary_sizes(env.as_ref())?; + print_binary_sizes(&env)?; + + Ok(()) +} + +// Copy rustc-perf from the given path into the environment and build it. +fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { + copy_directory(dir, &env.rustc_perf_dir())?; + build_rustc_perf(env) +} + +// Download and build rustc-perf into the given environment. +fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { + // FIXME: add some mechanism for synchronization of this commit SHA with + // Linux (which builds rustc-perf in a Dockerfile) + // rustc-perf version from 2023-05-30 + const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; + + let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(60 * 2)) + .connect_timeout(Duration::from_secs(60 * 2)) + .build()?; + let response = retry_action( + || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), + "Download rustc-perf archive", + 5, + )?; + + let mut archive = ZipArchive::new(Cursor::new(response))?; + archive.extract(env.rustc_perf_dir())?; + move_directory( + &env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), + &env.rustc_perf_dir(), + )?; + + build_rustc_perf(env) +} +fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> { + cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"]) + .workdir(&env.rustc_perf_dir()) + .env("RUSTC", &env.rustc_stage_0().into_string()) + .env("RUSTC_BOOTSTRAP", "1") + .run()?; Ok(()) } diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index e6b8df9c86203..54f7185f88a87 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -5,7 +5,7 @@ use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; /// Run tests on optimized dist artifacts. -pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { +pub fn run_tests(env: &Environment) -> anyhow::Result<()> { // After `dist` is executed, we extract its archived components into a sysroot directory, // and then use that extracted rustc as a stage0 compiler. // Then we run a subset of tests using that compiler, to have a basic smoke test which checks diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 950dd1127a9ba..dd8aa532408e2 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -30,7 +30,7 @@ const RUSTC_PGO_CRATES: &[&str] = &[ const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES; fn init_compiler_benchmarks( - env: &dyn Environment, + env: &Environment, profiles: &[&str], scenarios: &[&str], crates: &[&str], @@ -75,7 +75,7 @@ enum LlvmProfdata { } fn merge_llvm_profiles( - env: &dyn Environment, + env: &Environment, merged_path: &Utf8Path, profile_dir: &Utf8Path, profdata: LlvmProfdata, @@ -116,7 +116,7 @@ fn log_profile_stats( pub struct LlvmPGOProfile(pub Utf8PathBuf); pub fn gather_llvm_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented LLVM"); @@ -127,7 +127,7 @@ pub fn gather_llvm_profiles( .context("Cannot gather LLVM PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata"); + let merged_profile = env.artifact_dir().join("llvm-pgo.profdata"); log::info!("Merging LLVM PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?; @@ -143,7 +143,7 @@ pub fn gather_llvm_profiles( pub struct RustcPGOProfile(pub Utf8PathBuf); pub fn gather_rustc_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented rustc"); @@ -163,7 +163,7 @@ pub fn gather_rustc_profiles( .context("Cannot gather rustc PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata"); + let merged_profile = env.artifact_dir().join("rustc-pgo.profdata"); log::info!("Merging Rustc PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?; @@ -178,7 +178,7 @@ pub fn gather_rustc_profiles( pub struct LlvmBoltProfile(pub Utf8PathBuf); -pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result { +pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result { log::info!("Running benchmarks with BOLT instrumented LLVM"); with_log_group("Running benchmarks", || { @@ -187,7 +187,7 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result anyhow::Result<()> { Ok(()) } -pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { +pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> { use std::fmt::Write; let root = env.build_artifacts().join("stage2"); @@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { Ok(()) } -pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> { +pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> { // Bootstrap currently doesn't support rebuilding LLVM when PGO options // change (or any other llvm-related options); so just clear out the relevant // directories ourselves. From a2ed508f547e5538ba6bd882ef76687aad06f4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 19:32:46 +0200 Subject: [PATCH 06/22] Fix `reset_directory` function Before it was not deleting non-empty directories. --- src/tools/opt-dist/src/utils/io.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/opt-dist/src/utils/io.rs b/src/tools/opt-dist/src/utils/io.rs index 8bd516fa349b0..d24a1dc2d108e 100644 --- a/src/tools/opt-dist/src/utils/io.rs +++ b/src/tools/opt-dist/src/utils/io.rs @@ -7,7 +7,7 @@ use std::path::Path; /// Delete and re-create the directory. pub fn reset_directory(path: &Utf8Path) -> anyhow::Result<()> { log::info!("Resetting directory {path}"); - let _ = std::fs::remove_dir(path); + let _ = std::fs::remove_dir_all(path); std::fs::create_dir_all(path)?; Ok(()) } From 11f9283da91ca4ab7fb634f3f586156f699007c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 19:33:09 +0200 Subject: [PATCH 07/22] Add a Local environment to `opt-dist` This makes it easier to build a PGO/BOLT optimized `rustc` locally, outside of CI. --- src/tools/opt-dist/src/environment.rs | 7 +-- src/tools/opt-dist/src/main.rs | 75 ++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs index e730e61060431..ed43f469c5e91 100644 --- a/src/tools/opt-dist/src/environment.rs +++ b/src/tools/opt-dist/src/environment.rs @@ -7,7 +7,8 @@ pub struct Environment { python_binary: String, /// The rustc checkout, where the compiler source is located. checkout_dir: Utf8PathBuf, - /// The main directory where the build occurs. + /// The main directory where the build occurs. Stage0 rustc and cargo have to be available in + /// this directory before `opt-dist` is started. build_dir: Utf8PathBuf, /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) /// will be stored. @@ -15,7 +16,7 @@ pub struct Environment { /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. host_llvm_dir: Utf8PathBuf, /// List of test paths that should be skipped when testing the optimized artifacts. - skipped_tests: Vec<&'static str>, + skipped_tests: Vec, use_bolt: bool, shared_llvm: bool, } @@ -83,7 +84,7 @@ impl Environment { self.shared_llvm } - pub fn skipped_tests(&self) -> &[&'static str] { + pub fn skipped_tests(&self) -> &[String] { &self.skipped_tests } } diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 6a3bdd7cc75b4..b5e123360f875 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -42,10 +42,51 @@ struct SharedArgs { #[derive(clap::Parser, Clone, Debug)] enum EnvironmentCmd { + /// Perform a custom local PGO/BOLT optimized build. + Local { + /// Target triple of the host. + #[arg(long)] + target_triple: String, + + /// Checkout directory of `rustc`. + #[arg(long)] + checkout_dir: Utf8PathBuf, + + /// Host LLVM installation directory. + #[arg(long)] + llvm_dir: Utf8PathBuf, + + /// Python binary to use in bootstrap invocations. + #[arg(long, default_value = "python3")] + python: String, + + /// Directory where artifacts (like PGO profiles or rustc-perf) of this workflow + /// will be stored. + #[arg(long, default_value = "opt-artifacts")] + artifact_dir: Utf8PathBuf, + + /// Is LLVM for `rustc` built in shared library mode? + #[arg(long, default_value_t = true)] + llvm_shared: bool, + + /// Should BOLT optimization be used? If yes, host LLVM must have BOLT binaries + /// (`llvm-bolt` and `merge-fdata`) available. + #[arg(long, default_value_t = false)] + use_bolt: bool, + + /// Tests that should be skipped when testing the optimized compiler. + #[arg(long)] + skipped_tests: Vec, + + #[clap(flatten)] + shared: SharedArgs, + }, + /// Perform an optimized build on Linux CI, from inside Docker. LinuxCi { #[clap(flatten)] shared: SharedArgs, }, + /// Perform an optimized build on Windows CI, directly inside Github Actions. WindowsCi { #[clap(flatten)] shared: SharedArgs, @@ -58,6 +99,34 @@ fn is_try_build() -> bool { fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> { let (env, args) = match args.env { + EnvironmentCmd::Local { + target_triple, + checkout_dir, + llvm_dir, + python, + artifact_dir, + llvm_shared, + use_bolt, + skipped_tests, + shared, + } => { + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary(python) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(llvm_dir) + .artifact_dir(artifact_dir) + .build_dir(checkout_dir) + .shared_llvm(llvm_shared) + .use_bolt(use_bolt) + .skipped_tests(skipped_tests) + .build()?; + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) + })?; + + (env, shared.build_args) + } EnvironmentCmd::LinuxCi { shared } => { let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); @@ -74,7 +143,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(true) .skipped_tests(vec![ // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs", + "tests/ui/process/nofile-limit.rs".to_string(), ]) .build()?; // /tmp/rustc-perf comes from the x64 dist Dockerfile @@ -100,7 +169,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(false) .skipped_tests(vec![ // Fails as of June 2023. - "tests\\codegen\\vec-shrink-panik.rs", + "tests\\codegen\\vec-shrink-panik.rs".to_string(), ]) .build()?; @@ -309,6 +378,8 @@ fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { // Download and build rustc-perf into the given environment. fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { + reset_directory(&env.rustc_perf_dir())?; + // FIXME: add some mechanism for synchronization of this commit SHA with // Linux (which builds rustc-perf in a Dockerfile) // rustc-perf version from 2023-05-30 From 2962528b4accdf1d0c2b219568c4ef115f78a503 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Wed, 13 Sep 2023 15:04:34 +0800 Subject: [PATCH 08/22] Make AIX known by bootstrap --- src/bootstrap/bootstrap.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index e4c7622645402..bc8b5326d8d07 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -312,6 +312,8 @@ def default_build_triple(verbose): # non-standard string (e.g. gnuwin32 tools returns `windows32`). In # these cases, fall back to using sys.platform. return 'x86_64-pc-windows-msvc' + elif kernel == 'AIX': + return 'powerpc64-ibm-aix' else: err = "unknown OS type: {}".format(kernel) sys.exit(err) From fc17e09029a4f01fb59778e8575faa99a08f289a Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Wed, 13 Sep 2023 16:05:50 +0800 Subject: [PATCH 09/22] Add comment to elaborate --- src/bootstrap/bootstrap.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index bc8b5326d8d07..eca3a522f80b5 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -313,6 +313,12 @@ def default_build_triple(verbose): # these cases, fall back to using sys.platform. return 'x86_64-pc-windows-msvc' elif kernel == 'AIX': + # `uname -m` returns the machine ID rather than machine hardware on AIX, + # so we are unable to use cputype to form triple. Since AIX 7.2 and + # above supports 32-bit and 64-bit mode simultaneously and `uname -p` + # returns `powerpc`. Currently we only supports `powerpc64-ibm-aix` in + # rust. For above reasons, kerneltype_mapper and cputype_mapper are not + # used to infer AIX's triple. return 'powerpc64-ibm-aix' else: err = "unknown OS type: {}".format(kernel) From 5049a7167ca51be7f9bfbcf2f75eb9061722860b Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Wed, 13 Sep 2023 16:08:23 +0800 Subject: [PATCH 10/22] Adjust comment --- src/bootstrap/bootstrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index eca3a522f80b5..6e0326a7d10c2 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -317,8 +317,8 @@ def default_build_triple(verbose): # so we are unable to use cputype to form triple. Since AIX 7.2 and # above supports 32-bit and 64-bit mode simultaneously and `uname -p` # returns `powerpc`. Currently we only supports `powerpc64-ibm-aix` in - # rust. For above reasons, kerneltype_mapper and cputype_mapper are not - # used to infer AIX's triple. + # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper + # are not used to infer AIX's triple. return 'powerpc64-ibm-aix' else: err = "unknown OS type: {}".format(kernel) From 122d1cc8c332b4dd176e1475a063688e73d95168 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Wed, 13 Sep 2023 16:10:26 +0800 Subject: [PATCH 11/22] Adjust comment --- src/bootstrap/bootstrap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 6e0326a7d10c2..a9aa7524e8bc7 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -314,9 +314,9 @@ def default_build_triple(verbose): return 'x86_64-pc-windows-msvc' elif kernel == 'AIX': # `uname -m` returns the machine ID rather than machine hardware on AIX, - # so we are unable to use cputype to form triple. Since AIX 7.2 and + # so we are unable to use cputype to form triple. AIX 7.2 and # above supports 32-bit and 64-bit mode simultaneously and `uname -p` - # returns `powerpc`. Currently we only supports `powerpc64-ibm-aix` in + # returns `powerpc`, however we only supports `powerpc64-ibm-aix` in # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper # are not used to infer AIX's triple. return 'powerpc64-ibm-aix' From 6c718b5b8a98ba756c69e8a2019131c6a514e64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Sep 2023 18:11:34 +0200 Subject: [PATCH 12/22] Refactor rustc-perf building --- src/tools/opt-dist/src/environment.rs | 6 ++++++ src/tools/opt-dist/src/main.rs | 18 +++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs index ed43f469c5e91..f7b5c176375bb 100644 --- a/src/tools/opt-dist/src/environment.rs +++ b/src/tools/opt-dist/src/environment.rs @@ -17,6 +17,8 @@ pub struct Environment { host_llvm_dir: Utf8PathBuf, /// List of test paths that should be skipped when testing the optimized artifacts. skipped_tests: Vec, + /// Directory containing a pre-built rustc-perf checkout. + prebuilt_rustc_perf: Option, use_bolt: bool, shared_llvm: bool, } @@ -67,6 +69,10 @@ impl Environment { .join(format!("rustc{}", executable_extension())) } + pub fn prebuilt_rustc_perf(&self) -> Option { + self.prebuilt_rustc_perf.clone() + } + /// Path to the built rustc-perf benchmark suite. pub fn rustc_perf_dir(&self) -> Utf8PathBuf { self.artifact_dir.join("rustc-perf") diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index b5e123360f875..978e2dfa4e864 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -121,9 +121,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(use_bolt) .skipped_tests(skipped_tests) .build()?; - with_log_group("Building rustc-perf", || { - Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) - })?; (env, shared.build_args) } @@ -139,6 +136,8 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .host_llvm_dir(Utf8PathBuf::from("/rustroot")) .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) .build_dir(checkout_dir.join("obj")) + // /tmp/rustc-perf comes from the x64 dist Dockerfile + .prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf"))) .shared_llvm(true) .use_bolt(true) .skipped_tests(vec![ @@ -146,10 +145,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> "tests/ui/process/nofile-limit.rs".to_string(), ]) .build()?; - // /tmp/rustc-perf comes from the x64 dist Dockerfile - with_log_group("Building rustc-perf", || { - Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?) - })?; (env, shared.build_args) } @@ -173,10 +168,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> ]) .build()?; - with_log_group("Building rustc-perf", || { - Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) - })?; - (env, shared.build_args) } }; @@ -190,6 +181,11 @@ fn execute_pipeline( ) -> anyhow::Result<()> { reset_directory(&env.artifact_dir())?; + with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() { + Some(dir) => copy_rustc_perf(env, &dir), + None => download_rustc_perf(env), + })?; + // Stage 1: Build PGO instrumented rustc // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the // same time can cause issues, because the host and in-tree LLVM versions can diverge. From f13b54546bb7aea9beee7d3f9178882a7f10f45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Sep 2023 18:12:41 +0200 Subject: [PATCH 13/22] Resolve clippy warnings --- src/tools/opt-dist/src/tests.rs | 6 +++--- src/tools/opt-dist/src/training.rs | 2 +- src/tools/opt-dist/src/utils/io.rs | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 54f7185f88a87..31aabca09f39e 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -33,8 +33,8 @@ pub fn run_tests(env: &Environment) -> anyhow::Result<()> { // We need to manually copy libstd to the extracted rustc sysroot copy_directory( - &libstd_dir.join("lib").join("rustlib").join(&host_triple).join("lib"), - &rustc_dir.join("lib").join("rustlib").join(&host_triple).join("lib"), + &libstd_dir.join("lib").join("rustlib").join(host_triple).join("lib"), + &rustc_dir.join("lib").join("rustlib").join(host_triple).join("lib"), )?; // Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd @@ -109,6 +109,6 @@ fn find_dist_version(directory: &Utf8Path) -> anyhow::Result { .unwrap() .to_string(); let (version, _) = - archive.strip_prefix("reproducible-artifacts-").unwrap().split_once("-").unwrap(); + archive.strip_prefix("reproducible-artifacts-").unwrap().split_once('-').unwrap(); Ok(version.to_string()) } diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index dd8aa532408e2..274f4cea0ab5f 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -192,7 +192,7 @@ pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result = - glob::glob(&format!("{profile_root}*"))?.into_iter().collect::, _>>()?; + glob::glob(&format!("{profile_root}*"))?.collect::, _>>()?; let mut merge_args = vec!["merge-fdata"]; merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap())); diff --git a/src/tools/opt-dist/src/utils/io.rs b/src/tools/opt-dist/src/utils/io.rs index d24a1dc2d108e..d6bd5cb85e4af 100644 --- a/src/tools/opt-dist/src/utils/io.rs +++ b/src/tools/opt-dist/src/utils/io.rs @@ -63,7 +63,6 @@ pub fn get_files_from_dir( let path = format!("{dir}/*{}", suffix.unwrap_or("")); Ok(glob::glob(&path)? - .into_iter() .map(|p| p.map(|p| Utf8PathBuf::from_path_buf(p).unwrap())) .collect::, _>>()?) } @@ -74,9 +73,8 @@ pub fn find_file_in_dir( prefix: &str, suffix: &str, ) -> anyhow::Result { - let files = glob::glob(&format!("{directory}/{prefix}*{suffix}"))? - .into_iter() - .collect::, _>>()?; + let files = + glob::glob(&format!("{directory}/{prefix}*{suffix}"))?.collect::, _>>()?; match files.len() { 0 => Err(anyhow::anyhow!("No file with prefix {prefix} found in {directory}")), 1 => Ok(Utf8PathBuf::from_path_buf(files[0].clone()).unwrap()), From be9d7e0b948650fc93b1929b1558b68d479d2f08 Mon Sep 17 00:00:00 2001 From: lcnr Date: Thu, 14 Sep 2023 09:58:29 +0200 Subject: [PATCH 14/22] `GoalCandidate` to `Probe` --- .../rustc_middle/src/traits/solve/inspect.rs | 13 +++-- .../src/traits/solve/inspect/format.rs | 12 ++--- .../src/solve/eval_ctxt/probe.rs | 4 +- .../src/solve/inspect.rs | 52 +++++++++---------- 4 files changed, 41 insertions(+), 40 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve/inspect.rs b/compiler/rustc_middle/src/traits/solve/inspect.rs index d8b3a061b77d5..3ebb8ef5825fc 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect.rs @@ -52,19 +52,22 @@ pub struct GoalEvaluationStep<'tcx> { pub instantiated_goal: QueryInput<'tcx, ty::Predicate<'tcx>>, /// The actual evaluation of the goal, always `ProbeKind::Root`. - pub evaluation: GoalCandidate<'tcx>, + pub evaluation: Probe<'tcx>, } +/// A self-contained computation during trait solving. This either +/// corresponds to a `EvalCtxt::probe(_X)` call or the root evaluation +/// of a goal. #[derive(Eq, PartialEq)] -pub struct GoalCandidate<'tcx> { +pub struct Probe<'tcx> { pub added_goals_evaluations: Vec>, - pub candidates: Vec>, + pub nested_probes: Vec>, pub kind: ProbeKind<'tcx>, } -impl Debug for GoalCandidate<'_> { +impl Debug for Probe<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - ProofTreeFormatter::new(f).format_candidate(self) + ProofTreeFormatter::new(f).format_probe(self) } } diff --git a/compiler/rustc_middle/src/traits/solve/inspect/format.rs b/compiler/rustc_middle/src/traits/solve/inspect/format.rs index d916e80a625d0..8d63a245814db 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect/format.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect/format.rs @@ -92,11 +92,11 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { evaluation_step: &GoalEvaluationStep<'_>, ) -> std::fmt::Result { writeln!(self.f, "INSTANTIATED: {:?}", evaluation_step.instantiated_goal)?; - self.format_candidate(&evaluation_step.evaluation) + self.format_probe(&evaluation_step.evaluation) } - pub(super) fn format_candidate(&mut self, candidate: &GoalCandidate<'_>) -> std::fmt::Result { - match &candidate.kind { + pub(super) fn format_probe(&mut self, probe: &Probe<'_>) -> std::fmt::Result { + match &probe.kind { ProbeKind::Root { result } => { writeln!(self.f, "ROOT RESULT: {result:?}") } @@ -118,10 +118,10 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { }?; self.nested(|this| { - for candidate in &candidate.candidates { - this.format_candidate(candidate)?; + for probe in &probe.nested_probes { + this.format_probe(probe)?; } - for nested in &candidate.added_goals_evaluations { + for nested in &probe.added_goals_evaluations { this.format_added_goals_evaluation(nested)?; } Ok(()) diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs index f88cfbac3f3eb..6087b91679095 100644 --- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs +++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs @@ -24,13 +24,13 @@ where search_graph: outer_ecx.search_graph, nested_goals: outer_ecx.nested_goals.clone(), tainted: outer_ecx.tainted, - inspect: outer_ecx.inspect.new_goal_candidate(), + inspect: outer_ecx.inspect.new_probe(), }; let r = nested_ecx.infcx.probe(|_| f(&mut nested_ecx)); if !outer_ecx.inspect.is_noop() { let probe_kind = probe_kind(&r); nested_ecx.inspect.probe_kind(probe_kind); - outer_ecx.inspect.goal_candidate(nested_ecx.inspect); + outer_ecx.inspect.finish_probe(nested_ecx.inspect); } r } diff --git a/compiler/rustc_trait_selection/src/solve/inspect.rs b/compiler/rustc_trait_selection/src/solve/inspect.rs index 46025da768326..353f37ebe6038 100644 --- a/compiler/rustc_trait_selection/src/solve/inspect.rs +++ b/compiler/rustc_trait_selection/src/solve/inspect.rs @@ -87,7 +87,7 @@ impl<'tcx> WipAddedGoalsEvaluation<'tcx> { pub struct WipGoalEvaluationStep<'tcx> { pub instantiated_goal: QueryInput<'tcx, ty::Predicate<'tcx>>, - pub evaluation: WipGoalCandidate<'tcx>, + pub evaluation: WipProbe<'tcx>, } impl<'tcx> WipGoalEvaluationStep<'tcx> { @@ -102,21 +102,21 @@ impl<'tcx> WipGoalEvaluationStep<'tcx> { } #[derive(Eq, PartialEq, Debug)] -pub struct WipGoalCandidate<'tcx> { +pub struct WipProbe<'tcx> { pub added_goals_evaluations: Vec>, - pub candidates: Vec>, + pub nested_probes: Vec>, pub kind: Option>, } -impl<'tcx> WipGoalCandidate<'tcx> { - pub fn finalize(self) -> inspect::GoalCandidate<'tcx> { - inspect::GoalCandidate { +impl<'tcx> WipProbe<'tcx> { + pub fn finalize(self) -> inspect::Probe<'tcx> { + inspect::Probe { added_goals_evaluations: self .added_goals_evaluations .into_iter() .map(WipAddedGoalsEvaluation::finalize) .collect(), - candidates: self.candidates.into_iter().map(WipGoalCandidate::finalize).collect(), + nested_probes: self.nested_probes.into_iter().map(WipProbe::finalize).collect(), kind: self.kind.unwrap(), } } @@ -129,7 +129,7 @@ pub enum DebugSolver<'tcx> { CanonicalGoalEvaluation(WipCanonicalGoalEvaluation<'tcx>), AddedGoalsEvaluation(WipAddedGoalsEvaluation<'tcx>), GoalEvaluationStep(WipGoalEvaluationStep<'tcx>), - GoalCandidate(WipGoalCandidate<'tcx>), + Probe(WipProbe<'tcx>), } impl<'tcx> From> for DebugSolver<'tcx> { @@ -156,9 +156,9 @@ impl<'tcx> From> for DebugSolver<'tcx> { } } -impl<'tcx> From> for DebugSolver<'tcx> { - fn from(g: WipGoalCandidate<'tcx>) -> DebugSolver<'tcx> { - DebugSolver::GoalCandidate(g) +impl<'tcx> From> for DebugSolver<'tcx> { + fn from(p: WipProbe<'tcx>) -> DebugSolver<'tcx> { + DebugSolver::Probe(p) } } @@ -329,9 +329,9 @@ impl<'tcx> ProofTreeBuilder<'tcx> { ) -> ProofTreeBuilder<'tcx> { self.nested(|| WipGoalEvaluationStep { instantiated_goal, - evaluation: WipGoalCandidate { + evaluation: WipProbe { added_goals_evaluations: vec![], - candidates: vec![], + nested_probes: vec![], kind: None, }, }) @@ -350,10 +350,10 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } } - pub fn new_goal_candidate(&mut self) -> ProofTreeBuilder<'tcx> { - self.nested(|| WipGoalCandidate { + pub fn new_probe(&mut self) -> ProofTreeBuilder<'tcx> { + self.nested(|| WipProbe { added_goals_evaluations: vec![], - candidates: vec![], + nested_probes: vec![], kind: None, }) } @@ -361,7 +361,7 @@ impl<'tcx> ProofTreeBuilder<'tcx> { pub fn probe_kind(&mut self, probe_kind: ProbeKind<'tcx>) { if let Some(this) = self.as_mut() { match this { - DebugSolver::GoalCandidate(this) => { + DebugSolver::Probe(this) => { assert_eq!(this.kind.replace(probe_kind), None) } _ => unreachable!(), @@ -369,17 +369,17 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } } - pub fn goal_candidate(&mut self, candidate: ProofTreeBuilder<'tcx>) { + pub fn finish_probe(&mut self, probe: ProofTreeBuilder<'tcx>) { if let Some(this) = self.as_mut() { - match (this, candidate.state.unwrap().tree) { + match (this, probe.state.unwrap().tree) { ( - DebugSolver::GoalCandidate(WipGoalCandidate { candidates, .. }) + DebugSolver::Probe(WipProbe { nested_probes, .. }) | DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { - evaluation: WipGoalCandidate { candidates, .. }, + evaluation: WipProbe { nested_probes, .. }, .. }), - DebugSolver::GoalCandidate(candidate), - ) => candidates.push(candidate), + DebugSolver::Probe(probe), + ) => nested_probes.push(probe), _ => unreachable!(), } } @@ -416,12 +416,10 @@ impl<'tcx> ProofTreeBuilder<'tcx> { match (this, added_goals_evaluation.state.unwrap().tree) { ( DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { - evaluation: WipGoalCandidate { added_goals_evaluations, .. }, + evaluation: WipProbe { added_goals_evaluations, .. }, .. }) - | DebugSolver::GoalCandidate(WipGoalCandidate { - added_goals_evaluations, .. - }), + | DebugSolver::Probe(WipProbe { added_goals_evaluations, .. }), DebugSolver::AddedGoalsEvaluation(added_goals_evaluation), ) => added_goals_evaluations.push(added_goals_evaluation), _ => unreachable!(), From a3f9530b303b497750d0aebab689471dad9c7592 Mon Sep 17 00:00:00 2001 From: lcnr Date: Thu, 14 Sep 2023 10:20:23 +0200 Subject: [PATCH 15/22] order `added_goals_evaluation` and `nested_probes` --- .../rustc_middle/src/traits/solve/inspect.rs | 9 +++- .../src/traits/solve/inspect/format.rs | 10 ++-- .../src/solve/inspect.rs | 49 ++++++++++--------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve/inspect.rs b/compiler/rustc_middle/src/traits/solve/inspect.rs index 3ebb8ef5825fc..a014e8769d8f8 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect.rs @@ -60,8 +60,7 @@ pub struct GoalEvaluationStep<'tcx> { /// of a goal. #[derive(Eq, PartialEq)] pub struct Probe<'tcx> { - pub added_goals_evaluations: Vec>, - pub nested_probes: Vec>, + pub steps: Vec>, pub kind: ProbeKind<'tcx>, } @@ -71,6 +70,12 @@ impl Debug for Probe<'_> { } } +#[derive(Eq, PartialEq)] +pub enum ProbeStep<'tcx> { + EvaluateGoals(AddedGoalsEvaluation<'tcx>), + NestedProbe(Probe<'tcx>), +} + #[derive(Debug, PartialEq, Eq)] pub enum ProbeKind<'tcx> { /// The root inference context while proving a goal. diff --git a/compiler/rustc_middle/src/traits/solve/inspect/format.rs b/compiler/rustc_middle/src/traits/solve/inspect/format.rs index 8d63a245814db..faaf791bf895d 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect/format.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect/format.rs @@ -118,11 +118,11 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { }?; self.nested(|this| { - for probe in &probe.nested_probes { - this.format_probe(probe)?; - } - for nested in &probe.added_goals_evaluations { - this.format_added_goals_evaluation(nested)?; + for step in &probe.steps { + match step { + ProbeStep::EvaluateGoals(eval) => this.format_added_goals_evaluation(eval)?, + ProbeStep::NestedProbe(probe) => this.format_probe(probe)?, + } } Ok(()) }) diff --git a/compiler/rustc_trait_selection/src/solve/inspect.rs b/compiler/rustc_trait_selection/src/solve/inspect.rs index 353f37ebe6038..ca922b64e28ca 100644 --- a/compiler/rustc_trait_selection/src/solve/inspect.rs +++ b/compiler/rustc_trait_selection/src/solve/inspect.rs @@ -103,25 +103,34 @@ impl<'tcx> WipGoalEvaluationStep<'tcx> { #[derive(Eq, PartialEq, Debug)] pub struct WipProbe<'tcx> { - pub added_goals_evaluations: Vec>, - pub nested_probes: Vec>, + pub steps: Vec>, pub kind: Option>, } impl<'tcx> WipProbe<'tcx> { pub fn finalize(self) -> inspect::Probe<'tcx> { inspect::Probe { - added_goals_evaluations: self - .added_goals_evaluations - .into_iter() - .map(WipAddedGoalsEvaluation::finalize) - .collect(), - nested_probes: self.nested_probes.into_iter().map(WipProbe::finalize).collect(), + steps: self.steps.into_iter().map(WipProbeStep::finalize).collect(), kind: self.kind.unwrap(), } } } +#[derive(Eq, PartialEq, Debug)] +pub enum WipProbeStep<'tcx> { + EvaluateGoals(WipAddedGoalsEvaluation<'tcx>), + NestedProbe(WipProbe<'tcx>), +} + +impl<'tcx> WipProbeStep<'tcx> { + pub fn finalize(self) -> inspect::ProbeStep<'tcx> { + match self { + WipProbeStep::EvaluateGoals(eval) => inspect::ProbeStep::EvaluateGoals(eval.finalize()), + WipProbeStep::NestedProbe(probe) => inspect::ProbeStep::NestedProbe(probe.finalize()), + } + } +} + #[derive(Debug)] pub enum DebugSolver<'tcx> { Root, @@ -329,11 +338,7 @@ impl<'tcx> ProofTreeBuilder<'tcx> { ) -> ProofTreeBuilder<'tcx> { self.nested(|| WipGoalEvaluationStep { instantiated_goal, - evaluation: WipProbe { - added_goals_evaluations: vec![], - nested_probes: vec![], - kind: None, - }, + evaluation: WipProbe { steps: vec![], kind: None }, }) } pub fn goal_evaluation_step(&mut self, goal_evaluation_step: ProofTreeBuilder<'tcx>) { @@ -351,11 +356,7 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } pub fn new_probe(&mut self) -> ProofTreeBuilder<'tcx> { - self.nested(|| WipProbe { - added_goals_evaluations: vec![], - nested_probes: vec![], - kind: None, - }) + self.nested(|| WipProbe { steps: vec![], kind: None }) } pub fn probe_kind(&mut self, probe_kind: ProbeKind<'tcx>) { @@ -373,13 +374,13 @@ impl<'tcx> ProofTreeBuilder<'tcx> { if let Some(this) = self.as_mut() { match (this, probe.state.unwrap().tree) { ( - DebugSolver::Probe(WipProbe { nested_probes, .. }) + DebugSolver::Probe(WipProbe { steps, .. }) | DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { - evaluation: WipProbe { nested_probes, .. }, + evaluation: WipProbe { steps, .. }, .. }), DebugSolver::Probe(probe), - ) => nested_probes.push(probe), + ) => steps.push(WipProbeStep::NestedProbe(probe)), _ => unreachable!(), } } @@ -416,12 +417,12 @@ impl<'tcx> ProofTreeBuilder<'tcx> { match (this, added_goals_evaluation.state.unwrap().tree) { ( DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { - evaluation: WipProbe { added_goals_evaluations, .. }, + evaluation: WipProbe { steps, .. }, .. }) - | DebugSolver::Probe(WipProbe { added_goals_evaluations, .. }), + | DebugSolver::Probe(WipProbe { steps, .. }), DebugSolver::AddedGoalsEvaluation(added_goals_evaluation), - ) => added_goals_evaluations.push(added_goals_evaluation), + ) => steps.push(WipProbeStep::EvaluateGoals(added_goals_evaluation)), _ => unreachable!(), } } From 1b141b6d73896e168abf3df8046a940c8928880e Mon Sep 17 00:00:00 2001 From: lcnr Date: Thu, 14 Sep 2023 10:41:36 +0200 Subject: [PATCH 16/22] inspect: explicitly store added goals --- .../rustc_middle/src/traits/solve/inspect.rs | 1 + .../src/traits/solve/inspect/format.rs | 1 + .../src/solve/eval_ctxt.rs | 2 +- .../rustc_trait_selection/src/solve/inspect.rs | 17 +++++++++++++++++ compiler/rustc_trait_selection/src/solve/mod.rs | 1 + 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_middle/src/traits/solve/inspect.rs b/compiler/rustc_middle/src/traits/solve/inspect.rs index a014e8769d8f8..9698d2e4790fd 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect.rs @@ -72,6 +72,7 @@ impl Debug for Probe<'_> { #[derive(Eq, PartialEq)] pub enum ProbeStep<'tcx> { + AddGoal(Goal<'tcx, ty::Predicate<'tcx>>), EvaluateGoals(AddedGoalsEvaluation<'tcx>), NestedProbe(Probe<'tcx>), } diff --git a/compiler/rustc_middle/src/traits/solve/inspect/format.rs b/compiler/rustc_middle/src/traits/solve/inspect/format.rs index faaf791bf895d..3741b933a6bc4 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect/format.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect/format.rs @@ -120,6 +120,7 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { self.nested(|this| { for step in &probe.steps { match step { + ProbeStep::AddGoal(goal) => writeln!(this.f, "ADDED GOAL: {goal:?}")?, ProbeStep::EvaluateGoals(eval) => this.format_added_goals_evaluation(eval)?, ProbeStep::NestedProbe(probe) => this.format_probe(probe)?, } diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs index 307c0516f703a..8079e40698a77 100644 --- a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs +++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs @@ -85,7 +85,7 @@ pub struct EvalCtxt<'a, 'tcx> { // evaluation code. tainted: Result<(), NoSolution>, - inspect: ProofTreeBuilder<'tcx>, + pub(super) inspect: ProofTreeBuilder<'tcx>, } #[derive(Debug, Clone)] diff --git a/compiler/rustc_trait_selection/src/solve/inspect.rs b/compiler/rustc_trait_selection/src/solve/inspect.rs index ca922b64e28ca..86608dae2beef 100644 --- a/compiler/rustc_trait_selection/src/solve/inspect.rs +++ b/compiler/rustc_trait_selection/src/solve/inspect.rs @@ -118,6 +118,7 @@ impl<'tcx> WipProbe<'tcx> { #[derive(Eq, PartialEq, Debug)] pub enum WipProbeStep<'tcx> { + AddGoal(Goal<'tcx, ty::Predicate<'tcx>>), EvaluateGoals(WipAddedGoalsEvaluation<'tcx>), NestedProbe(WipProbe<'tcx>), } @@ -125,6 +126,7 @@ pub enum WipProbeStep<'tcx> { impl<'tcx> WipProbeStep<'tcx> { pub fn finalize(self) -> inspect::ProbeStep<'tcx> { match self { + WipProbeStep::AddGoal(goal) => inspect::ProbeStep::AddGoal(goal), WipProbeStep::EvaluateGoals(eval) => inspect::ProbeStep::EvaluateGoals(eval.finalize()), WipProbeStep::NestedProbe(probe) => inspect::ProbeStep::NestedProbe(probe.finalize()), } @@ -370,6 +372,21 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } } + pub fn add_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) { + if let Some(this) = self.as_mut() { + match this { + DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { + evaluation: WipProbe { steps, .. }, + .. + }) + | DebugSolver::Probe(WipProbe { steps, .. }) => { + steps.push(WipProbeStep::AddGoal(goal)) + } + _ => unreachable!(), + } + } + } + pub fn finish_probe(&mut self, probe: ProofTreeBuilder<'tcx>) { if let Some(this) = self.as_mut() { match (this, probe.state.unwrap().tree) { diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs index c492408bc7630..558e23a5d36d9 100644 --- a/compiler/rustc_trait_selection/src/solve/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/mod.rs @@ -228,6 +228,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { #[instrument(level = "debug", skip(self))] fn add_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) { + self.inspect.add_goal(goal); self.nested_goals.goals.push(goal); } From 0cb800ec342c95478fbca4ae94bd5a6c10c9c093 Mon Sep 17 00:00:00 2001 From: lcnr Date: Thu, 14 Sep 2023 15:10:45 +0200 Subject: [PATCH 17/22] differentiate root and nested goals --- .../rustc_middle/src/traits/solve/inspect.rs | 12 ++++-- .../src/traits/solve/inspect/format.rs | 17 ++++---- .../src/solve/eval_ctxt.rs | 25 +++++++---- .../src/solve/inspect.rs | 42 +++++++++++++------ .../rustc_trait_selection/src/solve/mod.rs | 9 +++- .../src/solve/search_graph/mod.rs | 6 +-- 6 files changed, 76 insertions(+), 35 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve/inspect.rs b/compiler/rustc_middle/src/traits/solve/inspect.rs index 9698d2e4790fd..c3ed40867cf28 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect.rs @@ -14,10 +14,16 @@ pub enum CacheHit { Global, } +#[derive(Eq, PartialEq)] +pub enum GoalEvaluationKind { + Root, + Nested { is_normalizes_to_hack: IsNormalizesToHack }, +} + #[derive(Eq, PartialEq)] pub struct GoalEvaluation<'tcx> { pub uncanonicalized_goal: Goal<'tcx, ty::Predicate<'tcx>>, - pub is_normalizes_to_hack: IsNormalizesToHack, + pub kind: GoalEvaluationKind, pub evaluation: CanonicalGoalEvaluation<'tcx>, pub returned_goals: Vec>>, } @@ -25,12 +31,12 @@ pub struct GoalEvaluation<'tcx> { #[derive(Eq, PartialEq)] pub struct CanonicalGoalEvaluation<'tcx> { pub goal: CanonicalInput<'tcx>, - pub kind: GoalEvaluationKind<'tcx>, + pub kind: CanonicalGoalEvaluationKind<'tcx>, pub result: QueryResult<'tcx>, } #[derive(Eq, PartialEq)] -pub enum GoalEvaluationKind<'tcx> { +pub enum CanonicalGoalEvaluationKind<'tcx> { Overflow, CacheHit(CacheHit), Uncached { revisions: Vec> }, diff --git a/compiler/rustc_middle/src/traits/solve/inspect/format.rs b/compiler/rustc_middle/src/traits/solve/inspect/format.rs index 3741b933a6bc4..d33e83ae1edb3 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect/format.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect/format.rs @@ -40,9 +40,12 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { } pub(super) fn format_goal_evaluation(&mut self, eval: &GoalEvaluation<'_>) -> std::fmt::Result { - let goal_text = match eval.is_normalizes_to_hack { - IsNormalizesToHack::Yes => "NORMALIZES-TO HACK GOAL", - IsNormalizesToHack::No => "GOAL", + let goal_text = match eval.kind { + GoalEvaluationKind::Root => "ROOT GOAL", + GoalEvaluationKind::Nested { is_normalizes_to_hack } => match is_normalizes_to_hack { + IsNormalizesToHack::No => "GOAL", + IsNormalizesToHack::Yes => "NORMALIZES-TO HACK GOAL", + }, }; writeln!(self.f, "{}: {:?}", goal_text, eval.uncanonicalized_goal)?; self.nested(|this| this.format_canonical_goal_evaluation(&eval.evaluation))?; @@ -68,16 +71,16 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { writeln!(self.f, "GOAL: {:?}", eval.goal)?; match &eval.kind { - GoalEvaluationKind::Overflow => { + CanonicalGoalEvaluationKind::Overflow => { writeln!(self.f, "OVERFLOW: {:?}", eval.result) } - GoalEvaluationKind::CacheHit(CacheHit::Global) => { + CanonicalGoalEvaluationKind::CacheHit(CacheHit::Global) => { writeln!(self.f, "GLOBAL CACHE HIT: {:?}", eval.result) } - GoalEvaluationKind::CacheHit(CacheHit::Provisional) => { + CanonicalGoalEvaluationKind::CacheHit(CacheHit::Provisional) => { writeln!(self.f, "PROVISIONAL CACHE HIT: {:?}", eval.result) } - GoalEvaluationKind::Uncached { revisions } => { + CanonicalGoalEvaluationKind::Uncached { revisions } => { for (n, step) in revisions.iter().enumerate() { writeln!(self.f, "REVISION {n}")?; self.nested(|this| this.format_evaluation_step(step))?; diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs index 8079e40698a77..51cb192a1e841 100644 --- a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs +++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs @@ -28,8 +28,8 @@ use std::ops::ControlFlow; use crate::traits::vtable::{count_own_vtable_entries, prepare_vtable_segments, VtblSegment}; use super::inspect::ProofTreeBuilder; -use super::search_graph; use super::SolverMode; +use super::{search_graph, GoalEvaluationKind}; use super::{search_graph::SearchGraph, Goal}; pub use select::InferCtxtSelectExt; @@ -164,7 +164,7 @@ impl<'tcx> InferCtxtEvalExt<'tcx> for InferCtxt<'tcx> { Option>, ) { EvalCtxt::enter_root(self, generate_proof_tree, |ecx| { - ecx.evaluate_goal(IsNormalizesToHack::No, goal) + ecx.evaluate_goal(GoalEvaluationKind::Root, goal) }) } } @@ -340,11 +340,11 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { /// been constrained and the certainty of the result. fn evaluate_goal( &mut self, - is_normalizes_to_hack: IsNormalizesToHack, + goal_evaluation_kind: GoalEvaluationKind, goal: Goal<'tcx, ty::Predicate<'tcx>>, ) -> Result<(bool, Certainty, Vec>>), NoSolution> { let (orig_values, canonical_goal) = self.canonicalize_goal(goal); - let mut goal_evaluation = self.inspect.new_goal_evaluation(goal, is_normalizes_to_hack); + let mut goal_evaluation = self.inspect.new_goal_evaluation(goal, goal_evaluation_kind); let encountered_overflow = self.search_graph.encountered_overflow(); let canonical_response = EvalCtxt::evaluate_canonical_goal( self.tcx(), @@ -389,7 +389,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { // solver cycle. if cfg!(debug_assertions) && has_changed - && is_normalizes_to_hack == IsNormalizesToHack::No + && !matches!( + goal_evaluation_kind, + GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::Yes } + ) && !self.search_graph.in_cycle() { // The nested evaluation has to happen with the original state @@ -561,8 +564,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { }, ); - let (_, certainty, instantiate_goals) = - self.evaluate_goal(IsNormalizesToHack::Yes, unconstrained_goal)?; + let (_, certainty, instantiate_goals) = self.evaluate_goal( + GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::Yes }, + unconstrained_goal, + )?; self.add_goals(instantiate_goals); // Finally, equate the goal's RHS with the unconstrained var. @@ -596,8 +601,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { } for goal in goals.goals.drain(..) { - let (has_changed, certainty, instantiate_goals) = - self.evaluate_goal(IsNormalizesToHack::No, goal)?; + let (has_changed, certainty, instantiate_goals) = self.evaluate_goal( + GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::No }, + goal, + )?; self.add_goals(instantiate_goals); if has_changed { unchanged_certainty = None; diff --git a/compiler/rustc_trait_selection/src/solve/inspect.rs b/compiler/rustc_trait_selection/src/solve/inspect.rs index 86608dae2beef..749bba33c9b67 100644 --- a/compiler/rustc_trait_selection/src/solve/inspect.rs +++ b/compiler/rustc_trait_selection/src/solve/inspect.rs @@ -7,13 +7,13 @@ use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::DumpSolverProofTree; use super::eval_ctxt::UseGlobalCache; -use super::GenerateProofTree; +use super::{GenerateProofTree, GoalEvaluationKind}; #[derive(Eq, PartialEq, Debug)] pub struct WipGoalEvaluation<'tcx> { pub uncanonicalized_goal: Goal<'tcx, ty::Predicate<'tcx>>, + pub kind: WipGoalEvaluationKind, pub evaluation: Option>, - pub is_normalizes_to_hack: IsNormalizesToHack, pub returned_goals: Vec>>, } @@ -21,8 +21,13 @@ impl<'tcx> WipGoalEvaluation<'tcx> { pub fn finalize(self) -> inspect::GoalEvaluation<'tcx> { inspect::GoalEvaluation { uncanonicalized_goal: self.uncanonicalized_goal, + kind: match self.kind { + WipGoalEvaluationKind::Root => inspect::GoalEvaluationKind::Root, + WipGoalEvaluationKind::Nested { is_normalizes_to_hack } => { + inspect::GoalEvaluationKind::Nested { is_normalizes_to_hack } + } + }, evaluation: self.evaluation.unwrap().finalize(), - is_normalizes_to_hack: self.is_normalizes_to_hack, returned_goals: self.returned_goals, } } @@ -30,6 +35,12 @@ impl<'tcx> WipGoalEvaluation<'tcx> { #[derive(Eq, PartialEq, Debug)] pub enum WipGoalEvaluationKind { + Root, + Nested { is_normalizes_to_hack: IsNormalizesToHack }, +} + +#[derive(Eq, PartialEq, Debug)] +pub enum WipCanonicalGoalEvaluationKind { Overflow, CacheHit(CacheHit), } @@ -37,7 +48,7 @@ pub enum WipGoalEvaluationKind { #[derive(Eq, PartialEq, Debug)] pub struct WipCanonicalGoalEvaluation<'tcx> { pub goal: CanonicalInput<'tcx>, - pub kind: Option, + pub kind: Option, pub revisions: Vec>, pub result: Option>, } @@ -45,11 +56,13 @@ pub struct WipCanonicalGoalEvaluation<'tcx> { impl<'tcx> WipCanonicalGoalEvaluation<'tcx> { pub fn finalize(self) -> inspect::CanonicalGoalEvaluation<'tcx> { let kind = match self.kind { - Some(WipGoalEvaluationKind::Overflow) => inspect::GoalEvaluationKind::Overflow, - Some(WipGoalEvaluationKind::CacheHit(hit)) => { - inspect::GoalEvaluationKind::CacheHit(hit) + Some(WipCanonicalGoalEvaluationKind::Overflow) => { + inspect::CanonicalGoalEvaluationKind::Overflow + } + Some(WipCanonicalGoalEvaluationKind::CacheHit(hit)) => { + inspect::CanonicalGoalEvaluationKind::CacheHit(hit) } - None => inspect::GoalEvaluationKind::Uncached { + None => inspect::CanonicalGoalEvaluationKind::Uncached { revisions: self .revisions .into_iter() @@ -260,15 +273,20 @@ impl<'tcx> ProofTreeBuilder<'tcx> { self.state.is_none() } - pub fn new_goal_evaluation( + pub(super) fn new_goal_evaluation( &mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>, - is_normalizes_to_hack: IsNormalizesToHack, + kind: GoalEvaluationKind, ) -> ProofTreeBuilder<'tcx> { self.nested(|| WipGoalEvaluation { uncanonicalized_goal: goal, + kind: match kind { + GoalEvaluationKind::Root => WipGoalEvaluationKind::Root, + GoalEvaluationKind::Nested { is_normalizes_to_hack } => { + WipGoalEvaluationKind::Nested { is_normalizes_to_hack } + } + }, evaluation: None, - is_normalizes_to_hack, returned_goals: vec![], }) } @@ -297,7 +315,7 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } } - pub fn goal_evaluation_kind(&mut self, kind: WipGoalEvaluationKind) { + pub fn goal_evaluation_kind(&mut self, kind: WipCanonicalGoalEvaluationKind) { if let Some(this) = self.as_mut() { match this { DebugSolver::CanonicalGoalEvaluation(canonical_goal_evaluation) => { diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs index 558e23a5d36d9..bd612ce4778db 100644 --- a/compiler/rustc_trait_selection/src/solve/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/mod.rs @@ -19,7 +19,8 @@ use rustc_infer::infer::canonical::{Canonical, CanonicalVarValues}; use rustc_infer::traits::query::NoSolution; use rustc_middle::infer::canonical::CanonicalVarInfos; use rustc_middle::traits::solve::{ - CanonicalResponse, Certainty, ExternalConstraintsData, Goal, QueryResult, Response, + CanonicalResponse, Certainty, ExternalConstraintsData, Goal, IsNormalizesToHack, QueryResult, + Response, }; use rustc_middle::ty::{self, Ty, TyCtxt, UniverseIndex}; use rustc_middle::ty::{ @@ -59,6 +60,12 @@ enum SolverMode { Coherence, } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum GoalEvaluationKind { + Root, + Nested { is_normalizes_to_hack: IsNormalizesToHack }, +} + trait CanonicalResponseExt { fn has_no_inference_or_external_constraints(&self) -> bool; diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs index c816b51f67acd..16de518e8e0b0 100644 --- a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs @@ -187,7 +187,7 @@ impl<'tcx> SearchGraph<'tcx> { last.encountered_overflow = true; } - inspect.goal_evaluation_kind(inspect::WipGoalEvaluationKind::Overflow); + inspect.goal_evaluation_kind(inspect::WipCanonicalGoalEvaluationKind::Overflow); return Self::response_no_constraints(tcx, input, Certainty::OVERFLOW); }; @@ -203,7 +203,7 @@ impl<'tcx> SearchGraph<'tcx> { available_depth, ) { - inspect.goal_evaluation_kind(inspect::WipGoalEvaluationKind::CacheHit( + inspect.goal_evaluation_kind(inspect::WipCanonicalGoalEvaluationKind::CacheHit( CacheHit::Global, )); self.on_cache_hit(reached_depth, encountered_overflow); @@ -240,7 +240,7 @@ impl<'tcx> SearchGraph<'tcx> { // Finally we can return either the provisional response for that goal if we have a // coinductive cycle or an ambiguous result if the cycle is inductive. Entry::Occupied(entry_index) => { - inspect.goal_evaluation_kind(inspect::WipGoalEvaluationKind::CacheHit( + inspect.goal_evaluation_kind(inspect::WipCanonicalGoalEvaluationKind::CacheHit( CacheHit::Provisional, )); From 5b882acc2e48e02f11622cf7fb4c02c732a81bfc Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Tue, 12 Sep 2023 19:35:06 -0400 Subject: [PATCH 18/22] Bump to supported Ubuntu The 22.10 Ubuntu repositories were returning 404s in last stable build. --- src/ci/docker/host-x86_64/arm-android/Dockerfile | 2 +- src/ci/docker/host-x86_64/dist-android/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index b6b4fdc67a949..db11700af2815 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.10 +FROM ubuntu:23.04 ARG DEBIAN_FRONTEND=noninteractive COPY scripts/android-base-apt-get.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/dist-android/Dockerfile b/src/ci/docker/host-x86_64/dist-android/Dockerfile index 9c6f648896b51..b09b6edb01a6d 100644 --- a/src/ci/docker/host-x86_64/dist-android/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.10 +FROM ubuntu:23.04 COPY scripts/android-base-apt-get.sh /scripts/ RUN sh /scripts/android-base-apt-get.sh From abd265ed0ed4fff89f87772150da1f66c863d7e1 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Wed, 13 Sep 2023 14:43:08 -0400 Subject: [PATCH 19/22] Move to older, mirrored redox install --- .../docker/host-x86_64/dist-various-1/install-x86_64-redox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh b/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh index dad9792233847..f86402b0180bb 100755 --- a/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh +++ b/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh @@ -2,5 +2,5 @@ set -ex -curl https://static.redox-os.org/toolchain/x86_64-unknown-redox/relibc-install.tar.gz | \ +curl https://ci-mirrors.rust-lang.org/rustc/2022-11-27-relibc-install.tar.gz | \ tar --extract --gzip --directory /usr/local From c70ee688b3ff2d1c24eda6990576a2dda2b76053 Mon Sep 17 00:00:00 2001 From: Jack Huey <31162821+jackh726@users.noreply.github.com> Date: Sun, 17 Sep 2023 12:05:36 -0400 Subject: [PATCH 20/22] Add me as on vacation --- triagebot.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/triagebot.toml b/triagebot.toml index d9d523bef39f3..d35e81c277ea2 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -585,7 +585,7 @@ cc = ["@nnethercote"] [assign] warn_non_default_branch = true contributing_url = "https://rustc-dev-guide.rust-lang.org/getting-started.html" -users_on_vacation = ["jyn514"] +users_on_vacation = ["jyn514", "jackh726"] [assign.adhoc_groups] compiler-team = [ From 3b817b2810ff7ecacab02f901f2a1a2902e5e72f Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 17 Sep 2023 09:35:18 +0200 Subject: [PATCH 21/22] nop_lift macros: ensure that we are using the right interner --- compiler/rustc_middle/src/ty/context.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index f748404875759..fcb1a6971e9c8 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1214,6 +1214,25 @@ macro_rules! nop_lift { impl<'a, 'tcx> Lift<'tcx> for $ty { type Lifted = $lifted; fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option { + // Assert that the set has the right type. + // Given an argument that has an interned type, the return type has the type of + // the corresponding interner set. This won't actually return anything, we're + // just doing this to compute said type! + fn _intern_set_ty_from_interned_ty<'tcx, Inner>( + _x: Interned<'tcx, Inner>, + ) -> InternedSet<'tcx, Inner> { + unreachable!() + } + fn _type_eq(_x: &T, _y: &T) {} + fn _test<'tcx>(x: $lifted, tcx: TyCtxt<'tcx>) { + // If `x` is a newtype around an `Interned`, then `interner` is an + // interner of appropriate type. (Ideally we'd also check that `x` is a + // newtype with just that one field. Not sure how to do that.) + let interner = _intern_set_ty_from_interned_ty(x.0); + // Now check that this is the same type as `interners.$set`. + _type_eq(&interner, &tcx.interners.$set); + } + tcx.interners .$set .contains_pointer_to(&InternedInSet(&*self.0.0)) @@ -1230,6 +1249,11 @@ macro_rules! nop_list_lift { impl<'a, 'tcx> Lift<'tcx> for &'a List<$ty> { type Lifted = &'tcx List<$lifted>; fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option { + // Assert that the set has the right type. + if false { + let _x: &InternedSet<'tcx, List<$lifted>> = &tcx.interners.$set; + } + if self.is_empty() { return Some(List::empty()); } From a716c9620f0e2f2ce7ecd66f5328cd9a59ff9208 Mon Sep 17 00:00:00 2001 From: Tshepang Mbambo Date: Mon, 18 Sep 2023 03:42:35 +0200 Subject: [PATCH 22/22] address review comment See https://github.com/rust-lang/rust/pull/115558#issuecomment-1722601187 --- config.example.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.example.toml b/config.example.toml index 95f0c95bc5c46..8732b66931766 100644 --- a/config.example.toml +++ b/config.example.toml @@ -455,6 +455,7 @@ changelog-seen = 2 # Sets the number of codegen units to build the standard library with, # regardless of what the codegen-unit setting for the rest of the compiler is. +# NOTE: building with anything other than 1 is known to occasionally have bugs. #codegen-units-std = codegen-units # Whether or not debug assertions are enabled for the compiler and standard library.