diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3680136d89ff7..0c20e4dc5351b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -393,7 +393,7 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-2019-8core-32gb - name: dist-i686-msvc diff --git a/.mailmap b/.mailmap index eb82cf4de8d0b..7d455ebd45f1e 100644 --- a/.mailmap +++ b/.mailmap @@ -549,6 +549,8 @@ Timothy Maloney Tomas Koutsky Torsten Weber Torsten Weber +Trevor Gross +Trevor Gross Trevor Spiteri Tshepang Mbambo Ty Overby diff --git a/Cargo.lock b/Cargo.lock index 386b57e6a44e9..fe4118db1b7cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -858,14 +858,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.3", + "darling_macro 0.20.3", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", ] [[package]] @@ -882,13 +906,24 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ - "darling_core", + "darling_core 0.20.3", "quote", "syn 2.0.29", ] @@ -919,6 +954,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -938,7 +1004,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d" dependencies = [ - "darling", + "darling 0.20.3", "proc-macro2", "quote", "syn 2.0.29", @@ -2584,6 +2650,8 @@ dependencies = [ "anyhow", "build_helper", "camino", + "clap", + "derive_builder", "env_logger 0.10.0", "fs_extra", "glob", diff --git a/compiler/rustc_middle/src/traits/solve/inspect.rs b/compiler/rustc_middle/src/traits/solve/inspect.rs index d8b3a061b77d5..c3ed40867cf28 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect.rs @@ -14,10 +14,16 @@ pub enum CacheHit { Global, } +#[derive(Eq, PartialEq)] +pub enum GoalEvaluationKind { + Root, + Nested { is_normalizes_to_hack: IsNormalizesToHack }, +} + #[derive(Eq, PartialEq)] pub struct GoalEvaluation<'tcx> { pub uncanonicalized_goal: Goal<'tcx, ty::Predicate<'tcx>>, - pub is_normalizes_to_hack: IsNormalizesToHack, + pub kind: GoalEvaluationKind, pub evaluation: CanonicalGoalEvaluation<'tcx>, pub returned_goals: Vec>>, } @@ -25,12 +31,12 @@ pub struct GoalEvaluation<'tcx> { #[derive(Eq, PartialEq)] pub struct CanonicalGoalEvaluation<'tcx> { pub goal: CanonicalInput<'tcx>, - pub kind: GoalEvaluationKind<'tcx>, + pub kind: CanonicalGoalEvaluationKind<'tcx>, pub result: QueryResult<'tcx>, } #[derive(Eq, PartialEq)] -pub enum GoalEvaluationKind<'tcx> { +pub enum CanonicalGoalEvaluationKind<'tcx> { Overflow, CacheHit(CacheHit), Uncached { revisions: Vec> }, @@ -52,22 +58,31 @@ pub struct GoalEvaluationStep<'tcx> { pub instantiated_goal: QueryInput<'tcx, ty::Predicate<'tcx>>, /// The actual evaluation of the goal, always `ProbeKind::Root`. - pub evaluation: GoalCandidate<'tcx>, + pub evaluation: Probe<'tcx>, } +/// A self-contained computation during trait solving. This either +/// corresponds to a `EvalCtxt::probe(_X)` call or the root evaluation +/// of a goal. #[derive(Eq, PartialEq)] -pub struct GoalCandidate<'tcx> { - pub added_goals_evaluations: Vec>, - pub candidates: Vec>, +pub struct Probe<'tcx> { + pub steps: Vec>, pub kind: ProbeKind<'tcx>, } -impl Debug for GoalCandidate<'_> { +impl Debug for Probe<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - ProofTreeFormatter::new(f).format_candidate(self) + ProofTreeFormatter::new(f).format_probe(self) } } +#[derive(Eq, PartialEq)] +pub enum ProbeStep<'tcx> { + AddGoal(Goal<'tcx, ty::Predicate<'tcx>>), + EvaluateGoals(AddedGoalsEvaluation<'tcx>), + NestedProbe(Probe<'tcx>), +} + #[derive(Debug, PartialEq, Eq)] pub enum ProbeKind<'tcx> { /// The root inference context while proving a goal. diff --git a/compiler/rustc_middle/src/traits/solve/inspect/format.rs b/compiler/rustc_middle/src/traits/solve/inspect/format.rs index d916e80a625d0..d33e83ae1edb3 100644 --- a/compiler/rustc_middle/src/traits/solve/inspect/format.rs +++ b/compiler/rustc_middle/src/traits/solve/inspect/format.rs @@ -40,9 +40,12 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { } pub(super) fn format_goal_evaluation(&mut self, eval: &GoalEvaluation<'_>) -> std::fmt::Result { - let goal_text = match eval.is_normalizes_to_hack { - IsNormalizesToHack::Yes => "NORMALIZES-TO HACK GOAL", - IsNormalizesToHack::No => "GOAL", + let goal_text = match eval.kind { + GoalEvaluationKind::Root => "ROOT GOAL", + GoalEvaluationKind::Nested { is_normalizes_to_hack } => match is_normalizes_to_hack { + IsNormalizesToHack::No => "GOAL", + IsNormalizesToHack::Yes => "NORMALIZES-TO HACK GOAL", + }, }; writeln!(self.f, "{}: {:?}", goal_text, eval.uncanonicalized_goal)?; self.nested(|this| this.format_canonical_goal_evaluation(&eval.evaluation))?; @@ -68,16 +71,16 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { writeln!(self.f, "GOAL: {:?}", eval.goal)?; match &eval.kind { - GoalEvaluationKind::Overflow => { + CanonicalGoalEvaluationKind::Overflow => { writeln!(self.f, "OVERFLOW: {:?}", eval.result) } - GoalEvaluationKind::CacheHit(CacheHit::Global) => { + CanonicalGoalEvaluationKind::CacheHit(CacheHit::Global) => { writeln!(self.f, "GLOBAL CACHE HIT: {:?}", eval.result) } - GoalEvaluationKind::CacheHit(CacheHit::Provisional) => { + CanonicalGoalEvaluationKind::CacheHit(CacheHit::Provisional) => { writeln!(self.f, "PROVISIONAL CACHE HIT: {:?}", eval.result) } - GoalEvaluationKind::Uncached { revisions } => { + CanonicalGoalEvaluationKind::Uncached { revisions } => { for (n, step) in revisions.iter().enumerate() { writeln!(self.f, "REVISION {n}")?; self.nested(|this| this.format_evaluation_step(step))?; @@ -92,11 +95,11 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { evaluation_step: &GoalEvaluationStep<'_>, ) -> std::fmt::Result { writeln!(self.f, "INSTANTIATED: {:?}", evaluation_step.instantiated_goal)?; - self.format_candidate(&evaluation_step.evaluation) + self.format_probe(&evaluation_step.evaluation) } - pub(super) fn format_candidate(&mut self, candidate: &GoalCandidate<'_>) -> std::fmt::Result { - match &candidate.kind { + pub(super) fn format_probe(&mut self, probe: &Probe<'_>) -> std::fmt::Result { + match &probe.kind { ProbeKind::Root { result } => { writeln!(self.f, "ROOT RESULT: {result:?}") } @@ -118,11 +121,12 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> { }?; self.nested(|this| { - for candidate in &candidate.candidates { - this.format_candidate(candidate)?; - } - for nested in &candidate.added_goals_evaluations { - this.format_added_goals_evaluation(nested)?; + for step in &probe.steps { + match step { + ProbeStep::AddGoal(goal) => writeln!(this.f, "ADDED GOAL: {goal:?}")?, + ProbeStep::EvaluateGoals(eval) => this.format_added_goals_evaluation(eval)?, + ProbeStep::NestedProbe(probe) => this.format_probe(probe)?, + } } Ok(()) }) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 880d3029451af..e8d50a1c025af 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1214,6 +1214,25 @@ macro_rules! nop_lift { impl<'a, 'tcx> Lift<'tcx> for $ty { type Lifted = $lifted; fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option { + // Assert that the set has the right type. + // Given an argument that has an interned type, the return type has the type of + // the corresponding interner set. This won't actually return anything, we're + // just doing this to compute said type! + fn _intern_set_ty_from_interned_ty<'tcx, Inner>( + _x: Interned<'tcx, Inner>, + ) -> InternedSet<'tcx, Inner> { + unreachable!() + } + fn _type_eq(_x: &T, _y: &T) {} + fn _test<'tcx>(x: $lifted, tcx: TyCtxt<'tcx>) { + // If `x` is a newtype around an `Interned`, then `interner` is an + // interner of appropriate type. (Ideally we'd also check that `x` is a + // newtype with just that one field. Not sure how to do that.) + let interner = _intern_set_ty_from_interned_ty(x.0); + // Now check that this is the same type as `interners.$set`. + _type_eq(&interner, &tcx.interners.$set); + } + tcx.interners .$set .contains_pointer_to(&InternedInSet(&*self.0.0)) @@ -1230,6 +1249,11 @@ macro_rules! nop_list_lift { impl<'a, 'tcx> Lift<'tcx> for &'a List<$ty> { type Lifted = &'tcx List<$lifted>; fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option { + // Assert that the set has the right type. + if false { + let _x: &InternedSet<'tcx, List<$lifted>> = &tcx.interners.$set; + } + if self.is_empty() { return Some(List::empty()); } diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs index 307c0516f703a..51cb192a1e841 100644 --- a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs +++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs @@ -28,8 +28,8 @@ use std::ops::ControlFlow; use crate::traits::vtable::{count_own_vtable_entries, prepare_vtable_segments, VtblSegment}; use super::inspect::ProofTreeBuilder; -use super::search_graph; use super::SolverMode; +use super::{search_graph, GoalEvaluationKind}; use super::{search_graph::SearchGraph, Goal}; pub use select::InferCtxtSelectExt; @@ -85,7 +85,7 @@ pub struct EvalCtxt<'a, 'tcx> { // evaluation code. tainted: Result<(), NoSolution>, - inspect: ProofTreeBuilder<'tcx>, + pub(super) inspect: ProofTreeBuilder<'tcx>, } #[derive(Debug, Clone)] @@ -164,7 +164,7 @@ impl<'tcx> InferCtxtEvalExt<'tcx> for InferCtxt<'tcx> { Option>, ) { EvalCtxt::enter_root(self, generate_proof_tree, |ecx| { - ecx.evaluate_goal(IsNormalizesToHack::No, goal) + ecx.evaluate_goal(GoalEvaluationKind::Root, goal) }) } } @@ -340,11 +340,11 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { /// been constrained and the certainty of the result. fn evaluate_goal( &mut self, - is_normalizes_to_hack: IsNormalizesToHack, + goal_evaluation_kind: GoalEvaluationKind, goal: Goal<'tcx, ty::Predicate<'tcx>>, ) -> Result<(bool, Certainty, Vec>>), NoSolution> { let (orig_values, canonical_goal) = self.canonicalize_goal(goal); - let mut goal_evaluation = self.inspect.new_goal_evaluation(goal, is_normalizes_to_hack); + let mut goal_evaluation = self.inspect.new_goal_evaluation(goal, goal_evaluation_kind); let encountered_overflow = self.search_graph.encountered_overflow(); let canonical_response = EvalCtxt::evaluate_canonical_goal( self.tcx(), @@ -389,7 +389,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { // solver cycle. if cfg!(debug_assertions) && has_changed - && is_normalizes_to_hack == IsNormalizesToHack::No + && !matches!( + goal_evaluation_kind, + GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::Yes } + ) && !self.search_graph.in_cycle() { // The nested evaluation has to happen with the original state @@ -561,8 +564,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { }, ); - let (_, certainty, instantiate_goals) = - self.evaluate_goal(IsNormalizesToHack::Yes, unconstrained_goal)?; + let (_, certainty, instantiate_goals) = self.evaluate_goal( + GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::Yes }, + unconstrained_goal, + )?; self.add_goals(instantiate_goals); // Finally, equate the goal's RHS with the unconstrained var. @@ -596,8 +601,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { } for goal in goals.goals.drain(..) { - let (has_changed, certainty, instantiate_goals) = - self.evaluate_goal(IsNormalizesToHack::No, goal)?; + let (has_changed, certainty, instantiate_goals) = self.evaluate_goal( + GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::No }, + goal, + )?; self.add_goals(instantiate_goals); if has_changed { unchanged_certainty = None; diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs index f88cfbac3f3eb..6087b91679095 100644 --- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs +++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs @@ -24,13 +24,13 @@ where search_graph: outer_ecx.search_graph, nested_goals: outer_ecx.nested_goals.clone(), tainted: outer_ecx.tainted, - inspect: outer_ecx.inspect.new_goal_candidate(), + inspect: outer_ecx.inspect.new_probe(), }; let r = nested_ecx.infcx.probe(|_| f(&mut nested_ecx)); if !outer_ecx.inspect.is_noop() { let probe_kind = probe_kind(&r); nested_ecx.inspect.probe_kind(probe_kind); - outer_ecx.inspect.goal_candidate(nested_ecx.inspect); + outer_ecx.inspect.finish_probe(nested_ecx.inspect); } r } diff --git a/compiler/rustc_trait_selection/src/solve/inspect.rs b/compiler/rustc_trait_selection/src/solve/inspect.rs index 46025da768326..749bba33c9b67 100644 --- a/compiler/rustc_trait_selection/src/solve/inspect.rs +++ b/compiler/rustc_trait_selection/src/solve/inspect.rs @@ -7,13 +7,13 @@ use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::DumpSolverProofTree; use super::eval_ctxt::UseGlobalCache; -use super::GenerateProofTree; +use super::{GenerateProofTree, GoalEvaluationKind}; #[derive(Eq, PartialEq, Debug)] pub struct WipGoalEvaluation<'tcx> { pub uncanonicalized_goal: Goal<'tcx, ty::Predicate<'tcx>>, + pub kind: WipGoalEvaluationKind, pub evaluation: Option>, - pub is_normalizes_to_hack: IsNormalizesToHack, pub returned_goals: Vec>>, } @@ -21,8 +21,13 @@ impl<'tcx> WipGoalEvaluation<'tcx> { pub fn finalize(self) -> inspect::GoalEvaluation<'tcx> { inspect::GoalEvaluation { uncanonicalized_goal: self.uncanonicalized_goal, + kind: match self.kind { + WipGoalEvaluationKind::Root => inspect::GoalEvaluationKind::Root, + WipGoalEvaluationKind::Nested { is_normalizes_to_hack } => { + inspect::GoalEvaluationKind::Nested { is_normalizes_to_hack } + } + }, evaluation: self.evaluation.unwrap().finalize(), - is_normalizes_to_hack: self.is_normalizes_to_hack, returned_goals: self.returned_goals, } } @@ -30,6 +35,12 @@ impl<'tcx> WipGoalEvaluation<'tcx> { #[derive(Eq, PartialEq, Debug)] pub enum WipGoalEvaluationKind { + Root, + Nested { is_normalizes_to_hack: IsNormalizesToHack }, +} + +#[derive(Eq, PartialEq, Debug)] +pub enum WipCanonicalGoalEvaluationKind { Overflow, CacheHit(CacheHit), } @@ -37,7 +48,7 @@ pub enum WipGoalEvaluationKind { #[derive(Eq, PartialEq, Debug)] pub struct WipCanonicalGoalEvaluation<'tcx> { pub goal: CanonicalInput<'tcx>, - pub kind: Option, + pub kind: Option, pub revisions: Vec>, pub result: Option>, } @@ -45,11 +56,13 @@ pub struct WipCanonicalGoalEvaluation<'tcx> { impl<'tcx> WipCanonicalGoalEvaluation<'tcx> { pub fn finalize(self) -> inspect::CanonicalGoalEvaluation<'tcx> { let kind = match self.kind { - Some(WipGoalEvaluationKind::Overflow) => inspect::GoalEvaluationKind::Overflow, - Some(WipGoalEvaluationKind::CacheHit(hit)) => { - inspect::GoalEvaluationKind::CacheHit(hit) + Some(WipCanonicalGoalEvaluationKind::Overflow) => { + inspect::CanonicalGoalEvaluationKind::Overflow } - None => inspect::GoalEvaluationKind::Uncached { + Some(WipCanonicalGoalEvaluationKind::CacheHit(hit)) => { + inspect::CanonicalGoalEvaluationKind::CacheHit(hit) + } + None => inspect::CanonicalGoalEvaluationKind::Uncached { revisions: self .revisions .into_iter() @@ -87,7 +100,7 @@ impl<'tcx> WipAddedGoalsEvaluation<'tcx> { pub struct WipGoalEvaluationStep<'tcx> { pub instantiated_goal: QueryInput<'tcx, ty::Predicate<'tcx>>, - pub evaluation: WipGoalCandidate<'tcx>, + pub evaluation: WipProbe<'tcx>, } impl<'tcx> WipGoalEvaluationStep<'tcx> { @@ -102,26 +115,37 @@ impl<'tcx> WipGoalEvaluationStep<'tcx> { } #[derive(Eq, PartialEq, Debug)] -pub struct WipGoalCandidate<'tcx> { - pub added_goals_evaluations: Vec>, - pub candidates: Vec>, +pub struct WipProbe<'tcx> { + pub steps: Vec>, pub kind: Option>, } -impl<'tcx> WipGoalCandidate<'tcx> { - pub fn finalize(self) -> inspect::GoalCandidate<'tcx> { - inspect::GoalCandidate { - added_goals_evaluations: self - .added_goals_evaluations - .into_iter() - .map(WipAddedGoalsEvaluation::finalize) - .collect(), - candidates: self.candidates.into_iter().map(WipGoalCandidate::finalize).collect(), +impl<'tcx> WipProbe<'tcx> { + pub fn finalize(self) -> inspect::Probe<'tcx> { + inspect::Probe { + steps: self.steps.into_iter().map(WipProbeStep::finalize).collect(), kind: self.kind.unwrap(), } } } +#[derive(Eq, PartialEq, Debug)] +pub enum WipProbeStep<'tcx> { + AddGoal(Goal<'tcx, ty::Predicate<'tcx>>), + EvaluateGoals(WipAddedGoalsEvaluation<'tcx>), + NestedProbe(WipProbe<'tcx>), +} + +impl<'tcx> WipProbeStep<'tcx> { + pub fn finalize(self) -> inspect::ProbeStep<'tcx> { + match self { + WipProbeStep::AddGoal(goal) => inspect::ProbeStep::AddGoal(goal), + WipProbeStep::EvaluateGoals(eval) => inspect::ProbeStep::EvaluateGoals(eval.finalize()), + WipProbeStep::NestedProbe(probe) => inspect::ProbeStep::NestedProbe(probe.finalize()), + } + } +} + #[derive(Debug)] pub enum DebugSolver<'tcx> { Root, @@ -129,7 +153,7 @@ pub enum DebugSolver<'tcx> { CanonicalGoalEvaluation(WipCanonicalGoalEvaluation<'tcx>), AddedGoalsEvaluation(WipAddedGoalsEvaluation<'tcx>), GoalEvaluationStep(WipGoalEvaluationStep<'tcx>), - GoalCandidate(WipGoalCandidate<'tcx>), + Probe(WipProbe<'tcx>), } impl<'tcx> From> for DebugSolver<'tcx> { @@ -156,9 +180,9 @@ impl<'tcx> From> for DebugSolver<'tcx> { } } -impl<'tcx> From> for DebugSolver<'tcx> { - fn from(g: WipGoalCandidate<'tcx>) -> DebugSolver<'tcx> { - DebugSolver::GoalCandidate(g) +impl<'tcx> From> for DebugSolver<'tcx> { + fn from(p: WipProbe<'tcx>) -> DebugSolver<'tcx> { + DebugSolver::Probe(p) } } @@ -249,15 +273,20 @@ impl<'tcx> ProofTreeBuilder<'tcx> { self.state.is_none() } - pub fn new_goal_evaluation( + pub(super) fn new_goal_evaluation( &mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>, - is_normalizes_to_hack: IsNormalizesToHack, + kind: GoalEvaluationKind, ) -> ProofTreeBuilder<'tcx> { self.nested(|| WipGoalEvaluation { uncanonicalized_goal: goal, + kind: match kind { + GoalEvaluationKind::Root => WipGoalEvaluationKind::Root, + GoalEvaluationKind::Nested { is_normalizes_to_hack } => { + WipGoalEvaluationKind::Nested { is_normalizes_to_hack } + } + }, evaluation: None, - is_normalizes_to_hack, returned_goals: vec![], }) } @@ -286,7 +315,7 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } } - pub fn goal_evaluation_kind(&mut self, kind: WipGoalEvaluationKind) { + pub fn goal_evaluation_kind(&mut self, kind: WipCanonicalGoalEvaluationKind) { if let Some(this) = self.as_mut() { match this { DebugSolver::CanonicalGoalEvaluation(canonical_goal_evaluation) => { @@ -329,11 +358,7 @@ impl<'tcx> ProofTreeBuilder<'tcx> { ) -> ProofTreeBuilder<'tcx> { self.nested(|| WipGoalEvaluationStep { instantiated_goal, - evaluation: WipGoalCandidate { - added_goals_evaluations: vec![], - candidates: vec![], - kind: None, - }, + evaluation: WipProbe { steps: vec![], kind: None }, }) } pub fn goal_evaluation_step(&mut self, goal_evaluation_step: ProofTreeBuilder<'tcx>) { @@ -350,18 +375,14 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } } - pub fn new_goal_candidate(&mut self) -> ProofTreeBuilder<'tcx> { - self.nested(|| WipGoalCandidate { - added_goals_evaluations: vec![], - candidates: vec![], - kind: None, - }) + pub fn new_probe(&mut self) -> ProofTreeBuilder<'tcx> { + self.nested(|| WipProbe { steps: vec![], kind: None }) } pub fn probe_kind(&mut self, probe_kind: ProbeKind<'tcx>) { if let Some(this) = self.as_mut() { match this { - DebugSolver::GoalCandidate(this) => { + DebugSolver::Probe(this) => { assert_eq!(this.kind.replace(probe_kind), None) } _ => unreachable!(), @@ -369,17 +390,32 @@ impl<'tcx> ProofTreeBuilder<'tcx> { } } - pub fn goal_candidate(&mut self, candidate: ProofTreeBuilder<'tcx>) { + pub fn add_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) { if let Some(this) = self.as_mut() { - match (this, candidate.state.unwrap().tree) { + match this { + DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { + evaluation: WipProbe { steps, .. }, + .. + }) + | DebugSolver::Probe(WipProbe { steps, .. }) => { + steps.push(WipProbeStep::AddGoal(goal)) + } + _ => unreachable!(), + } + } + } + + pub fn finish_probe(&mut self, probe: ProofTreeBuilder<'tcx>) { + if let Some(this) = self.as_mut() { + match (this, probe.state.unwrap().tree) { ( - DebugSolver::GoalCandidate(WipGoalCandidate { candidates, .. }) + DebugSolver::Probe(WipProbe { steps, .. }) | DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { - evaluation: WipGoalCandidate { candidates, .. }, + evaluation: WipProbe { steps, .. }, .. }), - DebugSolver::GoalCandidate(candidate), - ) => candidates.push(candidate), + DebugSolver::Probe(probe), + ) => steps.push(WipProbeStep::NestedProbe(probe)), _ => unreachable!(), } } @@ -416,14 +452,12 @@ impl<'tcx> ProofTreeBuilder<'tcx> { match (this, added_goals_evaluation.state.unwrap().tree) { ( DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep { - evaluation: WipGoalCandidate { added_goals_evaluations, .. }, + evaluation: WipProbe { steps, .. }, .. }) - | DebugSolver::GoalCandidate(WipGoalCandidate { - added_goals_evaluations, .. - }), + | DebugSolver::Probe(WipProbe { steps, .. }), DebugSolver::AddedGoalsEvaluation(added_goals_evaluation), - ) => added_goals_evaluations.push(added_goals_evaluation), + ) => steps.push(WipProbeStep::EvaluateGoals(added_goals_evaluation)), _ => unreachable!(), } } diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs index c492408bc7630..bd612ce4778db 100644 --- a/compiler/rustc_trait_selection/src/solve/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/mod.rs @@ -19,7 +19,8 @@ use rustc_infer::infer::canonical::{Canonical, CanonicalVarValues}; use rustc_infer::traits::query::NoSolution; use rustc_middle::infer::canonical::CanonicalVarInfos; use rustc_middle::traits::solve::{ - CanonicalResponse, Certainty, ExternalConstraintsData, Goal, QueryResult, Response, + CanonicalResponse, Certainty, ExternalConstraintsData, Goal, IsNormalizesToHack, QueryResult, + Response, }; use rustc_middle::ty::{self, Ty, TyCtxt, UniverseIndex}; use rustc_middle::ty::{ @@ -59,6 +60,12 @@ enum SolverMode { Coherence, } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum GoalEvaluationKind { + Root, + Nested { is_normalizes_to_hack: IsNormalizesToHack }, +} + trait CanonicalResponseExt { fn has_no_inference_or_external_constraints(&self) -> bool; @@ -228,6 +235,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { #[instrument(level = "debug", skip(self))] fn add_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) { + self.inspect.add_goal(goal); self.nested_goals.goals.push(goal); } diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs index c816b51f67acd..16de518e8e0b0 100644 --- a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs @@ -187,7 +187,7 @@ impl<'tcx> SearchGraph<'tcx> { last.encountered_overflow = true; } - inspect.goal_evaluation_kind(inspect::WipGoalEvaluationKind::Overflow); + inspect.goal_evaluation_kind(inspect::WipCanonicalGoalEvaluationKind::Overflow); return Self::response_no_constraints(tcx, input, Certainty::OVERFLOW); }; @@ -203,7 +203,7 @@ impl<'tcx> SearchGraph<'tcx> { available_depth, ) { - inspect.goal_evaluation_kind(inspect::WipGoalEvaluationKind::CacheHit( + inspect.goal_evaluation_kind(inspect::WipCanonicalGoalEvaluationKind::CacheHit( CacheHit::Global, )); self.on_cache_hit(reached_depth, encountered_overflow); @@ -240,7 +240,7 @@ impl<'tcx> SearchGraph<'tcx> { // Finally we can return either the provisional response for that goal if we have a // coinductive cycle or an ambiguous result if the cycle is inductive. Entry::Occupied(entry_index) => { - inspect.goal_evaluation_kind(inspect::WipGoalEvaluationKind::CacheHit( + inspect.goal_evaluation_kind(inspect::WipCanonicalGoalEvaluationKind::CacheHit( CacheHit::Provisional, )); diff --git a/config.example.toml b/config.example.toml index 2e0558c3f1b0d..f3c2366d674ea 100644 --- a/config.example.toml +++ b/config.example.toml @@ -458,7 +458,6 @@ changelog-seen = 2 # Sets the number of codegen units to build the standard library with, # regardless of what the codegen-unit setting for the rest of the compiler is. # NOTE: building with anything other than 1 is known to occasionally have bugs. -# See https://github.com/rust-lang/rust/issues/83600. #codegen-units-std = codegen-units # Whether or not debug assertions are enabled for the compiler and standard library. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index e4c7622645402..a9aa7524e8bc7 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -312,6 +312,14 @@ def default_build_triple(verbose): # non-standard string (e.g. gnuwin32 tools returns `windows32`). In # these cases, fall back to using sys.platform. return 'x86_64-pc-windows-msvc' + elif kernel == 'AIX': + # `uname -m` returns the machine ID rather than machine hardware on AIX, + # so we are unable to use cputype to form triple. AIX 7.2 and + # above supports 32-bit and 64-bit mode simultaneously and `uname -p` + # returns `powerpc`, however we only supports `powerpc64-ibm-aix` in + # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper + # are not used to infer AIX's triple. + return 'powerpc64-ibm-aix' else: err = "unknown OS type: {}".format(kernel) sys.exit(err) diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index b6b4fdc67a949..db11700af2815 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.10 +FROM ubuntu:23.04 ARG DEBIAN_FRONTEND=noninteractive COPY scripts/android-base-apt-get.sh /scripts/ diff --git a/src/ci/docker/host-x86_64/dist-android/Dockerfile b/src/ci/docker/host-x86_64/dist-android/Dockerfile index 9c6f648896b51..b09b6edb01a6d 100644 --- a/src/ci/docker/host-x86_64/dist-android/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.10 +FROM ubuntu:23.04 COPY scripts/android-base-apt-get.sh /scripts/ RUN sh /scripts/android-base-apt-get.sh diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh b/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh index dad9792233847..f86402b0180bb 100755 --- a/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh +++ b/src/ci/docker/host-x86_64/dist-various-1/install-x86_64-redox.sh @@ -2,5 +2,5 @@ set -ex -curl https://static.redox-os.org/toolchain/x86_64-unknown-redox/relibc-install.tar.gz | \ +curl https://ci-mirrors.rust-lang.org/rustc/2022-11-27-relibc-install.tar.gz | \ tar --extract --gzip --directory /usr/local diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index f6954275ad440..6f1b2a6a638be 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \ --set rust.lto=thin ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ - ./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \ + ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ build-manifest bootstrap diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 89b82d59d3176..1712d65ece971 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -624,7 +624,7 @@ jobs: --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-8c diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index 3f7dba81c3a9f..f1c3dd6aa9bb0 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" glob = "0.3" tempfile = "3.5" +derive_builder = "0.12" +clap = { version = "4", features = ["derive"] } diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs new file mode 100644 index 0000000000000..f7b5c176375bb --- /dev/null +++ b/src/tools/opt-dist/src/environment.rs @@ -0,0 +1,107 @@ +use camino::Utf8PathBuf; +use derive_builder::Builder; + +#[derive(Builder)] +pub struct Environment { + host_triple: String, + python_binary: String, + /// The rustc checkout, where the compiler source is located. + checkout_dir: Utf8PathBuf, + /// The main directory where the build occurs. Stage0 rustc and cargo have to be available in + /// this directory before `opt-dist` is started. + build_dir: Utf8PathBuf, + /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) + /// will be stored. + artifact_dir: Utf8PathBuf, + /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. + host_llvm_dir: Utf8PathBuf, + /// List of test paths that should be skipped when testing the optimized artifacts. + skipped_tests: Vec, + /// Directory containing a pre-built rustc-perf checkout. + prebuilt_rustc_perf: Option, + use_bolt: bool, + shared_llvm: bool, +} + +impl Environment { + pub fn host_triple(&self) -> &str { + &self.host_triple + } + + pub fn python_binary(&self) -> &str { + &self.python_binary + } + + pub fn checkout_path(&self) -> Utf8PathBuf { + self.checkout_dir.clone() + } + + pub fn build_root(&self) -> Utf8PathBuf { + self.build_dir.clone() + } + + pub fn build_artifacts(&self) -> Utf8PathBuf { + self.build_root().join("build").join(&self.host_triple) + } + + pub fn artifact_dir(&self) -> Utf8PathBuf { + self.artifact_dir.clone() + } + + pub fn cargo_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("cargo{}", executable_extension())) + } + + pub fn rustc_stage_0(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage0") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + pub fn rustc_stage_2(&self) -> Utf8PathBuf { + self.build_artifacts() + .join("stage2") + .join("bin") + .join(format!("rustc{}", executable_extension())) + } + + pub fn prebuilt_rustc_perf(&self) -> Option { + self.prebuilt_rustc_perf.clone() + } + + /// Path to the built rustc-perf benchmark suite. + pub fn rustc_perf_dir(&self) -> Utf8PathBuf { + self.artifact_dir.join("rustc-perf") + } + + pub fn host_llvm_dir(&self) -> Utf8PathBuf { + self.host_llvm_dir.clone() + } + + pub fn use_bolt(&self) -> bool { + self.use_bolt + } + + pub fn supports_shared_llvm(&self) -> bool { + self.shared_llvm + } + + pub fn skipped_tests(&self) -> &[String] { + &self.skipped_tests + } +} + +/// What is the extension of binary executables on this platform? +#[cfg(target_family = "unix")] +pub fn executable_extension() -> &'static str { + "" +} + +#[cfg(target_family = "windows")] +pub fn executable_extension() -> &'static str { + ".exe" +} diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs deleted file mode 100644 index 58b7e6d23067c..0000000000000 --- a/src/tools/opt-dist/src/environment/linux.rs +++ /dev/null @@ -1,58 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::copy_directory; -use camino::{Utf8Path, Utf8PathBuf}; - -pub(super) struct LinuxEnvironment; - -impl Environment for LinuxEnvironment { - fn python_binary(&self) -> &'static str { - "python3" - } - - fn checkout_path(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/checkout") - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/rustroot") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path().join("obj") - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // /tmp/rustc-perf comes from the x64 dist Dockerfile - copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?; - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - Ok(()) - } - - fn supports_bolt(&self) -> bool { - true - } - - fn supports_shared_llvm(&self) -> bool { - true - } - - fn executable_extension(&self) -> &'static str { - "" - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs deleted file mode 100644 index a8650fad011d9..0000000000000 --- a/src/tools/opt-dist/src/environment/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -use camino::Utf8PathBuf; - -#[cfg(target_family = "unix")] -mod linux; -#[cfg(target_family = "windows")] -mod windows; - -pub trait Environment { - fn host_triple(&self) -> String { - std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing") - } - - fn python_binary(&self) -> &'static str; - - /// The rustc checkout, where the compiler source is located. - fn checkout_path(&self) -> Utf8PathBuf; - - /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. - fn host_llvm_dir(&self) -> Utf8PathBuf; - - /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) - /// will be stored. - fn opt_artifacts(&self) -> Utf8PathBuf; - - /// The main directory where the build occurs. - fn build_root(&self) -> Utf8PathBuf; - - fn build_artifacts(&self) -> Utf8PathBuf { - self.build_root().join("build").join(self.host_triple()) - } - - fn cargo_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("cargo{}", self.executable_extension())) - } - - fn rustc_stage_0(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage0") - .join("bin") - .join(format!("rustc{}", self.executable_extension())) - } - - fn rustc_stage_2(&self) -> Utf8PathBuf { - self.build_artifacts() - .join("stage2") - .join("bin") - .join(format!("rustc{}", self.executable_extension())) - } - - /// Path to the built rustc-perf benchmark suite. - fn rustc_perf_dir(&self) -> Utf8PathBuf { - self.opt_artifacts().join("rustc-perf") - } - - /// Download and/or compile rustc-perf. - fn prepare_rustc_perf(&self) -> anyhow::Result<()>; - - fn supports_bolt(&self) -> bool; - - fn supports_shared_llvm(&self) -> bool; - - /// What is the extension of binary executables in this environment? - fn executable_extension(&self) -> &'static str; - - /// List of test paths that should be skipped when testing the optimized artifacts. - fn skipped_tests(&self) -> &'static [&'static str]; -} - -pub fn create_environment() -> Box { - #[cfg(target_family = "unix")] - return Box::new(linux::LinuxEnvironment); - #[cfg(target_family = "windows")] - return Box::new(windows::WindowsEnvironment::new()); -} diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs deleted file mode 100644 index 79399391798cb..0000000000000 --- a/src/tools/opt-dist/src/environment/windows.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::environment::Environment; -use crate::exec::cmd; -use crate::utils::io::move_directory; -use crate::utils::retry_action; -use camino::Utf8PathBuf; -use std::io::Cursor; -use std::time::Duration; -use zip::ZipArchive; - -pub(super) struct WindowsEnvironment { - checkout_dir: Utf8PathBuf, -} - -impl WindowsEnvironment { - pub fn new() -> Self { - Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap() } - } -} - -impl Environment for WindowsEnvironment { - fn python_binary(&self) -> &'static str { - "python" - } - - fn checkout_path(&self) -> Utf8PathBuf { - self.checkout_dir.clone() - } - - fn host_llvm_dir(&self) -> Utf8PathBuf { - self.checkout_path().join("citools").join("clang-rust") - } - - fn opt_artifacts(&self) -> Utf8PathBuf { - self.checkout_path().join("opt-artifacts") - } - - fn build_root(&self) -> Utf8PathBuf { - self.checkout_path() - } - - fn prepare_rustc_perf(&self) -> anyhow::Result<()> { - // FIXME: add some mechanism for synchronization of this commit SHA with - // Linux (which builds rustc-perf in a Dockerfile) - // rustc-perf version from 2023-05-30 - const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; - - let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(60 * 2)) - .connect_timeout(Duration::from_secs(60 * 2)) - .build()?; - let response = retry_action( - || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), - "Download rustc-perf archive", - 5, - )?; - - let mut archive = ZipArchive::new(Cursor::new(response))?; - archive.extract(self.rustc_perf_dir())?; - move_directory( - &self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), - &self.rustc_perf_dir(), - )?; - - cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"]) - .workdir(&self.rustc_perf_dir()) - .env("RUSTC", &self.rustc_stage_0().into_string()) - .env("RUSTC_BOOTSTRAP", "1") - .run()?; - - Ok(()) - } - - fn supports_bolt(&self) -> bool { - false - } - - fn supports_shared_llvm(&self) -> bool { - false - } - - fn executable_extension(&self) -> &'static str { - ".exe" - } - - fn skipped_tests(&self) -> &'static [&'static str] { - &[ - // Fails as of June 2023. - "tests\\codegen\\vec-shrink-panik.rs", - ] - } -} diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs index 4765dceb5dddd..04e0184528a5f 100644 --- a/src/tools/opt-dist/src/exec.rs +++ b/src/tools/opt-dist/src/exec.rs @@ -96,7 +96,7 @@ pub struct Bootstrap { } impl Bootstrap { - pub fn build(env: &dyn Environment) -> Self { + pub fn build(env: &Environment) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&[ env.python_binary(), @@ -114,7 +114,7 @@ impl Bootstrap { Self { cmd, metrics_path } } - pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self { + pub fn dist(env: &Environment, dist_args: &[String]) -> Self { let metrics_path = env.build_root().join("build").join("metrics.json"); let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::>()) .env("RUST_BACKTRACE", "full"); diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 8ab19674d05b7..978e2dfa4e864 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -1,17 +1,22 @@ use crate::bolt::{bolt_optimize, with_bolt_instrumented}; use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; +use clap::Parser; use log::LevelFilter; +use std::io::Cursor; +use std::time::Duration; use utils::io; +use zip::ZipArchive; -use crate::environment::{create_environment, Environment}; -use crate::exec::Bootstrap; +use crate::environment::{Environment, EnvironmentBuilder}; +use crate::exec::{cmd, Bootstrap}; use crate::tests::run_tests; use crate::timer::Timer; use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles}; -use crate::utils::io::reset_directory; +use crate::utils::io::{copy_directory, move_directory, reset_directory}; use crate::utils::{ clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space, - with_log_group, + retry_action, with_log_group, }; mod bolt; @@ -23,24 +28,169 @@ mod timer; mod training; mod utils; +#[derive(clap::Parser, Debug)] +struct Args { + #[clap(subcommand)] + env: EnvironmentCmd, +} + +#[derive(clap::Parser, Clone, Debug)] +struct SharedArgs { + // Arguments passed to `x` to perform the final (dist) build. + build_args: Vec, +} + +#[derive(clap::Parser, Clone, Debug)] +enum EnvironmentCmd { + /// Perform a custom local PGO/BOLT optimized build. + Local { + /// Target triple of the host. + #[arg(long)] + target_triple: String, + + /// Checkout directory of `rustc`. + #[arg(long)] + checkout_dir: Utf8PathBuf, + + /// Host LLVM installation directory. + #[arg(long)] + llvm_dir: Utf8PathBuf, + + /// Python binary to use in bootstrap invocations. + #[arg(long, default_value = "python3")] + python: String, + + /// Directory where artifacts (like PGO profiles or rustc-perf) of this workflow + /// will be stored. + #[arg(long, default_value = "opt-artifacts")] + artifact_dir: Utf8PathBuf, + + /// Is LLVM for `rustc` built in shared library mode? + #[arg(long, default_value_t = true)] + llvm_shared: bool, + + /// Should BOLT optimization be used? If yes, host LLVM must have BOLT binaries + /// (`llvm-bolt` and `merge-fdata`) available. + #[arg(long, default_value_t = false)] + use_bolt: bool, + + /// Tests that should be skipped when testing the optimized compiler. + #[arg(long)] + skipped_tests: Vec, + + #[clap(flatten)] + shared: SharedArgs, + }, + /// Perform an optimized build on Linux CI, from inside Docker. + LinuxCi { + #[clap(flatten)] + shared: SharedArgs, + }, + /// Perform an optimized build on Windows CI, directly inside Github Actions. + WindowsCi { + #[clap(flatten)] + shared: SharedArgs, + }, +} + fn is_try_build() -> bool { std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0" } +fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> { + let (env, args) = match args.env { + EnvironmentCmd::Local { + target_triple, + checkout_dir, + llvm_dir, + python, + artifact_dir, + llvm_shared, + use_bolt, + skipped_tests, + shared, + } => { + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary(python) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(llvm_dir) + .artifact_dir(artifact_dir) + .build_dir(checkout_dir) + .shared_llvm(llvm_shared) + .use_bolt(use_bolt) + .skipped_tests(skipped_tests) + .build()?; + + (env, shared.build_args) + } + EnvironmentCmd::LinuxCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir = Utf8PathBuf::from("/checkout"); + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python3".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(Utf8PathBuf::from("/rustroot")) + .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) + .build_dir(checkout_dir.join("obj")) + // /tmp/rustc-perf comes from the x64 dist Dockerfile + .prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf"))) + .shared_llvm(true) + .use_bolt(true) + .skipped_tests(vec![ + // Fails because of linker errors, as of June 2023. + "tests/ui/process/nofile-limit.rs".to_string(), + ]) + .build()?; + + (env, shared.build_args) + } + EnvironmentCmd::WindowsCi { shared } => { + let target_triple = + std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + + let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?; + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary("python".to_string()) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(checkout_dir.join("citools").join("clang-rust")) + .artifact_dir(checkout_dir.join("opt-artifacts")) + .build_dir(checkout_dir) + .shared_llvm(false) + .use_bolt(false) + .skipped_tests(vec![ + // Fails as of June 2023. + "tests\\codegen\\vec-shrink-panik.rs".to_string(), + ]) + .build()?; + + (env, shared.build_args) + } + }; + Ok((env, args)) +} + fn execute_pipeline( - env: &dyn Environment, + env: &Environment, timer: &mut Timer, dist_args: Vec, ) -> anyhow::Result<()> { - reset_directory(&env.opt_artifacts())?; + reset_directory(&env.artifact_dir())?; - with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?; + with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() { + Some(dir) => copy_rustc_perf(env, &dir), + None => download_rustc_perf(env), + })?; // Stage 1: Build PGO instrumented rustc // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the // same time can cause issues, because the host and in-tree LLVM versions can diverge. let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| { - let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo"); + let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo"); stage.section("Build PGO instrumented rustc and LLVM", |section| { let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root); @@ -74,7 +224,7 @@ fn execute_pipeline( // Remove the previous, uninstrumented build of LLVM. clear_llvm_files(env)?; - let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo"); + let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo"); stage.section("Build PGO instrumented LLVM", |section| { Bootstrap::build(env) @@ -95,7 +245,7 @@ fn execute_pipeline( Ok(profile) })?; - let llvm_bolt_profile = if env.supports_bolt() { + let llvm_bolt_profile = if env.use_bolt() { // Stage 3: Build BOLT instrumented LLVM // We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles. // Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build. @@ -171,8 +321,9 @@ fn main() -> anyhow::Result<()> { .parse_default_env() .init(); - let mut build_args: Vec = std::env::args().skip(1).collect(); - println!("Running optimized build pipeline with args `{}`", build_args.join(" ")); + let args = Args::parse(); + + println!("Running optimized build pipeline with args `{:?}`", args); with_log_group("Environment values", || { println!("Environment values\n{}", format_env_variables()); @@ -184,6 +335,8 @@ fn main() -> anyhow::Result<()> { } }); + let (env, mut build_args) = create_environment(args).context("Cannot create environment")?; + // Skip components that are not needed for try builds to speed them up if is_try_build() { log::info!("Skipping building of unimportant components for a try build"); @@ -202,14 +355,58 @@ fn main() -> anyhow::Result<()> { } let mut timer = Timer::new(); - let env = create_environment(); - let result = execute_pipeline(env.as_ref(), &mut timer, build_args); + let result = execute_pipeline(&env, &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); print_free_disk_space()?; result.context("Optimized build pipeline has failed")?; - print_binary_sizes(env.as_ref())?; + print_binary_sizes(&env)?; + + Ok(()) +} + +// Copy rustc-perf from the given path into the environment and build it. +fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { + copy_directory(dir, &env.rustc_perf_dir())?; + build_rustc_perf(env) +} + +// Download and build rustc-perf into the given environment. +fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { + reset_directory(&env.rustc_perf_dir())?; + + // FIXME: add some mechanism for synchronization of this commit SHA with + // Linux (which builds rustc-perf in a Dockerfile) + // rustc-perf version from 2023-05-30 + const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1"; + + let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(60 * 2)) + .connect_timeout(Duration::from_secs(60 * 2)) + .build()?; + let response = retry_action( + || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), + "Download rustc-perf archive", + 5, + )?; + + let mut archive = ZipArchive::new(Cursor::new(response))?; + archive.extract(env.rustc_perf_dir())?; + move_directory( + &env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), + &env.rustc_perf_dir(), + )?; + + build_rustc_perf(env) +} +fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> { + cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"]) + .workdir(&env.rustc_perf_dir()) + .env("RUSTC", &env.rustc_stage_0().into_string()) + .env("RUSTC_BOOTSTRAP", "1") + .run()?; Ok(()) } diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 3dd1a3223f5a5..31aabca09f39e 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -1,11 +1,11 @@ -use crate::environment::Environment; +use crate::environment::{executable_extension, Environment}; use crate::exec::cmd; use crate::utils::io::{copy_directory, find_file_in_dir, unpack_archive}; use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; /// Run tests on optimized dist artifacts. -pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { +pub fn run_tests(env: &Environment) -> anyhow::Result<()> { // After `dist` is executed, we extract its archived components into a sysroot directory, // and then use that extracted rustc as a stage0 compiler. // Then we run a subset of tests using that compiler, to have a basic smoke test which checks @@ -33,8 +33,8 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { // We need to manually copy libstd to the extracted rustc sysroot copy_directory( - &libstd_dir.join("lib").join("rustlib").join(&host_triple).join("lib"), - &rustc_dir.join("lib").join("rustlib").join(&host_triple).join("lib"), + &libstd_dir.join("lib").join("rustlib").join(host_triple).join("lib"), + &rustc_dir.join("lib").join("rustlib").join(host_triple).join("lib"), )?; // Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd @@ -45,9 +45,9 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { &rustc_dir.join("lib").join("rustlib").join("src"), )?; - let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", env.executable_extension())); + let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", executable_extension())); assert!(rustc_path.is_file()); - let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", env.executable_extension())); + let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", executable_extension())); assert!(cargo_path.is_file()); // Specify path to a LLVM config so that LLVM is not rebuilt. @@ -56,7 +56,7 @@ pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> { .build_artifacts() .join("llvm") .join("bin") - .join(format!("llvm-config{}", env.executable_extension())); + .join(format!("llvm-config{}", executable_extension())); assert!(llvm_config.is_file()); let config_content = format!( @@ -109,6 +109,6 @@ fn find_dist_version(directory: &Utf8Path) -> anyhow::Result { .unwrap() .to_string(); let (version, _) = - archive.strip_prefix("reproducible-artifacts-").unwrap().split_once("-").unwrap(); + archive.strip_prefix("reproducible-artifacts-").unwrap().split_once('-').unwrap(); Ok(version.to_string()) } diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 59c73fbd6953e..274f4cea0ab5f 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -1,4 +1,4 @@ -use crate::environment::Environment; +use crate::environment::{executable_extension, Environment}; use crate::exec::{cmd, CmdBuilder}; use crate::utils::io::{count_files, delete_directory}; use crate::utils::with_log_group; @@ -30,7 +30,7 @@ const RUSTC_PGO_CRATES: &[&str] = &[ const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES; fn init_compiler_benchmarks( - env: &dyn Environment, + env: &Environment, profiles: &[&str], scenarios: &[&str], crates: &[&str], @@ -75,7 +75,7 @@ enum LlvmProfdata { } fn merge_llvm_profiles( - env: &dyn Environment, + env: &Environment, merged_path: &Utf8Path, profile_dir: &Utf8Path, profdata: LlvmProfdata, @@ -86,7 +86,7 @@ fn merge_llvm_profiles( .build_artifacts() .join("llvm") .join("build") - .join(format!("bin/llvm-profdata{}", env.executable_extension())), + .join(format!("bin/llvm-profdata{}", executable_extension())), }; cmd(&[llvm_profdata.as_str(), "merge", "-o", merged_path.as_str(), profile_dir.as_str()]) @@ -116,7 +116,7 @@ fn log_profile_stats( pub struct LlvmPGOProfile(pub Utf8PathBuf); pub fn gather_llvm_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented LLVM"); @@ -127,7 +127,7 @@ pub fn gather_llvm_profiles( .context("Cannot gather LLVM PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata"); + let merged_profile = env.artifact_dir().join("llvm-pgo.profdata"); log::info!("Merging LLVM PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?; @@ -143,7 +143,7 @@ pub fn gather_llvm_profiles( pub struct RustcPGOProfile(pub Utf8PathBuf); pub fn gather_rustc_profiles( - env: &dyn Environment, + env: &Environment, profile_root: &Utf8Path, ) -> anyhow::Result { log::info!("Running benchmarks with PGO instrumented rustc"); @@ -163,7 +163,7 @@ pub fn gather_rustc_profiles( .context("Cannot gather rustc PGO profiles") })?; - let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata"); + let merged_profile = env.artifact_dir().join("rustc-pgo.profdata"); log::info!("Merging Rustc PGO profiles to {merged_profile}"); merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?; @@ -178,7 +178,7 @@ pub fn gather_rustc_profiles( pub struct LlvmBoltProfile(pub Utf8PathBuf); -pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result { +pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result { log::info!("Running benchmarks with BOLT instrumented LLVM"); with_log_group("Running benchmarks", || { @@ -187,12 +187,12 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result = - glob::glob(&format!("{profile_root}*"))?.into_iter().collect::, _>>()?; + glob::glob(&format!("{profile_root}*"))?.collect::, _>>()?; let mut merge_args = vec!["merge-fdata"]; merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap())); diff --git a/src/tools/opt-dist/src/utils/io.rs b/src/tools/opt-dist/src/utils/io.rs index 8bd516fa349b0..d6bd5cb85e4af 100644 --- a/src/tools/opt-dist/src/utils/io.rs +++ b/src/tools/opt-dist/src/utils/io.rs @@ -7,7 +7,7 @@ use std::path::Path; /// Delete and re-create the directory. pub fn reset_directory(path: &Utf8Path) -> anyhow::Result<()> { log::info!("Resetting directory {path}"); - let _ = std::fs::remove_dir(path); + let _ = std::fs::remove_dir_all(path); std::fs::create_dir_all(path)?; Ok(()) } @@ -63,7 +63,6 @@ pub fn get_files_from_dir( let path = format!("{dir}/*{}", suffix.unwrap_or("")); Ok(glob::glob(&path)? - .into_iter() .map(|p| p.map(|p| Utf8PathBuf::from_path_buf(p).unwrap())) .collect::, _>>()?) } @@ -74,9 +73,8 @@ pub fn find_file_in_dir( prefix: &str, suffix: &str, ) -> anyhow::Result { - let files = glob::glob(&format!("{directory}/{prefix}*{suffix}"))? - .into_iter() - .collect::, _>>()?; + let files = + glob::glob(&format!("{directory}/{prefix}*{suffix}"))?.collect::, _>>()?; match files.len() { 0 => Err(anyhow::anyhow!("No file with prefix {prefix} found in {directory}")), 1 => Ok(Utf8PathBuf::from_path_buf(files[0].clone()).unwrap()), diff --git a/src/tools/opt-dist/src/utils/mod.rs b/src/tools/opt-dist/src/utils/mod.rs index 2af28077ed12e..6fc96592a8858 100644 --- a/src/tools/opt-dist/src/utils/mod.rs +++ b/src/tools/opt-dist/src/utils/mod.rs @@ -26,7 +26,7 @@ pub fn print_free_disk_space() -> anyhow::Result<()> { Ok(()) } -pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { +pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> { use std::fmt::Write; let root = env.build_artifacts().join("stage2"); @@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { Ok(()) } -pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> { +pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> { // Bootstrap currently doesn't support rebuilding LLVM when PGO options // change (or any other llvm-related options); so just clear out the relevant // directories ourselves. diff --git a/triagebot.toml b/triagebot.toml index d9d523bef39f3..d35e81c277ea2 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -585,7 +585,7 @@ cc = ["@nnethercote"] [assign] warn_non_default_branch = true contributing_url = "https://rustc-dev-guide.rust-lang.org/getting-started.html" -users_on_vacation = ["jyn514"] +users_on_vacation = ["jyn514", "jackh726"] [assign.adhoc_groups] compiler-team = [