diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index cb7529b527e8f..42415b48e6342 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -16,12 +16,11 @@ use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE}; use rustc_hir::Crate; use rustc_lint::LintStore; use rustc_metadata::creader::CStore; -use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; use rustc_middle::middle; use rustc_middle::middle::cstore::{MetadataLoader, MetadataLoaderDyn}; use rustc_middle::ty::query::Providers; -use rustc_middle::ty::{self, GlobalCtxt, ResolverOutputs, TyCtxt}; +use rustc_middle::ty::{self, GlobalCtxt, ResolverOutputs, TcxArena, TyCtxt}; use rustc_mir_build as mir_build; use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str}; use rustc_passes::{self, hir_stats, layout_test}; @@ -96,6 +95,7 @@ mod boxed_resolver { session: Lrc, resolver_arenas: Option>, resolver: Option>, + tcx_arena: TcxArena<'static>, _pin: PhantomPinned, } @@ -111,12 +111,18 @@ mod boxed_resolver { impl BoxedResolver { pub(super) fn new( session: Lrc, - make_resolver: impl for<'a> FnOnce(&'a Session, &'a ResolverArenas<'a>) -> Resolver<'a>, + tcx_arena: TcxArena<'static>, + make_resolver: impl for<'a> FnOnce( + &'a Session, + &'a ResolverArenas<'a>, + TcxArena<'a>, + ) -> Resolver<'a>, ) -> BoxedResolver { let mut boxed_resolver = Box::new(BoxedResolverInner { session, resolver_arenas: Some(Resolver::arenas()), resolver: None, + tcx_arena, _pin: PhantomPinned, }); // SAFETY: `make_resolver` takes a resolver arena with an arbitrary lifetime and @@ -128,6 +134,7 @@ mod boxed_resolver { std::mem::transmute::<&ResolverArenas<'_>, &ResolverArenas<'_>>( boxed_resolver.resolver_arenas.as_ref().unwrap(), ), + std::mem::transmute::, TcxArena<'_>>(boxed_resolver.tcx_arena), ); boxed_resolver.resolver = Some(resolver); BoxedResolver(Pin::new_unchecked(boxed_resolver)) @@ -166,10 +173,13 @@ pub fn create_resolver( metadata_loader: Box, krate: &ast::Crate, crate_name: &str, + tcx_arena: usize, ) -> BoxedResolver { + // FIXME: Solve the problem of tcx_arena vs sess lifetimes somehow. + let tcx_arena = unsafe { std::mem::transmute(tcx_arena) }; tracing::trace!("create_resolver"); - BoxedResolver::new(sess, move |sess, resolver_arenas| { - Resolver::new(sess, &krate, &crate_name, metadata_loader, &resolver_arenas) + BoxedResolver::new(sess, tcx_arena, move |sess, resolver_arenas, tcx_arena| { + Resolver::new(sess, &krate, &crate_name, metadata_loader, &resolver_arenas, tcx_arena) }) } @@ -789,7 +799,7 @@ pub fn create_global_ctxt<'tcx>( crate_name: &str, queries: &'tcx OnceCell>, global_ctxt: &'tcx OnceCell>, - arena: &'tcx WorkerLocal>, + arena: TcxArena<'tcx>, hir_arena: &'tcx WorkerLocal>, ) -> QueryContext<'tcx> { // We're constructing the HIR here; we don't care what we will diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs index 2f540395b2d79..d8f5026126dd9 100644 --- a/compiler/rustc_interface/src/queries.rs +++ b/compiler/rustc_interface/src/queries.rs @@ -179,6 +179,7 @@ impl<'tcx> Queries<'tcx> { self.codegen_backend().metadata_loader(), &krate, &crate_name, + &self.arena as *const _ as usize, ); let krate = resolver.access(|resolver| { passes::configure_and_expand(&sess, &lint_store, krate, &crate_name, resolver) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index eb3a9f576a735..6e1a33593aec5 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -28,7 +28,7 @@ use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState}; use rustc_middle::mir::{self, Body, Promoted}; use rustc_middle::thir; use rustc_middle::ty::codec::TyDecoder; -use rustc_middle::ty::{self, Ty, TyCtxt, Visibility}; +use rustc_middle::ty::{self, TcxArena, Ty, TyCtxt, Visibility}; use rustc_serialize::{opaque, Decodable, Decoder}; use rustc_session::Session; use rustc_span::hygiene::{ExpnIndex, MacroKind}; @@ -37,10 +37,10 @@ use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{self, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP}; use proc_macro::bridge::client::ProcMacro; -use std::io; -use std::mem; +use std::hash::Hash; use std::num::NonZeroUsize; use std::path::Path; +use std::{io, mem}; use tracing::debug; pub use cstore_impl::{provide, provide_extern}; @@ -135,8 +135,27 @@ crate struct CrateMetadata { /// Information about the `extern crate` item or path that caused this crate to be loaded. /// If this is `None`, then the crate was injected (e.g., by the allocator). extern_crate: Lock>, + + // cache + // FIXME: Replace static lifetimes with a lifetime parameter. + struct_field_names: Cache]>, + struct_field_visibilities: Cache, + ctor_def_ids_and_kinds: Cache>, + visibilities: Cache, + item_children: Cache, + associated_items: Cache, + spans: Cache, + def_kinds: Cache, + generics: Cache, + module_expansions: Cache, + proc_macro_quoted_spans: Cache, + item_attrs: Cache, } +type Cache = Lock>; +#[allow(dead_code)] +type CacheIndexVec = Lock>>; + /// Holds information about a rustc_span::SourceFile imported from another crate. /// See `imported_source_files()` for more information. struct ImportedSourceFile { @@ -164,6 +183,58 @@ pub(super) struct DecodeContext<'a, 'tcx> { alloc_decoding_session: Option>, } +fn get_or_insert_with( + cache: &mut FxHashMap, + k: K, + v: impl FnOnce() -> V, +) -> &V { + cache.entry(k).or_insert_with(v) +} + +fn steal_or_create_with( + cache: &mut FxHashMap, + k: K, + v: impl FnOnce() -> V, +) -> V { + cache.remove(&k).unwrap_or_else(v) +} + +fn get_or_create_with( + cache: &mut FxHashMap, + k: K, + v: impl FnOnce() -> V, +) -> V { + cache.get(&k).copied().unwrap_or_else(v) +} + +#[allow(dead_code)] +fn get_or_insert_with_index_vec( + cache: &mut IndexVec>, + k: K, + v: impl FnOnce() -> V, +) -> &V { + cache.ensure_contains_elem(k, || None); + match &mut cache[k] { + entry @ None => { + *entry = Some(v()); + entry.as_ref().unwrap() + } + Some(v) => v, + } +} + +#[allow(dead_code)] +fn steal_or_create_with_index_vec( + cache: &mut IndexVec>, + k: K, + v: impl FnOnce() -> V, +) -> V { + match cache.get_mut(k) { + Some(entry @ Some(..)) => entry.take().unwrap(), + _ => v(), + } +} + /// Abstract over the various ways one can create metadata decoders. pub(super) trait Metadata<'a, 'tcx>: Copy { fn blob(self) -> &'a MetadataBlob; @@ -758,26 +829,37 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } fn def_kind(&self, item_id: DefIndex) -> DefKind { - self.root.tables.def_kind.get(self, item_id).map(|k| k.decode(self)).unwrap_or_else(|| { - bug!( - "CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}", - item_id, - self.root.name, - self.cnum, + *get_or_insert_with(&mut self.def_kinds.lock(), item_id, || { + self.root.tables.def_kind.get(self, item_id).map(|k| k.decode(self)).unwrap_or_else( + || { + bug!( + "CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}", + item_id, + self.root.name, + self.cnum, + ) + }, ) }) } fn get_span(&self, index: DefIndex, sess: &Session) -> Span { - self.root - .tables - .span - .get(self, index) - .unwrap_or_else(|| panic!("Missing span for {:?}", index)) - .decode((self, sess)) + *get_or_insert_with(&mut self.spans.lock(), index, || { + self.root + .tables + .span + .get(self, index) + .unwrap_or_else(|| panic!("Missing span for {:?}", index)) + .decode((self, sess)) + }) } - fn load_proc_macro(&self, id: DefIndex, sess: &Session) -> SyntaxExtension { + fn load_proc_macro( + &self, + id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> SyntaxExtension { let (name, kind, helper_attrs) = match *self.raw_proc_macro(id) { ProcMacro::CustomDerive { trait_name, attributes, client } => { let helper_attrs = @@ -796,7 +878,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } }; - let attrs: Vec<_> = self.get_item_attrs(id, sess).collect(); SyntaxExtension::new( sess, kind, @@ -804,7 +885,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { helper_attrs, self.root.edition, Symbol::intern(name), - &attrs, + self.get_item_attrs_cache_fill(id, sess, tcx_arena), ) } @@ -956,7 +1037,24 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .unwrap_or_default() } - fn get_generics(&self, item_id: DefIndex, sess: &Session) -> ty::Generics { + fn get_generics_cache_steal(&self, item_id: DefIndex, sess: &Session) -> ty::Generics { + steal_or_create_with(&mut self.generics.lock(), item_id, || { + self.get_generics_uncached(item_id, sess) + }) + } + + fn get_generics_cache_fill( + &self, + item_id: DefIndex, + sess: &Session, + callback: impl FnOnce(&ty::Generics) -> R, + ) -> R { + callback(get_or_insert_with(&mut self.generics.lock(), item_id, || { + self.get_generics_uncached(item_id, sess) + })) + } + + fn get_generics_uncached(&self, item_id: DefIndex, sess: &Session) -> ty::Generics { self.root.tables.generics.get(self, item_id).unwrap().decode((self, sess)) } @@ -982,7 +1080,9 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } fn get_visibility(&self, id: DefIndex) -> ty::Visibility { - self.root.tables.visibility.get(self, id).unwrap().decode(self) + *get_or_insert_with(&mut self.visibilities.lock(), id, || { + self.root.tables.visibility.get(self, id).unwrap().decode(self) + }) } fn get_impl_data(&self, id: DefIndex) -> ImplData { @@ -1064,34 +1164,63 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } } - /// Iterates over each child of the given item. - fn each_child_of_item(&self, id: DefIndex, mut callback: impl FnMut(Export), sess: &Session) { + fn get_item_children_cache_nofill( + &self, + id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [Export] { + get_or_create_with(&mut self.item_children.lock(), id, || { + self.get_item_children_uncached(id, sess, tcx_arena) + }) + } + + fn get_item_children_cache_fill( + &self, + id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [Export] { + *get_or_insert_with(&mut self.item_children.lock(), id, || { + self.get_item_children_uncached(id, sess, tcx_arena) + }) + } + + /// Collects all children of the given item. + fn get_item_children_uncached( + &self, + id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'static [Export] { if let Some(data) = &self.root.proc_macro_data { /* If we are loading as a proc macro, we want to return the view of this crate * as a proc macro crate. */ - if id == CRATE_DEF_INDEX { - let macros = data.macros.decode(self); - for def_index in macros { + return if id == CRATE_DEF_INDEX { + let res = tcx_arena.alloc_from_iter(data.macros.decode(self).map(|def_index| { let raw_macro = self.raw_proc_macro(def_index); let res = Res::Def( DefKind::Macro(macro_kind(raw_macro)), self.local_def_id(def_index), ); let ident = self.item_ident(def_index, sess); - callback(Export { ident, res, vis: ty::Visibility::Public, span: ident.span }); - } - } - return; + Export { ident, res, vis: ty::Visibility::Public, span: ident.span } + })); + unsafe { mem::transmute(res) } + } else { + &[] + }; } // Find the item. let kind = match self.maybe_kind(id) { - None => return, + None => return &[], Some(kind) => kind, }; // Iterate over all children. + let mut result = Vec::new(); let macros_only = self.dep_kind.lock().macros_only(); if !macros_only { let children = self.root.tables.children.get(self, id).unwrap_or_else(Lazy::empty); @@ -1115,7 +1244,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .unwrap_or_else(Lazy::empty); for child_index in child_children.decode((self, sess)) { let kind = self.def_kind(child_index); - callback(Export { + result.push(Export { res: Res::Def(kind, self.local_def_id(child_index)), ident: self.item_ident(child_index, sess), vis: self.get_visibility(child_index), @@ -1147,19 +1276,20 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { // FIXME: Macros are currently encoded twice, once as items and once as // reexports. We ignore the items here and only use the reexports. if !matches!(kind, DefKind::Macro(..)) { - callback(Export { res, ident, vis, span }); + result.push(Export { res, ident, vis, span }); } // For non-re-export structs and variants add their constructors to children. // Re-export lists automatically contain constructors when necessary. match kind { DefKind::Struct => { - if let Some(ctor_def_id) = self.get_ctor_def_id(child_index) { - let ctor_kind = self.get_ctor_kind(child_index); + if let Some((ctor_def_id, ctor_kind)) = + self.get_ctor_def_id_and_kind(child_index) + { let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Struct, ctor_kind), ctor_def_id); let vis = self.get_visibility(ctor_def_id.index); - callback(Export { res: ctor_res, vis, ident, span }); + result.push(Export { res: ctor_res, vis, ident, span }); } } DefKind::Variant => { @@ -1167,8 +1297,9 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { // value namespace, they are reserved for possible future use. // It's ok to use the variant's id as a ctor id since an // error will be reported on any use of such resolution anyway. - let ctor_def_id = self.get_ctor_def_id(child_index).unwrap_or(def_id); - let ctor_kind = self.get_ctor_kind(child_index); + let (ctor_def_id, ctor_kind) = self + .get_ctor_def_id_and_kind(child_index) + .unwrap_or((def_id, CtorKind::Fictive)); let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Variant, ctor_kind), ctor_def_id); let mut vis = self.get_visibility(ctor_def_id.index); @@ -1177,13 +1308,16 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { // within the crate. We only need this for fictive constructors, // for other constructors correct visibilities // were already encoded in metadata. - let mut attrs = self.get_item_attrs(def_id.index, sess); - if attrs.any(|item| item.has_name(sym::non_exhaustive)) { + if self + .get_item_attrs_cache_fill(def_id.index, sess, tcx_arena) + .iter() + .any(|item| item.has_name(sym::non_exhaustive)) + { let crate_def_id = self.local_def_id(CRATE_DEF_INDEX); vis = ty::Visibility::Restricted(crate_def_id); } } - callback(Export { res: ctor_res, ident, vis, span }); + result.push(Export { res: ctor_res, ident, vis, span }); } _ => {} } @@ -1198,9 +1332,12 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { _ if macros_only => continue, _ => {} } - callback(exp); + result.push(exp); } } + + let res = tcx_arena.alloc_slice(&result); + unsafe { mem::transmute(res) } } fn is_ctfe_mir_available(&self, id: DefIndex) -> bool { @@ -1212,11 +1349,13 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } fn module_expansion(&self, id: DefIndex, sess: &Session) -> ExpnId { - if let EntryKind::Mod(m) = self.kind(id) { - m.decode((self, sess)).expansion - } else { - panic!("Expected module, found {:?}", self.local_def_id(id)) - } + *get_or_insert_with(&mut self.module_expansions.lock(), id, || { + if let EntryKind::Mod(m) = self.kind(id) { + m.decode((self, sess)).expansion + } else { + panic!("Expected module, found {:?}", self.local_def_id(id)) + } + }) } fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> Body<'tcx> { @@ -1289,64 +1428,80 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } fn get_associated_item(&self, id: DefIndex, sess: &Session) -> ty::AssocItem { - let def_key = self.def_key(id); - let parent = self.local_def_id(def_key.parent.unwrap()); - let ident = self.item_ident(id, sess); - - let (kind, container, has_self) = match self.kind(id) { - EntryKind::AssocConst(container, _, _) => (ty::AssocKind::Const, container, false), - EntryKind::AssocFn(data) => { - let data = data.decode(self); - (ty::AssocKind::Fn, data.container, data.has_self) - } - EntryKind::AssocType(container) => (ty::AssocKind::Type, container, false), - _ => bug!("cannot get associated-item of `{:?}`", def_key), - }; + *get_or_insert_with(&mut self.associated_items.lock(), id, || { + let def_key = self.def_key(id); + let parent = self.local_def_id(def_key.parent.unwrap()); + let ident = self.item_ident(id, sess); + + let (kind, container, has_self) = match self.kind(id) { + EntryKind::AssocConst(container, _, _) => (ty::AssocKind::Const, container, false), + EntryKind::AssocFn(data) => { + let data = data.decode(self); + (ty::AssocKind::Fn, data.container, data.has_self) + } + EntryKind::AssocType(container) => (ty::AssocKind::Type, container, false), + _ => bug!("cannot get associated-item of `{:?}`", def_key), + }; - ty::AssocItem { - ident, - kind, - vis: self.get_visibility(id), - defaultness: container.defaultness(), - def_id: self.local_def_id(id), - container: container.with_def_id(parent), - fn_has_self_parameter: has_self, - } + ty::AssocItem { + ident, + kind, + vis: self.get_visibility(id), + defaultness: container.defaultness(), + def_id: self.local_def_id(id), + container: container.with_def_id(parent), + fn_has_self_parameter: has_self, + } + }) } fn get_item_variances(&'a self, id: DefIndex) -> impl Iterator + 'a { self.root.tables.variances.get(self, id).unwrap_or_else(Lazy::empty).decode(self) } - fn get_ctor_kind(&self, node_id: DefIndex) -> CtorKind { - match self.kind(node_id) { - EntryKind::Struct(data, _) | EntryKind::Union(data, _) | EntryKind::Variant(data) => { - data.decode(self).ctor_kind + fn get_ctor_def_id_and_kind(&self, node_id: DefIndex) -> Option<(DefId, CtorKind)> { + *get_or_insert_with(&mut self.ctor_def_ids_and_kinds.lock(), node_id, || { + match self.kind(node_id) { + EntryKind::Struct(data, _) | EntryKind::Variant(data) => { + let vdata = data.decode(self); + vdata.ctor.map(|index| (self.local_def_id(index), vdata.ctor_kind)) + } + _ => None, } - _ => CtorKind::Fictive, - } + }) } - fn get_ctor_def_id(&self, node_id: DefIndex) -> Option { - match self.kind(node_id) { - EntryKind::Struct(data, _) => { - data.decode(self).ctor.map(|index| self.local_def_id(index)) - } - EntryKind::Variant(data) => { - data.decode(self).ctor.map(|index| self.local_def_id(index)) - } - _ => None, - } + fn get_item_attrs_cache_nofill( + &self, + node_id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [ast::Attribute] { + get_or_create_with(&mut self.item_attrs.lock(), node_id, || { + self.get_item_attrs_uncached(node_id, sess, tcx_arena) + }) + } + + fn get_item_attrs_cache_fill( + &self, + node_id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [ast::Attribute] { + *get_or_insert_with(&mut self.item_attrs.lock(), node_id, || { + self.get_item_attrs_uncached(node_id, sess, tcx_arena) + }) } - fn get_item_attrs( - &'a self, + fn get_item_attrs_uncached( + &self, node_id: DefIndex, - sess: &'a Session, - ) -> impl Iterator + 'a { - // The attributes for a tuple struct/variant are attached to the definition, not the ctor; - // we assume that someone passing in a tuple struct ctor is actually wanting to - // look at the definition + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'static [ast::Attribute] { + // The attributes for a tuple struct/variant are attached to the definition, + // not the ctor; we assume that someone passing in a tuple struct ctor is actually + // wanting to look at the definition. let def_key = self.def_key(node_id); let item_id = if def_key.disambiguated_data.data == DefPathData::Ctor { def_key.parent.unwrap() @@ -1354,34 +1509,71 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { node_id }; - self.root - .tables - .attributes - .get(self, item_id) - .unwrap_or_else(Lazy::empty) - .decode((self, sess)) + let res = tcx_arena.alloc_from_iter( + self.root + .tables + .attributes + .get(self, item_id) + .unwrap_or_else(Lazy::empty) + .decode((self, sess)), + ); + unsafe { mem::transmute(res) } } - fn get_struct_field_names(&self, id: DefIndex, sess: &Session) -> Vec> { - self.root - .tables - .children - .get(self, id) - .unwrap_or_else(Lazy::empty) - .decode(self) - .map(|index| respan(self.get_span(index, sess), self.item_ident(index, sess).name)) - .collect() + fn get_struct_field_names( + &self, + id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [Spanned] { + *get_or_insert_with(&mut self.struct_field_names.lock(), id, || { + self.get_struct_field_names_uncached(id, sess, tcx_arena) + }) } - fn get_struct_field_visibilities(&self, id: DefIndex) -> Vec { - self.root - .tables - .children - .get(self, id) - .unwrap_or_else(Lazy::empty) - .decode(self) - .map(|field_index| self.get_visibility(field_index)) - .collect() + fn get_struct_field_names_uncached( + &self, + id: DefIndex, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'static [Spanned] { + let res = tcx_arena.alloc_from_iter( + self.root + .tables + .children + .get(self, id) + .unwrap_or_else(Lazy::empty) + .decode(self) + .map(|index| respan(self.get_span(index, sess), self.item_ident(index, sess).name)), + ); + unsafe { mem::transmute(res) } + } + + fn get_struct_field_visibilities( + &self, + id: DefIndex, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [Visibility] { + *get_or_insert_with(&mut self.struct_field_visibilities.lock(), id, || { + self.get_struct_field_visibilities_uncached(id, tcx_arena) + }) + } + + fn get_struct_field_visibilities_uncached( + &self, + id: DefIndex, + tcx_arena: TcxArena<'tcx>, + ) -> &'static [Visibility] { + let res = tcx_arena.alloc_from_iter( + self.root + .tables + .children + .get(self, id) + .unwrap_or_else(Lazy::empty) + .decode(self) + .map(|field_index| self.get_visibility(field_index)), + ); + unsafe { mem::transmute(res) } } fn get_inherent_implementations_for_type( @@ -1459,12 +1651,14 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } fn get_proc_macro_quoted_span(&self, index: usize, sess: &Session) -> Span { - self.root - .tables - .proc_macro_quoted_spans - .get(self, index) - .unwrap_or_else(|| panic!("Missing proc macro quoted span: {:?}", index)) - .decode((self, sess)) + *get_or_insert_with(&mut self.proc_macro_quoted_spans.lock(), index, || { + self.root + .tables + .proc_macro_quoted_spans + .get(self, index) + .unwrap_or_else(|| panic!("Missing proc macro quoted span: {:?}", index)) + .decode((self, sess)) + }) } fn get_foreign_modules(&self, tcx: TyCtxt<'tcx>) -> Lrc> { @@ -1893,6 +2087,8 @@ impl CrateMetadata { // Pre-decode the DefPathHash->DefIndex table. This is a cheap operation // that does not copy any data. It just does some data verification. let def_path_hash_map = root.def_path_hash_map.decode(&blob); + // let num_def_ids = root.tables.def_keys.size(); + // let num_quoted_spans = root.tables.proc_macro_quoted_spans.size(); CrateMetadata { blob, @@ -1914,6 +2110,30 @@ impl CrateMetadata { hygiene_context: Default::default(), def_key_cache: Default::default(), def_path_hash_cache: Default::default(), + struct_field_names: Default::default(), + struct_field_visibilities: Default::default(), + ctor_def_ids_and_kinds: Default::default(), + visibilities: Default::default(), + item_children: Default::default(), + associated_items: Default::default(), + spans: Default::default(), + def_kinds: Default::default(), + generics: Default::default(), + module_expansions: Default::default(), + proc_macro_quoted_spans: Default::default(), + item_attrs: Default::default(), + // struct_field_names: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // struct_field_visibilities: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // ctor_def_ids_and_kinds: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // visibilities: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // item_children: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // associated_items: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // spans: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // def_kinds: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // generics: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // module_expansions: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), + // proc_macro_quoted_spans: Lock::new(IndexVec::from_elem_n(None, num_quoted_spans)), + // item_attrs: Lock::new(IndexVec::from_elem_n(None, num_def_ids)), } } diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 70952d388d52d..a7276dedc660a 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -14,7 +14,7 @@ use rustc_middle::middle::cstore::{CrateSource, CrateStore, EncodedMetadata}; use rustc_middle::middle::exported_symbols::ExportedSymbol; use rustc_middle::middle::stability::DeprecationEntry; use rustc_middle::ty::query::Providers; -use rustc_middle::ty::{self, TyCtxt, Visibility}; +use rustc_middle::ty::{self, TcxArena, TyCtxt, Visibility}; use rustc_session::utils::NativeLibKind; use rustc_session::{Session, StableCrateId}; use rustc_span::hygiene::{ExpnHash, ExpnId}; @@ -22,7 +22,6 @@ use rustc_span::source_map::{Span, Spanned}; use rustc_span::symbol::Symbol; use rustc_data_structures::sync::Lrc; -use smallvec::SmallVec; use std::any::Any; macro_rules! provide { @@ -87,7 +86,7 @@ impl IntoArgs for (CrateNum, DefId) { provide! { <'tcx> tcx, def_id, other, cdata, type_of => { cdata.get_type(def_id.index, tcx) } - generics_of => { cdata.get_generics(def_id.index, tcx.sess) } + generics_of => { cdata.get_generics_cache_steal(def_id.index, tcx.sess) } explicit_predicates_of => { cdata.get_explicit_predicates(def_id.index, tcx) } inferred_outlives_of => { cdata.get_inferred_outlives(def_id.index, tcx) } super_predicates_of => { cdata.get_super_predicates(def_id.index, tcx) } @@ -100,10 +99,8 @@ provide! { <'tcx> tcx, def_id, other, cdata, } variances_of => { tcx.arena.alloc_from_iter(cdata.get_item_variances(def_id.index)) } associated_item_def_ids => { - let mut result = SmallVec::<[_; 8]>::new(); - cdata.each_child_of_item(def_id.index, - |child| result.push(child.res.def_id()), tcx.sess); - tcx.arena.alloc_slice(&result) + tcx.arena.alloc_from_iter(cdata.get_item_children_cache_nofill(def_id.index, tcx.sess, tcx.arena) + .iter().map(|export| export.res.def_id())) } associated_item => { cdata.get_associated_item(def_id.index, tcx.sess) } impl_trait_ref => { cdata.get_impl_trait(def_id.index, tcx) } @@ -141,9 +138,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, lookup_deprecation_entry => { cdata.get_deprecation(def_id.index).map(DeprecationEntry::external) } - item_attrs => { tcx.arena.alloc_from_iter( - cdata.get_item_attrs(def_id.index, tcx.sess) - ) } + item_attrs => { cdata.get_item_attrs_cache_nofill(def_id.index, tcx.sess, tcx.arena) } fn_arg_names => { cdata.get_fn_param_names(tcx, def_id.index) } rendered_const => { cdata.get_rendered_const(def_id.index) } impl_parent => { cdata.get_parent_impl(def_id.index) } @@ -205,9 +200,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, r } item_children => { - let mut result = SmallVec::<[_; 8]>::new(); - cdata.each_child_of_item(def_id.index, |child| result.push(child), tcx.sess); - tcx.arena.alloc_slice(&result) + cdata.get_item_children_cache_nofill(def_id.index, tcx.sess, tcx.arena) } defined_lib_features => { cdata.get_lib_features(tcx) } defined_lang_items => { cdata.get_lang_items(tcx) } @@ -374,54 +367,65 @@ pub fn provide(providers: &mut Providers) { } impl CStore { - pub fn struct_field_names_untracked(&self, def: DefId, sess: &Session) -> Vec> { - self.get_crate_data(def.krate).get_struct_field_names(def.index, sess) + pub fn struct_field_names_untracked( + &self, + def: DefId, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [Spanned] { + self.get_crate_data(def.krate).get_struct_field_names(def.index, sess, tcx_arena) } - pub fn struct_field_visibilities_untracked(&self, def: DefId) -> Vec { - self.get_crate_data(def.krate).get_struct_field_visibilities(def.index) + pub fn struct_field_visibilities_untracked<'tcx>( + &self, + def: DefId, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [Visibility] { + self.get_crate_data(def.krate).get_struct_field_visibilities(def.index, tcx_arena) } pub fn ctor_def_id_and_kind_untracked(&self, def: DefId) -> Option<(DefId, CtorKind)> { - self.get_crate_data(def.krate).get_ctor_def_id(def.index).map(|ctor_def_id| { - (ctor_def_id, self.get_crate_data(def.krate).get_ctor_kind(def.index)) - }) + self.get_crate_data(def.krate).get_ctor_def_id_and_kind(def.index) } pub fn visibility_untracked(&self, def: DefId) -> Visibility { self.get_crate_data(def.krate).get_visibility(def.index) } - pub fn item_children_untracked(&self, def_id: DefId, sess: &Session) -> Vec { - let mut result = vec![]; - self.get_crate_data(def_id.krate).each_child_of_item( + pub fn item_children_untracked<'tcx>( + &self, + def_id: DefId, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [Export] { + self.get_crate_data(def_id.krate).get_item_children_cache_fill( def_id.index, - |child| result.push(child), sess, - ); - result + tcx_arena, + ) } - pub fn load_macro_untracked(&self, id: DefId, sess: &Session) -> LoadedMacro { + pub fn load_macro_untracked( + &self, + id: DefId, + sess: &Session, + tcx_arena: TcxArena<'_>, + ) -> LoadedMacro { let _prof_timer = sess.prof.generic_activity("metadata_load_macro"); let data = self.get_crate_data(id.krate); if data.root.is_proc_macro_crate() { - return LoadedMacro::ProcMacro(data.load_proc_macro(id.index, sess)); + return LoadedMacro::ProcMacro(data.load_proc_macro(id.index, sess, tcx_arena)); } let span = data.get_span(id.index, sess); - let attrs = data.get_item_attrs(id.index, sess).collect(); - - let ident = data.item_ident(id.index, sess); - LoadedMacro::MacroDef( ast::Item { - ident, + ident: data.item_ident(id.index, sess), id: ast::DUMMY_NODE_ID, span, - attrs, + attrs: data.get_item_attrs_cache_fill(id.index, sess, tcx_arena).to_vec(), kind: ast::ItemKind::MacroDef(data.get_macro(id.index, sess)), vis: ast::Visibility { span: span.shrink_to_lo(), @@ -434,8 +438,8 @@ impl CStore { ) } - pub fn associated_item_cloned_untracked(&self, def: DefId, sess: &Session) -> ty::AssocItem { - self.get_crate_data(def.krate).get_associated_item(def.index, sess) + pub fn fn_has_self_parameter_untracked(&self, def: DefId, sess: &Session) -> bool { + self.get_crate_data(def.krate).get_associated_item(def.index, sess).fn_has_self_parameter } pub fn crate_source_untracked(&self, cnum: CrateNum) -> CrateSource { @@ -457,7 +461,8 @@ impl CStore { } pub fn item_generics_num_lifetimes(&self, def_id: DefId, sess: &Session) -> usize { - self.get_crate_data(def_id.krate).get_generics(def_id.index, sess).own_counts().lifetimes + self.get_crate_data(def_id.krate) + .get_generics_cache_fill(def_id.index, sess, |generics| generics.own_counts().lifetimes) } pub fn module_expansion_untracked(&self, def_id: DefId, sess: &Session) -> ExpnId { @@ -471,8 +476,13 @@ impl CStore { self.get_crate_data(cnum).num_def_ids() } - pub fn item_attrs(&self, def_id: DefId, sess: &Session) -> Vec { - self.get_crate_data(def_id.krate).get_item_attrs(def_id.index, sess).collect() + pub fn item_attrs_untracked<'tcx>( + &self, + def_id: DefId, + sess: &Session, + tcx_arena: TcxArena<'tcx>, + ) -> &'tcx [ast::Attribute] { + self.get_crate_data(def_id.krate).get_item_attrs_cache_fill(def_id.index, sess, tcx_arena) } pub fn get_proc_macro_quoted_span_untracked( diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 72b8d7cce7142..c58d1d775ccbf 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -978,6 +978,8 @@ impl<'tcx> Deref for TyCtxt<'tcx> { } } +pub type TcxArena<'tcx> = &'tcx WorkerLocal>; + pub struct GlobalCtxt<'tcx> { pub arena: &'tcx WorkerLocal>, diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 777c6035be831..482871babf4a1 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -65,7 +65,7 @@ pub use self::consts::{Const, ConstInt, ConstKind, InferConst, ScalarInt, Uneval pub use self::context::{ tls, CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CtxtInterners, DelaySpanBugEmitted, FreeRegionInfo, GeneratorInteriorTypeCause, GlobalCtxt, - Lift, OnDiskCache, TyCtxt, TypeckResults, UserType, UserTypeAnnotationIndex, + Lift, OnDiskCache, TcxArena, TyCtxt, TypeckResults, UserType, UserTypeAnnotationIndex, }; pub use self::instance::{Instance, InstanceDef}; pub use self::list::List; diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs index 55f2b04c4f1c1..bf2350f17fb17 100644 --- a/compiler/rustc_resolve/src/build_reduced_graph.rs +++ b/compiler/rustc_resolve/src/build_reduced_graph.rs @@ -205,10 +205,12 @@ impl<'a> Resolver<'a> { return ext.clone(); } - let ext = Lrc::new(match self.cstore().load_macro_untracked(def_id, &self.session) { - LoadedMacro::MacroDef(item, edition) => self.compile_macro(&item, edition), - LoadedMacro::ProcMacro(ext) => ext, - }); + let ext = Lrc::new( + match self.cstore().load_macro_untracked(def_id, &self.session, self.tcx_arena) { + LoadedMacro::MacroDef(item, edition) => self.compile_macro(&item, edition), + LoadedMacro::ProcMacro(ext) => ext, + }, + ); self.macro_map.insert(def_id, ext.clone()); ext @@ -227,10 +229,10 @@ impl<'a> Resolver<'a> { crate fn build_reduced_graph_external(&mut self, module: Module<'a>) { let def_id = module.def_id().expect("unpopulated module without a def-id"); - for child in self.cstore().item_children_untracked(def_id, self.session) { + for child in self.cstore().item_children_untracked(def_id, self.session, self.tcx_arena) { let parent_scope = ParentScope::module(module, self); BuildReducedGraphVisitor { r: self, parent_scope } - .build_reduced_graph_for_external_crate_res(child); + .build_reduced_graph_for_external_crate_res(*child); } } } @@ -1011,12 +1013,16 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { let cstore = self.r.cstore(); match res { Res::Def(DefKind::Struct, def_id) => { - let field_names = cstore.struct_field_names_untracked(def_id, self.r.session); + let field_names = cstore + .struct_field_names_untracked(def_id, self.r.session, self.r.tcx_arena) + .to_vec(); let ctor = cstore.ctor_def_id_and_kind_untracked(def_id); if let Some((ctor_def_id, ctor_kind)) = ctor { let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Struct, ctor_kind), ctor_def_id); let ctor_vis = cstore.visibility_untracked(ctor_def_id); - let field_visibilities = cstore.struct_field_visibilities_untracked(def_id); + let field_visibilities = cstore + .struct_field_visibilities_untracked(def_id, self.r.tcx_arena) + .to_vec(); self.r .struct_constructors .insert(def_id, (ctor_res, ctor_vis, field_visibilities)); @@ -1024,14 +1030,13 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { self.insert_field_names(def_id, field_names); } Res::Def(DefKind::Union, def_id) => { - let field_names = cstore.struct_field_names_untracked(def_id, self.r.session); + let field_names = cstore + .struct_field_names_untracked(def_id, self.r.session, self.r.tcx_arena) + .to_vec(); self.insert_field_names(def_id, field_names); } Res::Def(DefKind::AssocFn, def_id) => { - if cstore - .associated_item_cloned_untracked(def_id, self.r.session) - .fn_has_self_parameter - { + if cstore.fn_has_self_parameter_untracked(def_id, self.r.session) { self.r.has_self.insert(def_id); } } diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs index d76ba80e42eab..b783917dba255 100644 --- a/compiler/rustc_resolve/src/lib.rs +++ b/compiler/rustc_resolve/src/lib.rs @@ -49,7 +49,7 @@ use rustc_middle::hir::exports::ExportMap; use rustc_middle::middle::cstore::{CrateStore, MetadataLoaderDyn}; use rustc_middle::span_bug; use rustc_middle::ty::query::Providers; -use rustc_middle::ty::{self, DefIdTree, MainDefinition, ResolverOutputs}; +use rustc_middle::ty::{self, DefIdTree, MainDefinition, ResolverOutputs, TcxArena}; use rustc_session::lint; use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer}; use rustc_session::Session; @@ -871,6 +871,7 @@ struct DeriveData { /// This is the visitor that walks the whole crate. pub struct Resolver<'a> { session: &'a Session, + tcx_arena: TcxArena<'a>, definitions: Definitions, @@ -1254,6 +1255,7 @@ impl<'a> Resolver<'a> { crate_name: &str, metadata_loader: Box, arenas: &'a ResolverArenas<'a>, + tcx_arena: TcxArena<'a>, ) -> Resolver<'a> { let root_local_def_id = LocalDefId { local_def_index: CRATE_DEF_INDEX }; let root_def_id = root_local_def_id.to_def_id(); @@ -1312,6 +1314,7 @@ impl<'a> Resolver<'a> { let mut resolver = Resolver { session, + tcx_arena, definitions, @@ -3391,29 +3394,26 @@ impl<'a> Resolver<'a> { return None; } + // The lookup is cached to avoid parsing attributes for an item multiple times. if let Some(v) = self.legacy_const_generic_args.get(&def_id) { return v.clone(); } - let parse_attrs = || { - let attrs = self.cstore().item_attrs(def_id, self.session); - let attr = - attrs.iter().find(|a| a.has_name(sym::rustc_legacy_const_generics))?; - let mut ret = vec![]; - for meta in attr.meta_item_list()? { - match meta.literal()?.kind { - LitKind::Int(a, _) => { - ret.push(a as usize); - } - _ => panic!("invalid arg index"), + let attr = self + .cstore() + .item_attrs_untracked(def_id, self.session, self.tcx_arena) + .iter() + .find(|a| a.has_name(sym::rustc_legacy_const_generics))?; + let mut ret = vec![]; + for meta in attr.meta_item_list()? { + match meta.literal()?.kind { + LitKind::Int(a, _) => { + ret.push(a as usize); } + _ => panic!("invalid arg index"), } - Some(ret) - }; - - // Cache the lookup to avoid parsing attributes for an iterm - // multiple times. - let ret = parse_attrs(); + } + let ret = Some(ret); self.legacy_const_generic_args.insert(def_id, ret.clone()); return ret; } diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs index 29834c82b3de3..e560df5205b91 100644 --- a/src/librustdoc/clean/inline.rs +++ b/src/librustdoc/clean/inline.rs @@ -179,7 +179,7 @@ crate fn record_extern_fqn(cx: &mut DocContext<'_>, did: DefId, kind: ItemType) let fqn = if let ItemType::Macro = kind { // Check to see if it is a macro 2.0 or built-in macro if matches!( - CStore::from_tcx(cx.tcx).load_macro_untracked(did, cx.sess()), + CStore::from_tcx(cx.tcx).load_macro_untracked(did, cx.sess(), cx.tcx.arena), LoadedMacro::MacroDef(def, _) if matches!(&def.kind, ast::ItemKind::MacroDef(ast_def) if !ast_def.macro_rules) @@ -556,7 +556,7 @@ fn build_macro( name: Symbol, import_def_id: Option, ) -> clean::ItemKind { - match CStore::from_tcx(cx.tcx).load_macro_untracked(def_id, cx.sess()) { + match CStore::from_tcx(cx.tcx).load_macro_untracked(def_id, cx.sess(), cx.tcx.arena) { LoadedMacro::MacroDef(item_def, _) => { if let ast::ItemKind::MacroDef(ref def) = item_def.kind { clean::MacroItem(clean::Macro {