diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index 04d96e362a465..3d44fda9a5eec 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -1926,6 +1926,7 @@ pub mod tls { /// to `value` during the call to `f`. It is restored to its previous value after. /// This is used to set the pointer to the new ImplicitCtxt. #[cfg(parallel_queries)] + #[inline] fn set_tlv R, R>(value: usize, f: F) -> R { rayon_core::tlv::with(value, f) } @@ -1933,6 +1934,7 @@ pub mod tls { /// Gets Rayon's thread local variable which is preserved for Rayon jobs. /// This is used to get the pointer to the current ImplicitCtxt. #[cfg(parallel_queries)] + #[inline] fn get_tlv() -> usize { rayon_core::tlv::get() } @@ -1945,6 +1947,7 @@ pub mod tls { /// It is restored to its previous value after. /// This is used to set the pointer to the new ImplicitCtxt. #[cfg(not(parallel_queries))] + #[inline] fn set_tlv R, R>(value: usize, f: F) -> R { let old = get_tlv(); let _reset = OnDrop(move || TLV.with(|tlv| tlv.set(old))); diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index a73b92ed713ec..005a6a21394dc 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -499,37 +499,48 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { // If -Zincremental-verify-ich is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. - if self.sess.opts.debugging_opts.incremental_verify_ich { - use rustc_data_structures::stable_hasher::{StableHasher, HashStable}; - use ich::Fingerprint; + if unlikely!(self.sess.opts.debugging_opts.incremental_verify_ich) { + self.incremental_verify_ich::(&result, dep_node, dep_node_index); + } - assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) == - self.dep_graph.prev_fingerprint_of(dep_node), - "Fingerprint for green query instance not loaded \ - from cache: {:?}", dep_node); + if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { + self.dep_graph.mark_loaded_from_cache(dep_node_index, true); + } - debug!("BEGIN verify_ich({:?})", dep_node); - let mut hcx = self.create_stable_hashing_context(); - let mut hasher = StableHasher::new(); + job.complete(&result, dep_node_index); - result.hash_stable(&mut hcx, &mut hasher); + Ok(result) + } - let new_hash: Fingerprint = hasher.finish(); - debug!("END verify_ich({:?})", dep_node); + #[inline(never)] + #[cold] + fn incremental_verify_ich>( + self, + result: &Q::Value, + dep_node: &DepNode, + dep_node_index: DepNodeIndex, + ) { + use rustc_data_structures::stable_hasher::{StableHasher, HashStable}; + use ich::Fingerprint; - let old_hash = self.dep_graph.fingerprint_of(dep_node_index); + assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) == + self.dep_graph.prev_fingerprint_of(dep_node), + "Fingerprint for green query instance not loaded \ + from cache: {:?}", dep_node); - assert!(new_hash == old_hash, "Found unstable fingerprints \ - for {:?}", dep_node); - } + debug!("BEGIN verify_ich({:?})", dep_node); + let mut hcx = self.create_stable_hashing_context(); + let mut hasher = StableHasher::new(); - if self.sess.opts.debugging_opts.query_dep_graph { - self.dep_graph.mark_loaded_from_cache(dep_node_index, true); - } + result.hash_stable(&mut hcx, &mut hasher); - job.complete(&result, dep_node_index); + let new_hash: Fingerprint = hasher.finish(); + debug!("END verify_ich({:?})", dep_node); - Ok(result) + let old_hash = self.dep_graph.fingerprint_of(dep_node_index); + + assert!(new_hash == old_hash, "Found unstable fingerprints \ + for {:?}", dep_node); } fn force_query_with_job>( @@ -574,7 +585,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { let ((result, dep_node_index), diagnostics) = res; - if self.sess.opts.debugging_opts.query_dep_graph { + if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { self.dep_graph.mark_loaded_from_cache(dep_node_index, false); } diff --git a/src/librustc_data_structures/lib.rs b/src/librustc_data_structures/lib.rs index bc2b8f1d6523e..9d0201a9e38b1 100644 --- a/src/librustc_data_structures/lib.rs +++ b/src/librustc_data_structures/lib.rs @@ -113,12 +113,14 @@ pub struct OnDrop(pub F); impl OnDrop { /// Forgets the function which prevents it from running. /// Ensure that the function owns no memory, otherwise it will be leaked. + #[inline] pub fn disable(self) { std::mem::forget(self); } } impl Drop for OnDrop { + #[inline] fn drop(&mut self) { (self.0)(); }