Add inline.

Author: cjgillot

compiler/rustc_query_system/src/dep_graph/dep_kind.rs

@@ -62,6 +62,7 @@ static_assert_size!(DepNode, 24);
 
 impl DepNode {
     /// Used in testing
+    #[inline]
     pub fn has_label_string(label: &str) -> bool {
         dep_kind_from_label_string(label).is_ok()
     }

compiler/rustc_query_system/src/dep_graph/dep_node.rs

@@ -61,6 +61,7 @@ impl DepNode {
     /// Creates a new, parameterless DepNode. This method will assert
     /// that the DepNode corresponding to the given DepKind actually
     /// does not require any parameters.
+    #[inline]
     pub fn new_no_params<Ctxt>(tcx: Ctxt, kind: DepKind) -> DepNode
     where
         Ctxt: super::DepContext,
@@ -92,6 +93,7 @@ impl DepNode {
 }
 
 impl fmt::Debug for DepNode {
+    #[inline]
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         (*NODE_DEBUG)(self, f)
     }
@@ -170,6 +172,7 @@ pub struct WorkProductId {
 }
 
 impl WorkProductId {
+    #[inline]
     pub fn from_cgu_name(cgu_name: &str) -> WorkProductId {
         let mut hasher = StableHasher::new();
         cgu_name.len().hash(&mut hasher);

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -140,6 +140,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub fn new_disabled() -> DepGraph {
         DepGraph { data: None, virtual_dep_node_index: Lrc::new(AtomicU32::new(0)) }
     }
@@ -156,6 +157,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub fn assert_ignored(&self) {
         if let Some(..) = self.data {
             crate::tls::read_deps(|task_deps| {
@@ -422,18 +424,21 @@ impl DepGraph {
         self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some()
     }
 
+    #[inline]
     pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> {
         self.data.as_ref().unwrap().previous.fingerprint_of(dep_node)
     }
 
     /// Checks whether a previous work product exists for `v` and, if
     /// so, return the path that leads to it. Used to skip doing work.
+    #[inline]
     pub fn previous_work_product(&self, v: &WorkProductId) -> Option<WorkProduct> {
         self.data.as_ref().and_then(|data| data.previous_work_products.get(v).cloned())
     }
 
     /// Access the map of work-products created during the cached run. Only
     /// used during saving of the dep-graph.
+    #[inline]
     pub fn previous_work_products(&self) -> &FxHashMap<WorkProductId, WorkProduct> {
         &self.data.as_ref().unwrap().previous_work_products
     }
@@ -452,10 +457,12 @@ impl DepGraph {
         dep_node_debug.borrow_mut().insert(dep_node, debug_str);
     }
 
+    #[inline]
     pub fn dep_node_debug_str(&self, dep_node: DepNode) -> Option<String> {
         self.data.as_ref()?.dep_node_debug.borrow().get(&dep_node).cloned()
     }
 
+    #[inline]
     fn node_color(&self, dep_node: &DepNode) -> Option<DepNodeColor> {
         if let Some(ref data) = self.data {
             if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) {
@@ -709,12 +716,14 @@ impl DepGraph {
 
     // Returns true if the given node has been marked as red during the
     // current compilation session. Used in various assertions
+    #[inline]
     pub fn is_red(&self, dep_node: &DepNode) -> bool {
         self.node_color(dep_node) == Some(DepNodeColor::Red)
     }
 
     // Returns true if the given node has been marked as green during the
     // current compilation session. Used in various assertions
+    #[inline]
     pub fn is_green(&self, dep_node: &DepNode) -> bool {
         self.node_color(dep_node).map_or(false, |c| c.is_green())
     }
@@ -755,6 +764,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub fn encode(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         if let Some(data) = &self.data {
             data.current.encoder.steal().finish(profiler)
@@ -763,6 +773,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex {
         let index = self.virtual_dep_node_index.fetch_add(1, Relaxed);
         DepNodeIndex::from_u32(index)
@@ -929,6 +940,7 @@ impl CurrentDepGraph {
     }
 
     #[cfg(debug_assertions)]
+    #[inline]
     fn record_edge(&self, dep_node_index: DepNodeIndex, key: DepNode) {
         if let Some(forbidden_edge) = &self.forbidden_edge {
             forbidden_edge.index_to_node.lock().insert(dep_node_index, key);
@@ -937,6 +949,7 @@ impl CurrentDepGraph {
 
     /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it.
     /// Assumes that this is a node that has no equivalent in the previous dep-graph.
+    #[inline]
     fn intern_new_node(
         &self,
         profiler: &SelfProfilerRef,
@@ -957,6 +970,7 @@ impl CurrentDepGraph {
         }
     }
 
+    #[inline]
     fn intern_node(
         &self,
         profiler: &SelfProfilerRef,
@@ -1059,6 +1073,7 @@ impl CurrentDepGraph {
         }
     }
 
+    #[inline]
     fn promote_node_and_deps_to_current(
         &self,
         profiler: &SelfProfilerRef,
@@ -1129,6 +1144,7 @@ const COMPRESSED_RED: u32 = 1;
 const COMPRESSED_FIRST_GREEN: u32 = 2;
 
 impl DepNodeColorMap {
+    #[inline]
     fn new(size: usize) -> DepNodeColorMap {
         DepNodeColorMap { values: (0..size).map(|_| AtomicU32::new(COMPRESSED_NONE)).collect() }
     }
@@ -1144,6 +1160,7 @@ impl DepNodeColorMap {
         }
     }
 
+    #[inline]
     fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) {
         self.values[index].store(
             match color {

compiler/rustc_query_system/src/dep_graph/serialized.rs

@@ -53,6 +53,7 @@ pub struct SerializedDepGraph {
 }
 
 impl Default for SerializedDepGraph {
+    #[inline]
     fn default() -> Self {
         SerializedDepGraph {
             nodes: Default::default(),
@@ -91,6 +92,7 @@ impl SerializedDepGraph {
         self.fingerprints[dep_node_index]
     }
 
+    #[inline]
     pub fn node_count(&self) -> usize {
         self.index.len()
     }
@@ -153,7 +155,7 @@ impl<'a> Decodable<opaque::Decoder<'a>> for SerializedDepGraph {
     }
 }
 
-#[derive(Debug, Encodable, Decodable)]
+#[derive(Debug, Encodable)]
 pub struct NodeInfo {
     node: DepNode,
     fingerprint: Fingerprint,
@@ -175,6 +177,7 @@ struct EncoderState {
 }
 
 impl EncoderState {
+    #[inline]
     fn new(encoder: FileEncoder, record_stats: bool) -> Self {
         Self {
             encoder,
@@ -247,6 +250,7 @@ pub struct GraphEncoder {
 }
 
 impl GraphEncoder {
+    #[inline]
     pub fn new(
         encoder: FileEncoder,
         prev_node_count: usize,
@@ -320,6 +324,7 @@ impl GraphEncoder {
         }
     }
 
+    #[inline]
     pub(crate) fn send(
         &self,
         profiler: &SelfProfilerRef,
@@ -332,6 +337,7 @@ impl GraphEncoder {
         self.status.lock().encode_node(&node, &self.record_graph)
     }
 
+    #[inline]
     pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph");
         self.status.into_inner().finish(profiler)

compiler/rustc_query_system/src/query/job.rs

@@ -53,25 +53,30 @@ pub struct QueryJobId {
 }
 
 impl QueryJobId {
+    #[inline]
     pub fn new(job: QueryShardJobId, shard: usize, kind: DepKind) -> Self {
         QueryJobId { job, shard: u16::try_from(shard).unwrap(), kind }
     }
 
+    #[inline]
     fn query(self, map: &QueryMap) -> QueryStackFrame {
         map.get(&self).unwrap().query.clone()
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     fn span(self, map: &QueryMap) -> Span {
         map.get(&self).unwrap().job.span
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     fn parent(self, map: &QueryMap) -> Option<QueryJobId> {
         map.get(&self).unwrap().job.parent
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     fn latch<'a>(self, map: &'a QueryMap) -> Option<&'a QueryLatch> {
         map.get(&self).unwrap().job.latch.as_ref()
     }
@@ -100,6 +105,7 @@ pub struct QueryJob {
 
 impl QueryJob {
     /// Creates a new query job.
+    #[inline]
     pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId>) -> Self {
         QueryJob {
             id,
@@ -111,6 +117,7 @@ impl QueryJob {
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     pub(super) fn latch(&mut self) -> QueryLatch {
         if self.latch.is_none() {
             self.latch = Some(QueryLatch::new());
@@ -122,6 +129,7 @@ impl QueryJob {
     ///
     /// This does nothing for single threaded rustc,
     /// as there are no concurrent jobs which could be waiting on us
+    #[inline]
     pub fn signal_complete(self) {
         #[cfg(parallel_compiler)]
         {
@@ -204,6 +212,7 @@ pub(super) struct QueryLatch {
 
 #[cfg(parallel_compiler)]
 impl QueryLatch {
+    #[inline]
     fn new() -> Self {
         QueryLatch {
             info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })),
@@ -214,6 +223,7 @@ impl QueryLatch {
 #[cfg(parallel_compiler)]
 impl QueryLatch {
     /// Awaits for the query job to complete.
+    #[inline]
     pub(super) fn wait_on(&self, query: Option<QueryJobId>, span: Span) -> Result<(), CycleError> {
         let waiter =
             Lrc::new(QueryWaiter { query, span, cycle: Lock::new(None), condvar: Condvar::new() });
@@ -229,6 +239,7 @@ impl QueryLatch {
     }
 
     /// Awaits the caller on this latch by blocking the current thread.
+    #[inline]
     fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter>) {
         let mut info = self.info.lock();
         if !info.complete {
@@ -251,6 +262,7 @@ impl QueryLatch {
     }
 
     /// Sets the latch and resumes all waiters on it
+    #[inline]
     fn set(&self) {
         let mut info = self.info.lock();
         debug_assert!(!info.complete);
@@ -263,6 +275,7 @@ impl QueryLatch {
 
     /// Removes a single waiter from the list of waiters.
     /// This is used to break query cycles.
+    #[inline]
     fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter> {
         let mut info = self.info.lock();
         debug_assert!(!info.complete);