Implement copy-on-write scheme for managing the incremental compilation cache.

Author: michaelwoerister

src/librustc/hir/svh.rs

@@ -17,6 +17,7 @@
 
 use std::fmt;
 use std::hash::{Hash, Hasher};
+use serialize::{Encodable, Decodable, Encoder, Decoder};
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub struct Svh {
@@ -51,3 +52,17 @@ impl fmt::Display for Svh {
         f.pad(&self.to_string())
     }
 }
+
+impl Encodable for Svh {
+    fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
+        s.emit_u64(self.as_u64().to_le())
+    }
+}
+
+impl Decodable for Svh {
+    fn decode<D: Decoder>(d: &mut D) -> Result<Svh, D::Error> {
+        d.read_u64()
+         .map(u64::from_le)
+         .map(Svh::new)
+    }
+}

src/librustc/session/mod.rs

@@ -33,10 +33,11 @@ use syntax::feature_gate::AttributeType;
 use syntax_pos::{Span, MultiSpan};
 
 use rustc_back::target::Target;
+use rustc_data_structures::flock;
 use llvm;
 
 use std::path::{Path, PathBuf};
-use std::cell::{Cell, RefCell};
+use std::cell::{self, Cell, RefCell};
 use std::collections::{HashMap, HashSet};
 use std::env;
 use std::ffi::CString;
@@ -101,6 +102,8 @@ pub struct Session {
     /// macro name and defintion span in the source crate.
     pub imported_macro_spans: RefCell<HashMap<Span, (String, Span)>>,
 
+    incr_comp_session: RefCell<IncrCompSession>,
+
     next_node_id: Cell<ast::NodeId>,
 }
 
@@ -331,6 +334,70 @@ impl Session {
             &self.opts.search_paths,
             kind)
     }
+
+    pub fn init_incr_comp_session(&self,
+                                  session_dir: PathBuf,
+                                  lock_file: flock::Lock) {
+        let mut incr_comp_session = self.incr_comp_session.borrow_mut();
+
+        if let IncrCompSession::NotInitialized = *incr_comp_session { } else {
+            bug!("Trying to initialize IncrCompSession `{:?}`", *incr_comp_session)
+        }
+
+        *incr_comp_session = IncrCompSession::Active {
+            session_directory: session_dir,
+            lock_file: lock_file,
+        };
+    }
+
+    pub fn finalize_incr_comp_session(&self, new_directory_path: PathBuf) {
+        let mut incr_comp_session = self.incr_comp_session.borrow_mut();
+
+        if let IncrCompSession::Active { .. } = *incr_comp_session { } else {
+            bug!("Trying to finalize IncrCompSession `{:?}`", *incr_comp_session)
+        }
+
+        // Note: This will also drop the lock file, thus unlocking the directory
+        *incr_comp_session = IncrCompSession::Finalized {
+            session_directory: new_directory_path,
+        };
+    }
+
+    pub fn mark_incr_comp_session_as_invalid(&self) {
+        let mut incr_comp_session = self.incr_comp_session.borrow_mut();
+
+        if let IncrCompSession::Active { .. } = *incr_comp_session { } else {
+            bug!("Trying to invalidate IncrCompSession `{:?}`", *incr_comp_session)
+        }
+
+        // Note: This will also drop the lock file, thus unlocking the directory
+        *incr_comp_session = IncrCompSession::InvalidBecauseOfErrors;
+    }
+
+    pub fn incr_comp_session_dir(&self) -> cell::Ref<PathBuf> {
+        let incr_comp_session = self.incr_comp_session.borrow();
+        cell::Ref::map(incr_comp_session, |incr_comp_session| {
+            match *incr_comp_session {
+                IncrCompSession::NotInitialized |
+                IncrCompSession::InvalidBecauseOfErrors => {
+                    bug!("Trying to get session directory from IncrCompSession `{:?}`",
+                        *incr_comp_session)
+                }
+                IncrCompSession::Active { ref session_directory, .. } |
+                IncrCompSession::Finalized { ref session_directory } => {
+                    session_directory
+                }
+            }
+        })
+    }
+
+    pub fn incr_comp_session_dir_opt(&self) -> Option<cell::Ref<PathBuf>> {
+        if self.opts.incremental.is_some() {
+            Some(self.incr_comp_session_dir())
+        } else {
+            None
+        }
+    }
 }
 
 pub fn build_session(sopts: config::Options,
@@ -446,13 +513,37 @@ pub fn build_session_(sopts: config::Options,
         injected_panic_runtime: Cell::new(None),
         available_macros: RefCell::new(HashSet::new()),
         imported_macro_spans: RefCell::new(HashMap::new()),
+        incr_comp_session: RefCell::new(IncrCompSession::NotInitialized),
     };
 
     init_llvm(&sess);
 
     sess
 }
 
+/// Holds data on the current incremental compilation session, if there is one.
+#[derive(Debug)]
+pub enum IncrCompSession {
+    // This is the state the session will be in until the incr. comp. dir is
+    // needed.
+    NotInitialized,
+    // This is the state during which the session directory is private and can
+    // be modified.
+    Active {
+        session_directory: PathBuf,
+        lock_file: flock::Lock,
+    },
+    // This is the state after the session directory has been finalized. In this
+    // state, the contents of the directory must not be modified any more.
+    Finalized {
+        session_directory: PathBuf,
+    },
+    // This is an error state that is reached when some compilation error has
+    // occurred. It indicates that the contents of the session directory must
+    // not be used, since they might be invalid.
+    InvalidBecauseOfErrors,
+}
+
 fn init_llvm(sess: &Session) {
     unsafe {
         // Before we touch LLVM, make sure that multithreading is enabled.

src/librustc/util/fs.rs

@@ -56,14 +56,49 @@ pub fn fix_windows_verbatim_for_gcc(p: &Path) -> PathBuf {
     }
 }
 
+pub enum LinkOrCopy {
+    Link,
+    Copy
+}
+
 /// Copy `p` into `q`, preferring to use hard-linking if possible. If
 /// `q` already exists, it is removed first.
-pub fn link_or_copy<P: AsRef<Path>, Q: AsRef<Path>>(p: P, q: Q) -> io::Result<()> {
+/// The result indicates which of the two operations has been performed.
+pub fn link_or_copy<P: AsRef<Path>, Q: AsRef<Path>>(p: P, q: Q) -> io::Result<LinkOrCopy> {
     let p = p.as_ref();
     let q = q.as_ref();
     if q.exists() {
         try!(fs::remove_file(&q));
     }
-    fs::hard_link(p, q)
-        .or_else(|_| fs::copy(p, q).map(|_| ()))
+
+    match fs::hard_link(p, q) {
+        Ok(()) => Ok(LinkOrCopy::Link),
+        Err(_) => {
+            match fs::copy(p, q) {
+                Ok(_) => Ok(LinkOrCopy::Copy),
+                Err(e) => Err(e)
+            }
+        }
+    }
+}
+
+// Like std::fs::create_dir_all, except handles concurrent calls among multiple
+// threads or processes.
+pub fn create_dir_racy(path: &Path) -> io::Result<()> {
+    match fs::create_dir(path) {
+        Ok(()) => return Ok(()),
+        Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => return Ok(()),
+        Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
+        Err(e) => return Err(e),
+    }
+    match path.parent() {
+        Some(p) => try!(create_dir_racy(p)),
+        None => return Err(io::Error::new(io::ErrorKind::Other,
+                                          "failed to create whole tree")),
+    }
+    match fs::create_dir(path) {
+        Ok(()) => Ok(()),
+        Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(()),
+        Err(e) => Err(e),
+    }
 }

src/librustc_data_structures/flock.rs

@@ -136,6 +136,7 @@ mod imp {
         pub const F_SETLKW: libc::c_int = 7;
     }
 
+    #[derive(Debug)]
     pub struct Lock {
         fd: libc::c_int,
     }
@@ -251,6 +252,7 @@ mod imp {
                       lpOverlapped: LPOVERLAPPED) -> BOOL;
     }
 
+    #[derive(Debug)]
     pub struct Lock {
         _file: File,
     }

src/librustc_driver/driver.rs

@@ -88,7 +88,7 @@ pub fn compile_input(sess: &Session,
     // We need nested scopes here, because the intermediate results can keep
     // large chunks of memory alive and we want to free them as soon as
     // possible to keep the peak memory usage low
-    let (outputs, trans, crate_name) = {
+    let (outputs, trans) = {
         let krate = match phase_1_parse_input(sess, cfg, input) {
             Ok(krate) => krate,
             Err(mut parse_error) => {
@@ -213,11 +213,11 @@ pub fn compile_input(sess: &Session,
             // Discard interned strings as they are no longer required.
             token::clear_ident_interner();
 
-            Ok((outputs, trans, crate_name.clone()))
+            Ok((outputs, trans))
         })??
     };
 
-    let phase5_result = phase_5_run_llvm_passes(sess, &crate_name, &trans, &outputs);
+    let phase5_result = phase_5_run_llvm_passes(sess, &trans, &outputs);
 
     controller_entry_point!(after_llvm,
                             sess,
@@ -229,6 +229,10 @@ pub fn compile_input(sess: &Session,
 
     phase_6_link_output(sess, &trans, &outputs);
 
+    // Now that we won't touch anything in the incremental compilation directory
+    // any more, we can finalize it (which involves renaming it)
+    rustc_incremental::finalize_session_directory(sess, trans.link.crate_hash);
+
     controller_entry_point!(compilation_done,
                             sess,
                             CompileState::state_when_compilation_done(input, sess, outdir, output),
@@ -1026,19 +1030,19 @@ pub fn phase_4_translate_to_llvm<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 
     time(time_passes,
          "assert dep graph",
-         move || rustc_incremental::assert_dep_graph(tcx));
+         || rustc_incremental::assert_dep_graph(tcx));
 
     time(time_passes,
          "serialize dep graph",
-         move || rustc_incremental::save_dep_graph(tcx, &incremental_hashes_map));
-
+         || rustc_incremental::save_dep_graph(tcx,
+                                              &incremental_hashes_map,
+                                              translation.link.crate_hash));
     translation
 }
 
 /// Run LLVM itself, producing a bitcode file, assembly file or object file
 /// as a side effect.
 pub fn phase_5_run_llvm_passes(sess: &Session,
-                               crate_name: &str,
                                trans: &trans::CrateTranslation,
                                outputs: &OutputFilenames) -> CompileResult {
     if sess.opts.cg.no_integrated_as {
@@ -1061,7 +1065,7 @@ pub fn phase_5_run_llvm_passes(sess: &Session,
 
     time(sess.time_passes(),
          "serialize work products",
-         move || rustc_incremental::save_work_products(sess, crate_name));
+         move || rustc_incremental::save_work_products(sess));
 
     if sess.err_count() > 0 {
         Err(sess.err_count())

src/librustc_incremental/lib.rs

@@ -22,6 +22,7 @@
 #![feature(question_mark)]
 #![feature(rustc_private)]
 #![feature(staged_api)]
+#![feature(rand)]
 
 extern crate graphviz;
 extern crate rbml;
@@ -45,3 +46,4 @@ pub use persist::save_dep_graph;
 pub use persist::save_trans_partition;
 pub use persist::save_work_products;
 pub use persist::in_incr_comp_dir;
+pub use persist::finalize_session_directory;