Implemented span expansion info encoding in metadata.

This change adds an encoded version of hygiene related datastructures to
the metadata of a crate. It is then decoded on-demand when spans and AST
items from a foreign crate are accessed. Some changes might still be
necessary to implement the correct behaviour, however.

Author: ibabushkin

src/doc/book

@@ -293,6 +293,7 @@ impl<'a> CrateLoader<'a> {
             // Initialize this with an empty set. The field is populated below
             // after we were able to deserialize its contents.
             dllimport_foreign_items: FxHashSet(),
+            hygiene_data_import_info: RefCell::new(None),
         };
 
         let dllimports: FxHashSet<_> = cmeta

src/doc/nomicon

@@ -27,7 +27,7 @@ use owning_ref::ErasedBoxRef;
 use syntax::{ast, attr};
 use syntax::ext::base::SyntaxExtension;
 use syntax::symbol::Symbol;
-use syntax_pos;
+use syntax_pos::{self, hygiene};
 
 pub use rustc::middle::cstore::{NativeLibrary, NativeLibraryKind, LinkagePreference};
 pub use rustc::middle::cstore::NativeLibraryKind::*;
@@ -87,6 +87,8 @@ pub struct CrateMetadata {
     pub proc_macros: Option<Vec<(ast::Name, Rc<SyntaxExtension>)>>,
     // Foreign items imported from a dylib (Windows only)
     pub dllimport_foreign_items: FxHashSet<DefIndex>,
+
+    pub hygiene_data_import_info: RefCell<Option<hygiene::ImportedHygieneData>>,
 }
 
 pub struct CStore {

src/doc/reference

@@ -45,7 +45,8 @@ use syntax::ast::{self, Ident};
 use syntax::codemap;
 use syntax::symbol::{InternedString, Symbol};
 use syntax::ext::base::MacroKind;
-use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP, NO_EXPANSION};
+use syntax_pos::{self, Span, SyntaxContext, BytePos, Pos, DUMMY_SP};
+use syntax_pos::hygiene;
 
 pub struct DecodeContext<'a, 'tcx: 'a> {
     opaque: opaque::Decoder<'a>,
@@ -237,15 +238,42 @@ impl<'a, 'tcx> SpecializedDecoder<CrateNum> for DecodeContext<'a, 'tcx> {
     }
 }
 
+impl<'a, 'tcx> SpecializedDecoder<hygiene::Mark> for DecodeContext<'a, 'tcx> {
+    fn specialized_decode(&mut self) -> Result<hygiene::Mark, Self::Error> {
+        let mark = u32::decode(self)?;
+        if self.cdata().hygiene_data_import_info.borrow().is_some() && mark != 0  {
+            let imported_hygiene = self.cdata().imported_hygiene_data();
+
+            Ok(hygiene::Mark::from_u32(mark + imported_hygiene.mark_translation_offset))
+        } else {
+            Ok(hygiene::Mark::from_u32(mark))
+        }
+    }
+}
+
+impl<'a, 'tcx> SpecializedDecoder<SyntaxContext> for DecodeContext<'a, 'tcx> {
+    fn specialized_decode(&mut self) -> Result<SyntaxContext, Self::Error> {
+        let ctxt = u32::decode(self)?;
+        if self.cdata().hygiene_data_import_info.borrow().is_some() && ctxt != 0 {
+            let imported_hygiene = self.cdata().imported_hygiene_data();
+
+            Ok(SyntaxContext::from_u32(ctxt + imported_hygiene.ctxt_translation_offset))
+        } else {
+            Ok(SyntaxContext::from_u32(ctxt))
+        }
+    }
+}
+
 impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
     fn specialized_decode(&mut self) -> Result<Span, Self::Error> {
         let lo = BytePos::decode(self)?;
         let hi = BytePos::decode(self)?;
+        let ctxt = SyntaxContext::decode(self)?;
 
         let sess = if let Some(sess) = self.sess {
             sess
         } else {
-            return Ok(Span::new(lo, hi, NO_EXPANSION));
+            return Ok(Span { lo, hi, ctxt });
         };
 
         let (lo, hi) = if lo > hi {
@@ -292,7 +320,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
         let lo = (lo - filemap.original_start_pos) + filemap.translated_filemap.start_pos;
         let hi = (hi - filemap.original_start_pos) + filemap.translated_filemap.start_pos;
 
-        Ok(Span::new(lo, hi, NO_EXPANSION))
+        Ok(Span::new(lo, hi, ctxt))
     }
 }
 
@@ -1224,4 +1252,20 @@ impl<'a, 'tcx> CrateMetadata {
         *self.codemap_import_info.borrow_mut() = imported_filemaps;
         self.codemap_import_info.borrow()
     }
+
+    pub fn imported_hygiene_data(&'a self) -> Ref<'a, hygiene::ImportedHygieneData> {
+        {
+            let hygiene_data = self.hygiene_data_import_info.borrow();
+            if hygiene_data.is_some() {
+                return Ref::map(hygiene_data, |d| d.as_ref().unwrap());
+            }
+        }
+
+        let external_hygiene_data = self.root.hygiene_data.decode(self);
+
+        // This shouldn't borrow twice, but there is no way to downgrade RefMut to Ref.
+        *self.hygiene_data_import_info.borrow_mut() =
+            Some(hygiene::extend_hygiene_data(external_hygiene_data));
+        Ref::map(self.hygiene_data_import_info.borrow(), |d| d.as_ref().unwrap())
+    }
 }

src/libcompiler_builtins

@@ -42,7 +42,7 @@ use syntax::ast::{self, CRATE_NODE_ID};
 use syntax::codemap::Spanned;
 use syntax::attr;
 use syntax::symbol::Symbol;
-use syntax_pos;
+use syntax_pos::{self, hygiene};
 
 use rustc::hir::{self, PatKind};
 use rustc::hir::itemlikevisit::ItemLikeVisitor;
@@ -323,6 +323,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         self.lazy_seq_ref(adapted.iter().map(|rc| &**rc))
     }
 
+    fn encode_hygiene_data(&mut self) -> Lazy<hygiene::HygieneData> {
+        hygiene::HygieneData::safe_with(|data| self.lazy(data))
+    }
+
     fn encode_crate_root(&mut self) -> Lazy<CrateRoot> {
         let mut i = self.position();
 
@@ -367,6 +371,11 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let codemap = self.encode_codemap();
         let codemap_bytes = self.position() - i;
 
+        // Encode hygiene data
+        i = self.position();
+        let hygiene_data = self.encode_hygiene_data();
+        let hygiene_data_bytes = self.position() - i;
+
         // Encode DefPathTable
         i = self.position();
         let def_path_table = self.encode_def_path_table();
@@ -421,13 +430,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             } else {
                 None
             },
-
             crate_deps,
             dylib_dependency_formats,
             lang_items,
             lang_items_missing,
             native_libraries,
             codemap,
+            hygiene_data,
             def_path_table,
             impls,
             exported_symbols,
@@ -456,6 +465,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             println!("         codemap bytes: {}", codemap_bytes);
             println!("            impl bytes: {}", impl_bytes);
             println!("    exp. symbols bytes: {}", exported_symbols_bytes);
+            println!("    hygiene data bytes: {}", hygiene_data_bytes);
             println!("  def-path table bytes: {}", def_path_table_bytes);
             println!("            item bytes: {}", item_bytes);
             println!("           index bytes: {}", index_bytes);

src/liblibc

@@ -24,7 +24,7 @@ use rustc_back::PanicStrategy;
 use rustc_serialize as serialize;
 use syntax::{ast, attr};
 use syntax::symbol::Symbol;
-use syntax_pos::{self, Span};
+use syntax_pos::{self, hygiene, Span};
 
 use std::marker::PhantomData;
 use std::mem;
@@ -204,6 +204,7 @@ pub struct CrateRoot {
     pub lang_items_missing: LazySeq<lang_items::LangItem>,
     pub native_libraries: LazySeq<NativeLibrary>,
     pub codemap: LazySeq<syntax_pos::FileMap>,
+    pub hygiene_data: Lazy<hygiene::HygieneData>,
     pub def_path_table: Lazy<hir::map::definitions::DefPathTable>,
     pub impls: LazySeq<TraitImpls>,
     pub exported_symbols: LazySeq<DefIndex>,