trans: Stop informing LLVM about dllexport

Rust's current compilation model makes it impossible on Windows to generate one
object file with a complete and final set of dllexport annotations. This is
because when an object is generated the compiler doesn't actually know if it
will later be included in a dynamic library or not. The compiler works around
this today by flagging *everything* as dllexport, but this has the drawback of
exposing too much.

Thankfully there are alternate methods of specifying the exported surface area
of a dll on Windows, one of which is passing a `*.def` file to the linker which
lists all public symbols of the dynamic library. This commit removes all
locations that add `dllexport` to LLVM variables and instead dynamically
generates a `*.def` file which is passed to the linker. This file will include
all the public symbols of the current object file as well as all upstream
libraries, and the crucial aspect is that it's only used when generating a
dynamic library. When generating an executable this file isn't generated, so all
the symbols aren't exported from an executable.

To ensure that statically included native libraries are reexported correctly,
the previously added support for the `#[linked_from]` attribute is used to
determine the set of FFI symbols that are exported from a dynamic library, and
this is required to get the compiler to link correctly.

Author: alexcrichton

mk/platform.mk

@@ -277,10 +277,15 @@ $(foreach target,$(CFG_TARGET), \
 # Fun times!
 #
 # [1]: https://msdn.microsoft.com/en-us/library/28d6s79h.aspx
+#
+# FIXME(stage0): remove this macro and the usage below (and the commments above)
+# 	         when a new snapshot is available. Also remove the
+# 	         RUSTFLAGS$(1)_.._T_ variable in mk/target.mk along with
+# 	         CUSTOM_DEPS (as they were only added for this)
 define ADD_RUSTC_LLVM_DEF_TO_MSVC
 ifeq ($$(findstring msvc,$(1)),msvc)
-RUSTFLAGS_rustc_llvm_T_$(1) += -C link-args="-DEF:$(1)/rt/rustc_llvm.def"
-CUSTOM_DEPS_rustc_llvm_T_$(1) += $(1)/rt/rustc_llvm.def
+RUSTFLAGS0_rustc_llvm_T_$(1) += -C link-args="-DEF:$(1)/rt/rustc_llvm.def"
+CUSTOM_DEPS0_rustc_llvm_T_$(1) += $(1)/rt/rustc_llvm.def
 
 $(1)/rt/rustc_llvm.def: $$(S)src/etc/mklldef.py $$(S)src/librustc_llvm/lib.rs
 	$$(CFG_PYTHON) $$^ $$@ rustc_llvm-$$(CFG_FILENAME_EXTRA)

mk/target.mk

@@ -40,7 +40,7 @@ CRATE_FULLDEPS_$(1)_T_$(2)_H_$(3)_$(4) := \
 		  $$(RT_OUTPUT_DIR_$(2))/$$(dep)) \
 		$$(foreach dep,$$(NATIVE_TOOL_DEPS_$(4)_T_$(2)), \
 		  $$(TBIN$(1)_T_$(3)_H_$(3))/$$(dep)) \
-		$$(CUSTOM_DEPS_$(4)_T_$(2))
+		$$(CUSTOM_DEPS$(1)_$(4)_T_$(2))
 endef
 
 $(foreach host,$(CFG_HOST), \
@@ -92,7 +92,7 @@ $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
 		$$(LLVM_LIBDIR_RUSTFLAGS_$(2)) \
 		$$(LLVM_STDCPP_RUSTFLAGS_$(2)) \
 		$$(RUSTFLAGS_$(4)) \
-		$$(RUSTFLAGS_$(4)_T_$(2)) \
+		$$(RUSTFLAGS$(1)_$(4)_T_$(2)) \
 		--out-dir $$(@D) \
 		-C extra-filename=-$$(CFG_FILENAME_EXTRA) \
 		$$<

src/compiletest/procsrv.rs

@@ -26,8 +26,7 @@ fn add_target_env(cmd: &mut Command, lib_path: &str, aux_path: Option<&str>) {
     // Add the new dylib search path var
     let var = DynamicLibrary::envvar();
     let newpath = DynamicLibrary::create_path(&path);
-    let newpath = newpath.to_str().unwrap().to_string();
-    cmd.env(var, &newpath);
+    cmd.env(var, newpath);
 }
 
 pub struct Result {pub status: ExitStatus, pub out: String, pub err: String}

src/librustc/metadata/common.rs

@@ -205,8 +205,8 @@ pub const tag_plugin_registrar_fn: usize = 0x10b; // top-level only
 pub const tag_method_argument_names: usize = 0x85;
 pub const tag_method_argument_name: usize = 0x86;
 
-pub const tag_reachable_extern_fns: usize = 0x10c; // top-level only
-pub const tag_reachable_extern_fn_id: usize = 0x87;
+pub const tag_reachable_ids: usize = 0x10c; // top-level only
+pub const tag_reachable_id: usize = 0x87;
 
 pub const tag_items_data_item_stability: usize = 0x88;
 

src/librustc/metadata/csearch.rs

@@ -352,11 +352,11 @@ pub fn get_method_arg_names(cstore: &cstore::CStore, did: ast::DefId)
     decoder::get_method_arg_names(&*cdata, did.node)
 }
 
-pub fn get_reachable_extern_fns(cstore: &cstore::CStore, cnum: ast::CrateNum)
+pub fn get_reachable_ids(cstore: &cstore::CStore, cnum: ast::CrateNum)
     -> Vec<ast::DefId>
 {
     let cdata = cstore.get_crate_data(cnum);
-    decoder::get_reachable_extern_fns(&*cdata)
+    decoder::get_reachable_ids(&*cdata)
 }
 
 pub fn is_typedef(cstore: &cstore::CStore, did: ast::DefId) -> bool {
@@ -400,3 +400,9 @@ pub fn is_default_impl(cstore: &cstore::CStore, impl_did: ast::DefId) -> bool {
     let cdata = cstore.get_crate_data(impl_did.krate);
     decoder::is_default_impl(&*cdata, impl_did.node)
 }
+
+pub fn is_extern_fn(cstore: &cstore::CStore, did: ast::DefId,
+                    tcx: &ty::ctxt) -> bool {
+    let cdata = cstore.get_crate_data(did.krate);
+    decoder::is_extern_fn(&*cdata, did.node, tcx)
+}

src/librustc/metadata/decoder.rs

@@ -45,6 +45,7 @@ use std::str;
 use rbml::reader;
 use rbml;
 use serialize::Decodable;
+use syntax::abi;
 use syntax::attr;
 use syntax::parse::token::{IdentInterner, special_idents};
 use syntax::parse::token;
@@ -1418,10 +1419,10 @@ pub fn get_method_arg_names(cdata: Cmd, id: ast::NodeId) -> Vec<String> {
     }
 }
 
-pub fn get_reachable_extern_fns(cdata: Cmd) -> Vec<ast::DefId> {
+pub fn get_reachable_ids(cdata: Cmd) -> Vec<ast::DefId> {
     let items = reader::get_doc(rbml::Doc::new(cdata.data()),
-                                tag_reachable_extern_fns);
-    reader::tagged_docs(items, tag_reachable_extern_fn_id).map(|doc| {
+                                tag_reachable_ids);
+    reader::tagged_docs(items, tag_reachable_id).map(|doc| {
         ast::DefId {
             krate: cdata.cnum,
             node: reader::doc_as_u32(doc),
@@ -1543,3 +1544,21 @@ pub fn get_imported_filemaps(metadata: &[u8]) -> Vec<codemap::FileMap> {
         Decodable::decode(&mut decoder).unwrap()
     }).collect()
 }
+
+pub fn is_extern_fn(cdata: Cmd, id: ast::NodeId, tcx: &ty::ctxt) -> bool {
+    let root_doc = rbml::Doc::new(cdata.data());
+    let items = reader::get_doc(root_doc, tag_items);
+    let item_doc = match maybe_find_item(id, items) {
+        Some(doc) => doc,
+        None => return false,
+    };
+    if let Fn = item_family(item_doc) {
+        let ty::TypeScheme { generics, ty } = get_type(cdata, id, tcx);
+        generics.types.is_empty() && match ty.sty {
+            ty::TyBareFn(_, fn_ty) => fn_ty.abi != abi::Rust,
+            _ => false,
+        }
+    } else {
+        false
+    }
+}

src/librustc/metadata/encoder.rs

@@ -1781,9 +1781,8 @@ fn encode_crate_deps(rbml_w: &mut Encoder, cstore: &cstore::CStore) {
     // FIXME (#2166): This is not nearly enough to support correct versioning
     // but is enough to get transitive crate dependencies working.
     rbml_w.start_tag(tag_crate_deps);
-    let r = get_ordered_deps(cstore);
-    for dep in &r {
-        encode_crate_dep(rbml_w, (*dep).clone());
+    for dep in &get_ordered_deps(cstore) {
+        encode_crate_dep(rbml_w, dep);
     }
     rbml_w.end_tag();
 }
@@ -1971,24 +1970,22 @@ fn encode_misc_info(ecx: &EncodeContext,
     rbml_w.end_tag();
 }
 
-fn encode_reachable_extern_fns(ecx: &EncodeContext, rbml_w: &mut Encoder) {
-    rbml_w.start_tag(tag_reachable_extern_fns);
-
+// Encodes all reachable symbols in this crate into the metadata.
+//
+// This pass is seeded off the reachability list calculated in the
+// middle::reachable module but filters out items that either don't have a
+// symbol associated with them (they weren't translated) or if they're an FFI
+// definition (as that's not defined in this crate).
+fn encode_reachable(ecx: &EncodeContext, rbml_w: &mut Encoder) {
+    rbml_w.start_tag(tag_reachable_ids);
     for id in ecx.reachable {
-        if let Some(ast_map::NodeItem(i)) = ecx.tcx.map.find(*id) {
-            if let ast::ItemFn(_, _, _, abi, ref generics, _) = i.node {
-                if abi != abi::Rust && !generics.is_type_parameterized() {
-                    rbml_w.wr_tagged_u32(tag_reachable_extern_fn_id, *id);
-                }
-            }
-        }
+        rbml_w.wr_tagged_u32(tag_reachable_id, *id);
     }
-
     rbml_w.end_tag();
 }
 
 fn encode_crate_dep(rbml_w: &mut Encoder,
-                    dep: decoder::CrateDep) {
+                    dep: &decoder::CrateDep) {
     rbml_w.start_tag(tag_crate_dep);
     rbml_w.wr_tagged_str(tag_crate_dep_crate_name, &dep.name);
     rbml_w.wr_tagged_str(tag_crate_dep_hash, dep.hash.as_str());
@@ -2170,7 +2167,7 @@ fn encode_metadata_inner(wr: &mut Cursor<Vec<u8>>,
     // Encode miscellaneous info.
     i = rbml_w.writer.seek(SeekFrom::Current(0)).unwrap();
     encode_misc_info(&ecx, krate, &mut rbml_w);
-    encode_reachable_extern_fns(&ecx, &mut rbml_w);
+    encode_reachable(&ecx, &mut rbml_w);
     stats.misc_bytes = rbml_w.writer.seek(SeekFrom::Current(0)).unwrap() - i;
 
     // Encode and index the items.

src/librustc_llvm/lib.rs

@@ -31,6 +31,7 @@
 #![feature(link_args)]
 #![feature(staged_api)]
 #![feature(vec_push_all)]
+#![cfg_attr(not(stage0), feature(linked_from))]
 
 extern crate libc;
 #[macro_use] #[no_link] extern crate rustc_bitflags;
@@ -598,6 +599,7 @@ pub mod debuginfo {
 // automatically updated whenever LLVM is updated to include an up-to-date
 // set of the libraries we need to link to LLVM for.
 #[link(name = "rustllvm", kind = "static")]
+#[cfg_attr(not(stage0), linked_from = "rustllvm")] // not quite true but good enough
 extern {
     /* Create and destroy contexts. */
     pub fn LLVMContextCreate() -> ContextRef;

src/librustc_trans/back/link.rs

@@ -902,6 +902,12 @@ fn link_args(cmd: &mut Linker,
     }
     cmd.output_filename(out_filename);
 
+    // If we're building a dynamic library then some platforms need to make sure
+    // that all symbols are exported correctly from the dynamic library.
+    if dylib {
+        cmd.export_symbols(sess, trans, tmpdir);
+    }
+
     // When linking a dynamic library, we put the metadata into a section of the
     // executable. This metadata is in a separate object file from the main
     // object file, so we link that in here.

src/librustc_trans/back/linker.rs

@@ -9,14 +9,21 @@
 // except according to those terms.
 
 use std::ffi::OsString;
+use std::fs::{self, File};
+use std::io::{self, BufWriter};
+use std::io::prelude::*;
 use std::path::{Path, PathBuf};
 use std::process::Command;
-use std::fs;
 
 use back::archive;
+use metadata::csearch;
+use metadata::cstore;
 use session::Session;
-use session::config;
 use session::config::DebugInfoLevel::{NoDebugInfo, LimitedDebugInfo, FullDebugInfo};
+use session::config::CrateTypeDylib;
+use session::config;
+use syntax::ast;
+use trans::CrateTranslation;
 
 /// Linker abstraction used by back::link to build up the command to invoke a
 /// linker.
@@ -48,6 +55,8 @@ pub trait Linker {
     fn hint_dynamic(&mut self);
     fn whole_archives(&mut self);
     fn no_whole_archives(&mut self);
+    fn export_symbols(&mut self, sess: &Session, trans: &CrateTranslation,
+                      tmpdir: &Path);
 }
 
 pub struct GnuLinker<'a> {
@@ -192,6 +201,10 @@ impl<'a> Linker for GnuLinker<'a> {
         if !self.takes_hints() { return }
         self.cmd.arg("-Wl,-Bdynamic");
     }
+
+    fn export_symbols(&mut self, _: &Session, _: &CrateTranslation, _: &Path) {
+        // noop, visibility in object files takes care of this
+    }
 }
 
 pub struct MsvcLinker<'a> {
@@ -301,4 +314,61 @@ impl<'a> Linker for MsvcLinker<'a> {
     // we do on Unix platforms.
     fn hint_static(&mut self) {}
     fn hint_dynamic(&mut self) {}
+
+    // Currently the compiler doesn't use `dllexport` (an LLVM attribute) to
+    // export symbols from a dynamic library. When building a dynamic library,
+    // however, we're going to want some symbols exported, so this function
+    // generates a DEF file which lists all the symbols.
+    //
+    // The linker will read this `*.def` file and export all the symbols from
+    // the dynamic library. Note that this is not as simple as just exporting
+    // all the symbols in the current crate (as specified by `trans.reachable`)
+    // but rather we also need to possibly export the symbols of upstream
+    // crates. Upstream rlibs may be linked statically to this dynamic library,
+    // in which case they may continue to transitively be used and hence need
+    // their symbols exported.
+    fn export_symbols(&mut self, sess: &Session, trans: &CrateTranslation,
+                      tmpdir: &Path) {
+        let path = tmpdir.join("lib.def");
+        let res = (|| -> io::Result<()> {
+            let mut f = BufWriter::new(try!(File::create(&path)));
+
+            // Start off with the standard module name header and then go
+            // straight to exports.
+            try!(writeln!(f, "LIBRARY"));
+            try!(writeln!(f, "EXPORTS"));
+
+            // Write out all our local symbols
+            for sym in trans.reachable.iter() {
+                try!(writeln!(f, "  {}", sym));
+            }
+
+            // Take a look at how all upstream crates are linked into this
+            // dynamic library. For all statically linked libraries we take all
+            // their reachable symbols and emit them as well.
+            let cstore = &sess.cstore;
+            let symbols = trans.crate_formats[&CrateTypeDylib].iter();
+            let symbols = symbols.enumerate().filter_map(|(i, f)| {
+                if let Some(cstore::RequireStatic) = *f {
+                    Some((i + 1) as ast::CrateNum)
+                } else {
+                    None
+                }
+            }).flat_map(|cnum| {
+                csearch::get_reachable_ids(cstore, cnum)
+            }).map(|did| {
+                csearch::get_symbol(cstore, did)
+            });
+            for symbol in symbols {
+                try!(writeln!(f, "  {}", symbol));
+            }
+            Ok(())
+        })();
+        if let Err(e) = res {
+            sess.fatal(&format!("failed to write lib.def file: {}", e));
+        }
+        let mut arg = OsString::from("/DEF:");
+        arg.push(path);
+        self.cmd.arg(&arg);
+    }
 }