Rollup merge of #141769 - bjorn3:codegen_metadata_module_rework, r=workingjubilee,saethlin

Move metadata object generation for dylibs to the linker code

This deduplicates some code between codegen backends and may in the future allow adding extra metadata that is only known at link time.

Prerequisite of #96708.

Author: fmease

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -11,15 +11,13 @@ use std::thread::JoinHandle;
 use cranelift_object::{ObjectBuilder, ObjectModule};
 use rustc_codegen_ssa::assert_module_sources::CguReuse;
 use rustc_codegen_ssa::back::link::ensure_removed;
-use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
 use rustc_codegen_ssa::base::determine_cgu_reuse;
 use rustc_codegen_ssa::{
     CodegenResults, CompiledModule, CrateInfo, ModuleKind, errors as ssa_errors,
 };
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{IntoDynSyncSend, par_map};
-use rustc_metadata::EncodedMetadata;
 use rustc_metadata::fs::copy_to_stdout;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
@@ -61,8 +59,6 @@ impl<HCX> HashStable<HCX> for OngoingModuleCodegen {
 pub(crate) struct OngoingCodegen {
     modules: Vec<OngoingModuleCodegen>,
     allocator_module: Option<CompiledModule>,
-    metadata_module: Option<CompiledModule>,
-    metadata: EncodedMetadata,
     crate_info: CrateInfo,
     concurrency_limiter: ConcurrencyLimiter,
 }
@@ -134,8 +130,6 @@ impl OngoingCodegen {
         let codegen_results = CodegenResults {
             modules,
             allocator_module: self.allocator_module,
-            metadata_module: self.metadata_module,
-            metadata: self.metadata,
             crate_info: self.crate_info,
         };
 
@@ -646,42 +640,6 @@ fn module_codegen(
     }))
 }
 
-fn emit_metadata_module(tcx: TyCtxt<'_>, metadata: &EncodedMetadata) -> CompiledModule {
-    use rustc_middle::mir::mono::CodegenUnitNameBuilder;
-
-    let _timer = tcx.sess.timer("write compressed metadata");
-
-    let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
-    let metadata_cgu_name = cgu_name_builder
-        .build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata"))
-        .as_str()
-        .to_string();
-
-    let tmp_file = tcx.output_filenames(()).temp_path_for_cgu(
-        OutputType::Metadata,
-        &metadata_cgu_name,
-        tcx.sess.invocation_temp.as_deref(),
-    );
-
-    let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx);
-    let obj = create_compressed_metadata_file(tcx.sess, metadata, &symbol_name);
-
-    if let Err(err) = std::fs::write(&tmp_file, obj) {
-        tcx.dcx().fatal(format!("error writing metadata object file: {}", err));
-    }
-
-    CompiledModule {
-        name: metadata_cgu_name,
-        kind: ModuleKind::Metadata,
-        object: Some(tmp_file),
-        dwarf_object: None,
-        bytecode: None,
-        assembly: None,
-        llvm_ir: None,
-        links_from_incr_cache: Vec::new(),
-    }
-}
-
 fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> {
     let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string());
     let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module);
@@ -706,11 +664,7 @@ fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> {
     }
 }
 
-pub(crate) fn run_aot(
-    tcx: TyCtxt<'_>,
-    metadata: EncodedMetadata,
-    need_metadata_module: bool,
-) -> Box<OngoingCodegen> {
+pub(crate) fn run_aot(tcx: TyCtxt<'_>) -> Box<OngoingCodegen> {
     // FIXME handle `-Ctarget-cpu=native`
     let target_cpu = match tcx.sess.opts.cg.target_cpu {
         Some(ref name) => name,
@@ -726,8 +680,6 @@ pub(crate) fn run_aot(
         return Box::new(OngoingCodegen {
             modules: vec![],
             allocator_module: None,
-            metadata_module: None,
-            metadata,
             crate_info: CrateInfo::new(tcx, target_cpu),
             concurrency_limiter: ConcurrencyLimiter::new(0),
         });
@@ -787,14 +739,9 @@ pub(crate) fn run_aot(
 
     let allocator_module = emit_allocator_module(tcx);
 
-    let metadata_module =
-        if need_metadata_module { Some(emit_metadata_module(tcx, &metadata)) } else { None };
-
     Box::new(OngoingCodegen {
         modules,
         allocator_module,
-        metadata_module,
-        metadata,
         crate_info: CrateInfo::new(tcx, target_cpu),
         concurrency_limiter: concurrency_limiter.0,
     })

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -46,7 +46,6 @@ use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::settings::{self, Configurable};
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::{CodegenResults, TargetConfig};
-use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_session::Session;
 use rustc_session::config::OutputFilenames;
@@ -238,12 +237,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
         println!("Cranelift version: {}", cranelift_codegen::VERSION);
     }
 
-    fn codegen_crate(
-        &self,
-        tcx: TyCtxt<'_>,
-        metadata: EncodedMetadata,
-        need_metadata_module: bool,
-    ) -> Box<dyn Any> {
+    fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box<dyn Any> {
         info!("codegen crate {}", tcx.crate_name(LOCAL_CRATE));
         let config = self.config.clone().unwrap_or_else(|| {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
@@ -256,7 +250,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
             #[cfg(not(feature = "jit"))]
             tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift");
         } else {
-            driver::aot::run_aot(tcx, metadata, need_metadata_module)
+            driver::aot::run_aot(tcx)
         }
     }
 

compiler/rustc_codegen_gcc/src/lib.rs

@@ -48,7 +48,6 @@ extern crate rustc_index;
 #[cfg(feature = "master")]
 extern crate rustc_interface;
 extern crate rustc_macros;
-extern crate rustc_metadata;
 extern crate rustc_middle;
 extern crate rustc_session;
 extern crate rustc_span;
@@ -106,7 +105,6 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen, TargetCon
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::IntoDynSyncSend;
 use rustc_errors::DiagCtxtHandle;
-use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::util::Providers;
@@ -230,20 +228,9 @@ impl CodegenBackend for GccCodegenBackend {
         providers.global_backend_features = |tcx, ()| gcc_util::global_gcc_features(tcx.sess, true)
     }
 
-    fn codegen_crate(
-        &self,
-        tcx: TyCtxt<'_>,
-        metadata: EncodedMetadata,
-        need_metadata_module: bool,
-    ) -> Box<dyn Any> {
+    fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box<dyn Any> {
         let target_cpu = target_cpu(tcx.sess);
-        let res = codegen_crate(
-            self.clone(),
-            tcx,
-            target_cpu.to_string(),
-            metadata,
-            need_metadata_module,
-        );
+        let res = codegen_crate(self.clone(), tcx, target_cpu.to_string());
 
         Box::new(res)
     }

compiler/rustc_codegen_llvm/src/lib.rs

@@ -340,18 +340,11 @@ impl CodegenBackend for LlvmCodegenBackend {
         target_config(sess)
     }
 
-    fn codegen_crate<'tcx>(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        metadata: EncodedMetadata,
-        need_metadata_module: bool,
-    ) -> Box<dyn Any> {
+    fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Box<dyn Any> {
         Box::new(rustc_codegen_ssa::base::codegen_crate(
             LlvmCodegenBackend(()),
             tcx,
             crate::llvm_util::target_cpu(tcx.sess).to_string(),
-            metadata,
-            need_metadata_module,
         ))
     }
 
@@ -376,14 +369,20 @@ impl CodegenBackend for LlvmCodegenBackend {
         (codegen_results, work_products)
     }
 
-    fn link(&self, sess: &Session, codegen_results: CodegenResults, outputs: &OutputFilenames) {
+    fn link(
+        &self,
+        sess: &Session,
+        codegen_results: CodegenResults,
+        metadata: EncodedMetadata,
+        outputs: &OutputFilenames,
+    ) {
         use rustc_codegen_ssa::back::link::link_binary;
 
         use crate::back::archive::LlvmArchiveBuilderBuilder;
 
         // Run the linker on any artifacts that resulted from the LLVM run.
         // This should produce either a finished executable or library.
-        link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs);
+        link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, metadata, outputs);
     }
 }
 

compiler/rustc_codegen_ssa/messages.ftl

@@ -198,8 +198,6 @@ codegen_ssa_linking_failed = linking with `{$linker_path}` failed: {$exit_status
 codegen_ssa_malformed_cgu_name =
     found malformed codegen unit name `{$user_path}`. codegen units names must always start with the name of the crate (`{$crate_name}` in this case).
 
-codegen_ssa_metadata_object_file_write = error writing metadata object file: {$error}
-
 codegen_ssa_missing_cpp_build_tool_component = or a necessary component may be missing from the "C++ build tools" workload
 
 codegen_ssa_missing_features = add the missing features in a `target_feature` attribute

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -14,11 +14,12 @@ use object::read::macho::FatArch;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::memmap::Mmap;
 use rustc_fs_util::TempDirBuilder;
+use rustc_metadata::EncodedMetadata;
 use rustc_session::Session;
 use rustc_span::Symbol;
 use tracing::trace;
 
-use super::metadata::search_for_section;
+use super::metadata::{create_compressed_metadata_file, search_for_section};
 use crate::common;
 // Re-exporting for rustc_codegen_llvm::back::archive
 pub use crate::errors::{ArchiveBuildFailure, ExtractBundledLibsError, UnknownArchiveKind};
@@ -58,6 +59,15 @@ impl From<ImportLibraryItem> for COFFShortExport {
 pub trait ArchiveBuilderBuilder {
     fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a>;
 
+    fn create_dylib_metadata_wrapper(
+        &self,
+        sess: &Session,
+        metadata: &EncodedMetadata,
+        symbol_name: &str,
+    ) -> Vec<u8> {
+        create_compressed_metadata_file(sess, metadata, symbol_name)
+    }
+
     /// Creates a DLL Import Library <https://docs.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation#creating-an-import-library>.
     /// and returns the path on disk to that import library.
     /// This functions doesn't take `self` so that it can be called from