Rollup merge of rust-lang#88161 - michaelwoerister:fix-whole-archive-no-bundle, r=petrochenkov

Fix handling of +whole-archive native link modifier.

This PR fixes a bug in `add_upstream_native_libraries` that led to the `+whole-archive` modifier being ignored when linking in native libs.

~~Note that the PR does not address the situation when `+whole-archive` is combined with `+bundle`.~~
`@wesleywiser's` commit adds validation code that turns combining `+whole-archive` with `+bundle` into an error.

Fixes rust-lang#88085.

r? `@petrochenkov`
cc `@wesleywiser` `@gcoakes`

Author: Dylan-DPC

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -36,6 +36,7 @@ use regex::Regex;
 use tempfile::Builder as TempFileBuilder;
 
 use std::ffi::OsString;
+use std::lazy::OnceCell;
 use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Output, Stdio};
 use std::{ascii, char, env, fmt, fs, io, mem, str};
@@ -254,6 +255,19 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(
     // metadata of the rlib we're generating somehow.
     for lib in codegen_results.crate_info.used_libraries.iter() {
         match lib.kind {
+            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
+                if flavor == RlibFlavor::Normal =>
+            {
+                // Don't allow mixing +bundle with +whole_archive since an rlib may contain
+                // multiple native libs, some of which are +whole-archive and some of which are
+                // -whole-archive and it isn't clear how we can currently handle such a
+                // situation correctly.
+                // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897
+                sess.err(
+                    "the linking modifiers `+bundle` and `+whole-archive` are not compatible \
+                        with each other when generating rlibs",
+                );
+            }
             NativeLibKind::Static { bundle: None | Some(true), .. } => {}
             NativeLibKind::Static { bundle: Some(false), .. }
             | NativeLibKind::Dylib { .. }
@@ -1222,6 +1236,7 @@ pub fn archive_search_paths(sess: &Session) -> Vec<PathBuf> {
     sess.target_filesearch(PathKind::Native).search_path_dirs()
 }
 
+#[derive(PartialEq)]
 enum RlibFlavor {
     Normal,
     StaticlibBase,
@@ -2328,6 +2343,7 @@ fn add_upstream_native_libraries(
         .find(|(ty, _)| *ty == crate_type)
         .expect("failed to find crate type in dependency format list");
 
+    let search_path = OnceCell::new();
     let crates = &codegen_results.crate_info.used_crates;
     let mut last = (NativeLibKind::Unspecified, None);
     for &cnum in crates {
@@ -2352,19 +2368,34 @@ fn add_upstream_native_libraries(
                 NativeLibKind::Framework { as_needed } => {
                     cmd.link_framework(name, as_needed.unwrap_or(true))
                 }
-                NativeLibKind::Static { bundle: Some(false), .. } => {
+                NativeLibKind::Static { bundle: Some(false), whole_archive } => {
                     // Link "static-nobundle" native libs only if the crate they originate from
                     // is being linked statically to the current crate.  If it's linked dynamically
                     // or is an rlib already included via some other dylib crate, the symbols from
                     // native libs will have already been included in that dylib.
                     if data[cnum.as_usize() - 1] == Linkage::Static {
-                        cmd.link_staticlib(name, verbatim)
+                        if whole_archive == Some(true) {
+                            cmd.link_whole_staticlib(
+                                name,
+                                verbatim,
+                                search_path.get_or_init(|| archive_search_paths(sess)),
+                            );
+                        } else {
+                            cmd.link_staticlib(name, verbatim)
+                        }
                     }
                 }
                 // ignore statically included native libraries here as we've
                 // already included them when we included the rust library
                 // previously
-                NativeLibKind::Static { bundle: None | Some(true), .. } => {}
+                NativeLibKind::Static { bundle: None | Some(true), whole_archive } => {
+                    if whole_archive == Some(true) {
+                        bug!(
+                            "Combining `+bundle` with `+whole-archive` is not allowed. \
+                              This should have been caught by an earlier validation step already."
+                        )
+                    }
+                }
                 NativeLibKind::RawDylib => {}
             }
         }

compiler/rustc_codegen_ssa/src/lib.rs

@@ -5,6 +5,7 @@
 #![feature(in_band_lifetimes)]
 #![feature(nll)]
 #![feature(associated_type_bounds)]
+#![feature(once_cell)]
 #![recursion_limit = "256"]
 
 //! This crate contains codegen code that is used by all codegen backends (LLVM and others).

src/test/run-make/native-link-modifier-whole-archive/Makefile

@@ -0,0 +1,37 @@
+# This test case makes sure that native libraries are linked with --whole-archive semantics
+# when the `-bundle,+whole-archive` modifiers are applied to them.
+#
+# The test works by checking that the resulting executables produce the expected output,
+# part of which is emitted by otherwise unreferenced C code. If +whole-archive didn't work
+# that code would never make it into the final executable and we'd thus be missing some
+# of the output.
+
+-include ../../run-make-fulldeps/tools.mk
+
+all: $(TMPDIR)/$(call BIN,directly_linked) $(TMPDIR)/$(call BIN,indirectly_linked) $(TMPDIR)/$(call BIN,indirectly_linked_via_attr)
+	$(call RUN,directly_linked) | $(CGREP) 'static-initializer.directly_linked.'
+	$(call RUN,indirectly_linked) | $(CGREP) 'static-initializer.indirectly_linked.'
+	$(call RUN,indirectly_linked_via_attr) | $(CGREP) 'static-initializer.native_lib_in_src.'
+
+# Native lib linked directly into executable
+$(TMPDIR)/$(call BIN,directly_linked): $(call NATIVE_STATICLIB,c_static_lib_with_constructor)
+	$(RUSTC) directly_linked.rs -Z unstable-options -l static:+whole-archive=c_static_lib_with_constructor
+
+# Native lib linked into RLIB via `-l static:-bundle,+whole-archive`, RLIB linked into executable
+$(TMPDIR)/$(call BIN,indirectly_linked): $(TMPDIR)/librlib_with_cmdline_native_lib.rlib
+	$(RUSTC) indirectly_linked.rs
+
+# Native lib linked into RLIB via #[link] attribute, RLIB linked into executable
+$(TMPDIR)/$(call BIN,indirectly_linked_via_attr): $(TMPDIR)/libnative_lib_in_src.rlib
+	$(RUSTC) indirectly_linked_via_attr.rs
+
+# Native lib linked into rlib with via commandline
+$(TMPDIR)/librlib_with_cmdline_native_lib.rlib: $(call NATIVE_STATICLIB,c_static_lib_with_constructor)
+	$(RUSTC) rlib_with_cmdline_native_lib.rs -Z unstable-options --crate-type=rlib -l static:-bundle,+whole-archive=c_static_lib_with_constructor
+
+# Native lib linked into rlib via `#[link()]` attribute on extern block.
+$(TMPDIR)/libnative_lib_in_src.rlib: $(call NATIVE_STATICLIB,c_static_lib_with_constructor)
+	$(RUSTC) native_lib_in_src.rs --crate-type=rlib
+
+$(TMPDIR)/libc_static_lib_with_constructor.o: c_static_lib_with_constructor.cpp
+	$(call COMPILE_OBJ_CXX,$@,$<)

src/test/run-make/native-link-modifier-whole-archive/c_static_lib_with_constructor.cpp

@@ -0,0 +1,11 @@
+#include <cstdio>
+
+// Since this is a global variable, its constructor will be called before
+// main() is executed. But only if the object file containing it actually
+// gets linked into the executable.
+struct Foo {
+    Foo() {
+        printf("static-initializer.");
+        fflush(stdout);
+    }
+} FOO;

src/test/run-make/native-link-modifier-whole-archive/directly_linked.rs

@@ -0,0 +1,6 @@
+use std::io::Write;
+
+fn main() {
+    print!("directly_linked.");
+    std::io::stdout().flush().unwrap();
+}

src/test/run-make/native-link-modifier-whole-archive/indirectly_linked.rs

@@ -0,0 +1,5 @@
+extern crate rlib_with_cmdline_native_lib;
+
+fn main() {
+    rlib_with_cmdline_native_lib::hello();
+}

src/test/run-make/native-link-modifier-whole-archive/indirectly_linked_via_attr.rs

@@ -0,0 +1,5 @@
+extern crate native_lib_in_src;
+
+fn main() {
+    native_lib_in_src::hello();
+}

src/test/run-make/native-link-modifier-whole-archive/native_lib_in_src.rs

@@ -0,0 +1,15 @@
+#![feature(native_link_modifiers_bundle)]
+#![feature(native_link_modifiers_whole_archive)]
+#![feature(native_link_modifiers)]
+
+use std::io::Write;
+
+#[link(name = "c_static_lib_with_constructor",
+       kind = "static",
+       modifiers = "-bundle,+whole-archive")]
+extern {}
+
+pub fn hello() {
+    print!("native_lib_in_src.");
+    std::io::stdout().flush().unwrap();
+}

src/test/run-make/native-link-modifier-whole-archive/rlib_with_cmdline_native_lib.rs

@@ -0,0 +1,6 @@
+use std::io::Write;
+
+pub fn hello() {
+    print!("indirectly_linked.");
+    std::io::stdout().flush().unwrap();
+}

src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.rs

@@ -0,0 +1,12 @@
+// compile-flags: -Zunstable-options --crate-type rlib
+// build-fail
+// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
+
+#![feature(native_link_modifiers)]
+#![feature(native_link_modifiers_bundle)]
+#![feature(native_link_modifiers_whole_archive)]
+
+#[link(name = "mylib", kind = "static", modifiers = "+bundle,+whole-archive")]
+extern "C" { }
+
+fn main() { }

src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive-link-attr.stderr

@@ -0,0 +1,6 @@
+error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
+
+error: could not find native static library `mylib`, perhaps an -L flag is missing?
+
+error: aborting due to 2 previous errors
+

src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.rs

@@ -0,0 +1,7 @@
+// Mixing +bundle and +whole-archive is not allowed
+
+// compile-flags: -l static:+bundle,+whole-archive=mylib -Zunstable-options --crate-type rlib
+// build-fail
+// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
+
+fn main() { }

src/test/ui/native-library-link-flags/mix-bundle-and-whole-archive.stderr

@@ -0,0 +1,6 @@
+error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
+
+error: could not find native static library `mylib`, perhaps an -L flag is missing?
+
+error: aborting due to 2 previous errors
+