Auto merge of #139244 - jieyouxu:exp/auto-cross-run-make, r=<try>

[WIP] Enable automatic cross-compilation in run-make tests

> [!CAUTION]
>
> Stacked on top of #142414, needs rebase.

Supersedes #138066.

Blocker for #141856.

Based on #138066 with #139242 + #139239 cherry-picked in, plus `rustdoc()` cross-compile changes.

r? `@ghost`

try-job: armhf-gnu
try-job: test-various

Author: rust-bors[bot]

src/tools/run-make-support/src/external_deps/rustc.rs

@@ -6,7 +6,7 @@ use crate::command::Command;
 use crate::env::env_var;
 use crate::path_helpers::cwd;
 use crate::util::set_host_compiler_dylib_path;
-use crate::{is_aix, is_darwin, is_msvc, is_windows, uname};
+use crate::{is_aix, is_darwin, is_msvc, is_windows, target, uname};
 
 /// Construct a new `rustc` invocation. This will automatically set the library
 /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
@@ -27,9 +27,15 @@ pub fn bare_rustc() -> Rustc {
 #[must_use]
 pub struct Rustc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustc, |rustc: &mut Rustc| {
+    if let Some(target) = &rustc.target {
+        rustc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 pub fn rustc_path() -> String {
     env_var("RUSTC")
@@ -46,19 +52,22 @@ impl Rustc {
     // `rustc` invocation constructor methods
 
     /// Construct a new `rustc` invocation. This will automatically set the library
-    /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
+    /// search path as `-L cwd()` and also the compilation target.
+    /// Use [`bare_rustc`] to avoid this.
     #[track_caller]
     pub fn new() -> Self {
         let mut cmd = setup_common();
         cmd.arg("-L").arg(cwd());
-        Self { cmd }
+
+        // Automatically default to cross-compilation
+        Self { cmd, target: Some(target()) }
     }
 
     /// Construct a bare `rustc` invocation with no flags set.
     #[track_caller]
     pub fn bare() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: None }
     }
 
     // Argument provider methods
@@ -234,8 +243,9 @@ impl Rustc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in Rustc::new().
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/external_deps/rustdoc.rs

@@ -3,21 +3,36 @@ use std::path::Path;
 
 use crate::command::Command;
 use crate::env::env_var;
+use crate::target;
 use crate::util::set_host_compiler_dylib_path;
 
-/// Construct a new `rustdoc` invocation. This will configure the host compiler runtime libs.
+/// Construct a new `rustdoc` invocation with target automatically set to cross-compile target and
+/// with host compiler runtime libs configured. Use [`bare_rustdoc`] to avoid automatically setting
+/// cross-compile target.
 #[track_caller]
 pub fn rustdoc() -> Rustdoc {
     Rustdoc::new()
 }
 
+/// Bare `rustdoc` invocation, no args set.
+#[track_caller]
+pub fn bare_rustdoc() -> Rustdoc {
+    Rustdoc::bare()
+}
+
 #[derive(Debug)]
 #[must_use]
 pub struct Rustdoc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustdoc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustdoc, |rustdoc: &mut Rustdoc| {
+    if let Some(target) = &rustdoc.target {
+        rustdoc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 #[track_caller]
 fn setup_common() -> Command {
@@ -28,11 +43,20 @@ fn setup_common() -> Command {
 }
 
 impl Rustdoc {
-    /// Construct a bare `rustdoc` invocation. This will configure the host compiler runtime libs.
+    /// Construct a new `rustdoc` invocation with target automatically set to cross-compile target
+    /// and with host compiler runtime libs configured. Use [`bare_rustdoc`] to avoid automatically
+    /// setting cross-compile target.
     #[track_caller]
     pub fn new() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: Some(target()) }
+    }
+
+    /// Bare `rustdoc` invocation, no args set.
+    #[track_caller]
+    pub fn bare() -> Self {
+        let cmd = setup_common();
+        Self { cmd, target: None }
     }
 
     /// Specify where an external library is located.
@@ -85,8 +109,9 @@ impl Rustdoc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in `Rustdoc::new()`.
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/lib.rs

@@ -68,7 +68,7 @@ pub use llvm::{
 };
 pub use python::python_command;
 pub use rustc::{bare_rustc, rustc, rustc_path, Rustc};
-pub use rustdoc::{rustdoc, Rustdoc};
+pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc};
 
 /// [`diff`][mod@diff] is implemented in terms of the [similar] library.
 ///

src/tools/run-make-support/src/macros.rs

@@ -23,10 +23,16 @@
 /// }
 /// ```
 ///
+/// You can pass an optional second parameter which should be a function that is passed
+/// `&mut self` just before the command is executed.
+///
 /// [`Command`]: crate::command::Command
 /// [`CompletedProcess`]: crate::command::CompletedProcess
 macro_rules! impl_common_helpers {
     ($wrapper: ident) => {
+        $crate::macros::impl_common_helpers!($wrapper, |_| {});
+    };
+    ($wrapper: ident, $before_exec: expr) => {
         impl $wrapper {
             /// In very rare circumstances, you may need a e.g. `bare_rustc()` or `bare_rustdoc()`
             /// with host runtime libs configured, but want the underlying raw
@@ -130,12 +136,14 @@ macro_rules! impl_common_helpers {
             /// Run the constructed command and assert that it is successfully run.
             #[track_caller]
             pub fn run(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run()
             }
 
             /// Run the constructed command and assert that it does not successfully run.
             #[track_caller]
             pub fn run_fail(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_fail()
             }
 
@@ -145,6 +153,7 @@ macro_rules! impl_common_helpers {
             /// whenever possible.
             #[track_caller]
             pub fn run_unchecked(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_unchecked()
             }
 

tests/run-make/apple-deployment-target/rmake.rs

@@ -41,7 +41,6 @@ fn main() {
 
     // Remove env vars to get `rustc`'s default
     let output = rustc()
-        .target(target())
         .env_remove("MACOSX_DEPLOYMENT_TARGET")
         .env_remove("IPHONEOS_DEPLOYMENT_TARGET")
         .env_remove("WATCHOS_DEPLOYMENT_TARGET")
@@ -58,7 +57,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("lib");
             rustc.emit("obj");
             rustc.input("foo.rs");
@@ -82,7 +80,6 @@ fn main() {
 
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("dylib");
             rustc.input("foo.rs");
             rustc.output("libfoo.dylib");
@@ -108,7 +105,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("bin");
             rustc.input("foo.rs");
             rustc.output("foo");
@@ -147,7 +143,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.incremental("incremental");
             rustc.crate_type("lib");
             rustc.emit("obj");

tests/run-make/apple-sdk-version/rmake.rs

@@ -24,8 +24,7 @@ fn has_sdk_version(file: &str, version: &str) {
 
 fn main() {
     // Fetch rustc's inferred deployment target.
-    let current_deployment_target =
-        rustc().target(target()).print("deployment-target").run().stdout_utf8();
+    let current_deployment_target = rustc().print("deployment-target").run().stdout_utf8();
     let current_deployment_target = current_deployment_target.split('=').last().unwrap().trim();
 
     // Fetch current SDK version via. xcrun.
@@ -45,15 +44,15 @@ fn main() {
     let current_sdk_version = current_sdk_version.trim();
 
     // Check the SDK version in the object file produced by the codegen backend.
-    rustc().target(target()).crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
+    rustc().crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
     // Set to 0, which means not set or "n/a".
     has_sdk_version("foo.o", "n/a");
 
     // Check the SDK version in the .rmeta file, as set in `create_object_file`.
     //
     // This is just to ensure that we don't set some odd version in `create_object_file`,
     // if the rmeta file is packed in a different way in the future, this can safely be removed.
-    rustc().target(target()).crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
+    rustc().crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
     // Extra .rmeta file (which is encoded as an object file).
     cmd("ar").arg("-x").arg("libfoo.rlib").arg("lib.rmeta").run();
     has_sdk_version("lib.rmeta", "n/a");
@@ -69,7 +68,6 @@ fn main() {
         // Test with clang
         let file_name = format!("foo_cc{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=gcc")
             .input("foo.rs")
@@ -80,7 +78,6 @@ fn main() {
         // Test with ld64
         let file_name = format!("foo_ld{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=ld")
             .input("foo.rs")

tests/run-make/doctests-merge/rmake.rs

@@ -1,4 +1,5 @@
-//@ needs-target-std
+//@ ignore-cross-compile (needs to run doctests)
+
 use std::path::Path;
 
 use run_make_support::{cwd, diff, rustc, rustdoc};

tests/run-make/doctests-runtool/rmake.rs

@@ -1,5 +1,5 @@
-//@ needs-target-std
-//
+//@ ignore-cross-compile (needs to run host tool binary)
+
 // Tests behavior of rustdoc `--test-runtool`.
 
 use std::path::PathBuf;

tests/run-make/embed-metadata/rmake.rs

@@ -1,5 +1,6 @@
 //@ needs-target-std
-//
+//@ needs-crate-type: dylib
+
 // Tests the -Zembed-metadata compiler flag.
 // Tracking issue: https://github.com/rust-lang/rust/issues/139165
 

tests/run-make/embed-source-dwarf/rmake.rs

@@ -1,6 +1,7 @@
 //@ needs-target-std
 //@ ignore-windows
 //@ ignore-apple
+//@ ignore-wasm (`object` doesn't handle wasm object files)
 
 // This test should be replaced with one in tests/debuginfo once we can easily
 // tell via GDB or LLDB if debuginfo contains source code. Cheap tricks in LLDB

tests/run-make/emit-shared-files/rmake.rs

@@ -5,6 +5,8 @@
 // `all-shared` should only emit files that can be shared between crates.
 // See https://github.com/rust-lang/rust/pull/83478
 
+//@ needs-target-std
+
 use run_make_support::{has_extension, has_prefix, path, rustdoc, shallow_find_files};
 
 fn main() {

tests/run-make/emit-stack-sizes/rmake.rs

@@ -7,8 +7,7 @@
 // See https://github.com/rust-lang/rust/pull/51946
 
 //@ needs-target-std
-//@ ignore-windows
-//@ ignore-apple
+//@ only-elf
 // Reason: this feature only works when the output object format is ELF.
 // This won't be the case on Windows/OSX - for example, OSX produces a Mach-O binary.
 

tests/run-make/env-dep-info/rmake.rs

@@ -1,5 +1,7 @@
 //@ needs-target-std
-//
+//@ needs-crate-type: proc-macro
+//@ ignore-musl (FIXME: can't find `-lunwind`)
+
 // Inside dep-info emit files, #71858 made it so all accessed environment
 // variables are usefully printed. This test checks that this feature works
 // as intended by checking if the environment variables used in compilation

tests/run-make/export/disambiguator/rmake.rs

@@ -1,4 +1,8 @@
 //@ needs-target-std
+
+// NOTE: `sdylib`'s platform support is basically just `dylib`'s platform support.
+//@ needs-crate-type: dylib
+
 use run_make_support::rustc;
 
 fn main() {

tests/run-make/export/extern-opt/rmake.rs

@@ -1,4 +1,8 @@
 //@ needs-target-std
+
+// NOTE: `sdylib`'s platform support is basically that of `dylib`.
+//@ needs-crate-type: dylib
+
 use run_make_support::{dynamic_lib_name, rustc};
 
 fn main() {

tests/run-make/export/simple/rmake.rs

@@ -1,4 +1,8 @@
 //@ needs-target-std
+
+// NOTE: `sdylib`'s platform support is basically that of `dylib`.
+//@ needs-crate-type: dylib
+
 use run_make_support::rustc;
 
 fn main() {

tests/run-make/ice-dep-cannot-find-dep/rmake.rs

@@ -9,6 +9,7 @@
 
 //@ only-x86_64
 //@ only-linux
+//@ ignore-cross-compile
 // Reason: This is a platform-independent issue, no need to waste time testing
 // everywhere.
 

tests/run-make/include-all-symbols-linking/rmake.rs

@@ -8,6 +8,8 @@
 // See https://github.com/rust-lang/rust/issues/47384
 
 //@ needs-target-std
+//@ needs-crate-type: cdylib
+//@ needs-dynamic-linking
 //@ ignore-wasm differences in object file formats causes errors in the llvm_objdump step.
 //@ ignore-windows differences in object file formats causes errors in the llvm_objdump step.
 

tests/run-make/invalid-so/rmake.rs

@@ -1,5 +1,7 @@
 //@ needs-target-std
-//
+//@ needs-crate-type: dylib
+//@ needs-dynamic-linking
+
 // When a fake library was given to the compiler, it would
 // result in an obscure and unhelpful error message. This test
 // creates a false "foo" dylib, and checks that the standard error

tests/run-make/link-args-order/rmake.rs

@@ -1,5 +1,6 @@
 //@ needs-target-std
-//
+//@ ignore-wasm (explicit linker invocations)
+
 // Passing linker arguments to the compiler used to be lost or reordered in a messy way
 // as they were passed further to the linker. This was fixed in #70665, and this test
 // checks that linker arguments remain intact and in the order they were originally passed in.