From 7b89445cc3d6ef5b8accd73517880d35f5745466 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 5 Apr 2024 19:05:13 +0200 Subject: [PATCH 1/9] check-aux: test core and alloc in Miri --- src/bootstrap/mk/Makefile.in | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 0b67079917c7f..e70391ff66969 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -52,6 +52,16 @@ check-aux: src/tools/cargo \ src/tools/cargotest \ $(BOOTSTRAP_ARGS) + # Run standard library tests in Miri. + $(Q)$(BOOTSTRAP) miri --stage 2 \ + library/core \ + library/alloc \ + --no-doc + $(Q)MIRIFLAGS="-Zmiri-ignore-leaks -Zmiri-disable-isolation" \ + $(BOOTSTRAP) miri --stage 2 \ + library/core \ + library/alloc \ + --doc dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) distcheck: From c0b564b767866a5485339efdc059353266471342 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 5 Apr 2024 20:45:06 +0200 Subject: [PATCH 2/9] disable benches in Miri --- library/alloc/benches/lib.rs | 2 ++ library/core/benches/lib.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/library/alloc/benches/lib.rs b/library/alloc/benches/lib.rs index 638f343fb244b..0561f49c967e5 100644 --- a/library/alloc/benches/lib.rs +++ b/library/alloc/benches/lib.rs @@ -1,6 +1,8 @@ // Disabling on android for the time being // See https://github.com/rust-lang/rust/issues/73535#event-3477699747 #![cfg(not(target_os = "android"))] +// Disabling in Miri as these would take too long. +#![cfg(not(miri))] #![feature(btree_extract_if)] #![feature(iter_next_chunk)] #![feature(repr_simd)] diff --git a/library/core/benches/lib.rs b/library/core/benches/lib.rs index 4d14b930e4171..32d15c386cb1b 100644 --- a/library/core/benches/lib.rs +++ b/library/core/benches/lib.rs @@ -1,5 +1,7 @@ // wasm32 does not support benches (no time). #![cfg(not(target_arch = "wasm32"))] +// Disabling in Miri as these would take too long. +#![cfg(not(miri))] #![feature(flt2dec)] #![feature(test)] #![feature(trusted_random_access)] From 70c2b89f9d85cc55f02e135bb7163516406fa7c5 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 5 Apr 2024 23:22:17 +0200 Subject: [PATCH 3/9] also test parts of std --- library/std/benches/lib.rs | 2 ++ library/std/tests/process_spawning.rs | 2 ++ library/std/tests/switch-stdout.rs | 2 ++ src/bootstrap/mk/Makefile.in | 10 ++++++++++ 4 files changed, 16 insertions(+) diff --git a/library/std/benches/lib.rs b/library/std/benches/lib.rs index 4d1cf7fab7b7b..1b21c230a0bf2 100644 --- a/library/std/benches/lib.rs +++ b/library/std/benches/lib.rs @@ -1,3 +1,5 @@ +// Disabling in Miri as these would take too long. +#![cfg(not(miri))] #![feature(test)] extern crate test; diff --git a/library/std/tests/process_spawning.rs b/library/std/tests/process_spawning.rs index 59f67f9901ffa..2b7997299c501 100644 --- a/library/std/tests/process_spawning.rs +++ b/library/std/tests/process_spawning.rs @@ -1,4 +1,6 @@ #![cfg(not(target_env = "sgx"))] +// Process spawning does not work in Miri. +#![cfg(not(miri))] use std::env; use std::fs; diff --git a/library/std/tests/switch-stdout.rs b/library/std/tests/switch-stdout.rs index 27f3e8a9b96e4..a80f24fcb191d 100644 --- a/library/std/tests/switch-stdout.rs +++ b/library/std/tests/switch-stdout.rs @@ -1,4 +1,6 @@ #![cfg(any(target_family = "unix", target_family = "windows"))] +// Calls functions that are not supported by Miri. +#![cfg(not(miri))] use std::fs::File; use std::io::{Read, Write}; diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index e70391ff66969..e8b8daa63893a 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -57,11 +57,21 @@ check-aux: library/core \ library/alloc \ --no-doc + # Some doctests have intentional memory leaks. $(Q)MIRIFLAGS="-Zmiri-ignore-leaks -Zmiri-disable-isolation" \ $(BOOTSTRAP) miri --stage 2 \ library/core \ library/alloc \ --doc + # In `std` we cannot test everything, so we test the most target-dependent modules. + $(Q)MIRIFLAGS="-Zmiri-disable-isolation" BOOTSTRAP_SKIP_TARGET_SANITY=1 \ + $(BOOTSTRAP) miri --stage 2 library/std \ + --no-doc -- \ + --skip fs:: --skip net:: --skip process:: --skip sys::pal:: + $(Q)MIRIFLAGS="-Zmiri-ignore-leaks -Zmiri-disable-isolation" BOOTSTRAP_SKIP_TARGET_SANITY=1 \ + $(BOOTSTRAP) miri --stage 2 library/std \ + --doc -- \ + --skip fs:: --skip net:: --skip process:: --skip sys::pal:: dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) distcheck: From 6f5e6f0b77b1a45ac9dc543bb7bd553151f30e53 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 6 Apr 2024 12:37:28 +0200 Subject: [PATCH 4/9] make a doctest less slow in Miri --- library/alloc/src/sync.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs index 4dea27221b73d..6ae52cc7827bd 100644 --- a/library/alloc/src/sync.rs +++ b/library/alloc/src/sync.rs @@ -1062,7 +1062,9 @@ impl Arc { /// /// // Create a long list and clone it /// let mut x = LinkedList::new(); - /// for i in 0..100000 { + /// let size = 100000; + /// # let size = if cfg!(miri) { 100 } else { size }; + /// for i in 0..size { /// x.push(i); // Adds i to the front of x /// } /// let y = x.clone(); From 8069c003f5a02cae477b3f5e1c6961743ecd96aa Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 6 Apr 2024 15:11:54 +0200 Subject: [PATCH 5/9] disable a test with a thread leak in Miri --- library/std/tests/thread.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/library/std/tests/thread.rs b/library/std/tests/thread.rs index 4ce81f2846ea9..79a981d0b0d18 100644 --- a/library/std/tests/thread.rs +++ b/library/std/tests/thread.rs @@ -5,7 +5,8 @@ use std::time::Duration; #[test] #[cfg_attr(target_os = "emscripten", ignore)] -fn sleep() { +#[cfg_attr(miri, ignore)] // Miri does not like the thread leak +fn sleep_very_long() { let finished = Arc::new(Mutex::new(false)); let t_finished = finished.clone(); thread::spawn(move || { From f4da5536d319105ccd763f0fdc7bd0d2c7dfec07 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 6 Apr 2024 22:46:45 +0200 Subject: [PATCH 6/9] run some std tests on more targets --- src/bootstrap/mk/Makefile.in | 8 +++++- src/bootstrap/src/core/build_steps/test.rs | 31 +++++++++++++++------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index e8b8daa63893a..720f53cd3efd0 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -63,7 +63,7 @@ check-aux: library/core \ library/alloc \ --doc - # In `std` we cannot test everything, so we test the most target-dependent modules. + # In `std` we cannot test everything. $(Q)MIRIFLAGS="-Zmiri-disable-isolation" BOOTSTRAP_SKIP_TARGET_SANITY=1 \ $(BOOTSTRAP) miri --stage 2 library/std \ --no-doc -- \ @@ -72,6 +72,12 @@ check-aux: $(BOOTSTRAP) miri --stage 2 library/std \ --doc -- \ --skip fs:: --skip net:: --skip process:: --skip sys::pal:: + # Also test some very target-specific modules on other targets. + $(Q)MIRIFLAGS="-Zmiri-disable-isolation" BOOTSTRAP_SKIP_TARGET_SANITY=1 \ + $(BOOTSTRAP) miri --stage 2 library/std \ + --target aarch64-apple-darwin,i686-pc-windows-gnu \ + --no-doc -- \ + time:: sync:: thread:: env:: dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) distcheck: diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 028e0f6d05f4f..1e68f8d276a51 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -2689,16 +2689,27 @@ impl Step for Crate { match mode { Mode::Std => { - compile::std_cargo(builder, target, compiler.stage, &mut cargo); - // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, - // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. - // Override it. - if builder.download_rustc() && compiler.stage > 0 { - let sysroot = builder - .out - .join(compiler.host.triple) - .join(format!("stage{}-test-sysroot", compiler.stage)); - cargo.env("RUSTC_SYSROOT", sysroot); + if builder.kind == Kind::Miri { + // We can't use `std_cargo` as that uses `optimized-compiler-builtins` which + // needs host tools for the given target. This is similar to what `compile::Std` + // does when `is_for_mir_opt_tests` is true. There's probably a chance for + // de-duplication here... `std_cargo` should support a mode that avoids needing + // host tools. + cargo + .arg("--manifest-path") + .arg(builder.src.join("library/sysroot/Cargo.toml")); + } else { + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, + // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. + // Override it. + if builder.download_rustc() && compiler.stage > 0 { + let sysroot = builder + .out + .join(compiler.host.triple) + .join(format!("stage{}-test-sysroot", compiler.stage)); + cargo.env("RUSTC_SYSROOT", sysroot); + } } } Mode::Rustc => { From a8810b39373ea6342ac12cbadda8021eb26daf11 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 6 Apr 2024 18:05:08 +0200 Subject: [PATCH 7/9] disable debug assertions to speed up the check-aux job --- src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile index e08c4e1e8b7de..a74db2250fc67 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile @@ -25,5 +25,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Miri is just too slow with full assertions +ENV NO_DEBUG_ASSERTIONS=1 +ENV NO_OVERFLOW_CHECKS=1 + ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu ENV RUST_CHECK_TARGET check-aux From 7e33fd356b0e2f21c1b24a01c377baacb9c11d98 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 7 Apr 2024 07:29:43 +0200 Subject: [PATCH 8/9] also test core+alloc on a 32bit big-endian target --- src/bootstrap/mk/Makefile.in | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 720f53cd3efd0..d6e60d52d6329 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -53,11 +53,15 @@ check-aux: src/tools/cargotest \ $(BOOTSTRAP_ARGS) # Run standard library tests in Miri. - $(Q)$(BOOTSTRAP) miri --stage 2 \ + # We use a 64bit little-endian and a 32bit big-endian target for max coverage. + $(Q)BOOTSTRAP_SKIP_TARGET_SANITY=1 \ + $(BOOTSTRAP) miri --stage 2 \ + --target x86_64-unknown-linux-gnu,mips-unknown-linux-gnu \ library/core \ library/alloc \ --no-doc # Some doctests have intentional memory leaks. + # Also, they work only on the host. $(Q)MIRIFLAGS="-Zmiri-ignore-leaks -Zmiri-disable-isolation" \ $(BOOTSTRAP) miri --stage 2 \ library/core \ From 24ea7bca7ac160ca985f49716160ffe25f6d187d Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 7 Apr 2024 09:59:46 +0200 Subject: [PATCH 9/9] experiment --- .github/workflows/ci.yml | 3 +++ src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile | 2 +- src/ci/github-actions/ci.yml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16b0512a88a46..5ccd829bb8159 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,9 @@ jobs: - name: x86_64-gnu-tools os: ubuntu-20.04-16core-64gb env: {} + - name: x86_64-gnu-aux + os: ubuntu-20.04-4core-16gb + env: {} defaults: run: shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile index a74db2250fc67..d6b451b3cdf37 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile @@ -26,7 +26,7 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh # Miri is just too slow with full assertions -ENV NO_DEBUG_ASSERTIONS=1 +#ENV NO_DEBUG_ASSERTIONS=1 ENV NO_OVERFLOW_CHECKS=1 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 9323bb093ad96..aa96817e60f07 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -366,6 +366,9 @@ jobs: - name: x86_64-gnu-tools <<: *job-linux-16c + - name: x86_64-gnu-aux + <<: *job-linux-4c + auto: <<: *base-ci-job name: auto - ${{ matrix.name }}