diff --git a/.cirrus.yml b/.cirrus.yml index 76b48d70a..7c966aa1a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -6,8 +6,6 @@ task: - pkg install -y curl git bash - curl https://sh.rustup.rs -sSf --output rustup.sh - sh rustup.sh --default-toolchain none -y --profile=minimal - cargo_bin_cache: - folder: ~/.cargo/bin target_cache: folder: target prepare_script: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b79406879..cc9ae19af 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,12 +62,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache cargo installed crates - uses: actions/cache@v3 - with: - path: ~/.cargo/bin - key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-installed-crates - - name: Cache cargo registry and index uses: actions/cache@v3 with: diff --git a/.github/workflows/nightly-cranelift.yml b/.github/workflows/nightly-cranelift.yml index 0565938ee..968cd43ef 100644 --- a/.github/workflows/nightly-cranelift.yml +++ b/.github/workflows/nightly-cranelift.yml @@ -13,12 +13,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache cargo installed crates - uses: actions/cache@v3 - with: - path: ~/.cargo/bin - key: ubuntu-latest-cargo-installed-crates - - name: Prepare dependencies run: | git config --global user.email "user@example.com" diff --git a/.github/workflows/rustc.yml b/.github/workflows/rustc.yml index bab81b9dc..2c7de86f9 100644 --- a/.github/workflows/rustc.yml +++ b/.github/workflows/rustc.yml @@ -10,12 +10,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache cargo installed crates - uses: actions/cache@v3 - with: - path: ~/.cargo/bin - key: ${{ runner.os }}-cargo-installed-crates - - name: Cache cargo registry and index uses: actions/cache@v3 with: @@ -44,12 +38,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache cargo installed crates - uses: actions/cache@v3 - with: - path: ~/.cargo/bin - key: ${{ runner.os }}-cargo-installed-crates - - name: Cache cargo registry and index uses: actions/cache@v3 with: diff --git a/.gitignore b/.gitignore index b443fd58a..8012e93f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -target +/target **/*.rs.bk *.rlib *.o @@ -11,9 +11,6 @@ perf.data.old /y.exe /y.pdb /build -/build_sysroot/sysroot_src -/build_sysroot/compiler-builtins -/build_sysroot/rustc_version /dist /rust /download diff --git a/.vscode/settings.json b/.vscode/settings.json index bc914e37d..d8650d1e3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -30,7 +30,7 @@ ] }, { - "sysroot_src": "./build_sysroot/sysroot_src/library", + "sysroot_src": "./download/sysroot/sysroot_src/library", "crates": [ { "root_module": "./example/std_example.rs", diff --git a/build_system/bench.rs b/build_system/bench.rs new file mode 100644 index 000000000..0ad886322 --- /dev/null +++ b/build_system/bench.rs @@ -0,0 +1,79 @@ +use std::env; +use std::fs; +use std::path::Path; + +use super::path::{Dirs, RelPath}; +use super::prepare::GitRepo; +use super::rustc_info::{get_file_name, get_wrapper_file_name}; +use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject}; + +pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github( + "ebobby", + "simple-raytracer", + "804a7a21b9e673a482797aa289a18ed480e4d813", + "", +); + +pub(crate) static SIMPLE_RAYTRACER: CargoProject = + CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer"); + +pub(crate) fn benchmark(dirs: &Dirs) { + benchmark_simple_raytracer(dirs); +} + +fn benchmark_simple_raytracer(dirs: &Dirs) { + if std::process::Command::new("hyperfine").output().is_err() { + eprintln!("Hyperfine not installed"); + eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine"); + std::process::exit(1); + } + + let run_runs = env::var("RUN_RUNS") + .unwrap_or(if is_ci() { "2" } else { "10" }.to_string()) + .parse() + .unwrap(); + + eprintln!("[BENCH COMPILE] ebobby/simple-raytracer"); + let cargo_clif = RelPath::DIST.to_path(dirs).join(get_wrapper_file_name("cargo-clif", "bin")); + let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs); + let target_dir = SIMPLE_RAYTRACER.target_dir(dirs); + + let clean_cmd = format!( + "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}", + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + let llvm_build_cmd = format!( + "cargo build --manifest-path {manifest_path} --target-dir {target_dir}", + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + let clif_build_cmd = format!( + "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}", + cargo_clif = cargo_clif.display(), + manifest_path = manifest_path.display(), + target_dir = target_dir.display(), + ); + + let bench_compile = + hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd); + + spawn_and_wait(bench_compile); + + eprintln!("[BENCH RUN] ebobby/simple-raytracer"); + fs::copy( + target_dir.join("debug").join(get_file_name("main", "bin")), + RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")), + ) + .unwrap(); + + let mut bench_run = hyperfine_command( + 0, + run_runs, + None, + Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(), + Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(), + ); + bench_run.current_dir(RelPath::BUILD.to_path(dirs)); + spawn_and_wait(bench_run); +} diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index cbbf09b9b..711d4ccc5 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -146,10 +146,11 @@ pub(crate) fn build_sysroot( } } -// FIXME move to download/ or dist/ -pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = RelPath::BUILD_SYSROOT.join("rustc_version"); -pub(crate) static SYSROOT_SRC: RelPath = RelPath::BUILD_SYSROOT.join("sysroot_src"); -static STANDARD_LIBRARY: CargoProject = CargoProject::new(&RelPath::BUILD_SYSROOT, "build_sysroot"); +pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); +pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot"); +pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version"); +pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src"); +static STANDARD_LIBRARY: CargoProject = CargoProject::new(&BUILD_SYSROOT, "build_sysroot"); fn build_clif_sysroot_for_triple( dirs: &Dirs, diff --git a/build_system/mod.rs b/build_system/mod.rs index 2f311aed7..76d1d013b 100644 --- a/build_system/mod.rs +++ b/build_system/mod.rs @@ -5,6 +5,7 @@ use std::process; use self::utils::is_ci; mod abi_cafe; +mod bench; mod build_backend; mod build_sysroot; mod config; @@ -20,6 +21,7 @@ USAGE: ./y.rs prepare [--out-dir DIR] ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] + ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] OPTIONS: --sysroot none|clif|llvm @@ -54,6 +56,7 @@ enum Command { Prepare, Build, Test, + Bench, } #[derive(Copy, Clone, Debug)] @@ -67,7 +70,7 @@ pub fn main() { if env::var("RUST_BACKTRACE").is_err() { env::set_var("RUST_BACKTRACE", "1"); } - env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); + env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); // FIXME disable this by default env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); if is_ci() { @@ -83,6 +86,7 @@ pub fn main() { Some("prepare") => Command::Prepare, Some("build") => Command::Build, Some("test") => Command::Test, + Some("bench") => Command::Bench, Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), Some(command) => arg_error!("Unknown command {}", command), None => { @@ -198,5 +202,16 @@ pub fn main() { &target_triple, ); } + Command::Bench => { + build_sysroot::build_sysroot( + &dirs, + channel, + sysroot_kind, + &cg_clif_dylib, + &host_triple, + &target_triple, + ); + bench::benchmark(&dirs); + } } } diff --git a/build_system/path.rs b/build_system/path.rs index e93981f1d..35ab6f111 100644 --- a/build_system/path.rs +++ b/build_system/path.rs @@ -42,7 +42,6 @@ impl RelPath { pub(crate) const DIST: RelPath = RelPath::Base(PathBase::Dist); pub(crate) const SCRIPTS: RelPath = RelPath::SOURCE.join("scripts"); - pub(crate) const BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); pub(crate) const PATCHES: RelPath = RelPath::SOURCE.join("patches"); pub(crate) const fn join(&'static self, suffix: &'static str) -> RelPath { diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 8ac67e8f9..9ad4ddc92 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -3,9 +3,11 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; +use crate::build_system::rustc_info::get_default_sysroot; + +use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version}; +use super::rustc_info::{get_file_name, get_rustc_version}; use super::utils::{copy_dir_recursively, spawn_and_wait, Compiler}; pub(crate) fn prepare(dirs: &Dirs) { @@ -31,14 +33,14 @@ pub(crate) fn prepare(dirs: &Dirs) { super::tests::RAND_REPO.fetch(dirs); super::tests::REGEX_REPO.fetch(dirs); super::tests::PORTABLE_SIMD_REPO.fetch(dirs); - super::tests::SIMPLE_RAYTRACER_REPO.fetch(dirs); + super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs); eprintln!("[LLVM BUILD] simple-raytracer"); let host_compiler = Compiler::host(); - let build_cmd = super::tests::SIMPLE_RAYTRACER.build(&host_compiler, dirs); + let build_cmd = super::bench::SIMPLE_RAYTRACER.build(&host_compiler, dirs); spawn_and_wait(build_cmd); fs::copy( - super::tests::SIMPLE_RAYTRACER + super::bench::SIMPLE_RAYTRACER .target_dir(dirs) .join(&host_compiler.triple) .join("debug") @@ -49,27 +51,27 @@ pub(crate) fn prepare(dirs: &Dirs) { } fn prepare_sysroot(dirs: &Dirs) { - let rustc_path = get_rustc_path(); - let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust"); - let sysroot_src = SYSROOT_SRC; - + let sysroot_src_orig = get_default_sysroot().join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); - sysroot_src.ensure_fresh(dirs); - fs::create_dir_all(sysroot_src.to_path(dirs).join("library")).unwrap(); eprintln!("[COPY] sysroot src"); + + BUILD_SYSROOT.ensure_fresh(dirs); + copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs)); + + fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap(); copy_dir_recursively( &sysroot_src_orig.join("library"), - &sysroot_src.to_path(dirs).join("library"), + &SYSROOT_SRC.to_path(dirs).join("library"), ); let rustc_version = get_rustc_version(); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); eprintln!("[GIT] init"); - init_git_repo(&sysroot_src.to_path(dirs)); + init_git_repo(&SYSROOT_SRC.to_path(dirs)); - apply_patches(dirs, "sysroot", &sysroot_src.to_path(dirs)); + apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs)); } pub(crate) struct GitRepo { diff --git a/build_system/tests.rs b/build_system/tests.rs index 37c35106a..5b8b6f2df 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -1,11 +1,9 @@ -use super::build_sysroot; +use super::bench::SIMPLE_RAYTRACER; +use super::build_sysroot::{self, SYSROOT_SRC}; use super::config; use super::path::{Dirs, RelPath}; use super::prepare::GitRepo; -use super::rustc_info::{get_file_name, get_wrapper_file_name}; -use super::utils::{ - hyperfine_command, is_ci, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, -}; +use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler}; use super::SysrootKind; use std::env; use std::ffi::OsStr; @@ -251,18 +249,8 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( static PORTABLE_SIMD: CargoProject = CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd"); -pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github( - "ebobby", - "simple-raytracer", - "804a7a21b9e673a482797aa289a18ed480e4d813", - "", -); - -pub(crate) static SIMPLE_RAYTRACER: CargoProject = - CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer"); - static LIBCORE_TESTS: CargoProject = - CargoProject::new(&RelPath::BUILD_SYSROOT.join("sysroot_src/library/core/tests"), "core_tests"); + CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests"); const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::new("test.rust-random/rand", &|runner| { @@ -280,67 +268,9 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ spawn_and_wait(build_cmd); } }), - TestCase::new("bench.simple-raytracer", &|runner| { - let run_runs = env::var("RUN_RUNS") - .unwrap_or(if is_ci() { "2" } else { "10" }.to_string()) - .parse() - .unwrap(); - - if runner.is_native { - eprintln!("[BENCH COMPILE] ebobby/simple-raytracer"); - let cargo_clif = RelPath::DIST - .to_path(&runner.dirs) - .join(get_wrapper_file_name("cargo-clif", "bin")); - let manifest_path = SIMPLE_RAYTRACER.manifest_path(&runner.dirs); - let target_dir = SIMPLE_RAYTRACER.target_dir(&runner.dirs); - - let clean_cmd = format!( - "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}", - manifest_path = manifest_path.display(), - target_dir = target_dir.display(), - ); - let llvm_build_cmd = format!( - "cargo build --manifest-path {manifest_path} --target-dir {target_dir}", - manifest_path = manifest_path.display(), - target_dir = target_dir.display(), - ); - let clif_build_cmd = format!( - "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}", - cargo_clif = cargo_clif.display(), - manifest_path = manifest_path.display(), - target_dir = target_dir.display(), - ); - - let bench_compile = - hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd); - - spawn_and_wait(bench_compile); - - eprintln!("[BENCH RUN] ebobby/simple-raytracer"); - fs::copy( - target_dir.join("debug").join(get_file_name("main", "bin")), - RelPath::BUILD - .to_path(&runner.dirs) - .join(get_file_name("raytracer_cg_clif", "bin")), - ) - .unwrap(); - - let mut bench_run = hyperfine_command( - 0, - run_runs, - None, - Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(), - Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(), - ); - bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs)); - spawn_and_wait(bench_run); - } else { - spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.target_compiler.cargo, &runner.dirs)); - eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)"); - eprintln!("[COMPILE] ebobby/simple-raytracer"); - spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs)); - eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)"); - } + TestCase::new("test.simple-raytracer", &|runner| { + spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.host_compiler.cargo, &runner.dirs)); + spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs)); }), TestCase::new("test.libcore", &|runner| { spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs)); diff --git a/clean_all.sh b/clean_all.sh index 1760e5836..cdfc2e143 100755 --- a/clean_all.sh +++ b/clean_all.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash set -e -rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version} -rm -rf target/ build/ dist/ perf.data{,.old} y.bin -rm -rf download/ +rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb # Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh # FIXME remove at some point in the future rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/ +rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version} diff --git a/config.txt b/config.txt index 258b67e93..d9912a815 100644 --- a/config.txt +++ b/config.txt @@ -44,7 +44,7 @@ aot.issue-72793 testsuite.extended_sysroot test.rust-random/rand -bench.simple-raytracer +test.simple-raytracer test.libcore test.regex-shootout-regex-dna test.regex diff --git a/scripts/rustup.sh b/scripts/rustup.sh index bc4c06ed7..6111c2054 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -17,10 +17,10 @@ case $1 in done ./clean_all.sh - ./y.rs prepare (cd build_sysroot && cargo update) + ./y.rs prepare ;; "commit") git add rust-toolchain build_sysroot/Cargo.lock diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index 6c64b7de7..88bc64455 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -51,7 +51,7 @@ popd # FIXME remove once inline asm is fully supported export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc" -export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd build_sysroot/sysroot_src; pwd)" +export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd download/sysroot/sysroot_src; pwd)" # Allow the testsuite to use llvm tools host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")