From 35726e909d44f41e090ac22e626bb4fa0e68afdb Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Thu, 23 Mar 2023 17:56:26 +0800 Subject: [PATCH 1/8] LIBPATH is used as dylib's path environment variable on AIX --- src/bootstrap/dylib_util.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/dylib_util.rs index 6d75272c50130..b14c0bed66c2c 100644 --- a/src/bootstrap/dylib_util.rs +++ b/src/bootstrap/dylib_util.rs @@ -12,6 +12,8 @@ pub fn dylib_path_var() -> &'static str { "DYLD_LIBRARY_PATH" } else if cfg!(target_os = "haiku") { "LIBRARY_PATH" + } else if cfg!(target_os = "aix") { + "LIBPATH" } else { "LD_LIBRARY_PATH" } From 88b3ae96904312c7c0fe9b40357613f9fc29dbc8 Mon Sep 17 00:00:00 2001 From: Lenko Donchev Date: Sun, 26 Mar 2023 15:43:23 -0500 Subject: [PATCH 2/8] check for missing codegen backeng config --- src/bootstrap/compile.rs | 44 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 54971af644c28..783061d8ee84b 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -20,7 +20,7 @@ use serde_derive::Deserialize; use crate::builder::crate_description; use crate::builder::Cargo; -use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; use crate::cache::{Interned, INTERNER}; use crate::config::{LlvmLibunwind, RustcLto, TargetSelection}; use crate::dist; @@ -995,6 +995,44 @@ pub struct CodegenBackend { pub backend: Interned, } +fn needs_codegen_config(run: &RunConfig<'_>) -> bool { + let mut needs_codegen_cfg = false; + for path_set in &run.paths { + needs_codegen_cfg = match path_set { + PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), + PathSet::Suite(suite) => is_codegen_cfg_needed(&suite, run), + } + } + needs_codegen_cfg +} + +const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; + +fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { + if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { + let mut needs_codegen_backend_config = true; + for &backend in &run.builder.config.rust_codegen_backends { + if path + .path + .to_str() + .unwrap() + .ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + &backend)) + { + needs_codegen_backend_config = false; + } + } + if needs_codegen_backend_config { + run.builder.info( + "Warning: no codegen-backends config matched the requested path to build a codegen backend. \ + Help: add backend to codegen-backends in config.toml.", + ); + return true; + } + } + + return false; +} + impl Step for CodegenBackend { type Output = (); const ONLY_HOSTS: bool = true; @@ -1006,6 +1044,10 @@ impl Step for CodegenBackend { } fn make_run(run: RunConfig<'_>) { + if needs_codegen_config(&run) { + return; + } + for &backend in &run.builder.config.rust_codegen_backends { if backend == "llvm" { continue; // Already built as part of rustc From b56fcb173bbf5ae0a52f26362639b7f2b922e7e9 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Tue, 28 Mar 2023 10:59:45 +0800 Subject: [PATCH 3/8] Set LIBPATH --- src/bootstrap/bootstrap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 013d1ab525b0c..d12781cc33af0 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -741,6 +741,9 @@ def build_bootstrap(self, color, verbose_count): env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ (os.pathsep + env["LIBRARY_PATH"]) \ if "LIBRARY_PATH" in env else "" + env["LIBPATH"] = os.path.join(self.bin_root(), "lib") + \ + (os.pathsep + env["LIBPATH"]) \ + if "LIBPATH" in env else "" # Export Stage0 snapshot compiler related env variables build_section = "target.{}".format(self.build) From d3cc2f76465b854129daa4dca2a10b137a8699bf Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Wed, 29 Mar 2023 14:55:30 +0800 Subject: [PATCH 4/8] Use LIBPATH in compiletest --- src/tools/compiletest/src/util.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tools/compiletest/src/util.rs b/src/tools/compiletest/src/util.rs index 5f6a27e536638..748240cc94bc6 100644 --- a/src/tools/compiletest/src/util.rs +++ b/src/tools/compiletest/src/util.rs @@ -156,6 +156,8 @@ pub fn dylib_env_var() -> &'static str { "DYLD_LIBRARY_PATH" } else if cfg!(target_os = "haiku") { "LIBRARY_PATH" + } else if cfg!(target_os = "aix") { + "LIBPATH" } else { "LD_LIBRARY_PATH" } From a9aaf3fedba1ddbfa4e60c47794bd78688297ec2 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Wed, 29 Mar 2023 15:29:36 +0200 Subject: [PATCH 5/8] Implement read_buf for RustHermit In principle, this PR extends rust-lang/rust#108326 for RustyHermit. Signed-off-by: Stefan Lankes --- library/std/src/sys/hermit/net.rs | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/library/std/src/sys/hermit/net.rs b/library/std/src/sys/hermit/net.rs index 5fb6281aa1e3d..d6f64a2971902 100644 --- a/library/std/src/sys/hermit/net.rs +++ b/library/std/src/sys/hermit/net.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] use crate::cmp; -use crate::io::{self, IoSlice, IoSliceMut}; +use crate::io::{self, BorrowedBuf, BorrowedCursor, IoSlice, IoSliceMut}; use crate::mem; use crate::net::{Shutdown, SocketAddr}; use crate::os::hermit::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, RawFd}; @@ -146,18 +146,35 @@ impl Socket { Ok(Socket(unsafe { FileDesc::from_raw_fd(fd) })) } - fn recv_with_flags(&self, buf: &mut [u8], flags: i32) -> io::Result { - let ret = - cvt(unsafe { netc::recv(self.0.as_raw_fd(), buf.as_mut_ptr(), buf.len(), flags) })?; - Ok(ret as usize) + fn recv_with_flags(&self, mut buf: BorrowedCursor<'_>, flags: i32) -> io::Result<()> { + let ret = cvt(unsafe { + netc::recv( + self.0.as_raw_fd(), + buf.as_mut().as_mut_ptr() as *mut u8, + buf.capacity(), + flags, + ) + })?; + unsafe { + buf.advance(ret as usize); + } + Ok(()) } pub fn read(&self, buf: &mut [u8]) -> io::Result { - self.recv_with_flags(buf, 0) + let mut buf = BorrowedBuf::from(buf); + self.recv_with_flags(buf.unfilled(), 0)?; + Ok(buf.len()) } pub fn peek(&self, buf: &mut [u8]) -> io::Result { - self.recv_with_flags(buf, netc::MSG_PEEK) + let mut buf = BorrowedBuf::from(buf); + self.recv_with_flags(buf.unfilled(), netc::MSG_PEEK)?; + Ok(buf.len()) + } + + pub fn read_buf(&self, buf: BorrowedCursor<'_>) -> io::Result<()> { + self.recv_with_flags(buf, 0) } pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result { From 01f49b350df806b0fbf9d379bd524f183da7517d Mon Sep 17 00:00:00 2001 From: "William D. Jones" Date: Wed, 29 Mar 2023 22:49:09 -0400 Subject: [PATCH 6/8] Update compiler-builtins to 0.1.91 to bring in msp430 shift primitive fixes. --- Cargo.lock | 4 ++-- library/std/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 561f29abec4a9..77d3b8cc2a1e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -868,9 +868,9 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc9c2080d347a2c316518840ac9194644a9993dfa1e9778ef38979a339f5d8b" +checksum = "571298a3cce7e2afbd3d61abb91a18667d5ab25993ec577a88ee8ac45f00cc3a" dependencies = [ "cc", "rustc-std-workspace-core", diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml index 6345db240548d..96c75f97f6e2b 100644 --- a/library/std/Cargo.toml +++ b/library/std/Cargo.toml @@ -16,7 +16,7 @@ panic_unwind = { path = "../panic_unwind", optional = true } panic_abort = { path = "../panic_abort" } core = { path = "../core" } libc = { version = "0.2.140", default-features = false, features = ['rustc-dep-of-std'] } -compiler_builtins = { version = "0.1.87" } +compiler_builtins = { version = "0.1.91" } profiler_builtins = { path = "../profiler_builtins", optional = true } unwind = { path = "../unwind" } hashbrown = { version = "0.12", default-features = false, features = ['rustc-dep-of-std'] } From 6f8e47092cfbfbc944db848928bc92ca982041ea Mon Sep 17 00:00:00 2001 From: ozkanonur Date: Sun, 2 Apr 2023 21:23:53 +0300 Subject: [PATCH 7/8] Download beta compiler toolchain in bootstrap if it doesn't yet exist This is needed for when the shell scripts bypass python altogether and run the downloaded bootstrap directly. Changes are mainly provided from @jyn514, I just fixed the review notes. Signed-off-by: ozkanonur --- src/bootstrap/config.rs | 21 +++++++++---- src/bootstrap/download.rs | 64 +++++++++++++++++++++++++++++++++------ src/bootstrap/test.rs | 2 +- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 0fee094c22231..ec41288d25c1b 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -223,25 +223,34 @@ pub struct Config { pub reuse: Option, pub cargo_native_static: bool, pub configure_args: Vec, + pub out: PathBuf, + pub rust_info: channel::GitInfo, // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, + #[cfg(not(test))] initial_rustfmt: RefCell, #[cfg(test)] pub initial_rustfmt: RefCell, - pub out: PathBuf, - pub rust_info: channel::GitInfo, } #[derive(Default, Deserialize)] #[cfg_attr(test, derive(Clone))] pub struct Stage0Metadata { + pub compiler: CompilerMetadata, pub config: Stage0Config, pub checksums_sha256: HashMap, pub rustfmt: Option, } +#[derive(Default, Deserialize)] +#[cfg_attr(test, derive(Clone))] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} + #[derive(Default, Deserialize)] #[cfg_attr(test, derive(Clone))] pub struct Stage0Config { @@ -989,10 +998,10 @@ impl Config { config.out = crate::util::absolute(&config.out); } - config.initial_rustc = build - .rustc - .map(PathBuf::from) - .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/rustc")); + config.initial_rustc = build.rustc.map(PathBuf::from).unwrap_or_else(|| { + config.download_beta_toolchain(); + config.out.join(config.build.triple).join("stage0/bin/rustc") + }); config.initial_cargo = build .cargo .map(PathBuf::from) diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index d1e2149d3f95f..fc77186d16610 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -367,26 +367,70 @@ impl Config { pub(crate) fn download_ci_rustc(&self, commit: &str) { self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + let version = self.artifact_version_part(commit); + // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the + // `rustc_private` crates for tools. + let extra_components = ["rustc-dev"]; + + self.download_toolchain( + &version, + "ci-rustc", + commit, + &extra_components, + Self::download_ci_component, + ); + } + + pub(crate) fn download_beta_toolchain(&self) { + self.verbose(&format!("downloading stage0 beta artifacts")); + + let date = &self.stage0_metadata.compiler.date; + let version = &self.stage0_metadata.compiler.version; + let extra_components = ["cargo"]; + + let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { + config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") + }; + + self.download_toolchain( + version, + "stage0", + date, + &extra_components, + download_beta_component, + ); + } + + fn download_toolchain( + &self, + // FIXME(ozkanonur) use CompilerMetadata instead of `version: &str` + version: &str, + sysroot: &str, + stamp_key: &str, + extra_components: &[&str], + download_component: fn(&Config, String, &str, &str), + ) { let host = self.build.triple; - let bin_root = self.out.join(host).join("ci-rustc"); + let bin_root = self.out.join(host).join(sysroot); let rustc_stamp = bin_root.join(".rustc-stamp"); - if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) + if !bin_root.join("bin").join("rustc").exists() + || program_out_of_date(&rustc_stamp, stamp_key) { if bin_root.exists() { t!(fs::remove_dir_all(&bin_root)); } let filename = format!("rust-std-{version}-{host}.tar.xz"); let pattern = format!("rust-std-{host}"); - self.download_ci_component(filename, &pattern, commit); + download_component(self, filename, &pattern, stamp_key); let filename = format!("rustc-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc", commit); - // download-rustc doesn't need its own cargo, it can just use beta's. - let filename = format!("rustc-dev-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc-dev", commit); - let filename = format!("rust-src-{version}.tar.xz"); - self.download_ci_component(filename, "rust-src", commit); + download_component(self, filename, "rustc", stamp_key); + + for component in extra_components { + let filename = format!("{component}-{version}-{host}.tar.xz"); + download_component(self, filename, component, stamp_key); + } if self.should_fix_bins_and_dylibs() { self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); @@ -403,7 +447,7 @@ impl Config { } } - t!(fs::write(rustc_stamp, commit)); + t!(fs::write(rustc_stamp, stamp_key)); } } diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index baddc9da48db8..171e1ccf0468a 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1133,7 +1133,7 @@ impl Step for Tidy { if builder.config.channel == "dev" || builder.config.channel == "nightly" { builder.info("fmt check"); if builder.initial_rustfmt().is_none() { - let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap(); + let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); eprintln!( "\ error: no `rustfmt` binary found in {PATH} From 423e76f1ddf2ffeec4235b09946f563d3ddb6004 Mon Sep 17 00:00:00 2001 From: jyn Date: Sun, 2 Apr 2023 17:48:12 -0400 Subject: [PATCH 8/8] Improve job names in Github Actions preview Before: `CI / PR (mingw-check, false, ubuntu-20.04-16core-64gb) (pull_request)` After: `CI / PR - mingw-check (pull_request)` --- .github/workflows/ci.yml | 6 +++--- src/ci/github-actions/ci.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b45246eb4ead8..a917d9a7d55dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: pr: permissions: actions: write - name: PR + name: "PR - ${{ matrix.name }}" env: CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse @@ -159,7 +159,7 @@ jobs: auto: permissions: actions: write - name: auto + name: "auto - ${{ matrix.name }}" env: CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse @@ -578,7 +578,7 @@ jobs: try: permissions: actions: write - name: try + name: "try - ${{ matrix.name }}" env: CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index c594288dcf81d..403953b5047d9 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -284,7 +284,7 @@ jobs: permissions: actions: write # for rust-lang/simpleinfra/github-actions/cancel-outdated-builds <<: *base-ci-job - name: PR + name: PR - ${{ matrix.name }} env: <<: [*shared-ci-variables, *public-variables] if: github.event_name == 'pull_request' @@ -312,7 +312,7 @@ jobs: permissions: actions: write # for rust-lang/simpleinfra/github-actions/cancel-outdated-builds <<: *base-ci-job - name: auto + name: auto - ${{ matrix.name }} env: <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' @@ -741,7 +741,7 @@ jobs: permissions: actions: write # for rust-lang/simpleinfra/github-actions/cancel-outdated-builds <<: *base-ci-job - name: try + name: try - ${{ matrix.name }} env: <<: [*shared-ci-variables, *prod-variables] if: github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'