Rollup merge of #142377 - Urgau:unremap-rustc-dev, r=jieyouxu

Try unremapping compiler sources

See [#t-compiler/help > Span pointing to wrong file location (`rustc-dev` component)](https://rust-lang.zulipchat.com/#narrow/channel/182449-t-compiler.2Fhelp/topic/Span.20pointing.20to.20wrong.20file.20location.20.28.60rustc-dev.60.20component.29/with/521087083).

This PR is a follow-up to #141751 regarding the compiler side.

Specifically we now take into account the `CFG_VIRTUAL_RUSTC_DEV_SOURCE_BASE_DIR` env from #141751 when trying to unremap sources from `$sysroot/lib/rustlib/rustc-src/rust` (the `rustc-dev` component install directory).

Best reviewed commit by commit.

cc `@samueltardieu`
r? `@jieyouxu`

Author: fmease

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -1,7 +1,7 @@
 // Decoding metadata from a single crate's metadata
 
 use std::iter::TrustedLen;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::{Arc, OnceLock};
 use std::{io, iter, mem};
 
@@ -1610,10 +1610,14 @@ impl<'a> CrateMetadataRef<'a> {
     /// Proc macro crates don't currently export spans, so this function does not have
     /// to work for them.
     fn imported_source_file(self, source_file_index: u32, sess: &Session) -> ImportedSourceFile {
-        fn filter<'a>(sess: &Session, path: Option<&'a Path>) -> Option<&'a Path> {
+        fn filter<'a>(
+            sess: &Session,
+            real_source_base_dir: &Option<PathBuf>,
+            path: Option<&'a Path>,
+        ) -> Option<&'a Path> {
             path.filter(|_| {
                 // Only spend time on further checks if we have what to translate *to*.
-                sess.opts.real_rust_source_base_dir.is_some()
+                real_source_base_dir.is_some()
                 // Some tests need the translation to be always skipped.
                 && sess.opts.unstable_opts.translate_remapped_path_to_local_path
             })
@@ -1625,57 +1629,92 @@ impl<'a> CrateMetadataRef<'a> {
             })
         }
 
-        let try_to_translate_virtual_to_real = |name: &mut rustc_span::FileName| {
-            // Translate the virtual `/rustc/$hash` prefix back to a real directory
-            // that should hold actual sources, where possible.
-            //
-            // NOTE: if you update this, you might need to also update bootstrap's code for generating
-            // the `rust-src` component in `Src::run` in `src/bootstrap/dist.rs`.
-            let virtual_rust_source_base_dir = [
-                filter(sess, option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR").map(Path::new)),
-                filter(sess, sess.opts.unstable_opts.simulate_remapped_rust_src_base.as_deref()),
-            ];
+        let try_to_translate_virtual_to_real =
+            |virtual_source_base_dir: Option<&str>,
+             real_source_base_dir: &Option<PathBuf>,
+             name: &mut rustc_span::FileName| {
+                let virtual_source_base_dir = [
+                    filter(sess, real_source_base_dir, virtual_source_base_dir.map(Path::new)),
+                    filter(
+                        sess,
+                        real_source_base_dir,
+                        sess.opts.unstable_opts.simulate_remapped_rust_src_base.as_deref(),
+                    ),
+                ];
 
-            debug!(
-                "try_to_translate_virtual_to_real(name={:?}): \
-                 virtual_rust_source_base_dir={:?}, real_rust_source_base_dir={:?}",
-                name, virtual_rust_source_base_dir, sess.opts.real_rust_source_base_dir,
-            );
+                debug!(
+                    "try_to_translate_virtual_to_real(name={:?}): \
+                     virtual_source_base_dir={:?}, real_source_base_dir={:?}",
+                    name, virtual_source_base_dir, real_source_base_dir,
+                );
 
-            for virtual_dir in virtual_rust_source_base_dir.iter().flatten() {
-                if let Some(real_dir) = &sess.opts.real_rust_source_base_dir
+                for virtual_dir in virtual_source_base_dir.iter().flatten() {
+                    if let Some(real_dir) = &real_source_base_dir
+                        && let rustc_span::FileName::Real(old_name) = name
+                        && let rustc_span::RealFileName::Remapped { local_path: _, virtual_name } =
+                            old_name
+                        && let Ok(rest) = virtual_name.strip_prefix(virtual_dir)
+                    {
+                        let new_path = real_dir.join(rest);
+
+                        debug!(
+                            "try_to_translate_virtual_to_real: `{}` -> `{}`",
+                            virtual_name.display(),
+                            new_path.display(),
+                        );
+
+                        // Check if the translated real path is affected by any user-requested
+                        // remaps via --remap-path-prefix. Apply them if so.
+                        // Note that this is a special case for imported rust-src paths specified by
+                        // https://rust-lang.github.io/rfcs/3127-trim-paths.html#handling-sysroot-paths.
+                        // Other imported paths are not currently remapped (see #66251).
+                        let (user_remapped, applied) =
+                            sess.source_map().path_mapping().map_prefix(&new_path);
+                        let new_name = if applied {
+                            rustc_span::RealFileName::Remapped {
+                                local_path: Some(new_path.clone()),
+                                virtual_name: user_remapped.to_path_buf(),
+                            }
+                        } else {
+                            rustc_span::RealFileName::LocalPath(new_path)
+                        };
+                        *old_name = new_name;
+                    }
+                }
+            };
+
+        let try_to_translate_real_to_virtual =
+            |virtual_source_base_dir: Option<&str>,
+             real_source_base_dir: &Option<PathBuf>,
+             subdir: &str,
+             name: &mut rustc_span::FileName| {
+                if let Some(virtual_dir) = &sess.opts.unstable_opts.simulate_remapped_rust_src_base
+                    && let Some(real_dir) = real_source_base_dir
                     && let rustc_span::FileName::Real(old_name) = name
-                    && let rustc_span::RealFileName::Remapped { local_path: _, virtual_name } =
-                        old_name
-                    && let Ok(rest) = virtual_name.strip_prefix(virtual_dir)
                 {
-                    let new_path = real_dir.join(rest);
-
-                    debug!(
-                        "try_to_translate_virtual_to_real: `{}` -> `{}`",
-                        virtual_name.display(),
-                        new_path.display(),
-                    );
-
-                    // Check if the translated real path is affected by any user-requested
-                    // remaps via --remap-path-prefix. Apply them if so.
-                    // Note that this is a special case for imported rust-src paths specified by
-                    // https://rust-lang.github.io/rfcs/3127-trim-paths.html#handling-sysroot-paths.
-                    // Other imported paths are not currently remapped (see #66251).
-                    let (user_remapped, applied) =
-                        sess.source_map().path_mapping().map_prefix(&new_path);
-                    let new_name = if applied {
-                        rustc_span::RealFileName::Remapped {
-                            local_path: Some(new_path.clone()),
-                            virtual_name: user_remapped.to_path_buf(),
+                    let relative_path = match old_name {
+                        rustc_span::RealFileName::LocalPath(local) => {
+                            local.strip_prefix(real_dir).ok()
+                        }
+                        rustc_span::RealFileName::Remapped { virtual_name, .. } => {
+                            virtual_source_base_dir
+                                .and_then(|virtual_dir| virtual_name.strip_prefix(virtual_dir).ok())
                         }
-                    } else {
-                        rustc_span::RealFileName::LocalPath(new_path)
                     };
-                    *old_name = new_name;
+                    debug!(
+                        ?relative_path,
+                        ?virtual_dir,
+                        ?subdir,
+                        "simulate_remapped_rust_src_base"
+                    );
+                    if let Some(rest) = relative_path.and_then(|p| p.strip_prefix(subdir).ok()) {
+                        *old_name = rustc_span::RealFileName::Remapped {
+                            local_path: None,
+                            virtual_name: virtual_dir.join(subdir).join(rest),
+                        };
+                    }
                 }
-            }
-        };
+            };
 
         let mut import_info = self.cdata.source_map_import_info.lock();
         for _ in import_info.len()..=(source_file_index as usize) {
@@ -1713,36 +1752,45 @@ impl<'a> CrateMetadataRef<'a> {
                 // This is useful for testing so that tests about the effects of
                 // `try_to_translate_virtual_to_real` don't have to worry about how the
                 // compiler is bootstrapped.
-                if let Some(virtual_dir) = &sess.opts.unstable_opts.simulate_remapped_rust_src_base
-                    && let Some(real_dir) = &sess.opts.real_rust_source_base_dir
-                    && let rustc_span::FileName::Real(ref mut old_name) = name
-                {
-                    let relative_path = match old_name {
-                        rustc_span::RealFileName::LocalPath(local) => {
-                            local.strip_prefix(real_dir).ok()
-                        }
-                        rustc_span::RealFileName::Remapped { virtual_name, .. } => {
-                            option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR")
-                                .and_then(|virtual_dir| virtual_name.strip_prefix(virtual_dir).ok())
-                        }
-                    };
-                    debug!(?relative_path, ?virtual_dir, "simulate_remapped_rust_src_base");
-                    for subdir in ["library", "compiler"] {
-                        if let Some(rest) = relative_path.and_then(|p| p.strip_prefix(subdir).ok())
-                        {
-                            *old_name = rustc_span::RealFileName::Remapped {
-                                local_path: None, // FIXME: maybe we should preserve this?
-                                virtual_name: virtual_dir.join(subdir).join(rest),
-                            };
-                            break;
-                        }
-                    }
-                }
+                try_to_translate_real_to_virtual(
+                    option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR"),
+                    &sess.opts.real_rust_source_base_dir,
+                    "library",
+                    &mut name,
+                );
+
+                // If this file is under $sysroot/lib/rustlib/rustc-src/
+                // and the user wish to simulate remapping with -Z simulate-remapped-rust-src-base,
+                // then we change `name` to a similar state as if the rust was bootstrapped
+                // with `remap-debuginfo = true`.
+                try_to_translate_real_to_virtual(
+                    option_env!("CFG_VIRTUAL_RUSTC_DEV_SOURCE_BASE_DIR"),
+                    &sess.opts.real_rustc_dev_source_base_dir,
+                    "compiler",
+                    &mut name,
+                );
 
                 // If this file's path has been remapped to `/rustc/$hash`,
-                // we might be able to reverse that (also see comments above,
-                // on `try_to_translate_virtual_to_real`).
-                try_to_translate_virtual_to_real(&mut name);
+                // we might be able to reverse that.
+                //
+                // NOTE: if you update this, you might need to also update bootstrap's code for generating
+                // the `rust-src` component in `Src::run` in `src/bootstrap/dist.rs`.
+                try_to_translate_virtual_to_real(
+                    option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR"),
+                    &sess.opts.real_rust_source_base_dir,
+                    &mut name,
+                );
+
+                // If this file's path has been remapped to `/rustc-dev/$hash`,
+                // we might be able to reverse that.
+                //
+                // NOTE: if you update this, you might need to also update bootstrap's code for generating
+                // the `rustc-dev` component in `Src::run` in `src/bootstrap/dist.rs`.
+                try_to_translate_virtual_to_real(
+                    option_env!("CFG_VIRTUAL_RUSTC_DEV_SOURCE_BASE_DIR"),
+                    &sess.opts.real_rustc_dev_source_base_dir,
+                    &mut name,
+                );
 
                 let local_version = sess.source_map().new_imported_source_file(
                     name,

compiler/rustc_session/src/config.rs

@@ -1364,6 +1364,7 @@ impl Default for Options {
             cli_forced_local_thinlto_off: false,
             remap_path_prefix: Vec::new(),
             real_rust_source_base_dir: None,
+            real_rustc_dev_source_base_dir: None,
             edition: DEFAULT_EDITION,
             json_artifact_notifications: false,
             json_unused_externs: JsonUnusedExterns::No,
@@ -2701,9 +2702,8 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
 
     let sysroot = filesearch::materialize_sysroot(sysroot_opt);
 
-    let real_rust_source_base_dir = {
-        // This is the location used by the `rust-src` `rustup` component.
-        let mut candidate = sysroot.join("lib/rustlib/src/rust");
+    let real_source_base_dir = |suffix: &str, confirm: &str| {
+        let mut candidate = sysroot.join(suffix);
         if let Ok(metadata) = candidate.symlink_metadata() {
             // Replace the symlink bootstrap creates, with its destination.
             // We could try to use `fs::canonicalize` instead, but that might
@@ -2716,9 +2716,17 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
         }
 
         // Only use this directory if it has a file we can expect to always find.
-        candidate.join("library/std/src/lib.rs").is_file().then_some(candidate)
+        candidate.join(confirm).is_file().then_some(candidate)
     };
 
+    let real_rust_source_base_dir =
+        // This is the location used by the `rust-src` `rustup` component.
+        real_source_base_dir("lib/rustlib/src/rust", "library/std/src/lib.rs");
+
+    let real_rustc_dev_source_base_dir =
+        // This is the location used by the `rustc-dev` `rustup` component.
+        real_source_base_dir("lib/rustlib/rustc-src/rust", "compiler/rustc/src/main.rs");
+
     let mut search_paths = vec![];
     for s in &matches.opt_strs("L") {
         search_paths.push(SearchPath::from_cli_opt(
@@ -2772,6 +2780,7 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
         cli_forced_local_thinlto_off: disable_local_thinlto,
         remap_path_prefix,
         real_rust_source_base_dir,
+        real_rustc_dev_source_base_dir,
         edition,
         json_artifact_notifications,
         json_unused_externs,

compiler/rustc_session/src/options.rs

@@ -395,15 +395,25 @@ top_level_options!(
 
         /// Remap source path prefixes in all output (messages, object files, debug, etc.).
         remap_path_prefix: Vec<(PathBuf, PathBuf)> [TRACKED_NO_CRATE_HASH],
-        /// Base directory containing the `src/` for the Rust standard library, and
-        /// potentially `rustc` as well, if we can find it. Right now it's always
-        /// `$sysroot/lib/rustlib/src/rust` (i.e. the `rustup` `rust-src` component).
+
+        /// Base directory containing the `library/` directory for the Rust standard library.
+        /// Right now it's always `$sysroot/lib/rustlib/src/rust`
+        /// (i.e. the `rustup` `rust-src` component).
         ///
         /// This directory is what the virtual `/rustc/$hash` is translated back to,
         /// if Rust was built with path remapping to `/rustc/$hash` enabled
         /// (the `rust.remap-debuginfo` option in `bootstrap.toml`).
         real_rust_source_base_dir: Option<PathBuf> [TRACKED_NO_CRATE_HASH],
 
+        /// Base directory containing the `compiler/` directory for the rustc sources.
+        /// Right now it's always `$sysroot/lib/rustlib/rustc-src/rust`
+        /// (i.e. the `rustup` `rustc-dev` component).
+        ///
+        /// This directory is what the virtual `/rustc-dev/$hash` is translated back to,
+        /// if Rust was built with path remapping to `/rustc/$hash` enabled
+        /// (the `rust.remap-debuginfo` option in `bootstrap.toml`).
+        real_rustc_dev_source_base_dir: Option<PathBuf> [TRACKED_NO_CRATE_HASH],
+
         edition: Edition [TRACKED],
 
         /// `true` if we're emitting JSON blobs about each artifact produced

src/doc/rustc-dev-guide/src/tests/ui.md

@@ -113,6 +113,8 @@ Compiletest makes the following replacements on the compiler output:
 - The base directory where the test's output goes is replaced with
   `$TEST_BUILD_DIR`. This only comes up in a few rare circumstances. Example:
   `/path/to/rust/build/x86_64-unknown-linux-gnu/test/ui`
+- The real directory to the standard library source is replaced with `$SRC_DIR_REAL`.
+- The real directory to the compiler source is replaced with `$COMPILER_DIR_REAL`.
 - Tabs are replaced with `\t`.
 - Backslashes (`\`) are converted to forward slashes (`/`) within paths (using a
   heuristic). This helps normalize differences with Windows-style paths.

src/tools/compiletest/src/runtest.rs

@@ -2371,6 +2371,12 @@ impl<'test> TestCx<'test> {
         let rust_src_dir = rust_src_dir.read_link_utf8().unwrap_or(rust_src_dir.to_path_buf());
         normalize_path(&rust_src_dir.join("library"), "$SRC_DIR_REAL");
 
+        // Real paths into the compiler
+        let rustc_src_dir = &self.config.sysroot_base.join("lib/rustlib/rustc-src/rust");
+        rustc_src_dir.try_exists().expect(&*format!("{} should exists", rustc_src_dir));
+        let rustc_src_dir = rustc_src_dir.read_link_utf8().unwrap_or(rustc_src_dir.to_path_buf());
+        normalize_path(&rustc_src_dir.join("compiler"), "$COMPILER_DIR_REAL");
+
         // eg.
         // /home/user/rust/build/x86_64-unknown-linux-gnu/test/ui/<test_dir>/$name.$revision.$mode/
         normalize_path(&self.output_base_dir(), "$TEST_BUILD_DIR");

tests/ui-fulldeps/rustc-dev-remap.only-remap.stderr

@@ -0,0 +1,15 @@
+error[E0277]: the trait bound `NotAValidResultType: VisitorResult` is not satisfied
+  --> $DIR/rustc-dev-remap.rs:LL:COL
+   |
+LL |     type Result = NotAValidResultType;
+   |                   ^^^^^^^^^^^^^^^^^^^ the trait `VisitorResult` is not implemented for `NotAValidResultType`
+   |
+   = help: the following other types implement trait `VisitorResult`:
+             ()
+             ControlFlow<T>
+note: required by a bound in `rustc_ast::visit::Visitor::Result`
+  --> /rustc-dev/xyz/compiler/rustc_ast/src/visit.rs:LL:COL
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0277`.

tests/ui-fulldeps/rustc-dev-remap.remap-unremap.stderr

@@ -0,0 +1,18 @@
+error[E0277]: the trait bound `NotAValidResultType: VisitorResult` is not satisfied
+  --> $DIR/rustc-dev-remap.rs:LL:COL
+   |
+LL |     type Result = NotAValidResultType;
+   |                   ^^^^^^^^^^^^^^^^^^^ the trait `VisitorResult` is not implemented for `NotAValidResultType`
+   |
+   = help: the following other types implement trait `VisitorResult`:
+             ()
+             ControlFlow<T>
+note: required by a bound in `rustc_ast::visit::Visitor::Result`
+  --> $COMPILER_DIR_REAL/rustc_ast/src/visit.rs:LL:COL
+   |
+LL |     type Result: VisitorResult = ();
+   |                  ^^^^^^^^^^^^^ required by this bound in `Visitor::Result`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0277`.

tests/ui-fulldeps/rustc-dev-remap.rs

@@ -0,0 +1,29 @@
+//@ check-fail
+//
+//@ ignore-cross-compile
+//@ ignore-remote
+//
+//@ revisions: only-remap remap-unremap
+//@ compile-flags: -Z simulate-remapped-rust-src-base=/rustc-dev/xyz
+//@ [remap-unremap]compile-flags: -Ztranslate-remapped-path-to-local-path=yes
+
+// The $SRC_DIR*.rs:LL:COL normalisation doesn't kick in automatically
+// as the remapped revision will begin with $COMPILER_DIR_REAL,
+// so we have to do it ourselves.
+//@ normalize-stderr: ".rs:\d+:\d+" -> ".rs:LL:COL"
+
+#![feature(rustc_private)]
+
+extern crate rustc_ast;
+
+use rustc_ast::visit::Visitor;
+
+struct MyStruct;
+struct NotAValidResultType;
+
+impl Visitor<'_> for MyStruct {
+    type Result = NotAValidResultType;
+    //~^ ERROR the trait bound `NotAValidResultType: VisitorResult` is not satisfied
+}
+
+fn main() {}