Skip to content

rustdoc: Add DocVisitor and use it where possible #90475

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Nov 4, 2021
Merged
62 changes: 38 additions & 24 deletions src/librustdoc/fold.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,24 +46,40 @@ crate trait DocFolder: Sized {
i.items = i.items.into_iter().filter_map(|x| self.fold_item(x)).collect();
ImplItem(i)
}
VariantItem(i) => {
let i2 = i.clone(); // this clone is small
match i {
Variant::Struct(mut j) => {
let num_fields = j.fields.len();
j.fields = j.fields.into_iter().filter_map(|x| self.fold_item(x)).collect();
j.fields_stripped |= num_fields != j.fields.len()
|| j.fields.iter().any(|f| f.is_stripped());
VariantItem(Variant::Struct(j))
}
Variant::Tuple(fields) => {
let fields = fields.into_iter().filter_map(|x| self.fold_item(x)).collect();
VariantItem(Variant::Tuple(fields))
}
_ => VariantItem(i2),
VariantItem(i) => match i {
Variant::Struct(mut j) => {
let num_fields = j.fields.len();
j.fields = j.fields.into_iter().filter_map(|x| self.fold_item(x)).collect();
j.fields_stripped |=
num_fields != j.fields.len() || j.fields.iter().any(|f| f.is_stripped());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know it was done this way before, but any idea why it isn't using a conditional, or a short-circuiting OR, for this?

                     if !j.fields_stripped {
                         j.fields_stripped =
                             num_fields != j.fields.len() || j.fields.iter().any(|f| f.is_stripped());
                     }

The length comparison probably isn't a problem, but unconditionally iterating over all of the fields seems like leaving performance on the table (or gambling on a compiler optimization pass that may or may not fire).

From a dive through Blame, this use of the |= operator dates back to a version of the code that was only doing the length comparison:

d6d31d7#diff-3ba434e74829d360308543dff0f880edbd86bdc9fd90f025d990d91319b7002cR82

The code that started doing the any() loop doesn't have any documentation that justifies why it isn't a perf problem, either. It looks like they just added the any() loop without reconsidering the use of a non-short-circuiting OR.

b1543a1#diff-3ba434e74829d360308543dff0f880edbd86bdc9fd90f025d990d91319b7002cR48-R49

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably the reason it doesn't use a conditional is that no one thought of it ;)

It's a good idea; do you want to open a followup PR for it? Honestly, I think we should just remove the fields_stripped fields altogether and compute this information on-demand, but that'll take some time.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I'll do that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can assign me on it if you like.

VariantItem(Variant::Struct(j))
}
}
x => x,
Variant::Tuple(fields) => {
let fields = fields.into_iter().filter_map(|x| self.fold_item(x)).collect();
VariantItem(Variant::Tuple(fields))
}
Variant::CLike => VariantItem(Variant::CLike),
},
ExternCrateItem { src: _ }
| ImportItem(_)
| FunctionItem(_)
| TypedefItem(_, _)
| OpaqueTyItem(_)
| StaticItem(_)
| ConstantItem(_)
| TraitAliasItem(_)
| TyMethodItem(_)
| MethodItem(_, _)
| StructFieldItem(_)
| ForeignFunctionItem(_)
| ForeignStaticItem(_)
| ForeignTypeItem
| MacroItem(_)
| ProcMacroItem(_)
| PrimitiveItem(_)
| AssocConstItem(_, _)
| AssocTypeItem(_, _)
| KeywordItem(_) => kind,
}
}

Expand All @@ -86,14 +102,12 @@ crate trait DocFolder: Sized {
fn fold_crate(&mut self, mut c: Crate) -> Crate {
c.module = self.fold_item(c.module).unwrap();

{
let external_traits = { std::mem::take(&mut *c.external_traits.borrow_mut()) };
for (k, mut v) in external_traits {
v.trait_.items =
v.trait_.items.into_iter().filter_map(|i| self.fold_item(i)).collect();
c.external_traits.borrow_mut().insert(k, v);
}
let external_traits = { std::mem::take(&mut *c.external_traits.borrow_mut()) };
for (k, mut v) in external_traits {
v.trait_.items = v.trait_.items.into_iter().filter_map(|i| self.fold_item(i)).collect();
c.external_traits.borrow_mut().insert(k, v);
}

c
}
}
6 changes: 3 additions & 3 deletions src/librustdoc/html/render/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -461,9 +461,9 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
}
}

let (mut krate, local_sources, matches) = collect_spans_and_sources(
let (local_sources, matches) = collect_spans_and_sources(
tcx,
krate,
&krate,
&src_root,
include_sources,
generate_link_to_definition,
Expand Down Expand Up @@ -522,7 +522,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
};

if emit_crate {
krate = sources::render(&mut cx, krate)?;
sources::render(&mut cx, &krate)?;
}

// Build our search index
Expand Down
10 changes: 5 additions & 5 deletions src/librustdoc/html/render/span_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,21 +37,21 @@ crate enum LinkFromSrc {
/// only keep the `lo` and `hi`.
crate fn collect_spans_and_sources(
tcx: TyCtxt<'_>,
krate: clean::Crate,
krate: &clean::Crate,
src_root: &Path,
include_sources: bool,
generate_link_to_definition: bool,
) -> (clean::Crate, FxHashMap<PathBuf, String>, FxHashMap<Span, LinkFromSrc>) {
) -> (FxHashMap<PathBuf, String>, FxHashMap<Span, LinkFromSrc>) {
let mut visitor = SpanMapVisitor { tcx, matches: FxHashMap::default() };

if include_sources {
if generate_link_to_definition {
tcx.hir().walk_toplevel_module(&mut visitor);
}
let (krate, sources) = sources::collect_local_sources(tcx, src_root, krate);
(krate, sources, visitor.matches)
let sources = sources::collect_local_sources(tcx, src_root, &krate);
(sources, visitor.matches)
} else {
(krate, Default::default(), Default::default())
(Default::default(), Default::default())
}
}

Expand Down
60 changes: 32 additions & 28 deletions src/librustdoc/html/sources.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
use crate::clean;
use crate::docfs::PathError;
use crate::error::Error;
use crate::fold::DocFolder;
use crate::html::format::Buffer;
use crate::html::highlight;
use crate::html::layout;
use crate::html::render::{Context, BASIC_KEYWORDS};
use crate::visit::DocVisitor;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_hir::def_id::LOCAL_CRATE;
use rustc_middle::ty::TyCtxt;
Expand All @@ -16,23 +16,25 @@ use std::ffi::OsStr;
use std::fs;
use std::path::{Component, Path, PathBuf};

crate fn render(cx: &mut Context<'_>, krate: clean::Crate) -> Result<clean::Crate, Error> {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 That's way cleaner. I like this change; the other comments really are just nitpicking, and the PR could easily be merged as-is.

crate fn render(cx: &mut Context<'_>, krate: &clean::Crate) -> Result<(), Error> {
info!("emitting source files");

let dst = cx.dst.join("src").join(&*krate.name(cx.tcx()).as_str());
cx.shared.ensure_dir(&dst)?;
let mut folder = SourceCollector { dst, cx, emitted_local_sources: FxHashSet::default() };
Ok(folder.fold_crate(krate))

let mut collector = SourceCollector { dst, cx, emitted_local_sources: FxHashSet::default() };
collector.visit_crate(krate);
Ok(())
}

crate fn collect_local_sources<'tcx>(
tcx: TyCtxt<'tcx>,
src_root: &Path,
krate: clean::Crate,
) -> (clean::Crate, FxHashMap<PathBuf, String>) {
krate: &clean::Crate,
) -> FxHashMap<PathBuf, String> {
let mut lsc = LocalSourcesCollector { tcx, local_sources: FxHashMap::default(), src_root };

let krate = lsc.fold_crate(krate);
(krate, lsc.local_sources)
lsc.visit_crate(krate);
lsc.local_sources
}

struct LocalSourcesCollector<'a, 'tcx> {
Expand All @@ -42,7 +44,7 @@ struct LocalSourcesCollector<'a, 'tcx> {
}

fn is_real_and_local(span: clean::Span, sess: &Session) -> bool {
span.filename(sess).is_real() && span.cnum(sess) == LOCAL_CRATE
span.cnum(sess) == LOCAL_CRATE && span.filename(sess).is_real()
}

impl LocalSourcesCollector<'_, '_> {
Expand All @@ -54,12 +56,13 @@ impl LocalSourcesCollector<'_, '_> {
return;
}
let filename = span.filename(sess);
let p = match filename {
FileName::Real(ref file) => match file.local_path() {
Some(p) => p.to_path_buf(),
_ => return,
},
_ => return,
let p = if let FileName::Real(file) = filename {
match file.into_local_path() {
Some(p) => p,
None => return,
}
} else {
return;
};
if self.local_sources.contains_key(&*p) {
// We've already emitted this source
Expand All @@ -79,13 +82,11 @@ impl LocalSourcesCollector<'_, '_> {
}
}

impl DocFolder for LocalSourcesCollector<'_, '_> {
fn fold_item(&mut self, item: clean::Item) -> Option<clean::Item> {
self.add_local_source(&item);
impl DocVisitor for LocalSourcesCollector<'_, '_> {
fn visit_item(&mut self, item: &clean::Item) {
self.add_local_source(item);

// FIXME: if `include_sources` isn't set and DocFolder didn't require consuming the crate by value,
// we could return None here without having to walk the rest of the crate.
Some(self.fold_item_recur(item))
self.visit_item_recur(item)
}
}

Expand All @@ -98,16 +99,20 @@ struct SourceCollector<'a, 'tcx> {
emitted_local_sources: FxHashSet<PathBuf>,
}

impl DocFolder for SourceCollector<'_, '_> {
fn fold_item(&mut self, item: clean::Item) -> Option<clean::Item> {
impl DocVisitor for SourceCollector<'_, '_> {
fn visit_item(&mut self, item: &clean::Item) {
if !self.cx.include_sources {
return;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This line of code cries out for a more explicit way to do control flow than this. This conditional seems to stop it from trying to recurse into child nodes, but it can't stop the parent from trying to visit all of the siblings (and that seems wasteful), and I really only figured that out by carefully reading the implementation.

The API used by TypeVisitor seems like a good source for plagiarizing, since ControlFlow::Break provides a way to "just bail out" like we want here.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, ControlFlow makes everything better 🙃

I'd be happy to try out switching DocVisitor to use ControlFlow, but that'd make this change even bigger, so I think let's save it for a future PR.

}

let tcx = self.cx.tcx();
let span = item.span(tcx);
let sess = tcx.sess;

// If we're not rendering sources, there's nothing to do.
// If we're including source files, and we haven't seen this file yet,
// then we need to render it out to the filesystem.
if self.cx.include_sources && is_real_and_local(span, sess) {
if is_real_and_local(span, sess) {
let filename = span.filename(sess);
let span = span.inner();
let pos = sess.source_map().lookup_source_file(span.lo());
Expand All @@ -132,9 +137,8 @@ impl DocFolder for SourceCollector<'_, '_> {
}
};
}
// FIXME: if `include_sources` isn't set and DocFolder didn't require consuming the crate by value,
// we could return None here without having to walk the rest of the crate.
Some(self.fold_item_recur(item))

self.visit_item_recur(item)
}
}

Expand Down
1 change: 1 addition & 0 deletions src/librustdoc/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ mod markdown;
mod passes;
mod scrape_examples;
mod theme;
mod visit;
mod visit_ast;
mod visit_lib;

Expand Down
13 changes: 7 additions & 6 deletions src/librustdoc/passes/bare_urls.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use super::Pass;
use crate::clean::*;
use crate::core::DocContext;
use crate::fold::DocFolder;
use crate::html::markdown::main_body_opts;
use crate::visit::DocVisitor;
use core::ops::Range;
use pulldown_cmark::{Event, Parser, Tag};
use regex::Regex;
Expand Down Expand Up @@ -53,16 +53,17 @@ impl<'a, 'tcx> BareUrlsLinter<'a, 'tcx> {
}

crate fn check_bare_urls(krate: Crate, cx: &mut DocContext<'_>) -> Crate {
BareUrlsLinter { cx }.fold_crate(krate)
BareUrlsLinter { cx }.visit_crate(&krate);
krate
}

impl<'a, 'tcx> DocFolder for BareUrlsLinter<'a, 'tcx> {
fn fold_item(&mut self, item: Item) -> Option<Item> {
impl<'a, 'tcx> DocVisitor for BareUrlsLinter<'a, 'tcx> {
fn visit_item(&mut self, item: &Item) {
let hir_id = match DocContext::as_local_hir_id(self.cx.tcx, item.def_id) {
Some(hir_id) => hir_id,
None => {
// If non-local, no need to check anything.
return Some(self.fold_item_recur(item));
return;
}
};
let dox = item.attrs.collapsed_doc_value().unwrap_or_default();
Expand Down Expand Up @@ -106,6 +107,6 @@ impl<'a, 'tcx> DocFolder for BareUrlsLinter<'a, 'tcx> {
}
}

Some(self.fold_item_recur(item))
self.visit_item_recur(item)
}
}
23 changes: 12 additions & 11 deletions src/librustdoc/passes/calculate_doc_coverage.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use crate::clean;
use crate::core::DocContext;
use crate::fold::{self, DocFolder};
use crate::html::markdown::{find_testable_code, ErrorCodes};
use crate::passes::check_doc_test_visibility::{should_have_doc_example, Tests};
use crate::passes::Pass;
use crate::visit::DocVisitor;
use rustc_hir as hir;
use rustc_lint::builtin::MISSING_DOCS;
use rustc_middle::lint::LintLevelSource;
Expand All @@ -23,7 +23,7 @@ crate const CALCULATE_DOC_COVERAGE: Pass = Pass {

fn calculate_doc_coverage(krate: clean::Crate, ctx: &mut DocContext<'_>) -> clean::Crate {
let mut calc = CoverageCalculator { items: Default::default(), ctx };
let krate = calc.fold_crate(krate);
calc.visit_crate(&krate);

calc.print_results();

Expand Down Expand Up @@ -182,17 +182,18 @@ impl<'a, 'b> CoverageCalculator<'a, 'b> {
}
}

impl<'a, 'b> fold::DocFolder for CoverageCalculator<'a, 'b> {
fn fold_item(&mut self, i: clean::Item) -> Option<clean::Item> {
impl<'a, 'b> DocVisitor for CoverageCalculator<'a, 'b> {
fn visit_item(&mut self, i: &clean::Item) {
if !i.def_id.is_local() {
// non-local items are skipped because they can be out of the users control,
// especially in the case of trait impls, which rustdoc eagerly inlines
return;
}

match *i.kind {
_ if !i.def_id.is_local() => {
// non-local items are skipped because they can be out of the users control,
// especially in the case of trait impls, which rustdoc eagerly inlines
return Some(i);
}
clean::StrippedItem(..) => {
// don't count items in stripped modules
return Some(i);
return;
}
// docs on `use` and `extern crate` statements are not displayed, so they're not
// worth counting
Expand Down Expand Up @@ -269,6 +270,6 @@ impl<'a, 'b> fold::DocFolder for CoverageCalculator<'a, 'b> {
}
}

Some(self.fold_item_recur(i))
self.visit_item_recur(i)
}
}
11 changes: 6 additions & 5 deletions src/librustdoc/passes/check_code_block_syntax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ use rustc_span::{hygiene::AstPass, ExpnData, ExpnKind, FileName, InnerSpan, DUMM

use crate::clean;
use crate::core::DocContext;
use crate::fold::DocFolder;
use crate::html::markdown::{self, RustCodeBlock};
use crate::passes::Pass;
use crate::visit::DocVisitor;

crate const CHECK_CODE_BLOCK_SYNTAX: Pass = Pass {
name: "check-code-block-syntax",
Expand All @@ -19,7 +19,8 @@ crate const CHECK_CODE_BLOCK_SYNTAX: Pass = Pass {
};

crate fn check_code_block_syntax(krate: clean::Crate, cx: &mut DocContext<'_>) -> clean::Crate {
SyntaxChecker { cx }.fold_crate(krate)
SyntaxChecker { cx }.visit_crate(&krate);
krate
}

struct SyntaxChecker<'a, 'tcx> {
Expand Down Expand Up @@ -141,8 +142,8 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
}
}

impl<'a, 'tcx> DocFolder for SyntaxChecker<'a, 'tcx> {
fn fold_item(&mut self, item: clean::Item) -> Option<clean::Item> {
impl<'a, 'tcx> DocVisitor for SyntaxChecker<'a, 'tcx> {
fn visit_item(&mut self, item: &clean::Item) {
if let Some(dox) = &item.attrs.collapsed_doc_value() {
let sp = item.attr_span(self.cx.tcx);
let extra = crate::html::markdown::ExtraInfo::new_did(
Expand All @@ -155,7 +156,7 @@ impl<'a, 'tcx> DocFolder for SyntaxChecker<'a, 'tcx> {
}
}

Some(self.fold_item_recur(item))
self.visit_item_recur(item)
}
}

Expand Down
Loading