initial implementation of mergable rustdoc cci

Author: EtomicBomb

src/librustdoc/Cargo.toml

@@ -16,7 +16,7 @@ minifier = "0.3.0"
 pulldown-cmark-old = { version = "0.9.6", package = "pulldown-cmark", default-features = false }
 regex = "1"
 rustdoc-json-types = { path = "../rustdoc-json-types" }
-serde_json = "1.0"
+serde_json = { version = "1.0", features = ["preserve_order"] }
 serde = { version = "1.0", features = ["derive"] }
 smallvec = "1.8.1"
 tempfile = "3"

src/librustdoc/clean/types.rs

@@ -133,7 +133,7 @@ pub(crate) struct ExternalCrate {
 }
 
 impl ExternalCrate {
-    const LOCAL: Self = Self { crate_num: LOCAL_CRATE };
+    pub(crate) const LOCAL: Self = Self { crate_num: LOCAL_CRATE };
 
     #[inline]
     pub(crate) fn def_id(&self) -> DefId {

src/librustdoc/config.rs

@@ -736,7 +736,6 @@ impl Options {
         let extern_html_root_takes_precedence =
             matches.opt_present("extern-html-root-takes-precedence");
         let html_no_source = matches.opt_present("html-no-source");
-
         if generate_link_to_definition && (show_coverage || output_format != OutputFormat::Html) {
             dcx.fatal(
                 "--generate-link-to-definition option can only be used with HTML output format",

src/librustdoc/html/render/context.rs

@@ -13,7 +13,6 @@ use rustc_span::edition::Edition;
 use rustc_span::{sym, FileName, Symbol};
 
 use super::print_item::{full_path, item_path, print_item};
-use super::search_index::build_index;
 use super::write_shared::write_shared;
 use super::{
     collect_spans_and_sources, scrape_examples_help,
@@ -576,13 +575,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
         }
 
         if !no_emit_shared {
-            // Build our search index
-            let index = build_index(&krate, &mut Rc::get_mut(&mut cx.shared).unwrap().cache, tcx);
-
-            // Write shared runs within a flock; disable thread dispatching of IO temporarily.
-            Rc::get_mut(&mut cx.shared).unwrap().fs.set_sync_only(true);
-            write_shared(&mut cx, &krate, index, &md_opts)?;
-            Rc::get_mut(&mut cx.shared).unwrap().fs.set_sync_only(false);
+            write_shared(&mut cx, &krate, &md_opts, tcx)?;
         }
 
         Ok((cx, krate))
@@ -732,6 +725,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
         );
         shared.fs.write(help_file, v)?;
 
+        // if to avoid writing files to doc root unless we're on the final invocation
         if shared.layout.scrape_examples_extension {
             page.title = "About scraped examples";
             page.description = "How the scraped examples feature works in Rustdoc";

src/librustdoc/html/render/mod.rs

@@ -31,6 +31,8 @@ mod tests;
 mod context;
 mod print_item;
 mod sidebar;
+mod sorted_json;
+mod sorted_template;
 mod span_map;
 mod type_layout;
 mod write_shared;

src/librustdoc/html/render/search_index.rs

@@ -17,6 +17,7 @@ use crate::formats::cache::{Cache, OrphanImplItem};
 use crate::formats::item_type::ItemType;
 use crate::html::format::join_with_double_colon;
 use crate::html::markdown::short_markdown_summary;
+use crate::html::render::sorted_json::SortedJson;
 use crate::html::render::{self, IndexItem, IndexItemFunctionType, RenderType, RenderTypeId};
 
 use encode::{bitmap_to_string, write_vlqhex_to_string};
@@ -47,7 +48,7 @@ use encode::{bitmap_to_string, write_vlqhex_to_string};
 /// [2]: https://en.wikipedia.org/wiki/Sliding_window_protocol#Basic_concept
 /// [3]: https://learn.microsoft.com/en-us/troubleshoot/windows-server/networking/description-tcp-features
 pub(crate) struct SerializedSearchIndex {
-    pub(crate) index: String,
+    pub(crate) index: SortedJson,
     pub(crate) desc: Vec<(usize, String)>,
 }
 
@@ -684,24 +685,19 @@ pub(crate) fn build_index<'tcx>(
     // The index, which is actually used to search, is JSON
     // It uses `JSON.parse(..)` to actually load, since JSON
     // parses faster than the full JavaScript syntax.
-    let index = format!(
-        r#"["{}",{}]"#,
-        krate.name(tcx),
-        serde_json::to_string(&CrateData {
-            items: crate_items,
-            paths: crate_paths,
-            aliases: &aliases,
-            associated_item_disambiguators: &associated_item_disambiguators,
-            desc_index,
-            empty_desc,
-        })
-        .expect("failed serde conversion")
-        // All these `replace` calls are because we have to go through JS string for JSON content.
-        .replace('\\', r"\\")
-        .replace('\'', r"\'")
-        // We need to escape double quotes for the JSON.
-        .replace("\\\"", "\\\\\"")
-    );
+    let crate_name = krate.name(tcx);
+    let data = CrateData {
+        items: crate_items,
+        paths: crate_paths,
+        aliases: &aliases,
+        associated_item_disambiguators: &associated_item_disambiguators,
+        desc_index,
+        empty_desc,
+    };
+    let index = SortedJson::array_unsorted([
+        SortedJson::serialize(crate_name.as_str()),
+        SortedJson::serialize(data),
+    ]);
     SerializedSearchIndex { index, desc }
 }
 

src/librustdoc/html/render/sorted_json.rs

@@ -0,0 +1,82 @@
+use itertools::Itertools as _;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::borrow::Borrow;
+use std::fmt;
+
+/// Prerenedered json.
+///
+/// Arrays are sorted by their stringified entries, and objects are sorted by their stringified
+/// keys.
+///
+/// Must use serde_json with the preserve_order feature.
+///
+/// Both the Display and serde_json::to_string implementations write the serialized json
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[serde(from = "Value")]
+#[serde(into = "Value")]
+pub(crate) struct SortedJson(String);
+
+impl SortedJson {
+    /// If you pass in an array, it will not be sorted.
+    pub(crate) fn serialize<T: Serialize>(item: T) -> Self {
+        SortedJson(serde_json::to_string(&item).unwrap())
+    }
+
+    /// Serializes and sorts
+    pub(crate) fn array<T: Borrow<SortedJson>, I: IntoIterator<Item = T>>(items: I) -> Self {
+        let items = items
+            .into_iter()
+            .sorted_unstable_by(|a, b| a.borrow().cmp(&b.borrow()))
+            .format_with(",", |item, f| f(item.borrow()));
+        SortedJson(format!("[{}]", items))
+    }
+
+    pub(crate) fn array_unsorted<T: Borrow<SortedJson>, I: IntoIterator<Item = T>>(
+        items: I,
+    ) -> Self {
+        let items = items.into_iter().format_with(",", |item, f| f(item.borrow()));
+        SortedJson(format!("[{items}]"))
+    }
+}
+
+impl fmt::Display for SortedJson {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl From<Value> for SortedJson {
+    fn from(value: Value) -> Self {
+        SortedJson(serde_json::to_string(&value).unwrap())
+    }
+}
+
+impl From<SortedJson> for Value {
+    fn from(json: SortedJson) -> Self {
+        serde_json::from_str(&json.0).unwrap()
+    }
+}
+
+/// For use in JSON.parse('{...}').
+///
+/// JSON.parse supposedly loads faster than raw JS source,
+/// so this is used for large objects.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct EscapedJson(SortedJson);
+
+impl From<SortedJson> for EscapedJson {
+    fn from(json: SortedJson) -> Self {
+        EscapedJson(json)
+    }
+}
+
+impl fmt::Display for EscapedJson {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // All these `replace` calls are because we have to go through JS string
+        // for JSON content.
+        // We need to escape double quotes for the JSON
+        let json = self.0.0.replace('\\', r"\\").replace('\'', r"\'").replace("\\\"", "\\\\\"");
+        write!(f, "{}", json)
+    }
+}

src/librustdoc/html/render/sorted_template.rs

@@ -0,0 +1,136 @@
+use std::collections::BTreeSet;
+use std::fmt;
+use std::marker::PhantomData;
+use std::str::FromStr;
+
+use serde::{Deserialize, Serialize};
+
+/// Append-only templates for sorted, deduplicated lists of items.
+///
+/// Last line of the rendered output is a comment encoding the next insertion point.
+#[derive(Debug, Clone)]
+pub(crate) struct SortedTemplate<F> {
+    format: PhantomData<F>,
+    before: String,
+    after: String,
+    contents: BTreeSet<String>,
+}
+
+/// Written to last line of file to specify the location of each fragment
+#[derive(Serialize, Deserialize, Debug, Clone)]
+struct Offset {
+    /// Index of the first byte in the template
+    start: usize,
+    /// The length of each fragment in the encoded template, including the separator
+    delta: Vec<usize>,
+}
+
+impl<F> SortedTemplate<F> {
+    /// Generate this template from arbitary text.
+    /// Will insert wherever the substring `magic` can be found.
+    /// Errors if it does not appear exactly once.
+    pub(crate) fn magic(template: &str, magic: &str) -> Result<Self, Error> {
+        let mut split = template.split(magic);
+        let before = split.next().ok_or(Error)?;
+        let after = split.next().ok_or(Error)?;
+        if split.next().is_some() {
+            return Err(Error);
+        }
+        Ok(Self::before_after(before, after))
+    }
+
+    /// Template will insert contents between `before` and `after`
+    pub(crate) fn before_after<S: ToString, T: ToString>(before: S, after: T) -> Self {
+        let before = before.to_string();
+        let after = after.to_string();
+        SortedTemplate { format: PhantomData, before, after, contents: Default::default() }
+    }
+}
+
+impl<F: FileFormat> SortedTemplate<F> {
+    /// Adds this text to the template
+    pub(crate) fn append(&mut self, insert: String) {
+        self.contents.insert(insert);
+    }
+}
+
+impl<F: FileFormat> fmt::Display for SortedTemplate<F> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut delta = Vec::default();
+        write!(f, "{}", self.before)?;
+        let contents: Vec<_> = self.contents.iter().collect();
+        let mut sep = "";
+        for content in contents {
+            delta.push(sep.len() + content.len());
+            write!(f, "{}{}", sep, content)?;
+            sep = F::SEPARATOR;
+        }
+        let offset = Offset { start: self.before.len(), delta };
+        let offset = serde_json::to_string(&offset).unwrap();
+        write!(f, "{}\n{}{}{}", self.after, F::COMMENT_START, offset, F::COMMENT_END)?;
+        Ok(())
+    }
+}
+
+fn checked_split_at(s: &str, index: usize) -> Option<(&str, &str)> {
+    s.is_char_boundary(index).then(|| s.split_at(index))
+}
+
+impl<F: FileFormat> FromStr for SortedTemplate<F> {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (s, offset) = s.rsplit_once("\n").ok_or(Error)?;
+        let offset = offset.strip_prefix(F::COMMENT_START).ok_or(Error)?;
+        let offset = offset.strip_suffix(F::COMMENT_END).ok_or(Error)?;
+        let offset: Offset = serde_json::from_str(&offset).map_err(|_| Error)?;
+        let (before, mut s) = checked_split_at(s, offset.start).ok_or(Error)?;
+        let mut contents = BTreeSet::default();
+        let mut sep = "";
+        for &index in offset.delta.iter() {
+            let (content, rest) = checked_split_at(s, index).ok_or(Error)?;
+            s = rest;
+            let content = content.strip_prefix(sep).ok_or(Error)?;
+            contents.insert(content.to_string());
+            sep = F::SEPARATOR;
+        }
+        Ok(SortedTemplate {
+            format: PhantomData,
+            before: before.to_string(),
+            after: s.to_string(),
+            contents,
+        })
+    }
+}
+
+pub(crate) trait FileFormat {
+    const COMMENT_START: &'static str;
+    const COMMENT_END: &'static str;
+    const SEPARATOR: &'static str;
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Html;
+
+impl FileFormat for Html {
+    const COMMENT_START: &'static str = "<!--";
+    const COMMENT_END: &'static str = "-->";
+    const SEPARATOR: &'static str = "";
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Js;
+
+impl FileFormat for Js {
+    const COMMENT_START: &'static str = "//";
+    const COMMENT_END: &'static str = "";
+    const SEPARATOR: &'static str = ",";
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Error;
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "invalid template")
+    }
+}