rustdoc-search: shard the search result descriptions

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -56,7 +56,7 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
            /scripts/validate-error-codes.sh && \
            reuse --include-submodules lint && \
            # Runs checks to ensure that there are no ES5 issues in our JS code.
-           es-check es6 ../src/librustdoc/html/static/js/*.js && \
+           es-check es8 ../src/librustdoc/html/static/js/*.js && \
            eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \
            eslint -c ../src/tools/rustdoc-js/.eslintrc.js ../src/tools/rustdoc-js/tester.js && \
            eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js

src/librustdoc/html/render/mod.rs

@@ -184,40 +184,15 @@ pub(crate) enum RenderTypeId {
 
 impl RenderTypeId {
     pub fn write_to_string(&self, string: &mut String) {
-        // (sign, value)
-        let (sign, id): (bool, u32) = match &self {
+        let id: i32 = match &self {
             // 0 is a sentinel, everything else is one-indexed
             // concrete type
-            RenderTypeId::Index(idx) if *idx >= 0 => (false, (idx + 1isize).try_into().unwrap()),
+            RenderTypeId::Index(idx) if *idx >= 0 => (idx + 1isize).try_into().unwrap(),
             // generic type parameter
-            RenderTypeId::Index(idx) => (true, (-*idx).try_into().unwrap()),
+            RenderTypeId::Index(idx) => (*idx).try_into().unwrap(),
             _ => panic!("must convert render types to indexes before serializing"),
         };
-        // zig-zag encoding
-        let value: u32 = (id << 1) | (if sign { 1 } else { 0 });
-        // Self-terminating hex use capital letters for everything but the
-        // least significant digit, which is lowercase. For example, decimal 17
-        // would be `` Aa `` if zig-zag encoding weren't used.
-        //
-        // Zig-zag encoding, however, stores the sign bit as the last bit.
-        // This means, in the last hexit, 1 is actually `c`, -1 is `b`
-        // (`a` is the imaginary -0), and, because all the bits are shifted
-        // by one, `` A` `` is actually 8 and `` Aa `` is -8.
-        //
-        // https://rust-lang.github.io/rustc-dev-guide/rustdoc-internals/search.html
-        // describes the encoding in more detail.
-        let mut shift: u32 = 28;
-        let mut mask: u32 = 0xF0_00_00_00;
-        while shift < 32 {
-            let hexit = (value & mask) >> shift;
-            if hexit != 0 || shift == 0 {
-                let hex =
-                    char::try_from(if shift == 0 { '`' } else { '@' } as u32 + hexit).unwrap();
-                string.push(hex);
-            }
-            shift = shift.wrapping_sub(4);
-            mask = mask >> 4;
-        }
+        search_index::write_vlqhex_to_string(id, string);
     }
 }
 

src/librustdoc/html/render/search_index.rs

@@ -17,12 +17,25 @@ use crate::html::format::join_with_double_colon;
 use crate::html::markdown::short_markdown_summary;
 use crate::html::render::{self, IndexItem, IndexItemFunctionType, RenderType, RenderTypeId};
 
+/// The serialized search description sharded version
+///
+/// The `index` is a JSON-encoded list of names and other information.
+///
+/// The desc has newlined descriptions, split up by size into 1MiB shards.
+/// For example, `(4, "foo\nbar\nbaz\nquux")`.
+pub(crate) struct SerializedSearchIndex {
+    pub(crate) index: String,
+    pub(crate) desc: Vec<(usize, String)>,
+}
+
+const DESC_INDEX_SHARD_LEN: usize = 1024 * 1024;
+
 /// Builds the search index from the collected metadata
 pub(crate) fn build_index<'tcx>(
     krate: &clean::Crate,
     cache: &mut Cache,
     tcx: TyCtxt<'tcx>,
-) -> String {
+) -> SerializedSearchIndex {
     let mut itemid_to_pathid = FxHashMap::default();
     let mut primitives = FxHashMap::default();
     let mut associated_types = FxHashMap::default();
@@ -318,7 +331,6 @@ pub(crate) fn build_index<'tcx>(
         .collect::<Vec<_>>();
 
     struct CrateData<'a> {
-        doc: String,
         items: Vec<&'a IndexItem>,
         paths: Vec<(ItemType, Vec<Symbol>)>,
         // The String is alias name and the vec is the list of the elements with this alias.
@@ -327,6 +339,9 @@ pub(crate) fn build_index<'tcx>(
         aliases: &'a BTreeMap<String, Vec<usize>>,
         // Used when a type has more than one impl with an associated item with the same name.
         associated_item_disambiguators: &'a Vec<(usize, String)>,
+        // A list of shard lengths encoded as vlqhex. See the comment in write_vlqhex_to_string
+        // for information on the format.
+        descindex: String,
     }
 
     struct Paths {
@@ -408,7 +423,6 @@ pub(crate) fn build_index<'tcx>(
             let mut names = Vec::with_capacity(self.items.len());
             let mut types = String::with_capacity(self.items.len());
             let mut full_paths = Vec::with_capacity(self.items.len());
-            let mut descriptions = Vec::with_capacity(self.items.len());
             let mut parents = Vec::with_capacity(self.items.len());
             let mut functions = String::with_capacity(self.items.len());
             let mut deprecated = Vec::with_capacity(self.items.len());
@@ -431,7 +445,6 @@ pub(crate) fn build_index<'tcx>(
                 parents.push(item.parent_idx.map(|x| x + 1).unwrap_or(0));
 
                 names.push(item.name.as_str());
-                descriptions.push(&item.desc);
 
                 if !item.path.is_empty() {
                     full_paths.push((index, &item.path));
@@ -454,14 +467,12 @@ pub(crate) fn build_index<'tcx>(
             let has_aliases = !self.aliases.is_empty();
             let mut crate_data =
                 serializer.serialize_struct("CrateData", if has_aliases { 9 } else { 8 })?;
-            crate_data.serialize_field("doc", &self.doc)?;
             crate_data.serialize_field("t", &types)?;
             crate_data.serialize_field("n", &names)?;
-            // Serialize as an array of item indices and full paths
             crate_data.serialize_field("q", &full_paths)?;
-            crate_data.serialize_field("d", &descriptions)?;
             crate_data.serialize_field("i", &parents)?;
             crate_data.serialize_field("f", &functions)?;
+            crate_data.serialize_field("D", &self.descindex)?;
             crate_data.serialize_field("c", &deprecated)?;
             crate_data.serialize_field("p", &paths)?;
             crate_data.serialize_field("b", &self.associated_item_disambiguators)?;
@@ -472,24 +483,92 @@ pub(crate) fn build_index<'tcx>(
         }
     }
 
-    // Collect the index into a string
-    format!(
+    let desc = {
+        let mut result = Vec::new();
+        let mut set = String::new();
+        let mut len: usize = 0;
+        for desc in std::iter::once(&crate_doc).chain(crate_items.iter().map(|item| &item.desc)) {
+            if set.len() >= DESC_INDEX_SHARD_LEN {
+                result.push((len, std::mem::replace(&mut set, String::new())));
+                len = 0;
+            } else if len != 0 {
+                set.push('\n');
+            }
+            set.push_str(&desc);
+            len += 1;
+        }
+        result.push((len, std::mem::replace(&mut set, String::new())));
+        result
+    };
+
+    let descindex = {
+        let mut descindex = String::with_capacity(desc.len() * 4);
+        for &(len, _) in desc.iter() {
+            write_vlqhex_to_string(len.try_into().unwrap(), &mut descindex);
+        }
+        descindex
+    };
+
+    assert_eq!(crate_items.len() + 1, desc.iter().map(|(len, _)| *len).sum::<usize>());
+
+    // The index, which is actually used to search, is JSON
+    // It uses `JSON.parse(..)` to actually load, since JSON
+    // parses faster than the full JavaScript syntax.
+    let index = format!(
         r#"["{}",{}]"#,
         krate.name(tcx),
         serde_json::to_string(&CrateData {
-            doc: crate_doc,
             items: crate_items,
             paths: crate_paths,
             aliases: &aliases,
             associated_item_disambiguators: &associated_item_disambiguators,
+            descindex,
         })
         .expect("failed serde conversion")
         // All these `replace` calls are because we have to go through JS string for JSON content.
         .replace('\\', r"\\")
         .replace('\'', r"\'")
         // We need to escape double quotes for the JSON.
         .replace("\\\"", "\\\\\"")
-    )
+    );
+    SerializedSearchIndex { index, desc }
+}
+
+pub(crate) fn write_vlqhex_to_string(n: i32, string: &mut String) {
+    let (sign, magnitude): (bool, u32) =
+        if n >= 0 { (false, n.try_into().unwrap()) } else { (true, (-n).try_into().unwrap()) };
+    // zig-zag encoding
+    let value: u32 = (magnitude << 1) | (if sign { 1 } else { 0 });
+    // Self-terminating hex use capital letters for everything but the
+    // least significant digit, which is lowercase. For example, decimal 17
+    // would be `` Aa `` if zig-zag encoding weren't used.
+    //
+    // Zig-zag encoding, however, stores the sign bit as the last bit.
+    // This means, in the last hexit, 1 is actually `c`, -1 is `b`
+    // (`a` is the imaginary -0), and, because all the bits are shifted
+    // by one, `` A` `` is actually 8 and `` Aa `` is -8.
+    //
+    // https://rust-lang.github.io/rustc-dev-guide/rustdoc-internals/search.html
+    // describes the encoding in more detail.
+    let mut shift: u32 = 28;
+    let mut mask: u32 = 0xF0_00_00_00;
+    // first skip leading zeroes
+    while shift < 32 {
+        let hexit = (value & mask) >> shift;
+        if hexit != 0 || shift == 0 {
+            break;
+        }
+        shift = shift.wrapping_sub(4);
+        mask = mask >> 4;
+    }
+    // now write the rest
+    while shift < 32 {
+        let hexit = (value & mask) >> shift;
+        let hex = char::try_from(if shift == 0 { '`' } else { '@' } as u32 + hexit).unwrap();
+        string.push(hex);
+        shift = shift.wrapping_sub(4);
+        mask = mask >> 4;
+    }
 }
 
 pub(crate) fn get_function_type_for_search<'tcx>(

src/librustdoc/html/render/write_shared.rs

@@ -24,6 +24,7 @@ use crate::formats::cache::Cache;
 use crate::formats::item_type::ItemType;
 use crate::formats::Impl;
 use crate::html::format::Buffer;
+use crate::html::render::search_index::SerializedSearchIndex;
 use crate::html::render::{AssocItemLink, ImplRenderingParameters};
 use crate::html::{layout, static_files};
 use crate::visit::DocVisitor;
@@ -46,7 +47,7 @@ use crate::{try_err, try_none};
 pub(super) fn write_shared(
     cx: &mut Context<'_>,
     krate: &Crate,
-    search_index: String,
+    search_index: SerializedSearchIndex,
     options: &RenderOptions,
 ) -> Result<(), Error> {
     // Write out the shared files. Note that these are shared among all rustdoc
@@ -312,7 +313,7 @@ pub(super) fn write_shared(
     let dst = cx.dst.join(&format!("search-index{}.js", cx.shared.resource_suffix));
     let (mut all_indexes, mut krates) =
         try_err!(collect_json(&dst, krate.name(cx.tcx()).as_str()), &dst);
-    all_indexes.push(search_index);
+    all_indexes.push(search_index.index);
     krates.push(krate.name(cx.tcx()).to_string());
     krates.sort();
 
@@ -335,6 +336,32 @@ else if (window.initSearch) window.initSearch(searchIndex);
         Ok(v.into_bytes())
     })?;
 
+    let search_desc_dir = cx.dst.join(format!("search.desc/{krate}", krate = krate.name(cx.tcx())));
+    if Path::new(&search_desc_dir).exists() {
+        try_err!(std::fs::remove_dir_all(&search_desc_dir), &search_desc_dir);
+    }
+    try_err!(std::fs::create_dir_all(&search_desc_dir), &search_desc_dir);
+    let kratename = krate.name(cx.tcx()).to_string();
+    for (i, (_, data)) in search_index.desc.into_iter().enumerate() {
+        let output_filename = static_files::suffix_path(
+            &format!("{kratename}-desc-{i}-.js"),
+            &cx.shared.resource_suffix,
+        );
+        let path = search_desc_dir.join(output_filename);
+        try_err!(
+            std::fs::write(
+                &path,
+                &format!(
+                    r##"searchState.loadedDescShard({kratename}, {i}, {data})"##,
+                    kratename = serde_json::to_string(&kratename).unwrap(),
+                    data = serde_json::to_string(&data).unwrap(),
+                )
+                .into_bytes()
+            ),
+            &path
+        );
+    }
+
     write_invocation_specific("crates.js", &|| {
         let krates = krates.iter().map(|k| format!("\"{k}\"")).join(",");
         Ok(format!("window.ALL_CRATES = [{krates}];").into_bytes())

src/librustdoc/html/static/.eslintrc.js

@@ -5,7 +5,7 @@ module.exports = {
     },
     "extends": "eslint:recommended",
     "parserOptions": {
-        "ecmaVersion": 2015,
+        "ecmaVersion": 8,
         "sourceType": "module"
     },
     "rules": {

src/librustdoc/html/static/js/main.js

@@ -329,6 +329,26 @@ function preLoadCss(cssUrl) {
             search.innerHTML = "<h3 class=\"search-loading\">" + searchState.loadingText + "</h3>";
             searchState.showResults(search);
         },
+        descShards: new Map(),
+        loadDesc: async function({descShard, descIndex}) {
+            if (descShard.promise === null) {
+                descShard.promise = new Promise((resolve, reject) => {
+                    descShard.resolve = resolve;
+                    const ds = descShard;
+                    const fname = `${ds.crate}-desc-${ds.shard}-`;
+                    const url = resourcePath(
+                        `search.desc/${descShard.crate}/${fname}`,
+                        ".js",
+                    );
+                    loadScript(url, reject);
+                });
+            }
+            const list = await descShard.promise;
+            return list[descIndex];
+        },
+        loadedDescShard: function(crate, shard, data) {
+            this.descShards.get(crate)[shard].resolve(data.split("\n"));
+        },
     };
 
     const toggleAllDocsId = "toggle-all-docs";
@@ -381,7 +401,7 @@ function preLoadCss(cssUrl) {
                                     window.location.replace("#" + item.id);
                                 }, 0);
                             }
-                        }
+                        },
                     );
                 }
             }
@@ -585,7 +605,7 @@ function preLoadCss(cssUrl) {
         const script = document
             .querySelector("script[data-ignore-extern-crates]");
         const ignoreExternCrates = new Set(
-            (script ? script.getAttribute("data-ignore-extern-crates") : "").split(",")
+            (script ? script.getAttribute("data-ignore-extern-crates") : "").split(","),
         );
         for (const lib of libs) {
             if (lib === window.currentCrate || ignoreExternCrates.has(lib)) {
@@ -1098,7 +1118,7 @@ function preLoadCss(cssUrl) {
         } else {
             wrapper.style.setProperty(
                 "--popover-arrow-offset",
-                (wrapperPos.right - pos.right + 4) + "px"
+                (wrapperPos.right - pos.right + 4) + "px",
             );
         }
         wrapper.style.visibility = "";
@@ -1680,7 +1700,7 @@ href="https://doc.rust-lang.org/${channel}/rustdoc/read-documentation/search.htm
                 pendingSidebarResizingFrame = false;
                 document.documentElement.style.setProperty(
                     "--resizing-sidebar-width",
-                    desiredSidebarSize + "px"
+                    desiredSidebarSize + "px",
                 );
             }, 100);
         }