rust-lang · onur · Jul 6, 2018 · Jul 3, 2018 · Jul 3, 2018 · Jul 3, 2018
@@ -28,6 +28,7 @@ badge = { version = "0", path = "src/web/badge" }
 error-chain = "0.10"
 comrak = { version = "0.2.10", default-features = false }
 toml = "0.4"
+html5ever = "0.22"
 
 # iron dependencies
 iron = "0.5"

@@ -47,7 +47,7 @@ Vagrant.configure("2") do |config|
     ############################################################
     lxc-attach -n cratesfyi-container -- apt-get update
     lxc-attach -n cratesfyi-container -- apt-get install -y --no-install-recommends curl ca-certificates binutils gcc libc6-dev libmagic1
-    lxc-attach -n cratesfyi-container -- su - cratesfyi -c 'curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly-2018-03-25'
+    lxc-attach -n cratesfyi-container -- su - cratesfyi -c 'curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly-2018-06-03'
 
     ############################################################
     # Creating rustc links for cratesfyi user                  #

@@ -10,7 +10,7 @@ use std::path::PathBuf;
 use std::fs::remove_dir_all;
 use postgres::Connection;
 use rustc_serialize::json::Json;
-use error::Result;
+use error::{Result, ResultExt};
 
 
 /// List of targets supported by docs.rs
@@ -398,13 +398,15 @@ impl DocBuilder {
             let file_name = format!("{}-{}.{}", spl[0], rustc_version, spl[1]);
             let source_path = source.join(&file_name);
             let destination_path = destination.join(&file_name);
-            try!(copy(source_path, destination_path));
+            try!(copy(&source_path, &destination_path)
+                .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display())));
         }
 
         for file in files.1.iter() {
             let source_path = source.join(file);
             let destination_path = destination.join(file);
-            try!(copy(source_path, destination_path));
+            try!(copy(&source_path, &destination_path)
+                .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display())));
         }
 
         let conn = try!(connect_db());

@@ -28,6 +28,7 @@ extern crate badge;
 extern crate crates_index_diff;
 extern crate git2;
 extern crate toml;
+extern crate html5ever;
 
 pub use self::docbuilder::DocBuilder;
 pub use self::docbuilder::ChrootBuilderResult;

@@ -0,0 +1,58 @@
+use error::{Result, Error};
+
+use html5ever::serialize::{serialize, SerializeOpts};
+use html5ever::rcdom::{RcDom, NodeData, Handle};
+use html5ever::driver::{parse_document, ParseOpts};
+use html5ever::tendril::TendrilSink;
+
+/// Extracts the contents of the `<head>` and `<body>` tags from an HTML document.
+pub fn extract_head_and_body(html: &str) -> Result<(String, String)> {
+    let parser = parse_document(RcDom::default(), ParseOpts::default());
+    let dom = parser.one(html);
+
+    let (head, body) = extract_from_rcdom(&dom)?;
+
+    Ok((stringify(head), stringify(body)))
+}
+
+fn extract_from_rcdom(dom: &RcDom) -> Result<(Handle, Handle)> {
+    let mut worklist = vec![dom.document.clone()];
+    let (mut head, mut body) = (None, None);
+
+    while let Some(handle) = worklist.pop() {
+        match handle.data {
+            NodeData::Element { ref name, .. } => match name.local.as_ref() {
+                "head" => {
+                    if head.is_some() {
+                        return Err("duplicate <head> tag".into());
+                    } else {
+                        head = Some(handle.clone());
+                    }
+                }
+                "body" => {
+                    if body.is_some() {
+                        return Err("duplicate <body> tag".into());
+                    } else {
+                        body = Some(handle.clone());
+                    }
+                }
+                _ => {}  // do nothing
+            }
+            _ => {}  // do nothing
+        }
+
+        worklist.extend(handle.children.borrow().iter().cloned());
+    }
+
+    let head = head.ok_or_else(|| Error::from("couldn't find <head> tag in rustdoc output"))?;
+    let body = body.ok_or_else(|| Error::from("couldn't find <body> tag in rustdoc output"))?;
+    Ok((head, body))
+}
+
+fn stringify(node: Handle) -> String {
+    let mut vec = Vec::new();
+    serialize(&mut vec, &node, SerializeOpts::default())
+        .expect("serializing into buffer failed");
+
+    String::from_utf8(vec).expect("html5ever returned non-utf8 data")
+}
@@ -7,6 +7,7 @@ pub use self::github_updater::github_updater;
 pub use self::release_activity_updater::update_release_activity;
 pub use self::daemon::start_daemon;
 pub use self::rustc_version::{parse_rustc_version, get_current_versions, command_result};
+pub use self::html::extract_head_and_body;
 
 mod github_updater;
 mod build_doc;
@@ -15,3 +16,4 @@ mod release_activity_updater;
 mod daemon;
 mod pubsubhubbub;
 mod rustc_version;
+mod html;
@@ -69,6 +69,7 @@ const OPENSEARCH_XML: &'static [u8] = include_bytes!("opensearch.xml");
 
 
 struct CratesfyiHandler {
+    shared_resource_handler: Box<Handler>,
     router_handler: Box<Handler>,
     database_file_handler: Box<Handler>,
     static_handler: Box<Handler>,
@@ -186,12 +187,14 @@ impl CratesfyiHandler {
                    rustdoc::rustdoc_html_server_handler,
                    "crate_version_target_html");
 
+        let shared_resources = Self::chain(rustdoc::SharedResourceHandler);
         let router_chain = Self::chain(router);
         let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX").unwrap()).join("public_html");
         let static_handler = Static::new(prefix)
             .cache(Duration::from_secs(STATIC_FILE_CACHE_DURATION));
 
         CratesfyiHandler {
+            shared_resource_handler: Box::new(shared_resources),
             router_handler: Box::new(router_chain),
             database_file_handler: Box::new(file::DatabaseFileHandler),
             static_handler: Box::new(static_handler),
@@ -202,10 +205,13 @@ impl CratesfyiHandler {
 
 impl Handler for CratesfyiHandler {
     fn handle(&self, req: &mut Request) -> IronResult<Response> {
-        // try router first then db/static file handler
+        // try serving shared rustdoc resources first, then router, then db/static file handler
         // return 404 if none of them return Ok
-        self.router_handler
+        self.shared_resource_handler
             .handle(req)
+            .or_else(|e| {
+                self.router_handler.handle(req).or(Err(e))
+            })
             .or_else(|e| {
                 // if router fails try to serve files from database first
                 self.database_file_handler.handle(req).or(Err(e))

@@ -16,7 +16,8 @@ use rustc_serialize::json::{Json, ToJson};
 use std::collections::BTreeMap;
 use iron::headers::{Expires, HttpDate, CacheControl, CacheDirective};
 use time;
-
+use iron::Handler;
+use utils;
 
 
 #[derive(Debug)]
@@ -110,6 +111,10 @@ pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult<Response> {
 }
 
 
+/// Serves documentation generated by rustdoc.
+///
+/// This includes all HTML files for an individual crate, as well as the `search-index.js`, which is
+/// also crate-specific.
 pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult<Response> {
 
     let router = extension!(req, Router);
@@ -151,34 +156,13 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult<Response> {
         return Ok(file.serve());
     }
 
-    let (mut in_head, mut in_body) = (false, false);
-
     let mut content = RustdocPage::default();
 
     let file_content = ctry!(String::from_utf8(file.content));
 
-    for line in file_content.lines() {
-
-        if line.starts_with("<head") {
-            in_head = true;
-            continue;
-        } else if line.starts_with("</head") {
-            in_head = false;
-        } else if line.starts_with("<body") {
-            in_body = true;
-            continue;
-        } else if line.starts_with("</body") {
-            in_body = false;
-        }
-
-        if in_head {
-            content.head.push_str(&line[..]);
-            content.head.push('\n');
-        } else if in_body {
-            content.body.push_str(&line[..]);
-            content.body.push('\n');
-        }
-    }
+    let (head, body) = ctry!(utils::extract_head_and_body(&file_content));
+    content.head = head;
+    content.body = body;
 
     content.full = file_content;
     let crate_details = cexpect!(CrateDetails::new(&conn, &name, &version));
@@ -251,3 +235,28 @@ pub fn badge_handler(req: &mut Request) -> IronResult<Response> {
                                        CacheDirective::MustRevalidate]));
     Ok(resp)
 }
+
+/// Serves shared web resources used by rustdoc-generated documentation.
+///
+/// This includes common `css` and `js` files that only change when the compiler is updated, but are
+/// otherwise the same for all crates documented with that compiler. Those have a custom handler to
+/// deduplicate them and save space.
+pub struct SharedResourceHandler;
+
+impl Handler for SharedResourceHandler {
+    fn handle(&self, req: &mut Request) -> IronResult<Response> {
+        let path = req.url.path();
+        let filename = path.last().unwrap();  // unwrap is fine: vector is non-empty
+        let suffix = filename.split('.').last().unwrap();  // unwrap is fine: split always works
+        if ["js", "css", "woff", "svg"].contains(&suffix) {
+            let conn = extension!(req, Pool);
+
+            if let Some(file) = File::from_path(conn, filename) {
+                return Ok(file.serve());
+            }
+        }
+
+        // Just always return a 404 here - the main handler will then try the other handlers
+        Err(IronError::new(Nope::ResourceNotFound, status::NotFound))
+    }
+}
@@ -1 +1,17 @@
-{{{content.rustdoc_full}}}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    {{{content.rustdoc_head}}}
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/pure/0.6.0/menus-min.css" type="text/css" media="all" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/pure/0.6.0/grids-min.css" type="text/css" media="all" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css" type="text/css" media="all" />
+    <link rel="stylesheet" href="/style.css?{{cratesfyi_version_safe}}" type="text/css" media="all" />
+    <link rel="search" href="/opensearch.xml" type="application/opensearchdescription+xml" title="Docs.rs">
+</head>
+<body>
+{{> navigation_rustdoc}}
+<div class="rustdoc container-rustdoc">
+    {{{content.rustdoc_body}}}
+</div>
+</body>
+</html>