Autogenerate stubs and the summary of the unstable book

Author: est31

src/Cargo.lock

@@ -9,6 +9,7 @@ members = [
   "tools/error_index_generator",
   "tools/linkchecker",
   "tools/rustbook",
+  "tools/unstable-book-gen",
   "tools/tidy",
   "tools/build-manifest",
   "tools/remote-test-client",

src/Cargo.toml

@@ -27,18 +27,26 @@ use {Build, Compiler, Mode};
 use util::{cp_r, symlink_dir};
 use build_helper::up_to_date;
 
-/// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
-/// `name` into the `out` path.
+/// Invoke `rustbook` for `target` for the doc book `name`.
 ///
 /// This will not actually generate any documentation if the documentation has
 /// already been generated.
 pub fn rustbook(build: &Build, target: &str, name: &str) {
+    let src = build.src.join("src/doc");
+    rustbook_src(build, target, name, &src);
+}
+
+/// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.
+///
+/// This will not actually generate any documentation if the documentation has
+/// already been generated.
+pub fn rustbook_src(build: &Build, target: &str, name: &str, src: &Path) {
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
 
     let out = out.join(name);
     let compiler = Compiler::new(0, &build.config.build);
-    let src = build.src.join("src/doc").join(name);
+    let src = src.join(name);
     let index = out.join("index.html");
     let rustbook = build.tool(&compiler, "rustbook");
     if up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
@@ -354,6 +362,19 @@ pub fn error_index(build: &Build, target: &str) {
     build.run(&mut index);
 }
 
+pub fn unstable_book_gen(build: &Build, target: &str) {
+    println!("Generating unstable book md files ({})", target);
+    let out = build.md_doc_out(target).join("unstable-book");
+    t!(fs::create_dir_all(&out));
+    t!(fs::remove_dir_all(&out));
+    let compiler = Compiler::new(0, &build.config.build);
+    let mut cmd = build.tool_cmd(&compiler, "unstable-book-gen");
+    cmd.arg(build.src.join("src"));
+    cmd.arg(out);
+
+    build.run(&mut cmd);
+}
+
 fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
     if let Ok(m) = fs::symlink_metadata(dst) {
         if m.file_type().is_dir() {

src/bootstrap/doc.rs

@@ -677,6 +677,11 @@ impl Build {
         self.out.join(target).join("doc")
     }
 
+    /// Output directory for some generated md crate documentation for a target (temporary)
+    fn md_doc_out(&self, target: &str) -> PathBuf {
+        self.out.join(target).join("md-doc")
+    }
+
     /// Output directory for all crate documentation for a target (temporary)
     ///
     /// The artifacts here are then copied into `doc_out` above.

src/bootstrap/lib.rs

@@ -548,6 +548,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|s| s.name("maybe-clean-tools"))
          .dep(|s| s.name("librustc-tool"))
          .run(move |s| compile::tool(build, s.stage, s.target, "error_index_generator"));
+    rules.build("tool-unstable-book-gen", "src/tools/unstable-book-gen")
+         .dep(|s| s.name("maybe-clean-tools"))
+         .dep(|s| s.name("libstd-tool"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "unstable-book-gen"));
     rules.build("tool-tidy", "src/tools/tidy")
          .dep(|s| s.name("maybe-clean-tools"))
          .dep(|s| s.name("libstd-tool"))
@@ -662,8 +666,17 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
               .target(&build.config.build)
               .stage(0)
          })
+         .dep(move |s| {
+             s.name("doc-unstable-book-gen")
+              .host(&build.config.build)
+              .target(&build.config.build)
+              .stage(0)
+         })
          .default(build.config.docs)
-         .run(move |s| doc::rustbook(build, s.target, "unstable-book"));
+         .run(move |s| doc::rustbook_src(build,
+                                         s.target,
+                                         "unstable-book",
+                                         &build.md_doc_out(s.target)));
     rules.doc("doc-standalone", "src/doc")
          .dep(move |s| {
              s.name("rustc")
@@ -679,6 +692,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .default(build.config.docs)
          .host(true)
          .run(move |s| doc::error_index(build, s.target));
+    rules.doc("doc-unstable-book-gen", "src/tools/unstable-book-gen")
+         .dep(move |s| s.name("tool-unstable-book-gen").target(&build.config.build).stage(0))
+         .dep(move |s| s.name("librustc-link"))
+         .default(build.config.docs)
+         .host(true)
+         .run(move |s| doc::unstable_book_gen(build, s.target));
     for (krate, path, default) in krates("std") {
         rules.doc(&krate.doc_step, path)
              .dep(|s| s.name("libstd-link"))

src/bootstrap/step.rs

@@ -190,4 +190,4 @@ constraints, etc.
 [llvm-docs]: http://llvm.org/docs/LangRef.html#inline-assembler-expressions
 
 If you need more power and don't mind losing some of the niceties of
-`asm!`, check out [global_asm](language-features/global_asm.html).
+`asm!`, check out [global_asm](language-features/global-asm.html).

src/doc/unstable-book/src/language-features/asm.md

@@ -24,7 +24,7 @@ use std::fs::File;
 use std::io::prelude::*;
 use std::path::Path;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum Status {
     Stable,
     Removed,
@@ -42,13 +42,16 @@ impl fmt::Display for Status {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Feature {
     pub level: Status,
     pub since: String,
     pub has_gate_test: bool,
+    pub tracking_issue: Option<u32>,
 }
 
+pub type Features = HashMap<String, Feature>;
+
 pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
     let mut features = collect_lang_features(path);
     assert!(!features.is_empty());
@@ -168,8 +171,7 @@ fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
         .map(|(i, j)| &line[i..j])
 }
 
-fn test_filen_gate(filen_underscore: &str,
-                   features: &mut HashMap<String, Feature>) -> bool {
+fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
     if filen_underscore.starts_with("feature_gate") {
         for (n, f) in features.iter_mut() {
             if filen_underscore == format!("feature_gate_{}", n) {
@@ -181,7 +183,7 @@ fn test_filen_gate(filen_underscore: &str,
     return false;
 }
 
-pub fn collect_lang_features(base_src_path: &Path) -> HashMap<String, Feature> {
+pub fn collect_lang_features(base_src_path: &Path) -> Features {
     let mut contents = String::new();
     let path = base_src_path.join("libsyntax/feature_gate.rs");
     t!(t!(File::open(path)).read_to_string(&mut contents));
@@ -197,20 +199,28 @@ pub fn collect_lang_features(base_src_path: &Path) -> HashMap<String, Feature> {
             };
             let name = parts.next().unwrap().trim();
             let since = parts.next().unwrap().trim().trim_matches('"');
+            let issue_str = parts.next().unwrap().trim();
+            let tracking_issue = if issue_str.starts_with("None") {
+                None
+            } else {
+                let s = issue_str.split("(").nth(1).unwrap().split(")").nth(0).unwrap();
+                Some(s.parse().unwrap())
+            };
             Some((name.to_owned(),
                 Feature {
-                    level: level,
+                    level,
                     since: since.to_owned(),
                     has_gate_test: false,
+                    tracking_issue,
                 }))
         })
         .collect()
 }
 
 pub fn collect_lib_features(base_src_path: &Path,
                             bad: &mut bool,
-                            features: &HashMap<String, Feature>) -> HashMap<String, Feature> {
-    let mut lib_features = HashMap::<String, Feature>::new();
+                            features: &Features) -> Features {
+    let mut lib_features = Features::new();
     let mut contents = String::new();
     super::walk(base_src_path,
                 &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
@@ -224,10 +234,32 @@ pub fn collect_lib_features(base_src_path: &Path,
         contents.truncate(0);
         t!(t!(File::open(&file), &file).read_to_string(&mut contents));
 
+        let mut becoming_feature: Option<(String, Feature)> = None;
         for (i, line) in contents.lines().enumerate() {
             let mut err = |msg: &str| {
                 tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
             };
+            if let Some((ref name, ref mut f)) = becoming_feature {
+                if f.tracking_issue.is_none() {
+                    f.tracking_issue = find_attr_val(line, "issue")
+                    .map(|s| s.parse().unwrap());
+                }
+                if line.ends_with("]") {
+                    lib_features.insert(name.to_owned(), f.clone());
+                } else if !line.ends_with(",") && !line.ends_with("\\") {
+                    // We need to bail here because we might have missed the
+                    // end of a stability attribute above because the "]"
+                    // might not have been at the end of the line.
+                    // We could then get into the very unfortunate situation that
+                    // we continue parsing the file assuming the current stability
+                    // attribute has not ended, and ignoring possible feature
+                    // attributes in the process.
+                    err("malformed stability attribute");
+                } else {
+                    continue;
+                }
+            }
+            becoming_feature = None;
             let level = if line.contains("[unstable(") {
                 Status::Unstable
             } else if line.contains("[stable(") {
@@ -250,6 +282,7 @@ pub fn collect_lib_features(base_src_path: &Path,
                 }
                 None => "None",
             };
+            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());
 
             if features.contains_key(feature_name) {
                 err("duplicating a lang feature");
@@ -263,12 +296,17 @@ pub fn collect_lib_features(base_src_path: &Path,
                 }
                 continue;
             }
-            lib_features.insert(feature_name.to_owned(),
-                                Feature {
-                                    level: level,
-                                    since: since.to_owned(),
-                                    has_gate_test: false,
-                                });
+            let feature = Feature {
+                level,
+                since: since.to_owned(),
+                has_gate_test: false,
+                tracking_issue,
+            };
+            if line.contains("]") {
+                lib_features.insert(feature_name.to_owned(), feature);
+            } else {
+                becoming_feature = Some((feature_name.to_owned(), feature));
+            }
         }
     });
     lib_features

src/doc/unstable-book/src/language-features/global_asm.md renamed to src/doc/unstable-book/src/language-features/global-asm.md

@@ -11,26 +11,28 @@
 use std::collections::HashSet;
 use std::fs;
 use std::path;
-use features::{collect_lang_features, collect_lib_features, Status};
+use features::{collect_lang_features, collect_lib_features, Features, Status};
 
-const PATH_STR: &'static str = "doc/unstable-book/src";
+pub const PATH_STR: &str = "doc/unstable-book/src";
 
-const LANG_FEATURES_DIR: &'static str = "language-features";
+pub const COMPILER_FLAGS_DIR: &str = "compiler-flags";
 
-const LIB_FEATURES_DIR: &'static str = "library-features";
+pub const LANG_FEATURES_DIR: &str = "language-features";
+
+pub const LIB_FEATURES_DIR: &str = "library-features";
 
 /// Build the path to the Unstable Book source directory from the Rust 'src' directory
-fn unstable_book_path(base_src_path: &path::Path) -> path::PathBuf {
+pub fn unstable_book_path(base_src_path: &path::Path) -> path::PathBuf {
     base_src_path.join(PATH_STR)
 }
 
 /// Directory where the features are documented within the Unstable Book source directory
-fn unstable_book_lang_features_path(base_src_path: &path::Path) -> path::PathBuf {
+pub fn unstable_book_lang_features_path(base_src_path: &path::Path) -> path::PathBuf {
     unstable_book_path(base_src_path).join(LANG_FEATURES_DIR)
 }
 
 /// Directory where the features are documented within the Unstable Book source directory
-fn unstable_book_lib_features_path(base_src_path: &path::Path) -> path::PathBuf {
+pub fn unstable_book_lib_features_path(base_src_path: &path::Path) -> path::PathBuf {
     unstable_book_path(base_src_path).join(LIB_FEATURES_DIR)
 }
 
@@ -42,27 +44,16 @@ fn dir_entry_is_file(dir_entry: &fs::DirEntry) -> bool {
         .is_file()
 }
 
-/// Retrieve names of all lang-related unstable features
-fn collect_unstable_lang_feature_names(base_src_path: &path::Path) -> HashSet<String> {
-    collect_lang_features(base_src_path)
-        .into_iter()
-        .filter(|&(_, ref f)| f.level == Status::Unstable)
-        .map(|(ref name, _)| name.to_owned())
-        .collect()
-}
-
-/// Retrieve names of all lib-related unstable features
-fn collect_unstable_lib_feature_names(base_src_path: &path::Path) -> HashSet<String> {
-    let mut bad = true;
-    let lang_features = collect_lang_features(base_src_path);
-    collect_lib_features(base_src_path, &mut bad, &lang_features)
-        .into_iter()
+/// Retrieve names of all unstable features
+pub fn collect_unstable_feature_names(features: &Features) -> HashSet<String> {
+    features
+        .iter()
         .filter(|&(_, ref f)| f.level == Status::Unstable)
-        .map(|(ref name, _)| name.to_owned())
+        .map(|(name, _)| name.to_owned())
         .collect()
 }
 
-fn collect_unstable_book_section_file_names(dir: &path::Path) -> HashSet<String> {
+pub fn collect_unstable_book_section_file_names(dir: &path::Path) -> HashSet<String> {
     fs::read_dir(dir)
         .expect("could not read directory")
         .into_iter()
@@ -95,7 +86,10 @@ pub fn check(path: &path::Path, bad: &mut bool) {
 
     // Library features
 
-    let unstable_lib_feature_names = collect_unstable_lib_feature_names(path);
+    let lang_features = collect_lang_features(path);
+    let lib_features = collect_lib_features(path, bad, &lang_features);
+
+    let unstable_lib_feature_names = collect_unstable_feature_names(&lib_features);
     let unstable_book_lib_features_section_file_names =
         collect_unstable_book_lib_features_section_file_names(path);
 
@@ -119,7 +113,7 @@ pub fn check(path: &path::Path, bad: &mut bool) {
 
     // Language features
 
-    let unstable_lang_feature_names = collect_unstable_lang_feature_names(path);
+    let unstable_lang_feature_names = collect_unstable_feature_names(&lang_features);
     let unstable_book_lang_features_section_file_names =
         collect_unstable_book_lang_features_section_file_names(path);
 

src/doc/unstable-book/src/library-features/from_utf8_error_as_bytes.md renamed to src/doc/unstable-book/src/library-features/from-utf8-error-as-bytes.md

@@ -0,0 +1,9 @@
+[package]
+authors = ["est31 <MTest31@outlook.com>",
+           "The Rust Project Developers"]
+name = "unstable-book-gen"
+version = "0.1.0"
+license = "MIT/Apache-2.0"
+
+[dependencies]
+tidy = { path = "../tidy" }

src/tools/tidy/src/features.rs

@@ -0,0 +1,8 @@
+[The Unstable Book](the-unstable-book.md)
+
+- [Compiler flags](compiler-flags.md)
+{compiler_flags}
+- [Language features](language-features.md)
+{language_features}
+- [Library Features](library-features.md)
+{library_features}