Replace client-side highlight.js by server-side syntect highlighting

This is setup to only support the same languages we targeted before:
Rust, Markdown and TOML.

Author: Nemo157

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "assets/syntaxes/Packages"]
+	path = assets/syntaxes/Packages
+	url = https://github.com/sublimehq/Packages
+[submodule "assets/syntaxes/Extras/TOML"]
+	path = assets/syntaxes/Extras/TOML
+	url = https://github.com/jasonwilliams/sublime_toml_highlighting

Cargo.lock

@@ -44,6 +44,7 @@ backtrace = "0.3.61"
 failure = "0.1.8"
 thiserror = "1.0.26"
 comrak = { version = "0.14.0", default-features = false }
+syntect = { version = "5.0.0", default-features = false, features = ["parsing", "html", "dump-load", "regex-fancy"] }
 toml = "0.5"
 schemamama = "0.3"
 schemamama_postgres = "0.3"
@@ -128,6 +129,7 @@ walkdir = "2"
 anyhow = { version = "1.0.42", features = ["backtrace"] }
 grass = { version = "0.11.0", default-features = false }
 once_cell = { version = "1.4.0", features = ["parking_lot"] }
+syntect = { version = "5.0.0", default-features = false, features = ["parsing", "dump-create", "yaml-load", "regex-fancy"] }
 
 [[bench]]
 name = "compression"

Cargo.toml

@@ -36,6 +36,13 @@ mod tracked {
         Ok(())
     }
 
+    pub(crate) fn track_recursive(path: impl AsRef<Path>) -> Result<()> {
+        for entry in walkdir::WalkDir::new(path) {
+            track(entry?.path())?;
+        }
+        Ok(())
+    }
+
     pub(crate) fn read(path: impl AsRef<Path>) -> Result<Vec<u8>> {
         let path = path.as_ref();
         track(path)?;
@@ -72,6 +79,7 @@ fn main() -> Result<()> {
     write_git_version(out_dir)?;
     compile_sass(out_dir)?;
     write_known_targets(out_dir)?;
+    compile_syntax(out_dir)?;
     Ok(())
 }
 
@@ -175,3 +183,32 @@ fn write_known_targets(out_dir: &Path) -> Result<()> {
 
     Ok(())
 }
+
+fn compile_syntax(out_dir: &Path) -> Result<()> {
+    use syntect::{dumps::dump_to_uncompressed_file, parsing::SyntaxSetBuilder};
+
+    fn tracked_add_from_folder(
+        builder: &mut SyntaxSetBuilder,
+        path: impl AsRef<Path>,
+    ) -> Result<()> {
+        // There's no easy way to know exactly which files matter, so just track everything in the
+        // folder
+        tracked::track_recursive(&path)?;
+        builder.add_from_folder(path, true)?;
+        Ok(())
+    }
+
+    let mut builder = SyntaxSetBuilder::new();
+    builder.add_plain_text_syntax();
+    tracked_add_from_folder(&mut builder, "assets/syntaxes/Packages/Rust/")?;
+    // Some of the extended syntaxes fail to compile, so only load the primary markdown syntax
+    tracked_add_from_folder(
+        &mut builder,
+        "assets/syntaxes/Packages/Markdown/Markdown.sublime-syntax",
+    )?;
+    tracked_add_from_folder(&mut builder, "assets/syntaxes/Extras/TOML/")?;
+
+    dump_to_uncompressed_file(&builder.build(), out_dir.join("syntect.packdump"))?;
+
+    Ok(())
+}

assets/syntaxes/Extras/TOML

@@ -50,6 +50,7 @@ COPY src src/
 RUN find src -name "*.rs" -exec touch {} \;
 COPY templates/style templates/style
 COPY vendor vendor/
+COPY assets assets/
 
 RUN cargo build --release
 

assets/syntaxes/Packages

@@ -1,4 +1,4 @@
-use super::{match_version, redirect_base, render_markdown, MatchSemver, MetaData};
+use super::{markdown, match_version, redirect_base, MatchSemver, MetaData};
 use crate::utils::{get_correct_docsrs_style_file, report_error};
 use crate::{
     db::Pool,
@@ -69,7 +69,7 @@ where
 {
     markdown
         .as_ref()
-        .map(|markdown| render_markdown(markdown))
+        .map(|markdown| markdown::render(markdown))
         .serialize(serializer)
 }
 

build.rs

@@ -0,0 +1,106 @@
+use crate::error::Result;
+use comrak::{
+    adapters::SyntaxHighlighterAdapter, ComrakExtensionOptions, ComrakOptions, ComrakPlugins,
+    ComrakRenderPlugins,
+};
+use once_cell::sync::Lazy;
+use std::collections::HashMap;
+use std::fmt::Write;
+
+#[derive(Debug)]
+struct CodeAdapter;
+
+impl SyntaxHighlighterAdapter for CodeAdapter {
+    fn highlight(&self, lang: Option<&str>, code: &str) -> String {
+        highlight_code(lang, code)
+    }
+
+    fn build_pre_tag(&self, attributes: &HashMap<String, String>) -> String {
+        build_opening_tag("pre", attributes)
+    }
+
+    fn build_code_tag(&self, attributes: &HashMap<String, String>) -> String {
+        build_opening_tag("code", attributes)
+    }
+}
+
+fn build_opening_tag(tag: &str, attributes: &HashMap<String, String>) -> String {
+    let mut tag_parts = format!("<{tag}");
+    for (attr, val) in attributes {
+        write!(tag_parts, " {attr}=\"{val}\"").unwrap();
+    }
+    tag_parts.push('>');
+    tag_parts
+}
+
+pub fn try_highlight_code(lang: Option<&str>, code: &str) -> Result<String> {
+    use syntect::{
+        html::{ClassStyle, ClassedHTMLGenerator},
+        parsing::SyntaxSet,
+        util::LinesWithEndings,
+    };
+
+    static SYNTAX_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/syntect.packdump"));
+    static SYNTAXES: Lazy<SyntaxSet> = Lazy::new(|| {
+        let syntaxes: SyntaxSet = syntect::dumps::from_uncompressed_data(SYNTAX_DATA).unwrap();
+        let names = syntaxes
+            .syntaxes()
+            .iter()
+            .map(|s| &s.name)
+            .collect::<Vec<_>>();
+        log::debug!("known syntaxes {names:?}");
+        syntaxes
+    });
+
+    let syntax = lang
+        .and_then(|lang| SYNTAXES.find_syntax_by_token(lang))
+        .or_else(|| SYNTAXES.find_syntax_by_first_line(code))
+        .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text());
+
+    log::trace!("Using syntax {:?} for language {lang:?}", syntax.name);
+
+    let mut html_generator = ClassedHTMLGenerator::new_with_class_style(
+        syntax,
+        &SYNTAXES,
+        ClassStyle::SpacedPrefixed { prefix: "syntax-" },
+    );
+
+    for line in LinesWithEndings::from(code) {
+        html_generator.parse_html_for_line_which_includes_newline(line)?;
+    }
+
+    Ok(html_generator.finalize())
+}
+
+pub fn highlight_code(lang: Option<&str>, code: &str) -> String {
+    match try_highlight_code(lang, code) {
+        Ok(highlighted) => highlighted,
+        Err(err) => {
+            log::error!("failed while highlighting code: {err:?}");
+            code.to_owned()
+        }
+    }
+}
+
+/// Wrapper around the Markdown parser and renderer to render markdown
+pub(crate) fn render(text: &str) -> String {
+    comrak::markdown_to_html_with_plugins(
+        text,
+        &ComrakOptions {
+            extension: ComrakExtensionOptions {
+                superscript: true,
+                table: true,
+                autolink: true,
+                tasklist: true,
+                strikethrough: true,
+                ..ComrakExtensionOptions::default()
+            },
+            ..ComrakOptions::default()
+        },
+        &ComrakPlugins {
+            render: ComrakRenderPlugins {
+                codefence_syntax_highlighter: Some(&CodeAdapter),
+            },
+        },
+    )
+}

dockerfiles/Dockerfile

@@ -81,6 +81,7 @@ mod error;
 mod extensions;
 mod features;
 mod file;
+mod markdown;
 pub(crate) mod metrics;
 mod releases;
 mod routes;
@@ -413,25 +414,6 @@ fn match_version(
     Err(Nope::VersionNotFound)
 }
 
-/// Wrapper around the Markdown parser and renderer to render markdown
-fn render_markdown(text: &str) -> String {
-    use comrak::{markdown_to_html, ComrakExtensionOptions, ComrakOptions};
-
-    let options = ComrakOptions {
-        extension: ComrakExtensionOptions {
-            superscript: true,
-            table: true,
-            autolink: true,
-            tasklist: true,
-            strikethrough: true,
-            ..ComrakExtensionOptions::default()
-        },
-        ..ComrakOptions::default()
-    };
-
-    markdown_to_html(text, &options)
-}
-
 #[must_use = "`Server` blocks indefinitely when dropped"]
 pub struct Server {
     inner: Listening,