Add class based syntax higlighting + line numbers (#1531)

* Add class based syntax higlighting + line numbers

* Use fork of syntect for now

* Fix tests

* Fix diff background on inline highlighter

Co-authored-by: evan-brass <evan-brass@protonmail.com>

.gitignore

@@ -3,6 +3,7 @@ target
 test_site/public
 test_site_i18n/public
 docs/public
+docs/out
 
 small-blog
 medium-blog

Cargo.lock

@@ -2549,9 +2549,8 @@ dependencies = [
 
 [[package]]
 name = "syntect"
-version = "4.5.0"
+version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/Keats/syntect.git?branch=scopestack#6b36f5eb406d57e57ddb6eb51df3a5e36e52c955"
-checksum = "2bfac2b23b4d049dc9a89353b4e06bbc85a8f42020cccbe5409a115cf19031e5"
 dependencies = [
  "bincode",
  "bitflags",

components/config/Cargo.toml

@@ -12,7 +12,8 @@ serde_derive = "1"
 chrono = "0.4"
 globset = "0.4"
 lazy_static = "1"
-syntect = "4.1"
+# TODO: go back to version 4/5 once https://github.com/trishume/syntect/pull/337 is merged
+syntect = { git = "https://github.com/Keats/syntect.git", branch = "scopestack" }
 unic-langid = "0.9"
 
 errors = { path = "../errors" }

components/config/src/config/languages.rs

@@ -16,6 +16,9 @@ pub struct LanguageOptions {
     pub description: Option<String>,
     /// Whether to generate a feed for that language, defaults to `false`
     pub generate_feed: bool,
+    /// The filename to use for feeds. Used to find the template, too.
+    /// Defaults to "atom.xml", with "rss.xml" also having a template provided out of the box.
+    pub feed_filename: String,
     pub taxonomies: Vec<taxonomies::Taxonomy>,
     /// Whether to generate search index for that language, defaults to `false`
     pub build_search_index: bool,
@@ -34,6 +37,7 @@ impl Default for LanguageOptions {
             title: None,
             description: None,
             generate_feed: false,
+            feed_filename: String::new(),
             build_search_index: false,
             taxonomies: Vec::new(),
             search: search::Search::default(),

components/config/src/config/markup.rs

@@ -7,6 +7,21 @@ use errors::Result;
 
 pub const DEFAULT_HIGHLIGHT_THEME: &str = "base16-ocean-dark";
 
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(default)]
+pub struct ThemeCss {
+    /// Which theme are we generating the CSS from
+    pub theme: String,
+    /// In which file are we going to output the CSS
+    pub filename: String,
+}
+
+impl Default for ThemeCss {
+    fn default() -> ThemeCss {
+        ThemeCss { theme: String::new(), filename: String::new() }
+    }
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(default)]
 pub struct Markdown {
@@ -15,6 +30,8 @@ pub struct Markdown {
     /// Which themes to use for code highlighting. See Readme for supported themes
     /// Defaults to "base16-ocean-dark"
     pub highlight_theme: String,
+    /// Generate CSS files for Themes out of syntect
+    pub highlight_themes_css: Vec<ThemeCss>,
     /// Whether to render emoji aliases (e.g.: :smile: => 😄) in the markdown files
     pub render_emoji: bool,
     /// Whether external links are to be opened in a new tab
@@ -87,12 +104,13 @@ impl Default for Markdown {
         Markdown {
             highlight_code: false,
             highlight_theme: DEFAULT_HIGHLIGHT_THEME.to_owned(),
+            highlight_themes_css: Vec::new(),
             render_emoji: false,
             external_links_target_blank: false,
             external_links_no_follow: false,
             external_links_no_referrer: false,
             smart_punctuation: false,
-            extra_syntaxes: vec![],
+            extra_syntaxes: Vec::new(),
             extra_syntax_set: None,
         }
     }

components/config/src/config/mod.rs

@@ -98,6 +98,7 @@ pub struct SerializedConfig<'a> {
     description: &'a Option<String>,
     languages: HashMap<&'a String, &'a languages::LanguageOptions>,
     generate_feed: bool,
+    feed_filename: &'a str,
     taxonomies: &'a [taxonomies::Taxonomy],
     build_search_index: bool,
     extra: &'a HashMap<String, Toml>,
@@ -116,12 +117,14 @@ impl Config {
             bail!("A base URL is required in config.toml with key `base_url`");
         }
 
+        if config.markdown.highlight_theme != "css" {
             if !THEME_SET.themes.contains_key(&config.markdown.highlight_theme) {
                 bail!(
                     "Highlight theme {} defined in config does not exist.",
                     config.markdown.highlight_theme
                 );
             }
+        }
 
         languages::validate_code(&config.default_language)?;
         for code in config.languages.keys() {
@@ -201,6 +204,7 @@ impl Config {
                     title: self.title.clone(),
                     description: self.description.clone(),
                     generate_feed: self.generate_feed,
+                    feed_filename: self.feed_filename.clone(),
                     build_search_index: self.build_search_index,
                     taxonomies: self.taxonomies.clone(),
                     search: self.search.clone(),
@@ -288,6 +292,7 @@ impl Config {
             description: &options.description,
             languages: self.languages.iter().filter(|(k, _)| k.as_str() != lang).collect(),
             generate_feed: options.generate_feed,
+            feed_filename: &options.feed_filename,
             taxonomies: &options.taxonomies,
             build_search_index: options.build_search_index,
             extra: &self.extra,

components/config/src/highlighting.rs

@@ -1,10 +1,10 @@
 use lazy_static::lazy_static;
 use syntect::dumps::from_binary;
-use syntect::easy::HighlightLines;
+use syntect::highlighting::{Theme, ThemeSet};
-use syntect::highlighting::ThemeSet;
+use syntect::parsing::{SyntaxReference, SyntaxSet};
-use syntect::parsing::SyntaxSet;
 
 use crate::config::Config;
+use syntect::html::{css_for_theme_with_class_style, ClassStyle};
 
 lazy_static! {
     pub static ref SYNTAX_SET: SyntaxSet = {
@@ -16,24 +16,47 @@ lazy_static! {
         from_binary(include_bytes!("../../../sublime/themes/all.themedump"));
 }
 
+pub const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: "z-" };
+
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub enum HighlightSource {
-    Theme,
+    /// One of the built-in Zola syntaxes
+    BuiltIn,
+    /// Found in the extra syntaxes
     Extra,
+    /// No language specified
     Plain,
+    /// We didn't find the language in built-in and extra syntaxes
     NotFound,
 }
 
-/// Returns the highlighter and whether it was found in the extra or not
+pub struct SyntaxAndTheme<'config> {
-pub fn get_highlighter(
+    pub syntax: &'config SyntaxReference,
-    language: Option<&str>,
+    pub syntax_set: &'config SyntaxSet,
-    config: &Config,
+    /// None if highlighting via CSS
-) -> (HighlightLines<'static>, HighlightSource) {
+    pub theme: Option<&'config Theme>,
-    let theme = &THEME_SET.themes[&config.markdown.highlight_theme];
+    pub source: HighlightSource,
+}
+
+pub fn resolve_syntax_and_theme<'config>(
+    language: Option<&'_ str>,
+    config: &'config Config,
+) -> SyntaxAndTheme<'config> {
+    let theme = if config.markdown.highlight_theme != "css" {
+        Some(&THEME_SET.themes[&config.markdown.highlight_theme])
+    } else {
+        None
+    };
 
     if let Some(ref lang) = language {
         if let Some(ref extra_syntaxes) = config.markdown.extra_syntax_set {
             if let Some(syntax) = extra_syntaxes.find_syntax_by_token(lang) {
-                return (HighlightLines::new(syntax, theme), HighlightSource::Extra);
+                return SyntaxAndTheme {
+                    syntax,
+                    syntax_set: extra_syntaxes,
+                    theme,
+                    source: HighlightSource::Extra,
+                };
             }
         }
         // The JS syntax hangs a lot... the TS syntax is probably better anyway.
@@ -42,14 +65,31 @@ pub fn get_highlighter(
         // https://github.com/getzola/zola/issues/1174
         let hacked_lang = if *lang == "js" || *lang == "javascript" { "ts" } else { lang };
         if let Some(syntax) = SYNTAX_SET.find_syntax_by_token(hacked_lang) {
-            (HighlightLines::new(syntax, theme), HighlightSource::Theme)
+            SyntaxAndTheme {
-        } else {
+                syntax,
-            (
+                syntax_set: &SYNTAX_SET as &SyntaxSet,
-                HighlightLines::new(SYNTAX_SET.find_syntax_plain_text(), theme),
+                theme,
-                HighlightSource::NotFound,
+                source: HighlightSource::BuiltIn,
-            )
             }
         } else {
-        (HighlightLines::new(SYNTAX_SET.find_syntax_plain_text(), theme), HighlightSource::Plain)
+            SyntaxAndTheme {
+                syntax: SYNTAX_SET.find_syntax_plain_text(),
+                syntax_set: &SYNTAX_SET as &SyntaxSet,
+                theme,
+                source: HighlightSource::NotFound,
             }
         }
+    } else {
+        SyntaxAndTheme {
+            syntax: SYNTAX_SET.find_syntax_plain_text(),
+            syntax_set: &SYNTAX_SET as &SyntaxSet,
+            theme,
+            source: HighlightSource::Plain,
+        }
+    }
+}
+
+pub fn export_theme_css(theme_name: &str) -> String {
+    let theme = &THEME_SET.themes[theme_name];
+    css_for_theme_with_class_style(theme, CLASS_STYLE)
+}

components/errors/Cargo.toml

@@ -8,4 +8,5 @@ edition = "2018"
 tera = "1"
 toml = "0.5"
 image = "0.23"
-syntect = "4.4"
+# TODO: go back to version 4/5 once https://github.com/trishume/syntect/pull/337 is merged
+syntect = { git = "https://github.com/Keats/syntect.git", branch = "scopestack" }

components/rendering/Cargo.toml

@@ -7,7 +7,8 @@ include = ["src/**/*"]
 
 [dependencies]
 tera = { version = "1", features = ["preserve_order"] }
-syntect = "4.1"
+# TODO: go back to version 4/5 once https://github.com/trishume/syntect/pull/337 is merged
+syntect = { git = "https://github.com/Keats/syntect.git", branch = "scopestack" }
 pulldown-cmark = { version = "0.8", default-features = false }
 serde = "1"
 serde_derive = "1"

components/rendering/src/codeblock/fence.rs

@@ -1,11 +1,6 @@
-#[derive(Copy, Clone, Debug)]
+use std::ops::RangeInclusive;
-pub struct Range {
-    pub from: usize,
-    pub to: usize,
-}
 
-impl Range {
+fn parse_range(s: &str) -> Option<RangeInclusive<usize>> {
-    fn parse(s: &str) -> Option<Range> {
     match s.find('-') {
         Some(dash) => {
             let mut from = s[..dash].parse().ok()?;
@@ -13,12 +8,11 @@ impl Range {
             if to < from {
                 std::mem::swap(&mut from, &mut to);
             }
-                Some(Range { from, to })
+            Some(from..=to)
         }
         None => {
             let val = s.parse().ok()?;
-                Some(Range { from: val, to: val })
+            Some(val..=val)
-            }
         }
     }
 }
@@ -27,14 +21,17 @@ impl Range {
 pub struct FenceSettings<'a> {
     pub language: Option<&'a str>,
     pub line_numbers: bool,
-    pub highlight_lines: Vec<Range>,
+    pub line_number_start: usize,
-    pub hide_lines: Vec<Range>,
+    pub highlight_lines: Vec<RangeInclusive<usize>>,
+    pub hide_lines: Vec<RangeInclusive<usize>>,
 }
+
 impl<'a> FenceSettings<'a> {
     pub fn new(fence_info: &'a str) -> Self {
         let mut me = Self {
             language: None,
             line_numbers: false,
+            line_number_start: 1,
             highlight_lines: Vec::new(),
             hide_lines: Vec::new(),
         };
@@ -43,6 +40,7 @@ impl<'a> FenceSettings<'a> {
             match token {
                 FenceToken::Language(lang) => me.language = Some(lang),
                 FenceToken::EnableLineNumbers => me.line_numbers = true,
+                FenceToken::InitialLineNumber(l) => me.line_number_start = l,
                 FenceToken::HighlightLines(lines) => me.highlight_lines.extend(lines),
                 FenceToken::HideLines(lines) => me.hide_lines.extend(lines),
             }
@@ -56,22 +54,24 @@ impl<'a> FenceSettings<'a> {
 enum FenceToken<'a> {
     Language(&'a str),
     EnableLineNumbers,
-    HighlightLines(Vec<Range>),
+    InitialLineNumber(usize),
-    HideLines(Vec<Range>),
+    HighlightLines(Vec<RangeInclusive<usize>>),
+    HideLines(Vec<RangeInclusive<usize>>),
 }
 
 struct FenceIter<'a> {
     split: std::str::Split<'a, char>,
 }
+
 impl<'a> FenceIter<'a> {
     fn new(fence_info: &'a str) -> Self {
         Self { split: fence_info.split(',') }
     }
 
-    fn parse_ranges(token: Option<&str>) -> Vec<Range> {
+    fn parse_ranges(token: Option<&str>) -> Vec<RangeInclusive<usize>> {
         let mut ranges = Vec::new();
         for range in token.unwrap_or("").split(' ') {
-            if let Some(range) = Range::parse(range) {
+            if let Some(range) = parse_range(range) {
                 ranges.push(range);
             }
         }
@@ -89,6 +89,11 @@ impl<'a> Iterator for FenceIter<'a> {
             let mut tok_split = tok.split('=');
             match tok_split.next().unwrap_or("").trim() {
                 "" => continue,
+                "linenostart" => {
+                    if let Some(l) = tok_split.next().and_then(|s| s.parse().ok()) {
+                        return Some(FenceToken::InitialLineNumber(l));
+                    }
+                }
                 "linenos" => return Some(FenceToken::EnableLineNumbers),
                 "hl_lines" => {
                     let ranges = Self::parse_ranges(tok_split.next());

components/rendering/src/codeblock/highlight.rs

@@ -0,0 +1,226 @@
+use std::fmt::Write;
+
+use config::highlighting::{SyntaxAndTheme, CLASS_STYLE};
+use syntect::easy::HighlightLines;
+use syntect::highlighting::{Color, Theme};
+use syntect::html::{
+    styled_line_to_highlighted_html, tokens_to_classed_spans, ClassStyle, IncludeBackground,
+};
+use syntect::parsing::{ParseState, ScopeStack, SyntaxReference, SyntaxSet};
+
+/// Not public, but from syntect::html
+fn write_css_color(s: &mut String, c: Color) {
+    if c.a != 0xFF {
+        write!(s, "#{:02x}{:02x}{:02x}{:02x}", c.r, c.g, c.b, c.a).unwrap();
+    } else {
+        write!(s, "#{:02x}{:02x}{:02x}", c.r, c.g, c.b).unwrap();
+    }
+}
+
+pub(crate) struct ClassHighlighter<'config> {
+    syntax_set: &'config SyntaxSet,
+    open_spans: isize,
+    parse_state: ParseState,
+    scope_stack: ScopeStack,
+}
+
+impl<'config> ClassHighlighter<'config> {
+    pub fn new(syntax: &'config SyntaxReference, syntax_set: &'config SyntaxSet) -> Self {
+        let parse_state = ParseState::new(syntax);
+        Self { syntax_set, open_spans: 0, parse_state, scope_stack: ScopeStack::new() }
+    }
+
+    /// Parse the line of code and update the internal HTML buffer with tagged HTML
+    ///
+    /// *Note:* This function requires `line` to include a newline at the end and
+    /// also use of the `load_defaults_newlines` version of the syntaxes.
+    pub fn highlight_line(&mut self, line: &str) -> String {
+        debug_assert!(line.ends_with("\n"));
+        let parsed_line = self.parse_state.parse_line(line, &self.syntax_set);
+        let (formatted_line, delta) = tokens_to_classed_spans(
+            line,
+            parsed_line.as_slice(),
+            CLASS_STYLE,
+            &mut self.scope_stack,
+        );
+        self.open_spans += delta;
+        formatted_line
+    }
+
+    /// Close all open `<span>` tags and return the finished HTML string
+    pub fn finalize(&mut self) -> String {
+        let mut html = String::with_capacity((self.open_spans * 7) as usize);
+        for _ in 0..self.open_spans {
+            html.push_str("</span>");
+        }
+        html
+    }
+}
+
+pub(crate) struct InlineHighlighter<'config> {
+    theme: &'config Theme,
+    fg_color: String,
+    bg_color: Color,
+    syntax_set: &'config SyntaxSet,
+    h: HighlightLines<'config>,
+}
+
+impl<'config> InlineHighlighter<'config> {
+    pub fn new(
+        syntax: &'config SyntaxReference,
+        syntax_set: &'config SyntaxSet,
+        theme: &'config Theme,
+    ) -> Self {
+        let h = HighlightLines::new(syntax, theme);
+        let mut color = String::new();
+        write_css_color(&mut color, theme.settings.foreground.unwrap_or(Color::BLACK));
+        let fg_color = format!(r#" style="color:{};""#, color);
+        let bg_color = theme.settings.background.unwrap_or(Color::WHITE);
+        Self { theme, fg_color, bg_color, syntax_set, h }
+    }
+
+    pub fn highlight_line(&mut self, line: &str) -> String {
+        let regions = self.h.highlight(line, &self.syntax_set);
+        // TODO: add a param like `IncludeBackground` for `IncludeForeground` in syntect
+        let highlighted = styled_line_to_highlighted_html(&regions, IncludeBackground::IfDifferent(self.bg_color));
+        highlighted.replace(&self.fg_color, "")
+    }
+}
+
+pub(crate) enum SyntaxHighlighter<'config> {
+    Inlined(InlineHighlighter<'config>),
+    Classed(ClassHighlighter<'config>),
+    /// We might not want highlighting but we want line numbers or to hide some lines
+    NoHighlight,
+}
+
+impl<'config> SyntaxHighlighter<'config> {
+    pub fn new(highlight_code: bool, s: SyntaxAndTheme<'config>) -> Self {
+        if highlight_code {
+            if let Some(theme) = s.theme {
+                SyntaxHighlighter::Inlined(InlineHighlighter::new(s.syntax, s.syntax_set, theme))
+            } else {
+                SyntaxHighlighter::Classed(ClassHighlighter::new(s.syntax, s.syntax_set))
+            }
+        } else {
+            SyntaxHighlighter::NoHighlight
+        }
+    }
+
+    pub fn highlight_line(&mut self, line: &str) -> String {
+        use SyntaxHighlighter::*;
+
+        match self {
+            Inlined(h) => h.highlight_line(line),
+            Classed(h) => h.highlight_line(line),
+            NoHighlight => line.to_owned(),
+        }
+    }
+
+    pub fn finalize(&mut self) -> Option<String> {
+        use SyntaxHighlighter::*;
+
+        match self {
+            Inlined(_) | NoHighlight => None,
+            Classed(h) => Some(h.finalize()),
+        }
+    }
+
+    /// Inlined needs to set the background/foreground colour on <pre>
+    pub fn pre_style(&self) -> Option<String> {
+        use SyntaxHighlighter::*;
+
+        match self {
+            Classed(_) | NoHighlight => None,
+            Inlined(h) => {
+                let mut styles = String::from("background-color:");
+                write_css_color(&mut styles, h.theme.settings.background.unwrap_or(Color::WHITE));
+                styles.push_str(";color:");
+                write_css_color(&mut styles, h.theme.settings.foreground.unwrap_or(Color::BLACK));
+                styles.push(';');
+                Some(styles)
+            }
+        }
+    }
+
+    /// Classed needs to set a class on the pre
+    pub fn pre_class(&self) -> Option<String> {
+        use SyntaxHighlighter::*;
+
+        match self {
+            Classed(_) => {
+                if let ClassStyle::SpacedPrefixed { prefix } = CLASS_STYLE {
+                    Some(format!("{}code", prefix))
+                } else {
+                    unreachable!()
+                }
+            }
+            Inlined(_) | NoHighlight => None,
+        }
+    }
+
+    /// Inlined needs to set the background/foreground colour
+    pub fn mark_style(&self) -> Option<String> {
+        use SyntaxHighlighter::*;
+
+        match self {
+            Classed(_) | NoHighlight => None,
+            Inlined(h) => {
+                let mut styles = String::from("background-color:");
+                write_css_color(
+                    &mut styles,
+                    h.theme.settings.line_highlight.unwrap_or(Color { r: 255, g: 255, b: 0, a: 0 }),
+                );
+                styles.push_str(";");
+                Some(styles)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use config::highlighting::resolve_syntax_and_theme;
+    use config::Config;
+    use syntect::util::LinesWithEndings;
+
+    #[test]
+    fn can_highlight_with_classes() {
+        let mut config = Config::default();
+        config.markdown.highlight_code = true;
+        let code = "import zen\nz = x + y\nprint('hello')\n";
+        let syntax_and_theme = resolve_syntax_and_theme(Some("py"), &config);
+        let mut highlighter =
+            ClassHighlighter::new(syntax_and_theme.syntax, syntax_and_theme.syntax_set);
+        let mut out = String::new();
+        for line in LinesWithEndings::from(&code) {
+            out.push_str(&highlighter.highlight_line(line));
+        }
+        out.push_str(&highlighter.finalize());
+
+        assert!(out.starts_with("<span class"));
+        assert!(out.ends_with("</span>"));
+        assert!(out.contains("z-"));
+    }
+
+    #[test]
+    fn can_highlight_inline() {
+        let mut config = Config::default();
+        config.markdown.highlight_code = true;
+        let code = "import zen\nz = x + y\nprint('hello')\n";
+        let syntax_and_theme = resolve_syntax_and_theme(Some("py"), &config);
+        let mut highlighter = InlineHighlighter::new(
+            syntax_and_theme.syntax,
+            syntax_and_theme.syntax_set,
+            syntax_and_theme.theme.unwrap(),
+        );
+        let mut out = String::new();
+        for line in LinesWithEndings::from(&code) {
+            out.push_str(&highlighter.highlight_line(line));
+        }
+
+        assert!(out.starts_with(r#"<span style="color"#));
+        assert!(out.ends_with("</span>"));
+    }
+}

components/rendering/src/codeblock/mod.rs

@@ -0,0 +1,186 @@
+mod fence;
+mod highlight;
+
+use std::ops::RangeInclusive;
+
+use syntect::util::LinesWithEndings;
+
+use crate::codeblock::highlight::SyntaxHighlighter;
+use config::highlighting::{resolve_syntax_and_theme, HighlightSource};
+use config::Config;
+pub(crate) use fence::FenceSettings;
+
+fn opening_html(
+    language: Option<&str>,
+    pre_style: Option<String>,
+    pre_class: Option<String>,
+    line_numbers: bool,
+) -> String {
+    let mut html = String::from("<pre");
+    if line_numbers {
+        html.push_str(" data-linenos");
+    }
+    let mut classes = String::new();
+
+    if let Some(lang) = language {
+        classes.push_str("language-");
+        classes.push_str(&lang);
+        classes.push_str(" ");
+
+        html.push_str(" data-lang=\"");
+        html.push_str(lang);
+        html.push('"');
+    }
+
+    if let Some(styles) = pre_style {
+        html.push_str(" style=\"");
+        html.push_str(styles.as_str());
+        html.push('"');
+    }
+
+    if let Some(c) = pre_class {
+        classes.push_str(&c);
+    }
+
+    if !classes.is_empty() {
+        html.push_str(" class=\"");
+        html.push_str(&classes);
+        html.push('"');
+    }
+
+    html.push_str("><code");
+    if let Some(lang) = language {
+        html.push_str(" class=\"language-");
+        html.push_str(lang);
+        html.push_str("\" data-lang=\"");
+        html.push_str(lang);
+        html.push('"');
+    }
+    html.push('>');
+    html
+}
+
+pub struct CodeBlock<'config> {
+    highlighter: SyntaxHighlighter<'config>,
+    // fence options
+    line_numbers: bool,
+    line_number_start: usize,
+    highlight_lines: Vec<RangeInclusive<usize>>,
+    hide_lines: Vec<RangeInclusive<usize>>,
+}
+
+impl<'config> CodeBlock<'config> {
+    pub fn new<'fence_info>(
+        fence: FenceSettings<'fence_info>,
+        config: &'config Config,
+        // path to the current file if there is one, to point where the error is
+        path: Option<&'config str>,
+    ) -> (Self, String) {
+        let syntax_and_theme = resolve_syntax_and_theme(fence.language, config);
+        if syntax_and_theme.source == HighlightSource::NotFound {
+            let lang = fence.language.unwrap();
+            if let Some(p) = path {
+                eprintln!("Warning: Highlight language {} not found in {}", lang, p);
+            } else {
+                eprintln!("Warning: Highlight language {} not found", lang);
+            }
+        }
+        let highlighter = SyntaxHighlighter::new(config.markdown.highlight_code, syntax_and_theme);
+
+        let html_start = opening_html(
+            fence.language,
+            highlighter.pre_style(),
+            highlighter.pre_class(),
+            fence.line_numbers,
+        );
+        (
+            Self {
+                highlighter,
+                line_numbers: fence.line_numbers,
+                line_number_start: fence.line_number_start,
+                highlight_lines: fence.highlight_lines,
+                hide_lines: fence.hide_lines,
+            },
+            html_start,
+        )
+    }
+
+    pub fn highlight(&mut self, content: &str) -> String {
+        let mut buffer = String::new();
+        let mark_style = self.highlighter.mark_style();
+
+        if self.line_numbers {
+            buffer.push_str("<table><tbody>");
+        }
+
+        // syntect leaking here in this file
+        for (i, line) in LinesWithEndings::from(&content).enumerate() {
+            let one_indexed = i + 1;
+            // first do we need to skip that line?
+            let mut skip = false;
+            for range in &self.hide_lines {
+                if range.contains(&one_indexed) {
+                    skip = true;
+                    break;
+                }
+            }
+            if skip {
+                continue;
+            }
+
+            // Next is it supposed to be higlighted?
+            let mut is_higlighted = false;
+            for range in &self.highlight_lines {
+                if range.contains(&one_indexed) {
+                    is_higlighted = true;
+                }
+            }
+
+            if self.line_numbers {
+                buffer.push_str("<tr><td>");
+                let num = format!("{}", self.line_number_start + i);
+                if is_higlighted {
+                    buffer.push_str("<mark");
+                    if let Some(ref s) = mark_style {
+                        buffer.push_str(" style=\"");
+                        buffer.push_str(&s);
+                        buffer.push_str("\">");
+                    } else {
+                        buffer.push_str(">")
+                    }
+                    buffer.push_str(&num);
+                    buffer.push_str("</mark>");
+                } else {
+                    buffer.push_str(&num);
+                }
+                buffer.push_str("</td><td>");
+            }
+
+            let highlighted_line = self.highlighter.highlight_line(line);
+            if is_higlighted {
+                buffer.push_str("<mark");
+                if let Some(ref s) = mark_style {
+                    buffer.push_str(" style=\"");
+                    buffer.push_str(&s);
+                    buffer.push_str("\">");
+                } else {
+                    buffer.push_str(">")
+                }
+                buffer.push_str(&highlighted_line);
+                buffer.push_str("</mark>");
+            } else {
+                buffer.push_str(&highlighted_line);
+            }
+        }
+
+        if let Some(rest) = self.highlighter.finalize() {
+            buffer.push_str(&rest);
+        }
+
+        if self.line_numbers {
+            buffer.push_str("</tr></tbody></table>");
+        }
+
+        buffer
+    }
+}

components/rendering/src/lib.rs

@@ -1,3 +1,4 @@
+mod codeblock;
 mod context;
 mod markdown;
 mod shortcode;

components/rendering/src/markdown.rs

@@ -1,11 +1,9 @@
 use lazy_static::lazy_static;
 use pulldown_cmark as cmark;
 use regex::Regex;
-use syntect::html::{start_highlighted_html_snippet, IncludeBackground};
 
 use crate::context::RenderContext;
 use crate::table_of_contents::{make_table_of_contents, Heading};
-use config::highlighting::THEME_SET;
 use errors::{Error, Result};
 use front_matter::InsertAnchor;
 use utils::site::resolve_internal_link;
@@ -13,10 +11,7 @@ use utils::slugs::slugify_anchors;
 use utils::vec::InsertMany;
 
 use self::cmark::{Event, LinkType, Options, Parser, Tag};
-
+use crate::codeblock::{CodeBlock, FenceSettings};
-mod codeblock;
-mod fence;
-use self::codeblock::CodeBlock;
 
 const CONTINUE_READING: &str = "<span id=\"continue-reading\"></span>";
 const ANCHOR_LINK_TEMPLATE: &str = "anchor-link.html";
@@ -32,7 +27,7 @@ pub struct Rendered {
     pub external_links: Vec<String>,
 }
 
-// tracks a heading in a slice of pulldown-cmark events
+/// Tracks a heading in a slice of pulldown-cmark events
 #[derive(Debug)]
 struct HeadingRef {
     start_idx: usize,
@@ -64,13 +59,13 @@ fn find_anchor(anchors: &[String], name: String, level: u16) -> String {
     find_anchor(anchors, name, level + 1)
 }
 
-// Returns whether the given string starts with a schema.
+/// Returns whether the given string starts with a schema.
-//
+///
-// Although there exists [a list of registered URI schemes][uri-schemes], a link may use arbitrary,
+/// Although there exists [a list of registered URI schemes][uri-schemes], a link may use arbitrary,
-// private schemes. This function checks if the given string starts with something that just looks
+/// private schemes. This function checks if the given string starts with something that just looks
-// like a scheme, i.e., a case-insensitive identifier followed by a colon.
+/// like a scheme, i.e., a case-insensitive identifier followed by a colon.
-//
+///
-// [uri-schemes]: https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
+/// [uri-schemes]: https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
 fn starts_with_schema(s: &str) -> bool {
     lazy_static! {
         static ref PATTERN: Regex = Regex::new(r"^[0-9A-Za-z\-]+:").unwrap();
@@ -79,14 +74,14 @@ fn starts_with_schema(s: &str) -> bool {
     PATTERN.is_match(s)
 }
 
-// Colocated asset links refers to the files in the same directory,
+/// Colocated asset links refers to the files in the same directory,
-// there it should be a filename only
+/// there it should be a filename only
 fn is_colocated_asset_link(link: &str) -> bool {
     !link.contains('/')  // http://, ftp://, ../ etc
         && !starts_with_schema(link)
 }
 
-// Returns whether a link starts with an HTTP(s) scheme.
+/// Returns whether a link starts with an HTTP(s) scheme.
 fn is_external_link(link: &str) -> bool {
     link.starts_with("http:") || link.starts_with("https:")
 }
@@ -165,12 +160,17 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
         static ref EMOJI_REPLACER: gh_emoji::Replacer = gh_emoji::Replacer::new();
     }
 
+    let path = context
+        .tera_context
+        .get("page")
+        .or(context.tera_context.get("section"))
+        .map(|x| x.as_object().unwrap().get("relative_path").unwrap().as_str().unwrap());
     // the rendered html
     let mut html = String::with_capacity(content.len());
     // Set while parsing
     let mut error = None;
 
-    let mut highlighter: Option<CodeBlock> = None;
+    let mut code_block: Option<CodeBlock> = None;
 
     let mut inserted_anchors: Vec<String> = vec![];
     let mut headings: Vec<Heading> = vec![];
@@ -195,7 +195,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
                 match event {
                     Event::Text(text) => {
                         // if we are in the middle of a highlighted code block
-                        if let Some(ref mut code_block) = highlighter {
+                        if let Some(ref mut code_block) = code_block {
                             let html = code_block.highlight(&text);
                             Event::Html(html.into())
                         } else if context.config.markdown.render_emoji {
@@ -207,74 +207,20 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
                         }
                     }
                     Event::Start(Tag::CodeBlock(ref kind)) => {
-                        let language = match kind {
+                        let fence = match kind {
                             cmark::CodeBlockKind::Fenced(fence_info) => {
-                                let fence_info = fence::FenceSettings::new(fence_info);
+                                FenceSettings::new(fence_info)
-                                fence_info.language
                             }
-                            _ => None,
+                            _ => FenceSettings::new(""),
                         };
-
+                        let (block, begin) = CodeBlock::new(fence, &context.config, path);
-                        if !context.config.markdown.highlight_code {
+                        code_block = Some(block);
-                            if let Some(lang) = language {
+                        Event::Html(begin.into())
-                                let html = format!(
-                                    r#"<pre><code class="language-{}" data-lang="{}">"#,
-                                    lang, lang
-                                );
-                                return Event::Html(html.into());
-                            }
-                            return Event::Html("<pre><code>".into());
-                        }
-
-                        let theme = &THEME_SET.themes[&context.config.markdown.highlight_theme];
-                        match kind {
-                            cmark::CodeBlockKind::Indented => (),
-                            cmark::CodeBlockKind::Fenced(fence_info) => {
-                                // This selects the background color the same way that
-                                // start_coloured_html_snippet does
-                                let color = theme
-                                    .settings
-                                    .background
-                                    .unwrap_or(::syntect::highlighting::Color::WHITE);
-
-                                highlighter = Some(CodeBlock::new(
-                                    fence_info,
-                                    &context.config,
-                                    IncludeBackground::IfDifferent(color),
-                                    context
-                                        .tera_context
-                                        .get("page")
-                                        .or(context.tera_context.get("section"))
-                                        .map(|x| {
-                                            x.as_object()
-                                                .unwrap()
-                                                .get("relative_path")
-                                                .unwrap()
-                                                .as_str()
-                                                .unwrap()
-                                        }),
-                                ));
-                            }
-                        };
-                        let snippet = start_highlighted_html_snippet(theme);
-                        let mut html = snippet.0;
-                        if let Some(lang) = language {
-                            html.push_str(&format!(
-                                r#"<code class="language-{}" data-lang="{}">"#,
-                                lang, lang
-                            ));
-                        } else {
-                            html.push_str("<code>");
-                        }
-                        Event::Html(html.into())
                     }
                     Event::End(Tag::CodeBlock(_)) => {
-                        if !context.config.markdown.highlight_code {
-                            return Event::Html("</code></pre>\n".into());
-                        }
                         // reset highlight and close the code block
-                        highlighter = None;
+                        code_block = None;
-                        Event::Html("</code></pre>".into())
+                        Event::Html("</code></pre>\n".into())
                     }
                     Event::Start(Tag::Image(link_type, src, title)) => {
                         if is_colocated_asset_link(&src) {

components/rendering/src/markdown/codeblock.rs

@@ -1,238 +0,0 @@
-use config::highlighting::{get_highlighter, HighlightSource, SYNTAX_SET, THEME_SET};
-use config::Config;
-use std::cmp::min;
-use std::collections::HashSet;
-use syntect::easy::HighlightLines;
-use syntect::highlighting::{Color, Style, Theme};
-use syntect::html::{styled_line_to_highlighted_html, IncludeBackground};
-use syntect::parsing::SyntaxSet;
-
-use super::fence::{FenceSettings, Range};
-
-pub struct CodeBlock<'config> {
-    highlighter: HighlightLines<'static>,
-    extra_syntax_set: Option<&'config SyntaxSet>,
-    background: IncludeBackground,
-    theme: &'static Theme,
-
-    /// List of ranges of lines to highlight.
-    highlight_lines: Vec<Range>,
-    /// List of ranges of lines to hide.
-    hide_lines: Vec<Range>,
-    /// The number of lines in the code block being processed.
-    num_lines: usize,
-}
-
-impl<'config> CodeBlock<'config> {
-    pub fn new(
-        fence_info: &str,
-        config: &'config Config,
-        background: IncludeBackground,
-        path: Option<&'config str>,
-    ) -> Self {
-        let fence_info = FenceSettings::new(fence_info);
-        let theme = &THEME_SET.themes[&config.markdown.highlight_theme];
-        let (highlighter, highlight_source) = get_highlighter(fence_info.language, config);
-        let extra_syntax_set = match highlight_source {
-            HighlightSource::Extra => config.markdown.extra_syntax_set.as_ref(),
-            HighlightSource::NotFound => {
-                // Language was not found, so it exists (safe unwrap)
-                let lang = fence_info.language.unwrap();
-                if let Some(path) = path {
-                    eprintln!("Warning: Highlight language {} not found in {}", lang, path);
-                } else {
-                    eprintln!("Warning: Highlight language {} not found", lang);
-                }
-                None
-            }
-            _ => None,
-        };
-        Self {
-            highlighter,
-            extra_syntax_set,
-            background,
-            theme,
-
-            highlight_lines: fence_info.highlight_lines,
-            hide_lines: fence_info.hide_lines,
-            num_lines: 0,
-        }
-    }
-
-    pub fn highlight(&mut self, text: &str) -> String {
-        let highlighted =
-            self.highlighter.highlight(text, self.extra_syntax_set.unwrap_or(&SYNTAX_SET));
-        let line_boundaries = self.find_line_boundaries(&highlighted);
-
-        // First we make sure that `highlighted` is split at every line
-        // boundary. The `styled_line_to_highlighted_html` function will
-        // merge split items with identical styles, so this is not a
-        // problem.
-        //
-        // Note that this invalidates the values in `line_boundaries`.
-        // The `perform_split` function takes it by value to ensure that
-        // we don't use it later.
-        let mut highlighted = perform_split(&highlighted, line_boundaries);
-
-        let hl_background =
-            self.theme.settings.line_highlight.unwrap_or(Color { r: 255, g: 255, b: 0, a: 0 });
-
-        let hl_lines = self.get_highlighted_lines();
-        color_highlighted_lines(&mut highlighted, &hl_lines, hl_background);
-
-        let hide_lines = self.get_hidden_lines();
-        let highlighted = hide_hidden_lines(highlighted, &hide_lines);
-
-        styled_line_to_highlighted_html(&highlighted, self.background)
-    }
-
-    fn find_line_boundaries(&mut self, styled: &[(Style, &str)]) -> Vec<StyledIdx> {
-        let mut boundaries = Vec::new();
-        for (vec_idx, (_style, s)) in styled.iter().enumerate() {
-            for (str_idx, character) in s.char_indices() {
-                if character == '\n' {
-                    boundaries.push(StyledIdx { vec_idx, str_idx });
-                }
-            }
-        }
-        self.num_lines = boundaries.len() + 1;
-        boundaries
-    }
-
-    fn get_highlighted_lines(&self) -> HashSet<usize> {
-        self.ranges_to_lines(&self.highlight_lines)
-    }
-
-    fn get_hidden_lines(&self) -> HashSet<usize> {
-        self.ranges_to_lines(&self.hide_lines)
-    }
-
-    fn ranges_to_lines(&self, range: &Vec<Range>) -> HashSet<usize> {
-        let mut lines = HashSet::new();
-        for range in range {
-            for line in range.from..=min(range.to, self.num_lines) {
-                // Ranges are one-indexed
-                lines.insert(line.saturating_sub(1));
-            }
-        }
-        lines
-    }
-}
-
-/// This is an index of a character in a `&[(Style, &'b str)]`. The `vec_idx` is the
-/// index in the slice, and `str_idx` is the byte index of the character in the
-/// corresponding string slice.
-///
-/// The `Ord` impl on this type sorts lexiographically on `vec_idx`, and then `str_idx`.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
-struct StyledIdx {
-    vec_idx: usize,
-    str_idx: usize,
-}
-
-/// This is a utility used by `perform_split`. If the `vec_idx` in the `StyledIdx` is
-/// equal to the provided value, return the `str_idx`, otherwise return `None`.
-fn get_str_idx_if_vec_idx_is(idx: Option<&StyledIdx>, vec_idx: usize) -> Option<usize> {
-    match idx {
-        Some(idx) if idx.vec_idx == vec_idx => Some(idx.str_idx),
-        _ => None,
-    }
-}
-
-/// This function assumes that `line_boundaries` is sorted according to the `Ord` impl on
-/// the `StyledIdx` type.
-fn perform_split<'b>(
-    split: &[(Style, &'b str)],
-    line_boundaries: Vec<StyledIdx>,
-) -> Vec<(Style, &'b str)> {
-    let mut result = Vec::new();
-
-    let mut idxs_iter = line_boundaries.into_iter().peekable();
-
-    for (split_idx, item) in split.iter().enumerate() {
-        let mut last_split = 0;
-
-        // Since `line_boundaries` is sorted, we know that any remaining indexes in
-        // `idxs_iter` have `vec_idx >= split_idx`, and that if there are any with
-        // `vec_idx == split_idx`, they will be first.
-        //
-        // Using the `get_str_idx_if_vec_idx_is` utility, this loop will keep consuming
-        // indexes from `idxs_iter` as long as `vec_idx == split_idx` holds. Once
-        // `vec_idx` becomes larger than `split_idx`, the loop will finish without
-        // consuming that index.
-        //
-        // If `idxs_iter` is empty, or there are no indexes with `vec_idx == split_idx`,
-        // the loop does nothing.
-        while let Some(str_idx) = get_str_idx_if_vec_idx_is(idxs_iter.peek(), split_idx) {
-            // Consume the value we just peeked.
-            idxs_iter.next();
-
-            // This consumes the index to split at. We add one to include the newline
-            // together with its own line, rather than as the first character in the next
-            // line.
-            let split_at = min(str_idx + 1, item.1.len());
-
-            // This will fail if `line_boundaries` is not sorted.
-            debug_assert!(split_at >= last_split);
-
-            // Skip splitting if the string slice would be empty.
-            if last_split != split_at {
-                result.push((item.0, &item.1[last_split..split_at]));
-                last_split = split_at;
-            }
-        }
-
-        // Now append the remainder. If the current item was not split, this will
-        // append the entire item.
-        if last_split != item.1.len() {
-            result.push((item.0, &item.1[last_split..]));
-        }
-    }
-
-    result
-}
-
-fn color_highlighted_lines(data: &mut [(Style, &str)], lines: &HashSet<usize>, background: Color) {
-    if lines.is_empty() {
-        return;
-    }
-
-    let mut current_line = 0;
-
-    for item in data {
-        if lines.contains(&current_line) {
-            item.0.background = background;
-        }
-
-        // We split the lines such that every newline is at the end of an item.
-        if item.1.ends_with('\n') {
-            current_line += 1;
-        }
-    }
-}
-
-fn hide_hidden_lines<'a>(
-    data: Vec<(Style, &'a str)>,
-    lines: &HashSet<usize>,
-) -> Vec<(Style, &'a str)> {
-    if lines.is_empty() {
-        return data;
-    }
-
-    let mut current_line = 0;
-
-    let mut to_keep = Vec::new();
-
-    for item in data {
-        if !lines.contains(&current_line) {
-            to_keep.push(item);
-        }
-
-        // We split the lines such that every newline is at the end of an item.
-        if item.1.ends_with('\n') {
-            current_line += 1;
-        }
-    }
-
-    to_keep
-}

components/rendering/tests/codeblock_hide_lines.rs

@@ -8,7 +8,7 @@ use rendering::{render_content, RenderContext};
 
 macro_rules! colored_html_line {
     ( $s:expr ) => {{
-        let mut result = "<span style=\"color:#c0c5ce;\">".to_string();
+        let mut result = "<span>".to_string();
         result.push_str($s);
         result.push_str("\n</span>");
         result
@@ -17,11 +17,11 @@ macro_rules! colored_html_line {
 
 macro_rules! colored_html {
     ( $($s:expr),* $(,)* ) => {{
-        let mut result = "<pre style=\"background-color:#2b303b;\">\n<code>".to_string();
+        let mut result = "<pre style=\"background-color:#2b303b;color:#c0c5ce;\"><code>".to_string();
         $(
             result.push_str(colored_html_line!($s).as_str());
         )*
-        result.push_str("</code></pre>");
+        result.push_str("</code></pre>\n");
         result
     }};
 }
@@ -52,5 +52,5 @@ bat
         &context,
     )
     .unwrap();
-    assert_eq!(res.body, colored_html!("foo\nbaz\nbat",));
+    assert_eq!(res.body, colored_html!("foo", "baz", "bat"));
 }

components/rendering/tests/codeblock_hl_lines.rs

@@ -8,26 +8,28 @@ use rendering::{render_content, RenderContext};
 
 macro_rules! colored_html_line {
     ( @no $s:expr ) => {{
-        let mut result = "<span style=\"color:#c0c5ce;\">".to_string();
+        let mut result = "<span>".to_string();
         result.push_str($s);
         result.push_str("\n</span>");
         result
     }};
     ( @hl $s:expr ) => {{
-        let mut result = "<span style=\"background-color:#65737e30;color:#c0c5ce;\">".to_string();
+        let mut result = "<mark style=\"background-color:#65737e30;\">".to_string();
+        result.push_str("<span>");
         result.push_str($s);
         result.push_str("\n</span>");
+        result.push_str("</mark>");
         result
     }};
 }
 
 macro_rules! colored_html {
     ( $(@$kind:tt $s:expr),* $(,)* ) => {{
-        let mut result = "<pre style=\"background-color:#2b303b;\">\n<code>".to_string();
+        let mut result = "<pre style=\"background-color:#2b303b;color:#c0c5ce;\"><code>".to_string();
         $(
             result.push_str(colored_html_line!(@$kind $s).as_str());
         )*
-        result.push_str("</code></pre>");
+        result.push_str("</code></pre>\n");
         result
     }};
 }
@@ -63,7 +65,8 @@ baz
         colored_html!(
             @no "foo",
             @hl "bar",
-            @no "bar\nbaz",
+            @no "bar",
+            @no "baz",
         )
     );
 }
@@ -98,7 +101,8 @@ baz
         res.body,
         colored_html!(
             @no "foo",
-            @hl "bar\nbar",
+            @hl "bar",
+            @hl "bar",
             @no "baz",
         )
     );
@@ -133,7 +137,10 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @hl "foo\nbar\nbar\nbaz",
+            @hl "foo",
+            @hl "bar",
+            @hl "bar",
+            @hl "baz",
         )
     );
 }
@@ -167,7 +174,9 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @hl "foo\nbar\nbar",
+            @hl "foo",
+            @hl "bar",
+            @hl "bar",
             @no "baz",
         )
     );
@@ -202,7 +211,9 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @hl "foo\nbar\nbar",
+            @hl "foo",
+            @hl "bar",
+            @hl "bar",
             @no "baz",
         )
     );
@@ -237,8 +248,10 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @no "foo\nbar",
+            @no "foo",
-            @hl "bar\nbaz",
+            @no "bar",
+            @hl "bar",
+            @hl "baz",
         )
     );
 }
@@ -272,8 +285,10 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @no "foo\nbar",
+            @no "foo",
-            @hl "bar\nbaz",
+            @no "bar",
+            @hl "bar",
+            @hl "baz",
         )
     );
 }
@@ -307,7 +322,9 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @hl "foo\nbar\nbar",
+            @hl "foo",
+            @hl "bar",
+            @hl "bar",
             @no "baz",
         )
     );
@@ -341,7 +358,9 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @hl "foo\nbar\nbar",
+            @hl "foo",
+            @hl "bar",
+            @hl "bar",
             @no "baz",
         )
     );
@@ -376,7 +395,9 @@ baz
     assert_eq!(
         res.body,
         colored_html!(
-            @hl "foo\nbar\nbar",
+            @hl "foo",
+            @hl "bar",
+            @hl "bar",
             @no "baz",
         )
     );
@@ -413,7 +434,8 @@ baz
         colored_html!(
             @hl "foo",
             @no "bar",
-            @hl "bar\nbaz",
+            @hl "bar",
+            @hl "baz",
         )
     );
 }
@@ -449,7 +471,8 @@ baz
         colored_html!(
             @no "foo",
             @hl "bar",
-            @no "bar\nbaz",
+            @no "bar",
+            @no "baz",
         )
     );
 }
@@ -484,7 +507,8 @@ baz
         res.body,
         colored_html!(
             @no "foo",
-            @hl "bar\nbar",
+            @hl "bar",
+            @hl "bar",
             @no "baz",
         )
     );

components/rendering/tests/codeblock_linenos.rs

@@ -0,0 +1,97 @@
+use std::collections::HashMap;
+
+use tera::Tera;
+
+use config::Config;
+use front_matter::InsertAnchor;
+use rendering::{render_content, RenderContext};
+
+#[test]
+fn can_add_line_numbers() {
+    let tera_ctx = Tera::default();
+    let permalinks_ctx = HashMap::new();
+    let mut config = Config::default_for_test();
+    config.markdown.highlight_code = true;
+    let context = RenderContext::new(
+        &tera_ctx,
+        &config,
+        &config.default_language,
+        "",
+        &permalinks_ctx,
+        InsertAnchor::None,
+    );
+    let res = render_content(
+        r#"
+```linenos
+foo
+bar
+```
+    "#,
+        &context,
+    )
+    .unwrap();
+    assert_eq!(
+        res.body,
+        "<pre data-linenos style=\"background-color:#2b303b;color:#c0c5ce;\"><code><table><tbody><tr><td>1</td><td><span>foo\n</span><tr><td>2</td><td><span>bar\n</span></tr></tbody></table></code></pre>\n"
+    );
+}
+
+#[test]
+fn can_add_line_numbers_with_linenostart() {
+    let tera_ctx = Tera::default();
+    let permalinks_ctx = HashMap::new();
+    let mut config = Config::default_for_test();
+    config.markdown.highlight_code = true;
+    let context = RenderContext::new(
+        &tera_ctx,
+        &config,
+        &config.default_language,
+        "",
+        &permalinks_ctx,
+        InsertAnchor::None,
+    );
+    let res = render_content(
+        r#"
+```linenos, linenostart=40
+foo
+bar
+```
+    "#,
+        &context,
+    )
+    .unwrap();
+    assert_eq!(
+        res.body,
+        "<pre data-linenos style=\"background-color:#2b303b;color:#c0c5ce;\"><code><table><tbody><tr><td>40</td><td><span>foo\n</span><tr><td>41</td><td><span>bar\n</span></tr></tbody></table></code></pre>\n"
+    );
+}
+
+#[test]
+fn can_add_line_numbers_with_highlight() {
+    let tera_ctx = Tera::default();
+    let permalinks_ctx = HashMap::new();
+    let mut config = Config::default_for_test();
+    config.markdown.highlight_code = true;
+    let context = RenderContext::new(
+        &tera_ctx,
+        &config,
+        &config.default_language,
+        "",
+        &permalinks_ctx,
+        InsertAnchor::None,
+    );
+    let res = render_content(
+        r#"
+```linenos, hl_lines=2
+foo
+bar
+```
+    "#,
+        &context,
+    )
+    .unwrap();
+    assert_eq!(
+        res.body,
+        "<pre data-linenos style=\"background-color:#2b303b;color:#c0c5ce;\"><code><table><tbody><tr><td>1</td><td><span>foo\n</span><tr><td><mark style=\"background-color:#65737e30;\">2</mark></td><td><mark style=\"background-color:#65737e30;\"><span>bar\n</span></mark></tr></tbody></table></code></pre>\n"
+    );
+}

components/rendering/tests/markdown.rs

@@ -60,7 +60,7 @@ fn can_highlight_code_block_no_lang() {
     let res = render_content("```\n$ gutenberg server\n$ ping\n```", &context).unwrap();
     assert_eq!(
         res.body,
-        "<pre style=\"background-color:#2b303b;\">\n<code><span style=\"color:#c0c5ce;\">$ gutenberg server\n$ ping\n</span></code></pre>"
+        "<pre style=\"background-color:#2b303b;color:#c0c5ce;\"><code><span>$ gutenberg server\n</span><span>$ ping\n</span></code></pre>\n"
     );
 }
 
@@ -81,7 +81,7 @@ fn can_highlight_code_block_with_lang() {
     let res = render_content("```python\nlist.append(1)\n```", &context).unwrap();
     assert_eq!(
         res.body,
-        "<pre style=\"background-color:#2b303b;\">\n<code class=\"language-python\" data-lang=\"python\"><span style=\"color:#c0c5ce;\">list.</span><span style=\"color:#bf616a;\">append</span><span style=\"color:#c0c5ce;\">(</span><span style=\"color:#d08770;\">1</span><span style=\"color:#c0c5ce;\">)\n</span></code></pre>"
+        "<pre data-lang=\"python\" style=\"background-color:#2b303b;color:#c0c5ce;\" class=\"language-python \"><code class=\"language-python\" data-lang=\"python\"><span>list.</span><span style=\"color:#bf616a;\">append</span><span>(</span><span style=\"color:#d08770;\">1</span><span>)\n</span></code></pre>\n"
     );
 }
 
@@ -103,7 +103,7 @@ fn can_higlight_code_block_with_unknown_lang() {
     // defaults to plain text
     assert_eq!(
         res.body,
-        "<pre style=\"background-color:#2b303b;\">\n<code class=\"language-yolo\" data-lang=\"yolo\"><span style=\"color:#c0c5ce;\">list.append(1)\n</span></code></pre>"
+        "<pre data-lang=\"yolo\" style=\"background-color:#2b303b;color:#c0c5ce;\" class=\"language-yolo \"><code class=\"language-yolo\" data-lang=\"yolo\"><span>list.append(1)\n</span></code></pre>\n"
     );
 }
 

components/site/src/lib.rs

@@ -14,6 +14,7 @@ use rayon::prelude::*;
 use tera::{Context, Tera};
 use walkdir::{DirEntry, WalkDir};
 
+use config::highlighting::export_theme_css;
 use config::{get_config, Config};
 use errors::{bail, Error, Result};
 use front_matter::InsertAnchor;
@@ -666,7 +667,8 @@ impl Site {
             self.render_feed(pages, Some(&PathBuf::from(code)), &code, |c| c)?;
             start = log_time(start, "Generated feed in other language");
         }
-
+        self.render_themes_css()?;
+        start = log_time(start, "Rendered themes css");
         self.render_404()?;
         start = log_time(start, "Rendered 404");
         self.render_robots()?;
@@ -684,6 +686,20 @@ impl Site {
         Ok(())
     }
 
+    pub fn render_themes_css(&self) -> Result<()> {
+        ensure_directory_exists(&self.static_path)?;
+
+        for t in &self.config.markdown.highlight_themes_css {
+            let p = self.static_path.join(&t.filename);
+            if !p.exists() {
+                let content = export_theme_css(&t.theme);
+                create_file(&p, &content)?;
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn build_search_index(&self) -> Result<()> {
         ensure_directory_exists(&self.output_path)?;
         // TODO: add those to the SITE_CONTENT map

components/templates/src/filters.rs

@@ -178,7 +178,7 @@ mod tests {
             .unwrap()
             .filter(&to_value(&md).unwrap(), &HashMap::new());
         assert!(result.is_ok());
-        assert!(result.unwrap().as_str().unwrap().contains("<pre style"));
+        assert!(result.unwrap().as_str().unwrap().contains("style"));
     }
 
     #[test]

docs/config.toml

@@ -8,6 +8,11 @@ build_search_index = true
 [markdown]
 highlight_code = true
 highlight_theme = "kronuz"
+#highlight_theme = "css"
+#highlight_themes_css = [
+#  { theme = "base16-ocean-dark", filename = "syntax-theme-dark.css" },
+#  { theme = "base16-ocean-light", filename = "syntax-theme-light.css" },
+#]
 
 
 [extra]

docs/content/documentation/content/syntax-highlighting.md

@@ -171,6 +171,79 @@ If your site source is laid out as follows:
 
 you would set your `extra_syntaxes` to `["syntaxes", "syntaxes/Sublime-Language1"]` to load `lang1.sublime-syntax` and `lang2.sublime-syntax`.
 
+## Inline VS classed highlighting
+
+If you use a highlighting scheme like
+
+```toml
+highlight_theme = "base16-ocean-dark"
+```
+
+for a code block like
+
+````md
+```rs
+let highlight = true;
+```
+````
+
+you get the colors directly encoded in the html file.
+
+```html
+<pre class="language-rs" style="background-color:#2b303b;">
+    <code class="language-rs">
+        <span style="color:#b48ead;">let</span>
+        <span style="color:#c0c5ce;"> highlight = </span>
+        <span style="color:#d08770;">true</span>
+        <span style="color:#c0c5ce;">;
+    </span>
+  </code>
+</pre>
+```
+
+This is nice, because your page will load faster if everything is in one file.
+But if you would like to have the user choose a theme from a
+list, or use different color schemes for dark/light color schemes, you need a
+different solution.
+
+If you use the special `css` color scheme
+
+```toml
+highlight_theme = "css"
+```
+
+you get CSS class definitions, instead.
+
+```html
+<pre class="language-rs">
+    <code class="language-rs">
+        <span class="z-source z-rust">
+            <span class="z-storage z-type z-rust">let</span> highlight
+            <span class="z-keyword z-operator z-assignment z-rust">=</span>
+            <span class="z-constant z-language z-rust">true</span>
+            <span class="z-punctuation z-terminator z-rust">;</span>
+        </span>
+    </code>
+</pre>
+```
+
+Zola can output a css file for a theme in the `static` directory using the `highlighting_themes_css` option.
+
+```toml
+highlight_themes_css = [
+  { theme = "base16-ocean-dark", filename = "syntax-theme-dark.css" },
+  { theme = "base16-ocean-light", filename = "syntax-theme-light.css" },
+]
+```
+
+You can then support light and dark mode like so:
+
+```css
+@import url("syntax-theme-dark.css") (prefers-color-scheme: dark);
+@import url("syntax-theme-light.css") (prefers-color-scheme: light);
+```
+
+
 ## Annotations
 
 You can use additional annotations to customize how code blocks are displayed:
@@ -185,18 +258,28 @@ highlight(code);
 ```
 ````
 
-- `hl_lines` to highlight lines. You must specify a list of ranges of lines to highlight,
+- `linenostart` to specify the number for the first line (defaults to 1)
-separated by ` `. Ranges are 1-indexed.
   
 ````
-```rust,hl_lines=3
+```rust,linenos,linenostart=20
 use highlighter::highlight;
 let code = "...";
 highlight(code);
 ```
 ````
 
-- `hide_lines` to hide lines. You must specify a list of ranges of lines to hide,
+- `hl_lines` to highlight lines. You must specify a list of inclusive ranges of lines to highlight,
+separated by whitespaces. Ranges are 1-indexed and `linenostart` doesn't influence the values, it always refers to the codeblock line number.
+  
+````
+```rust,hl_lines=1 3-5 9
+use highlighter::highlight;
+let code = "...";
+highlight(code);
+```
+````
+
+- `hide_lines` to hide lines. You must specify a list of inclusive ranges of lines to hide,
 separated by ` `. Ranges are 1-indexed.
   
 ````
@@ -206,3 +289,61 @@ let code = "...";
 highlight(code);
 ```
 ````
+
+## Styling codeblocks
+
+Depending on the annotations used, some codeblocks will be hard to read without any CSS. We recommend using the following
+snippet in your sites:
+
+```scss
+pre {
+  padding: 1rem;
+  overflow: auto;
+}
+// The line numbers already provide some kind of left/right padding
+pre[data-linenos] {
+  padding: 1rem 0;
+}
+pre table td {
+  padding: 0;
+}
+// The line number cells
+pre table td:nth-of-type(1) {
+  text-align: center;
+  user-select: none;
+}
+pre mark {
+  // If you want your highlights to take the full width.
+  display: block;
+  // The default background colour of a mark is bright yellow
+  background-color: rgba(254, 252, 232, 0.9);
+}
+pre table {
+  width: 100%;
+  border-collapse: collapse;
+}
+```
+
+This snippet makes the highlighting work on the full width and ensures that a user can copy the content without
+selecting the line numbers. Obviously you will probably need to adjust it to fit your site style.
+
+Here's an example with all the options used: `scss, linenos, linenostart=10, hl_lines=3-4 8-9, hide_lines=2 7` with the
+snippet above.
+
+```scss, linenos, linenostart=10, hl_lines=3-4 8-9, hide_lines=2 7
+pre mark {
+  // If you want your highlights to take the full width.
+  display: block;
+  color: currentcolor;
+}
+pre table td:nth-of-type(1) {
+  // Select a colour matching your theme
+  color: #6b6b6b;
+  font-style: italic;
+}
+```
+
+Line 2 and 7 are comments that are not shown in the final output.
+
+When line numbers are active, the code block is turned into a table with one row and two cells. The first cell contains the line number and the second cell contains the code.
+Highlights are done via the `<mark>` HTML tag. When a line with line number is highlighted two `<mark>` tags are created: one around the line number(s) and one around the code.

docs/sass/_base.scss

@@ -82,6 +82,26 @@ pre {
   padding: 1rem;
   overflow: auto;
 }
+pre[data-linenos] {
+  padding: 1rem 0;
+}
+pre mark {
+  // If you want your highlights to take the full width.
+  display: block;
+  // The default background colour of a mark is bright yellow
+  background-color: rgba(254, 252, 232, 0.9);
+}
+pre table {
+  width: 100%;
+  border-collapse: collapse;
+}
+pre table td {
+  padding: 0;
+}
+pre table td:nth-of-type(1) {
+  text-align: center;
+  user-select: none;
+}
 
 p code, li code {
   background-color: #f5f5f5;

docs/sass/site.scss

@@ -17,3 +17,6 @@ $link-color: #007CBC;
 @import "docs";
 @import "themes";
 @import "search";
+
+//@import url("syntax-theme-dark.css") (prefers-color-scheme: dark);
+//@import url("syntax-theme-light.css") (prefers-color-scheme: light);