Back to pest (#1665)

Cargo.lock

@@ -107,12 +107,6 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
-[[package]]
-name = "beef"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bed554bd50246729a1ec158d08aa3235d1b69d94ad120ebe187e28894787e736"
-
 [[package]]
 name = "bincode"
 version = "1.3.3"
@@ -1408,30 +1402,6 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
-[[package]]
-name = "logos"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "427e2abca5be13136da9afdbf874e6b34ad9001dd70f2b103b083a85daa7b345"
-dependencies = [
- "logos-derive",
-]
-
-[[package]]
-name = "logos-derive"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56a7d287fd2ac3f75b11f19a1c8a874a7d55744bd91f7a1b3e7cf87d4343c36d"
-dependencies = [
- "beef",
- "fnv",
- "proc-macro2",
- "quote",
- "regex-syntax 0.6.25",
- "syn",
- "utf8-ranges",
-]
-
 [[package]]
 name = "mac"
 version = "0.1.1"
@@ -2366,7 +2336,8 @@ dependencies = [
  "gh-emoji",
  "lazy_static",
  "link_checker",
- "logos",
+ "pest",
+ "pest_derive",
  "pulldown-cmark",
  "regex",
  "serde",

components/rendering/Cargo.toml

@@ -11,7 +11,8 @@ syntect = "4"
 pulldown-cmark = { version = "0.8", default-features = false }
 serde = "1"
 serde_derive = "1"
-logos = "0.12"
+pest = "2"
+pest_derive = "2"
 regex = "1"
 lazy_static = "1"
 gh-emoji = "1.0"

components/rendering/src/content.pest

@@ -25,7 +25,7 @@ string = @{
 
 boolean = { "true" | "false" }
 
-literal = { boolean | string | float | int }
+literal = { boolean | string | float | int | array }
 array = { "[" ~ (literal  ~ ",")* ~ literal? ~ "]"}
 
 /// Idents
@@ -40,7 +40,7 @@ ident = @{
 
 // shortcode is abbreviated to sc to keep things short
 
-kwarg   = { ident ~ "=" ~ (literal | array) }
+kwarg   = { ident ~ "=" ~ literal }
 kwargs  = _{ kwarg ~ ("," ~ kwarg )* }
 sc_def  = _{ ident ~ "(" ~ kwargs* ~ ")" }
 

components/rendering/src/markdown.rs

@@ -11,7 +11,7 @@ use utils::vec::InsertMany;
 
 use self::cmark::{Event, LinkType, Options, Parser, Tag};
 use crate::codeblock::{CodeBlock, FenceSettings};
-use crate::shortcode::parser::{Shortcode, SHORTCODE_PLACEHOLDER};
+use crate::shortcode::{Shortcode, SHORTCODE_PLACEHOLDER};
 
 const CONTINUE_READING: &str = "<span id=\"continue-reading\"></span>";
 const ANCHOR_LINK_TEMPLATE: &str = "anchor-link.html";
@@ -211,6 +211,7 @@ pub fn markdown_to_html(
                                     }
 
                                     let shortcode = next_shortcode.take().unwrap();
+
                                     match shortcode.render(&context.tera, &context.tera_context) {
                                         Ok(s) => {
                                             events.push(Event::Html(s.into()));
@@ -317,7 +318,6 @@ pub fn markdown_to_html(
                     });
                 }
                 Event::Html(text) => {
-                    println!("Got text: {:?}", text);
                     if text.contains("<!-- more -->") {
                         has_summary = true;
                         events.push(Event::Html(CONTINUE_READING.into()));

components/rendering/src/shortcode/lexer.rs

@@ -1,111 +0,0 @@
-use logos::Logos;
-use std::fmt::Formatter;
-
-fn replace_string_markers(input: &str, marker: char) -> String {
-    input.replace(marker, "")
-}
-
-#[derive(Logos, Debug, PartialEq, Clone)]
-pub enum Content {
-    #[token("{{")]
-    InlineShortcodeStart,
-    #[token("{{/*")]
-    IgnoredInlineShortcodeStart,
-    #[token("{%")]
-    ShortcodeWithBodyStart,
-    #[token("{%/*")]
-    IgnoredShortcodeWithBodyStart,
-
-    #[token("}}")]
-    InlineShortcodeEnd,
-    #[token("*/}}")]
-    IgnoredInlineShortcodeEnd,
-    #[token("%}")]
-    ShortcodeWithBodyEnd,
-    #[token("*/%}")]
-    IgnoredShortcodeWithBodyEnd,
-    #[token("{% end %}")]
-    ShortcodeWithBodyClosing,
-    #[token("{%/* end */%}")]
-    IgnoredShortcodeWithBodyClosing,
-
-    #[regex(r"[^{\*%\}]+", logos::skip)]
-    #[error]
-    Error,
-}
-
-impl std::fmt::Display for Content {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        let val = match self {
-            Content::InlineShortcodeStart => "`{{`",
-            Content::IgnoredInlineShortcodeStart => "`{{/*`",
-            Content::ShortcodeWithBodyStart => "`{%`",
-            Content::IgnoredShortcodeWithBodyStart => "`{%/*",
-            Content::InlineShortcodeEnd => "`}}`",
-            Content::IgnoredInlineShortcodeEnd => "`*/}}`",
-            Content::ShortcodeWithBodyEnd => "`%}`",
-            Content::IgnoredShortcodeWithBodyEnd => "`*/%}`",
-            Content::ShortcodeWithBodyClosing => "`{% end %}`",
-            Content::IgnoredShortcodeWithBodyClosing => "`{%/* end */%}`",
-            Content::Error => "`error`",
-        };
-
-        write!(f, "{}", val)
-    }
-}
-
-#[derive(Logos, Debug, PartialEq, Clone)]
-pub enum InnerShortcode {
-    #[token("(")]
-    OpenParenthesis,
-    #[token(")")]
-    CloseParenthesis,
-    #[token("[")]
-    OpenBracket,
-    #[token("]")]
-    CloseBracket,
-    #[token(",")]
-    Comma,
-    #[token("=")]
-    Equals,
-    #[regex("[a-zA-Z][a-zA-Z0-9_]*")]
-    Ident,
-    #[regex("-?[0-9]+", |lex| lex.slice().parse())]
-    Integer(i64),
-    #[regex("-?[0-9]+\\.[0-9]+", |lex| lex.slice().parse())]
-    Float(f64),
-    #[token("true", |_| true)]
-    #[token("True", |_| true)]
-    #[token("false", |_| false)]
-    #[token("False", |_| false)]
-    Bool(bool),
-    #[regex(r#"'([^'\\]*(\\.[^'\\]*)*)'"#, |lex| replace_string_markers(lex.slice(), '\''))]
-    #[regex(r#""([^"\\]*(\\.[^"\\]*)*)""#, |lex| replace_string_markers(lex.slice(), '"'))]
-    #[regex(r#"`([^`\\]*(\\.[^`\\]*)*)`"#, |lex| replace_string_markers(lex.slice(), '`'))]
-    Str(String),
-
-    #[regex(r"[ \t\n\f]+", logos::skip)]
-    #[error]
-    Error,
-}
-
-impl std::fmt::Display for InnerShortcode {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        let val = match self {
-            InnerShortcode::OpenParenthesis => "`(`",
-            InnerShortcode::CloseParenthesis => "`)`",
-            InnerShortcode::OpenBracket => "`[`",
-            InnerShortcode::CloseBracket => "`]`",
-            InnerShortcode::Comma => "`,`",
-            InnerShortcode::Equals => "`=`",
-            InnerShortcode::Ident => "`identifier`",
-            InnerShortcode::Integer(_) => "`integer`",
-            InnerShortcode::Float(_) => "`float`",
-            InnerShortcode::Bool(_) => "`boolean`",
-            InnerShortcode::Str(_) => "`string`",
-            InnerShortcode::Error => "`error`",
-        };
-
-        write!(f, "{}", val)
-    }
-}

components/rendering/src/shortcode/mod.rs

@@ -3,17 +3,16 @@ use std::collections::HashMap;
 use errors::{Error, Result};
 use utils::templates::{ShortcodeDefinition, ShortcodeFileType};
 
-mod lexer;
+mod parser;
-pub(crate) mod parser;
 
-use parser::{Shortcode, ShortcodeExtractor};
+pub(crate) use parser::{parse_for_shortcodes, Shortcode, SHORTCODE_PLACEHOLDER};
 
 /// Extracts the shortcodes present in the source, check if we know them and errors otherwise
 pub fn extract_shortcodes(
     source: &str,
     definitions: &HashMap<String, ShortcodeDefinition>,
 ) -> Result<(String, Vec<Shortcode>)> {
-    let (out, mut shortcodes) = ShortcodeExtractor::parse(source)?;
+    let (out, mut shortcodes) = parse_for_shortcodes(source)?;
 
     for sc in &mut shortcodes {
         if let Some(def) = definitions.get(&sc.name) {
@@ -58,7 +57,7 @@ pub fn insert_md_shortcodes(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::shortcode::parser::SHORTCODE_PLACEHOLDER;
+    use crate::shortcode::SHORTCODE_PLACEHOLDER;
     use tera::to_value;
 
     #[test]

components/rendering/src/shortcode/parser.rs

@@ -1,11 +1,11 @@
-use std::collections::HashMap;
 use std::ops::Range;
 
-use errors::{Error, Result};
+use errors::{bail, Result};
-use logos::{Lexer, Logos};
+use pest::iterators::Pair;
-use tera::{to_value, Context, Tera, Value};
+use pest::Parser;
-
+use pest_derive::Parser;
-use crate::shortcode::lexer::{Content, InnerShortcode};
+use std::collections::HashMap;
+use tera::{to_value, Context, Map, Tera, Value};
 use utils::templates::ShortcodeFileType;
 
 pub const SHORTCODE_PLACEHOLDER: &str = "||ZOLA_SC_PLACEHOLDER||";
@@ -16,9 +16,9 @@ pub struct Shortcode {
     pub(crate) args: Value,
     pub(crate) span: Range<usize>,
     pub(crate) body: Option<String>,
+    pub(crate) nth: usize,
     // set later down the line, for quick access without needing the definitions
     pub(crate) tera_name: String,
-    pub(crate) nth: usize,
 }
 
 impl Shortcode {
@@ -43,7 +43,8 @@ impl Shortcode {
         new_context.extend(context.clone());
 
         let res = utils::templates::render_template(&tpl_name, tera, new_context, &None)
-            .map_err(|e| errors::Error::chain(format!("Failed to render {} shortcode", name), e))?.replace("\r\n", "\n");
+            .map_err(|e| errors::Error::chain(format!("Failed to render {} shortcode", name), e))?
+            .replace("\r\n", "\n");
 
         Ok(res)
     }
@@ -65,180 +66,145 @@ impl Shortcode {
     }
 }
 
-struct InnerShortcodeParser<'a, 'b> {
+// This include forces recompiling this source file if the grammar file changes.
-    name: String,
+// Uncomment it when doing changes to the .pest file
-    source: &'a str,
+const _GRAMMAR: &str = include_str!("../content.pest");
-    lexer: &'b mut Lexer<'a, InnerShortcode>,
-}
 
-impl<'a, 'b> InnerShortcodeParser<'a, 'b> {
+#[derive(Parser)]
-    fn next_or_eof(&mut self) -> Result<InnerShortcode> {
+#[grammar = "content.pest"]
-        match self.lexer.next() {
+pub struct ContentParser;
-            None => Err(Error::msg(format!(
+
-                "Unexpected end of content while parsing shortcode {}",
+fn replace_string_markers(input: &str) -> String {
-                self.name
+    match input.chars().next().unwrap() {
-            ))),
+        '"' => input.replace('"', ""),
-            Some(t) => Ok(t),
+        '\'' => input.replace('\'', ""),
+        '`' => input.replace('`', ""),
+        _ => unreachable!("How did you even get there"),
     }
 }
 
-    fn expect_token(&mut self, expected_token: InnerShortcode) -> Result<()> {
+fn parse_kwarg_value(pair: Pair<Rule>) -> Value {
-        let token = self.next_or_eof()?;
+    let mut val = None;
-        if token != expected_token {
+    for p in pair.into_inner() {
-            Err(Error::msg(format!(
+        match p.as_rule() {
-                "Unexpected token {} while looking for {} of shortcode {}",
+            Rule::boolean => match p.as_str() {
-                token, expected_token, self.name
+                "true" => val = Some(Value::Bool(true)),
-            )))
+                "false" => val = Some(Value::Bool(false)),
-        } else {
+                _ => unreachable!(),
-            Ok(())
+            },
+            Rule::string => val = Some(Value::String(replace_string_markers(p.as_str()))),
+            Rule::float => {
+                val = Some(to_value(p.as_str().parse::<f64>().unwrap()).unwrap());
+            }
+            Rule::int => {
+                val = Some(to_value(p.as_str().parse::<i64>().unwrap()).unwrap());
+            }
+            Rule::array => {
+                let mut vals = vec![];
+                for p2 in p.into_inner() {
+                    match p2.as_rule() {
+                        Rule::literal => vals.push(parse_kwarg_value(p2)),
+                        _ => unreachable!("Got something other than literal in an array: {:?}", p2),
                     }
                 }
-
+                val = Some(Value::Array(vals));
-    fn parse_array(&mut self) -> Result<Value> {
-        let mut values = Vec::new();
-        loop {
-            values.push(self.parse_value()?);
-            match self.next_or_eof()? {
-                InnerShortcode::Comma => {
-                    continue;
-                }
-                InnerShortcode::CloseBracket => break,
-                t => {
-                    return Err(Error::msg(format!(
-                        "Unexpected token {} while looking for `,` or `]` in shortcode {}",
-                        t, self.name
-                    )));
             }
+            _ => unreachable!("Unknown literal: {:?}", p),
         };
     }
 
-        Ok(Value::Array(values))
+    val.unwrap()
 }
 
-    fn parse_value(&mut self) -> Result<Value> {
+/// Returns (shortcode_name, kwargs)
-        match self.next_or_eof()? {
+fn parse_shortcode_call(pair: Pair<Rule>) -> (String, Value) {
-            InnerShortcode::Bool(b) => Ok(Value::Bool(b)),
+    let mut name = None;
-            InnerShortcode::Str(s) => Ok(Value::String(s)),
+    let mut args = Map::new();
-            InnerShortcode::Integer(i) => Ok(to_value(i).expect("valid i64")),
+
-            InnerShortcode::Float(f) => Ok(to_value(f).expect("valid f64")),
+    for p in pair.into_inner() {
-            InnerShortcode::OpenBracket => self.parse_array(),
+        match p.as_rule() {
-            t => {
+            Rule::ident => {
-                return Err(Error::msg(format!(
+                name = Some(p.as_span().as_str().to_string());
-                    "Unexpected token {} while parsing arguments of shortcode {}",
-                    t, self.name
-                )));
             }
+            Rule::kwarg => {
+                let mut arg_name = None;
+                let mut arg_val = None;
+                for p2 in p.into_inner() {
+                    match p2.as_rule() {
+                        Rule::ident => {
+                            arg_name = Some(p2.as_span().as_str().to_string());
+                        }
+                        Rule::literal => {
+                            arg_val = Some(parse_kwarg_value(p2));
+                        }
+                        _ => unreachable!("Got something unexpected in a kwarg: {:?}", p2),
                     }
                 }
 
-    pub fn parse(
+                args.insert(arg_name.unwrap(), arg_val.unwrap());
-        source: &'a str,
-        lexer: &'b mut Lexer<'a, InnerShortcode>,
-    ) -> Result<(String, Value)> {
-        let mut parser = Self { source, lexer, name: String::new() };
-
-        parser.expect_token(InnerShortcode::Ident)?;
-        parser.name = parser.source[parser.lexer.span()].to_string();
-        parser.expect_token(InnerShortcode::OpenParenthesis)?;
-        let mut arguments = HashMap::new();
-
-        loop {
-            match parser.next_or_eof()? {
-                InnerShortcode::CloseParenthesis => {
-                    break;
             }
-                InnerShortcode::Comma => {
+            _ => unreachable!("Got something unexpected in a shortcode: {:?}", p),
-                    continue;
-                }
-                InnerShortcode::Ident => {
-                    let ident = source[parser.lexer.span()].to_string();
-                    parser.expect_token(InnerShortcode::Equals)?;
-                    let value = parser.parse_value()?;
-                    arguments.insert(ident, value);
-                }
-                t => {
-                    return Err(Error::msg(format!(
-                        "Unexpected token {} while parsing arguments of shortcode {}",
-                        t, parser.name
-                    )));
         }
     }
+    (name.unwrap(), Value::Object(args))
 }
 
-        Ok((parser.name, to_value(arguments).unwrap()))
+pub fn parse_for_shortcodes(content: &str) -> Result<(String, Vec<Shortcode>)> {
-    }
-}
-
-pub(crate) struct ShortcodeExtractor<'a> {
-    source: &'a str,
-    output: String,
-    last_lex_end: usize,
-    lexer: Lexer<'a, Content>,
-}
-
-impl<'a> ShortcodeExtractor<'a> {
-    /// Only called if there was a `{{` or a `{%` in the source input
-    pub fn parse(source: &'a str) -> Result<(String, Vec<Shortcode>)> {
-        let sc = Self {
-            source,
-            output: String::with_capacity(source.len()),
-            last_lex_end: 0,
-            lexer: Content::lexer(source),
-        };
-
-        sc.process()
-    }
-
-    fn expect_token(&mut self, expected_token: Content) -> Result<()> {
-        let token = self.next_or_eof()?;
-        if token != expected_token {
-            Err(Error::msg(format!(
-                "Unexpected token {} while looking for {}",
-                token, expected_token
-            )))
-        } else {
-            Ok(())
-        }
-    }
-
-    fn next_or_eof(&mut self) -> Result<Content> {
-        match self.lexer.next() {
-            None => Err(Error::msg("Unexpected end of content while parsing shortcodes")),
-            Some(t) => Ok(t),
-        }
-    }
-
-    fn parse_until(&mut self, token: Content) -> Result<()> {
-        loop {
-            let tok = self.next_or_eof()?;
-            if tok == token {
-                return Ok(());
-            }
-        }
-    }
-
-    fn process(mut self) -> Result<(String, Vec<Shortcode>)> {
     let mut shortcodes = Vec::new();
     let mut nths = HashMap::new();
+    let mut get_invocation_count = |name: &str| {
+        let nth = nths.entry(String::from(name)).or_insert(0);
+        *nth += 1;
+        *nth
+    };
+    let mut output = String::with_capacity(content.len());
 
-        loop {
+    let mut pairs = match ContentParser::parse(Rule::page, content) {
-            // TODO: some code duplications here but nothing major
+        Ok(p) => p,
-            match self.lexer.next() {
+        Err(e) => {
-                None => {
+            let fancy_e = e.renamed_rules(|rule| match *rule {
-                    // We're done, pushing whatever's left
+                Rule::int => "an integer".to_string(),
-                    self.output.push_str(&self.source[self.last_lex_end..]);
+                Rule::float => "a float".to_string(),
-                    break;
+                Rule::string => "a string".to_string(),
+                Rule::literal => "a literal (int, float, string, bool)".to_string(),
+                Rule::array => "an array".to_string(),
+                Rule::kwarg => "a keyword argument".to_string(),
+                Rule::ident => "an identifier".to_string(),
+                Rule::inline_shortcode => "an inline shortcode".to_string(),
+                Rule::ignored_inline_shortcode => "an ignored inline shortcode".to_string(),
+                Rule::sc_body_start => "the start of a shortcode".to_string(),
+                Rule::ignored_sc_body_start => "the start of an ignored shortcode".to_string(),
+                Rule::text => "some text".to_string(),
+                Rule::EOI => "end of input".to_string(),
+                Rule::double_quoted_string => "double quoted string".to_string(),
+                Rule::single_quoted_string => "single quoted string".to_string(),
+                Rule::backquoted_quoted_string => "backquoted quoted string".to_string(),
+                Rule::boolean => "a boolean (true, false)".to_string(),
+                Rule::all_chars => "a alphanumerical character".to_string(),
+                Rule::kwargs => "a list of keyword arguments".to_string(),
+                Rule::sc_def => "a shortcode definition".to_string(),
+                Rule::shortcode_with_body => "a shortcode with body".to_string(),
+                Rule::ignored_shortcode_with_body => "an ignored shortcode with body".to_string(),
+                Rule::sc_body_end => "{% end %}".to_string(),
+                Rule::ignored_sc_body_end => "{%/* end */%}".to_string(),
+                Rule::text_in_body_sc => "text in a shortcode body".to_string(),
+                Rule::text_in_ignored_body_sc => "text in an ignored shortcode body".to_string(),
+                Rule::content => "some content".to_string(),
+                Rule::page => "a page".to_string(),
+                Rule::WHITESPACE => "whitespace".to_string(),
+            });
+            bail!("{}", fancy_e);
         }
-                Some(Content::InlineShortcodeStart) => {
+    };
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
+
-                    let start = self.output.len();
+    // We have at least a `page` pair
-                    let mut inner_lexer = self.lexer.morph();
+    for p in pairs.next().unwrap().into_inner() {
-                    let (name, args) = InnerShortcodeParser::parse(self.source, &mut inner_lexer)?;
+        match p.as_rule() {
-                    self.lexer = inner_lexer.morph();
+            Rule::text => output.push_str(p.as_span().as_str()),
-                    self.expect_token(Content::InlineShortcodeEnd)?;
+            Rule::inline_shortcode => {
-                    self.output.push_str(SHORTCODE_PLACEHOLDER);
+                let start = output.len();
-                    self.last_lex_end = self.lexer.span().end;
+                let (name, args) = parse_shortcode_call(p);
-                    let nth = *nths.entry(name.to_owned()).and_modify(|e| *e += 1).or_insert(1);
+                let nth = get_invocation_count(&name);
                 shortcodes.push(Shortcode {
                     name,
                     args,
@@ -247,96 +213,156 @@ impl<'a> ShortcodeExtractor<'a> {
                     nth,
                     tera_name: String::new(),
                 });
+                output.push_str(SHORTCODE_PLACEHOLDER);
             }
-                Some(Content::IgnoredInlineShortcodeStart) => {
+            Rule::shortcode_with_body => {
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
+                let start = output.len();
-                    self.output.push_str("{{");
+                let mut inner = p.into_inner();
-                    self.last_lex_end = self.lexer.span().end;
+                // 3 items in inner: call, body, end
-                    self.parse_until(Content::IgnoredInlineShortcodeEnd)?;
+                // we don't care about the closing tag
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
+                let (name, args) = parse_shortcode_call(inner.next().unwrap());
-                    self.output.push_str("}}");
+                let body = inner.next().unwrap().as_span().as_str().trim();
-                    self.last_lex_end = self.lexer.span().end;
+                let nth = get_invocation_count(&name);
-                }
-                Some(Content::ShortcodeWithBodyStart) => {
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
-                    let start = self.output.len();
-                    let mut inner_lexer = self.lexer.morph();
-                    let (name, args) = InnerShortcodeParser::parse(self.source, &mut inner_lexer)?;
-                    self.lexer = inner_lexer.morph();
-                    self.expect_token(Content::ShortcodeWithBodyEnd)?;
-                    let body_start = self.lexer.span().end;
-                    self.parse_until(Content::ShortcodeWithBodyClosing)?;
-                    // We trim the start to avoid newlines that users would have put to make the shortcode pretty in md, eg
-                    // {% hello() %}
-                    // body
-                    // {% end %}
-                    // it's unlikely that the user wanted/expected a newline before or after "body"
-                    let body = self.source[body_start..self.lexer.span().start].trim().to_owned();
-                    self.last_lex_end = self.lexer.span().end;
-                    self.output.push_str(SHORTCODE_PLACEHOLDER);
-                    let nth = *nths.entry(name.to_owned()).and_modify(|e| *e += 1).or_insert(1);
                 shortcodes.push(Shortcode {
                     name,
                     args,
                     span: start..(start + SHORTCODE_PLACEHOLDER.len()),
-                        body: Some(body),
+                    body: Some(body.to_string()),
                     nth,
                     tera_name: String::new(),
                 });
+                output.push_str(SHORTCODE_PLACEHOLDER)
             }
-                Some(Content::IgnoredShortcodeWithBodyStart) => {
+            Rule::ignored_inline_shortcode => {
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
+                output.push_str(
-                    self.output.push_str("{%");
+                    &p.as_span().as_str().replacen("{{/*", "{{", 1).replacen("*/}}", "}}", 1),
-                    self.last_lex_end = self.lexer.span().end;
+                );
-                    self.parse_until(Content::IgnoredShortcodeWithBodyEnd)?;
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
-                    self.output.push_str("%}");
-                    self.last_lex_end = self.lexer.span().end;
-                    self.parse_until(Content::IgnoredShortcodeWithBodyClosing)?;
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
-                    self.output.push_str("{% end %}");
-                    self.last_lex_end = self.lexer.span().end;
             }
-                Some(Content::Error) => {
+            Rule::ignored_shortcode_with_body => {
-                    // Likely just a `*`, `/`, `{`
+                for p2 in p.into_inner() {
-                    self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().end]);
+                    match p2.as_rule() {
-                    self.last_lex_end = self.lexer.span().end;
+                        Rule::ignored_sc_body_start | Rule::ignored_sc_body_end => {
+                            output.push_str(
+                                &p2.as_span()
+                                    .as_str()
+                                    .replacen("{%/*", "{%", 1)
+                                    .replacen("*/%}", "%}", 1),
+                            );
                         }
-                Some(c) => {
+                        Rule::text_in_ignored_body_sc => output.push_str(p2.as_span().as_str()),
-                    return Err(Error::msg(format!(
+                        _ => unreachable!("Got something weird in an ignored shortcode: {:?}", p2),
-                        "Unexpected token {} while parsing shortcodes",
-                        c
-                    )));
                     }
                 }
             }
+            Rule::EOI => (),
+            _ => unreachable!("unexpected page rule: {:?}", p.as_rule()),
+        }
+    }
 
-        Ok((self.output, shortcodes))
+    Ok((output, shortcodes))
-    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    // From maplit
+    macro_rules! assert_lex_rule {
-    macro_rules! hashmap {
+        ($rule: expr, $input: expr) => {
-    (@single $($x:tt)*) => (());
+            let res = ContentParser::parse($rule, $input);
-        (@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*]));
+            println!("{:?}", $input);
-
+            println!("{:#?}", res);
-        ($($key:expr => $value:expr,)+) => { hashmap!($($key => $value),+) };
+            if res.is_err() {
-        ($($key:expr => $value:expr),*) => {
+                println!("{}", res.unwrap_err());
-            {
+                panic!();
-                let _cap = hashmap!(@count $($key),*);
-                let mut _map = ::std::collections::HashMap::with_capacity(_cap);
-                $(
-                    let _ = _map.insert($key, $value);
-                )*
-                _map
             }
+            assert!(res.is_ok());
+            assert_eq!(res.unwrap().last().unwrap().as_span().end(), $input.len());
         };
     }
 
+    #[test]
+    fn lex_text() {
+        let inputs = vec!["Hello world", "HEllo \n world", "Hello 1 2 true false 'hey'"];
+        for i in inputs {
+            assert_lex_rule!(Rule::text, i);
+        }
+    }
+
+    #[test]
+    fn lex_inline_shortcode() {
+        let inputs = vec![
+            "{{ youtube() }}",
+            "{{ youtube(id=1, autoplay=true, url='hey') }}",
+            "{{ youtube(id=1, \nautoplay=true, url='hey', array=[]) }}",
+            "{{ youtube(id=1, \nautoplay=true, url='hey', multi_aray=[[]]) }}",
+        ];
+        for i in inputs {
+            assert_lex_rule!(Rule::inline_shortcode, i);
+        }
+    }
+
+    #[test]
+    fn lex_inline_ignored_shortcode() {
+        let inputs = vec![
+            "{{/* youtube() */}}",
+            "{{/* youtube(id=1, autoplay=true, url='hey') */}}",
+            "{{/* youtube(id=1, \nautoplay=true, \nurl='hey') */}}",
+        ];
+        for i in inputs {
+            assert_lex_rule!(Rule::ignored_inline_shortcode, i);
+        }
+    }
+
+    #[test]
+    fn lex_shortcode_with_body() {
+        let inputs = vec![
+            r#"{% youtube() %}
+            Some text
+            {% end %}"#,
+            r#"{% youtube(id=1,
+            autoplay=true, url='hey') %}
+            Some text
+            {% end %}"#,
+        ];
+        for i in inputs {
+            assert_lex_rule!(Rule::shortcode_with_body, i);
+        }
+    }
+
+    #[test]
+    fn lex_ignored_shortcode_with_body() {
+        let inputs = vec![
+            r#"{%/* youtube() */%}
+            Some text
+            {%/* end */%}"#,
+            r#"{%/* youtube(id=1,
+            autoplay=true, url='hey') */%}
+            Some text
+            {%/* end */%}"#,
+        ];
+        for i in inputs {
+            assert_lex_rule!(Rule::ignored_shortcode_with_body, i);
+        }
+    }
+
+    #[test]
+    fn lex_page() {
+        let inputs = vec![
+            "Some text and a shortcode `{{/* youtube() */}}`",
+            "{{ youtube(id=1, autoplay=true, url='hey') }}",
+            "{{ youtube(id=1, \nautoplay=true, url='hey') }} that's it",
+            r#"
+            This is a test
+            {% hello() %}
+            Body {{ var }}
+            {% end %}
+            "#,
+        ];
+        for i in inputs {
+            assert_lex_rule!(Rule::page, i);
+        }
+    }
+
     #[test]
     fn can_update_ranges() {
         let mut sc = Shortcode {
@@ -355,56 +381,24 @@ mod tests {
         assert_eq!(sc.span, 7..17);
     }
 
-    #[test]
-    fn can_parse_inner_shortcode() {
-        let input = vec![
-            // txt, name, args
-            ("hello() }}", "hello", HashMap::new()),
-            ("hello_lo1() }}", "hello_lo1", HashMap::new()),
-            (
-                " shortcode(name='bob', age=45) }}",
-                "shortcode",
-                hashmap!("name".to_owned() => Value::String("bob".to_owned()), "age".to_owned() => to_value(45).unwrap()),
-            ),
-            (
-                " shortcode(admin=true, age=45.1) }}",
-                "shortcode",
-                hashmap!("admin".to_owned() => Value::Bool(true), "age".to_owned() => to_value(45.1).unwrap()),
-            ),
-            (
-                "with_array(hello=['true', false]) }}",
-                "with_array",
-                hashmap!("hello".to_owned() => Value::Array(vec![Value::String("true".to_owned()), Value::Bool(false)])),
-            ),
-            (
-                "with_array(hello=['true', [true, false]]) }}",
-                "with_array",
-                hashmap!("hello".to_owned() => Value::Array(vec![Value::String("true".to_owned()), Value::Array(vec![Value::Bool(true), Value::Bool(false)])])),
-            ),
-        ];
-
-        for (txt, expected_name, expected_args) in input {
-            let mut lexer = InnerShortcode::lexer(txt);
-            let (name, args) = InnerShortcodeParser::parse(txt, &mut lexer).unwrap();
-            assert_eq!(&name, expected_name);
-            assert_eq!(args, to_value(expected_args).unwrap());
-        }
-    }
-
     #[test]
     fn can_extract_basic_inline_shortcode_with_args() {
-        let (out, shortcodes) = ShortcodeExtractor::parse(
+        let (out, shortcodes) = parse_for_shortcodes(
-            "Inline shortcode: {{ hello(string='hey', int=1, float=2.1, bool=true) }} hey",
+            "Inline shortcode: {{ hello(string='hey', int=1, float=2.1, bool=true, array=[true, false]) }} hey",
         )
         .unwrap();
         assert_eq!(out, format!("Inline shortcode: {} hey", SHORTCODE_PLACEHOLDER));
         assert_eq!(shortcodes.len(), 1);
         assert_eq!(shortcodes[0].name, "hello");
-        assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 4);
+        assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 5);
         assert_eq!(shortcodes[0].args["string"], Value::String("hey".to_string()));
         assert_eq!(shortcodes[0].args["bool"], Value::Bool(true));
         assert_eq!(shortcodes[0].args["int"], to_value(1).unwrap());
         assert_eq!(shortcodes[0].args["float"], to_value(2.1).unwrap());
+        assert_eq!(
+            shortcodes[0].args["array"],
+            Value::Array(vec![Value::Bool(true), Value::Bool(false)])
+        );
         assert_eq!(shortcodes[0].span, 18..(18 + SHORTCODE_PLACEHOLDER.len()));
         assert_eq!(shortcodes[0].nth, 1);
     }
@@ -412,22 +406,26 @@ mod tests {
     #[test]
     fn can_unignore_ignored_inline_shortcode() {
         let (out, shortcodes) =
-            ShortcodeExtractor::parse("Hello World {{/* youtube() */}} hey").unwrap();
+            parse_for_shortcodes("Hello World {{/* youtube() */}} hey").unwrap();
         assert_eq!(out, "Hello World {{ youtube() }} hey");
         assert_eq!(shortcodes.len(), 0);
     }
 
     #[test]
     fn can_extract_shortcode_with_body() {
-        let (out, shortcodes) = ShortcodeExtractor::parse(
+        let (out, shortcodes) = parse_for_shortcodes(
-            "Body shortcode\n {% quote(author='Bobby') %}DROP TABLES;{% end %} \n hey",
+            "Body shortcode\n {% quote(author='Bobby', array=[[true]]) %}DROP TABLES;{% end %} \n hey",
         )
         .unwrap();
         assert_eq!(out, format!("Body shortcode\n {} \n hey", SHORTCODE_PLACEHOLDER));
         assert_eq!(shortcodes.len(), 1);
         assert_eq!(shortcodes[0].name, "quote");
-        assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 1);
+        assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 2);
         assert_eq!(shortcodes[0].args["author"], Value::String("Bobby".to_string()));
+        assert_eq!(
+            shortcodes[0].args["array"],
+            Value::Array(vec![Value::Array(vec![Value::Bool(true)])])
+        );
         assert_eq!(shortcodes[0].body, Some("DROP TABLES;".to_owned()));
         assert_eq!(shortcodes[0].span, 16..(16 + SHORTCODE_PLACEHOLDER.len()));
         assert_eq!(shortcodes[0].nth, 1);
@@ -436,7 +434,7 @@ mod tests {
     #[test]
     fn can_unignore_ignored_shortcode_with_body() {
         let (out, shortcodes) =
-            ShortcodeExtractor::parse("Hello World {%/* youtube() */%} Somebody {%/* end */%} hey")
+            parse_for_shortcodes("Hello World {%/* youtube() */%} Somebody {%/* end */%} hey")
                 .unwrap();
         assert_eq!(out, "Hello World {% youtube() %} Somebody {% end %} hey");
         assert_eq!(shortcodes.len(), 0);
@@ -444,7 +442,7 @@ mod tests {
 
     #[test]
     fn can_extract_multiple_shortcodes_and_increment_nth() {
-        let (out, shortcodes) = ShortcodeExtractor::parse(
+        let (out, shortcodes) = parse_for_shortcodes(
             "Hello World {% youtube() %} Somebody {% end %} {{ hello() }}\n {{hello()}}",
         )
         .unwrap();
@@ -463,7 +461,7 @@ mod tests {
 
     #[test]
     fn can_handle_multiple_shortcodes() {
-        let (_, shortcodes) = ShortcodeExtractor::parse(
+        let (_, shortcodes) = parse_for_shortcodes(
             r#"
         {{ youtube(id="ub36ffWAqgQ") }}
         {{ youtube(id="ub36ffWAqgQ", autoplay=true) }}
@@ -474,44 +472,4 @@ mod tests {
         .unwrap();
         assert_eq!(shortcodes.len(), 5);
     }
-
-    #[test]
-    fn can_provide_good_error_messages() {
-        let tests = vec![
-            ("{{ hey()", "Unexpected end of content while parsing shortcodes"),
-            ("{% hey()", "Unexpected end of content while parsing shortcodes"),
-            ("{{ hey(=", "Unexpected token `=` while parsing arguments of shortcode hey"),
-            ("{{ hey(ho==", "Unexpected token `=` while parsing arguments of shortcode hey"),
-            ("{{ hey(ho=1h", "Unexpected end of content while parsing shortcode hey"),
-            ("{{ hey)", "Unexpected token `)` while looking for `(` of shortcode hey"),
-            ("{{ hey(ho=(", "Unexpected token `(` while parsing arguments of shortcode hey"),
-            ("hello }}", "Unexpected token `}}` while parsing shortcodes"),
-            ("hello %}", "Unexpected token `%}` while parsing shortcodes"),
-            ("hello {% end %}", "Unexpected token `{% end %}` while parsing shortcodes"),
-        ];
-
-        for (t, expected) in tests {
-            println!("Testing: {}", t);
-            let res = ShortcodeExtractor::parse(t);
-            assert!(res.is_err());
-            let err = res.unwrap_err();
-            assert_eq!(expected, err.to_string());
-        }
-    }
-
-    #[test]
-    fn can_parse_ok_with_problematic_chars() {
-        let inputs = vec![
-            ("* a\n* b", "* a\n* b"),
-            ("a regex {a,b}", "a regex {a,b}"),
-            ("a slash //", "a slash //"),
-            ("a slash */", "a slash */"),
-            ("%a percent%", "%a percent%"),
-        ];
-
-        for (input, expected) in inputs {
-            let (out, _) = ShortcodeExtractor::parse(input).unwrap();
-            assert_eq!(out, expected);
-        }
-    }
 }

docs/content/documentation/deployment/github-pages.md

@@ -18,7 +18,7 @@ We can use any continuous integration (CI) server to build and deploy our site.
 
 In either case, it seems to work best if you use `git submodule` to include your theme, e.g.:
 
-```shell
+```bash
 git submodule add https://github.com/getzola/after-dark.git themes/after-dark
 ```
 
@@ -110,7 +110,7 @@ Depending on how you added your theme, Travis may not know how to access
 it. The best way to ensure that it will have full access to the theme is to use git
 submodules. When doing this, ensure that you are using the `https` version of the URL.
 
-```shell
+```bash
 $ git submodule add {THEME_URL} themes/{THEME_NAME}
 ```
 

docs/content/documentation/deployment/gitlab-pages.md

@@ -21,12 +21,12 @@ This guide assumes that your Zola project is located in the root of your reposit
 Depending on how you added your theme, your repository may not contain it. The best way to ensure that the theme will
 be added is to use submodules. When doing this, ensure that you are using the `https` version of the URL.
 
-```shell
+```bash
 $ git submodule add {THEME_URL} themes/{THEME_NAME}
 ```
 
 For example, this could look like:
-```shell
+```bash
 $ git submodule add https://github.com/getzola/hyde.git themes/hyde
 ```
 

docs/content/documentation/getting-started/configuration.md

@@ -237,7 +237,7 @@ Zola uses the Sublime Text themes, making it very easy to add more.
 If you want a theme not listed above, please open an issue or a pull request on the [Zola repo](https://github.com/getzola/zola).
 
 Alternatively you can use the `extra_syntaxes_and_themes` configuration option to load your own custom themes from a .tmTheme file.
-See [Syntax Highlighting](@/syntax-highlighting.md) for more details.
+See [Syntax Highlighting](@/documentation/content/syntax-highlighting.md) for more details.
 
 ## Slugification strategies
 

docs/content/documentation/themes/installing-and-using-themes.md

@@ -35,7 +35,7 @@ in the configuration file is `simple-blog`. Also make sure to place the variable
 Any file from the theme can be overridden by creating a file with the same path and name in your `templates` or `static`
 directory. Here are a few examples of that, assuming that the theme name is `simple-blog`:
 
-```plain
+```
 templates/pages/post.html -> replace themes/simple-blog/templates/pages/post.html
 templates/macros.html -> replace themes/simple-blog/templates/macros.html
 static/js/site.js -> replace themes/simple-blog/static/js/site.js