Mirrors/zola
2
0
Fork 0
mirror of https://github.com/getzola/zola synced 2024-12-13 13:52:28 +00:00
Code Issues Projects Releases Packages Wiki Activity

Back to pest (#1665)

Browse source
This commit is contained in:
Vincent Prouillet 2021-11-23 22:58:51 +01:00 committed by GitHub
parent 48e4fa0ce5
commit 19125e8dd2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 314 additions and 496 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
Cargo.lock generated
View file

@ -107,12 +107,6 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "beef"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bed554bd50246729a1ec158d08aa3235d1b69d94ad120ebe187e28894787e736"
[[package]] [[package]]
name = "bincode" name = "bincode"
version = "1.3.3" version = "1.3.3"
@ -1408,30 +1402,6 @@ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
] ]
[[package]]
name = "logos"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "427e2abca5be13136da9afdbf874e6b34ad9001dd70f2b103b083a85daa7b345"
dependencies = [
"logos-derive",
]
[[package]]
name = "logos-derive"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56a7d287fd2ac3f75b11f19a1c8a874a7d55744bd91f7a1b3e7cf87d4343c36d"
dependencies = [
"beef",
"fnv",
"proc-macro2",
"quote",
"regex-syntax 0.6.25",
"syn",
"utf8-ranges",
]
[[package]] [[package]]
name = "mac" name = "mac"
version = "0.1.1" version = "0.1.1"
@ -2366,7 +2336,8 @@ dependencies = [
"gh-emoji", "gh-emoji",
"lazy_static", "lazy_static",
"link_checker", "link_checker",
"logos", "pest",
"pest_derive",
"pulldown-cmark", "pulldown-cmark",
"regex", "regex",
"serde", "serde",

3
components/rendering/Cargo.toml
View file

@ -11,7 +11,8 @@ syntect = "4"
pulldown-cmark = { version = "0.8", default-features = false } pulldown-cmark = { version = "0.8", default-features = false }
serde = "1" serde = "1"
serde_derive = "1" serde_derive = "1"
logos = "0.12" pest = "2"
pest_derive = "2"
regex = "1" regex = "1"
lazy_static = "1" lazy_static = "1"
gh-emoji = "1.0" gh-emoji = "1.0"

4
components/rendering/src/content.pest
View file

@ -25,7 +25,7 @@ string = @{
boolean = { "true" | "false" } boolean = { "true" | "false" }
literal = { boolean | string | float | int } literal = { boolean | string | float | int | array }
array = { "[" ~ (literal ~ ",")* ~ literal? ~ "]"} array = { "[" ~ (literal ~ ",")* ~ literal? ~ "]"}
/// Idents /// Idents
@ -40,7 +40,7 @@ ident = @{
// shortcode is abbreviated to sc to keep things short // shortcode is abbreviated to sc to keep things short
kwarg = { ident ~ "=" ~ (literal | array) } kwarg = { ident ~ "=" ~ literal }
kwargs = _{ kwarg ~ ("," ~ kwarg )* } kwargs = _{ kwarg ~ ("," ~ kwarg )* }
sc_def = _{ ident ~ "(" ~ kwargs* ~ ")" } sc_def = _{ ident ~ "(" ~ kwargs* ~ ")" }

4
components/rendering/src/markdown.rs
View file

@ -11,7 +11,7 @@ use utils::vec::InsertMany;
use self::cmark::{Event, LinkType, Options, Parser, Tag}; use self::cmark::{Event, LinkType, Options, Parser, Tag};
use crate::codeblock::{CodeBlock, FenceSettings}; use crate::codeblock::{CodeBlock, FenceSettings};
use crate::shortcode::parser::{Shortcode, SHORTCODE_PLACEHOLDER}; use crate::shortcode::{Shortcode, SHORTCODE_PLACEHOLDER};
const CONTINUE_READING: &str = "<span id=\"continue-reading\"></span>"; const CONTINUE_READING: &str = "<span id=\"continue-reading\"></span>";
const ANCHOR_LINK_TEMPLATE: &str = "anchor-link.html"; const ANCHOR_LINK_TEMPLATE: &str = "anchor-link.html";
@ -211,6 +211,7 @@ pub fn markdown_to_html(
} }
let shortcode = next_shortcode.take().unwrap(); let shortcode = next_shortcode.take().unwrap();
match shortcode.render(&context.tera, &context.tera_context) { match shortcode.render(&context.tera, &context.tera_context) {
Ok(s) => { Ok(s) => {
events.push(Event::Html(s.into())); events.push(Event::Html(s.into()));
@ -317,7 +318,6 @@ pub fn markdown_to_html(
}); });
} }
Event::Html(text) => { Event::Html(text) => {
println!("Got text: {:?}", text);
if text.contains("<!-- more -->") { if text.contains("<!-- more -->") {
has_summary = true; has_summary = true;
events.push(Event::Html(CONTINUE_READING.into())); events.push(Event::Html(CONTINUE_READING.into()));

111
components/rendering/src/shortcode/lexer.rs
View file

@ -1,111 +0,0 @@
use logos::Logos;
use std::fmt::Formatter;
fn replace_string_markers(input: &str, marker: char) -> String {
input.replace(marker, "")
}
#[derive(Logos, Debug, PartialEq, Clone)]
pub enum Content {
#[token("{{")]
InlineShortcodeStart,
#[token("{{/*")]
IgnoredInlineShortcodeStart,
#[token("{%")]
ShortcodeWithBodyStart,
#[token("{%/*")]
IgnoredShortcodeWithBodyStart,
#[token("}}")]
InlineShortcodeEnd,
#[token("*/}}")]
IgnoredInlineShortcodeEnd,
#[token("%}")]
ShortcodeWithBodyEnd,
#[token("*/%}")]
IgnoredShortcodeWithBodyEnd,
#[token("{% end %}")]
ShortcodeWithBodyClosing,
#[token("{%/* end */%}")]
IgnoredShortcodeWithBodyClosing,
#[regex(r"[^{\*%\}]+", logos::skip)]
#[error]
Error,
}
impl std::fmt::Display for Content {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let val = match self {
Content::InlineShortcodeStart => "`{{`",
Content::IgnoredInlineShortcodeStart => "`{{/*`",
Content::ShortcodeWithBodyStart => "`{%`",
Content::IgnoredShortcodeWithBodyStart => "`{%/*",
Content::InlineShortcodeEnd => "`}}`",
Content::IgnoredInlineShortcodeEnd => "`*/}}`",
Content::ShortcodeWithBodyEnd => "`%}`",
Content::IgnoredShortcodeWithBodyEnd => "`*/%}`",
Content::ShortcodeWithBodyClosing => "`{% end %}`",
Content::IgnoredShortcodeWithBodyClosing => "`{%/* end */%}`",
Content::Error => "`error`",
};
write!(f, "{}", val)
}
}
#[derive(Logos, Debug, PartialEq, Clone)]
pub enum InnerShortcode {
#[token("(")]
OpenParenthesis,
#[token(")")]
CloseParenthesis,
#[token("[")]
OpenBracket,
#[token("]")]
CloseBracket,
#[token(",")]
Comma,
#[token("=")]
Equals,
#[regex("[a-zA-Z][a-zA-Z0-9_]*")]
Ident,
#[regex("-?[0-9]+", |lex| lex.slice().parse())]
Integer(i64),
#[regex("-?[0-9]+\\.[0-9]+", |lex| lex.slice().parse())]
Float(f64),
#[token("true", |_| true)]
#[token("True", |_| true)]
#[token("false", |_| false)]
#[token("False", |_| false)]
Bool(bool),
#[regex(r#"'([^'\\]*(\\.[^'\\]*)*)'"#, |lex| replace_string_markers(lex.slice(), '\''))]
#[regex(r#""([^"\\]*(\\.[^"\\]*)*)""#, |lex| replace_string_markers(lex.slice(), '"'))]
#[regex(r#"`([^`\\]*(\\.[^`\\]*)*)`"#, |lex| replace_string_markers(lex.slice(), '`'))]
Str(String),
#[regex(r"[ \t\n\f]+", logos::skip)]
#[error]
Error,
}
impl std::fmt::Display for InnerShortcode {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let val = match self {
InnerShortcode::OpenParenthesis => "`(`",
InnerShortcode::CloseParenthesis => "`)`",
InnerShortcode::OpenBracket => "`[`",
InnerShortcode::CloseBracket => "`]`",
InnerShortcode::Comma => "`,`",
InnerShortcode::Equals => "`=`",
InnerShortcode::Ident => "`identifier`",
InnerShortcode::Integer(_) => "`integer`",
InnerShortcode::Float(_) => "`float`",
InnerShortcode::Bool(_) => "`boolean`",
InnerShortcode::Str(_) => "`string`",
InnerShortcode::Error => "`error`",
};
write!(f, "{}", val)
}
}

9
components/rendering/src/shortcode/mod.rs
View file

@ -3,17 +3,16 @@ use std::collections::HashMap;
use errors::{Error, Result}; use errors::{Error, Result};
use utils::templates::{ShortcodeDefinition, ShortcodeFileType}; use utils::templates::{ShortcodeDefinition, ShortcodeFileType};
mod lexer; mod parser;
pub(crate) mod parser;
use parser::{Shortcode, ShortcodeExtractor}; pub(crate) use parser::{parse_for_shortcodes, Shortcode, SHORTCODE_PLACEHOLDER};
/// Extracts the shortcodes present in the source, check if we know them and errors otherwise /// Extracts the shortcodes present in the source, check if we know them and errors otherwise
pub fn extract_shortcodes( pub fn extract_shortcodes(
source: &str, source: &str,
definitions: &HashMap<String, ShortcodeDefinition>, definitions: &HashMap<String, ShortcodeDefinition>,
) -> Result<(String, Vec<Shortcode>)> { ) -> Result<(String, Vec<Shortcode>)> {
let (out, mut shortcodes) = ShortcodeExtractor::parse(source)?; let (out, mut shortcodes) = parse_for_shortcodes(source)?;
for sc in &mut shortcodes { for sc in &mut shortcodes {
if let Some(def) = definitions.get(&sc.name) { if let Some(def) = definitions.get(&sc.name) {
@ -58,7 +57,7 @@ pub fn insert_md_shortcodes(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::shortcode::parser::SHORTCODE_PLACEHOLDER; use crate::shortcode::SHORTCODE_PLACEHOLDER;
use tera::to_value; use tera::to_value;
#[test] #[test]

634
components/rendering/src/shortcode/parser.rs
View file

@ -1,11 +1,11 @@
use std::collections::HashMap;
use std::ops::Range; use std::ops::Range;
use errors::{Error, Result}; use errors::{bail, Result};
use logos::{Lexer, Logos}; use pest::iterators::Pair;
use tera::{to_value, Context, Tera, Value}; use pest::Parser;
use pest_derive::Parser;
use crate::shortcode::lexer::{Content, InnerShortcode}; use std::collections::HashMap;
use tera::{to_value, Context, Map, Tera, Value};
use utils::templates::ShortcodeFileType; use utils::templates::ShortcodeFileType;
pub const SHORTCODE_PLACEHOLDER: &str = "||ZOLA_SC_PLACEHOLDER||"; pub const SHORTCODE_PLACEHOLDER: &str = "||ZOLA_SC_PLACEHOLDER||";
@ -16,9 +16,9 @@ pub struct Shortcode {
pub(crate) args: Value, pub(crate) args: Value,
pub(crate) span: Range<usize>, pub(crate) span: Range<usize>,
pub(crate) body: Option<String>, pub(crate) body: Option<String>,
pub(crate) nth: usize,
// set later down the line, for quick access without needing the definitions // set later down the line, for quick access without needing the definitions
pub(crate) tera_name: String, pub(crate) tera_name: String,
pub(crate) nth: usize,
} }
impl Shortcode { impl Shortcode {
@ -43,7 +43,8 @@ impl Shortcode {
new_context.extend(context.clone()); new_context.extend(context.clone());
let res = utils::templates::render_template(&tpl_name, tera, new_context, &None) let res = utils::templates::render_template(&tpl_name, tera, new_context, &None)
.map_err(|e| errors::Error::chain(format!("Failed to render {} shortcode", name), e))?.replace("\r\n", "\n"); .map_err(|e| errors::Error::chain(format!("Failed to render {} shortcode", name), e))?
.replace("\r\n", "\n");
Ok(res) Ok(res)
} }
@ -65,278 +66,303 @@ impl Shortcode {
} }
} }
struct InnerShortcodeParser<'a, 'b> { // This include forces recompiling this source file if the grammar file changes.
name: String, // Uncomment it when doing changes to the .pest file
source: &'a str, const _GRAMMAR: &str = include_str!("../content.pest");
lexer: &'b mut Lexer<'a, InnerShortcode>,
#[derive(Parser)]
#[grammar = "content.pest"]
pub struct ContentParser;
fn replace_string_markers(input: &str) -> String {
match input.chars().next().unwrap() {
'"' => input.replace('"', ""),
'\'' => input.replace('\'', ""),
'`' => input.replace('`', ""),
_ => unreachable!("How did you even get there"),
}
} }
impl<'a, 'b> InnerShortcodeParser<'a, 'b> { fn parse_kwarg_value(pair: Pair<Rule>) -> Value {
fn next_or_eof(&mut self) -> Result<InnerShortcode> { let mut val = None;
match self.lexer.next() { for p in pair.into_inner() {
None => Err(Error::msg(format!( match p.as_rule() {
"Unexpected end of content while parsing shortcode {}", Rule::boolean => match p.as_str() {
self.name "true" => val = Some(Value::Bool(true)),
))), "false" => val = Some(Value::Bool(false)),
Some(t) => Ok(t), _ => unreachable!(),
} },
} Rule::string => val = Some(Value::String(replace_string_markers(p.as_str()))),
Rule::float => {
fn expect_token(&mut self, expected_token: InnerShortcode) -> Result<()> { val = Some(to_value(p.as_str().parse::<f64>().unwrap()).unwrap());
let token = self.next_or_eof()?;
if token != expected_token {
Err(Error::msg(format!(
"Unexpected token {} while looking for {} of shortcode {}",
token, expected_token, self.name
)))
} else {
Ok(())
}
}
fn parse_array(&mut self) -> Result<Value> {
let mut values = Vec::new();
loop {
values.push(self.parse_value()?);
match self.next_or_eof()? {
InnerShortcode::Comma => {
continue;
}
InnerShortcode::CloseBracket => break,
t => {
return Err(Error::msg(format!(
"Unexpected token {} while looking for `,` or `]` in shortcode {}",
t, self.name
)));
}
};
}
Ok(Value::Array(values))
}
fn parse_value(&mut self) -> Result<Value> {
match self.next_or_eof()? {
InnerShortcode::Bool(b) => Ok(Value::Bool(b)),
InnerShortcode::Str(s) => Ok(Value::String(s)),
InnerShortcode::Integer(i) => Ok(to_value(i).expect("valid i64")),
InnerShortcode::Float(f) => Ok(to_value(f).expect("valid f64")),
InnerShortcode::OpenBracket => self.parse_array(),
t => {
return Err(Error::msg(format!(
"Unexpected token {} while parsing arguments of shortcode {}",
t, self.name
)));
} }
} Rule::int => {
} val = Some(to_value(p.as_str().parse::<i64>().unwrap()).unwrap());
pub fn parse(
source: &'a str,
lexer: &'b mut Lexer<'a, InnerShortcode>,
) -> Result<(String, Value)> {
let mut parser = Self { source, lexer, name: String::new() };
parser.expect_token(InnerShortcode::Ident)?;
parser.name = parser.source[parser.lexer.span()].to_string();
parser.expect_token(InnerShortcode::OpenParenthesis)?;
let mut arguments = HashMap::new();
loop {
match parser.next_or_eof()? {
InnerShortcode::CloseParenthesis => {
break;
}
InnerShortcode::Comma => {
continue;
}
InnerShortcode::Ident => {
let ident = source[parser.lexer.span()].to_string();
parser.expect_token(InnerShortcode::Equals)?;
let value = parser.parse_value()?;
arguments.insert(ident, value);
}
t => {
return Err(Error::msg(format!(
"Unexpected token {} while parsing arguments of shortcode {}",
t, parser.name
)));
}
} }
} Rule::array => {
let mut vals = vec![];
Ok((parser.name, to_value(arguments).unwrap())) for p2 in p.into_inner() {
} match p2.as_rule() {
} Rule::literal => vals.push(parse_kwarg_value(p2)),
_ => unreachable!("Got something other than literal in an array: {:?}", p2),
pub(crate) struct ShortcodeExtractor<'a> { }
source: &'a str, }
output: String, val = Some(Value::Array(vals));
last_lex_end: usize, }
lexer: Lexer<'a, Content>, _ => unreachable!("Unknown literal: {:?}", p),
}
impl<'a> ShortcodeExtractor<'a> {
/// Only called if there was a `{{` or a `{%` in the source input
pub fn parse(source: &'a str) -> Result<(String, Vec<Shortcode>)> {
let sc = Self {
source,
output: String::with_capacity(source.len()),
last_lex_end: 0,
lexer: Content::lexer(source),
}; };
sc.process()
} }
fn expect_token(&mut self, expected_token: Content) -> Result<()> { val.unwrap()
let token = self.next_or_eof()?; }
if token != expected_token {
Err(Error::msg(format!(
"Unexpected token {} while looking for {}",
token, expected_token
)))
} else {
Ok(())
}
}
fn next_or_eof(&mut self) -> Result<Content> { /// Returns (shortcode_name, kwargs)
match self.lexer.next() { fn parse_shortcode_call(pair: Pair<Rule>) -> (String, Value) {
None => Err(Error::msg("Unexpected end of content while parsing shortcodes")), let mut name = None;
Some(t) => Ok(t), let mut args = Map::new();
}
}
fn parse_until(&mut self, token: Content) -> Result<()> { for p in pair.into_inner() {
loop { match p.as_rule() {
let tok = self.next_or_eof()?; Rule::ident => {
if tok == token { name = Some(p.as_span().as_str().to_string());
return Ok(());
} }
Rule::kwarg => {
let mut arg_name = None;
let mut arg_val = None;
for p2 in p.into_inner() {
match p2.as_rule() {
Rule::ident => {
arg_name = Some(p2.as_span().as_str().to_string());
}
Rule::literal => {
arg_val = Some(parse_kwarg_value(p2));
}
_ => unreachable!("Got something unexpected in a kwarg: {:?}", p2),
}
}
args.insert(arg_name.unwrap(), arg_val.unwrap());
}
_ => unreachable!("Got something unexpected in a shortcode: {:?}", p),
} }
} }
(name.unwrap(), Value::Object(args))
}
fn process(mut self) -> Result<(String, Vec<Shortcode>)> { pub fn parse_for_shortcodes(content: &str) -> Result<(String, Vec<Shortcode>)> {
let mut shortcodes = Vec::new(); let mut shortcodes = Vec::new();
let mut nths = HashMap::new(); let mut nths = HashMap::new();
let mut get_invocation_count = |name: &str| {
let nth = nths.entry(String::from(name)).or_insert(0);
*nth += 1;
*nth
};
let mut output = String::with_capacity(content.len());
loop { let mut pairs = match ContentParser::parse(Rule::page, content) {
// TODO: some code duplications here but nothing major Ok(p) => p,
match self.lexer.next() { Err(e) => {
None => { let fancy_e = e.renamed_rules(|rule| match *rule {
// We're done, pushing whatever's left Rule::int => "an integer".to_string(),
self.output.push_str(&self.source[self.last_lex_end..]); Rule::float => "a float".to_string(),
break; Rule::string => "a string".to_string(),
} Rule::literal => "a literal (int, float, string, bool)".to_string(),
Some(Content::InlineShortcodeStart) => { Rule::array => "an array".to_string(),
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]); Rule::kwarg => "a keyword argument".to_string(),
let start = self.output.len(); Rule::ident => "an identifier".to_string(),
let mut inner_lexer = self.lexer.morph(); Rule::inline_shortcode => "an inline shortcode".to_string(),
let (name, args) = InnerShortcodeParser::parse(self.source, &mut inner_lexer)?; Rule::ignored_inline_shortcode => "an ignored inline shortcode".to_string(),
self.lexer = inner_lexer.morph(); Rule::sc_body_start => "the start of a shortcode".to_string(),
self.expect_token(Content::InlineShortcodeEnd)?; Rule::ignored_sc_body_start => "the start of an ignored shortcode".to_string(),
self.output.push_str(SHORTCODE_PLACEHOLDER); Rule::text => "some text".to_string(),
self.last_lex_end = self.lexer.span().end; Rule::EOI => "end of input".to_string(),
let nth = *nths.entry(name.to_owned()).and_modify(|e| *e += 1).or_insert(1); Rule::double_quoted_string => "double quoted string".to_string(),
shortcodes.push(Shortcode { Rule::single_quoted_string => "single quoted string".to_string(),
name, Rule::backquoted_quoted_string => "backquoted quoted string".to_string(),
args, Rule::boolean => "a boolean (true, false)".to_string(),
span: start..(start + SHORTCODE_PLACEHOLDER.len()), Rule::all_chars => "a alphanumerical character".to_string(),
body: None, Rule::kwargs => "a list of keyword arguments".to_string(),
nth, Rule::sc_def => "a shortcode definition".to_string(),
tera_name: String::new(), Rule::shortcode_with_body => "a shortcode with body".to_string(),
}); Rule::ignored_shortcode_with_body => "an ignored shortcode with body".to_string(),
} Rule::sc_body_end => "{% end %}".to_string(),
Some(Content::IgnoredInlineShortcodeStart) => { Rule::ignored_sc_body_end => "{%/* end */%}".to_string(),
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]); Rule::text_in_body_sc => "text in a shortcode body".to_string(),
self.output.push_str("{{"); Rule::text_in_ignored_body_sc => "text in an ignored shortcode body".to_string(),
self.last_lex_end = self.lexer.span().end; Rule::content => "some content".to_string(),
self.parse_until(Content::IgnoredInlineShortcodeEnd)?; Rule::page => "a page".to_string(),
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]); Rule::WHITESPACE => "whitespace".to_string(),
self.output.push_str("}}"); });
self.last_lex_end = self.lexer.span().end; bail!("{}", fancy_e);
} }
Some(Content::ShortcodeWithBodyStart) => { };
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]);
let start = self.output.len(); // We have at least a `page` pair
let mut inner_lexer = self.lexer.morph(); for p in pairs.next().unwrap().into_inner() {
let (name, args) = InnerShortcodeParser::parse(self.source, &mut inner_lexer)?; match p.as_rule() {
self.lexer = inner_lexer.morph(); Rule::text => output.push_str(p.as_span().as_str()),
self.expect_token(Content::ShortcodeWithBodyEnd)?; Rule::inline_shortcode => {
let body_start = self.lexer.span().end; let start = output.len();
self.parse_until(Content::ShortcodeWithBodyClosing)?; let (name, args) = parse_shortcode_call(p);
// We trim the start to avoid newlines that users would have put to make the shortcode pretty in md, eg let nth = get_invocation_count(&name);
// {% hello() %} shortcodes.push(Shortcode {
// body name,
// {% end %} args,
// it's unlikely that the user wanted/expected a newline before or after "body" span: start..(start + SHORTCODE_PLACEHOLDER.len()),
let body = self.source[body_start..self.lexer.span().start].trim().to_owned(); body: None,
self.last_lex_end = self.lexer.span().end; nth,
self.output.push_str(SHORTCODE_PLACEHOLDER); tera_name: String::new(),
let nth = *nths.entry(name.to_owned()).and_modify(|e| *e += 1).or_insert(1); });
shortcodes.push(Shortcode { output.push_str(SHORTCODE_PLACEHOLDER);
name, }
args, Rule::shortcode_with_body => {
span: start..(start + SHORTCODE_PLACEHOLDER.len()), let start = output.len();
body: Some(body), let mut inner = p.into_inner();
nth, // 3 items in inner: call, body, end
tera_name: String::new(), // we don't care about the closing tag
}); let (name, args) = parse_shortcode_call(inner.next().unwrap());
} let body = inner.next().unwrap().as_span().as_str().trim();
Some(Content::IgnoredShortcodeWithBodyStart) => { let nth = get_invocation_count(&name);
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]); shortcodes.push(Shortcode {
self.output.push_str("{%"); name,
self.last_lex_end = self.lexer.span().end; args,
self.parse_until(Content::IgnoredShortcodeWithBodyEnd)?; span: start..(start + SHORTCODE_PLACEHOLDER.len()),
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]); body: Some(body.to_string()),
self.output.push_str("%}"); nth,
self.last_lex_end = self.lexer.span().end; tera_name: String::new(),
self.parse_until(Content::IgnoredShortcodeWithBodyClosing)?; });
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().start]); output.push_str(SHORTCODE_PLACEHOLDER)
self.output.push_str("{% end %}"); }
self.last_lex_end = self.lexer.span().end; Rule::ignored_inline_shortcode => {
} output.push_str(
Some(Content::Error) => { &p.as_span().as_str().replacen("{{/*", "{{", 1).replacen("*/}}", "}}", 1),
// Likely just a `*`, `/`, `{` );
self.output.push_str(&self.source[self.last_lex_end..self.lexer.span().end]); }
self.last_lex_end = self.lexer.span().end; Rule::ignored_shortcode_with_body => {
} for p2 in p.into_inner() {
Some(c) => { match p2.as_rule() {
return Err(Error::msg(format!( Rule::ignored_sc_body_start | Rule::ignored_sc_body_end => {
"Unexpected token {} while parsing shortcodes", output.push_str(
c &p2.as_span()
))); .as_str()
.replacen("{%/*", "{%", 1)
.replacen("*/%}", "%}", 1),
);
}
Rule::text_in_ignored_body_sc => output.push_str(p2.as_span().as_str()),
_ => unreachable!("Got something weird in an ignored shortcode: {:?}", p2),
}
} }
} }
Rule::EOI => (),
_ => unreachable!("unexpected page rule: {:?}", p.as_rule()),
} }
Ok((self.output, shortcodes))
} }
Ok((output, shortcodes))
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
// From maplit macro_rules! assert_lex_rule {
macro_rules! hashmap { ($rule: expr, $input: expr) => {
(@single $($x:tt)*) => (()); let res = ContentParser::parse($rule, $input);
(@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*])); println!("{:?}", $input);
println!("{:#?}", res);
($($key:expr => $value:expr,)+) => { hashmap!($($key => $value),+) }; if res.is_err() {
($($key:expr => $value:expr),*) => { println!("{}", res.unwrap_err());
{ panic!();
let _cap = hashmap!(@count $($key),*);
let mut _map = ::std::collections::HashMap::with_capacity(_cap);
$(
let _ = _map.insert($key, $value);
)*
_map
} }
assert!(res.is_ok());
assert_eq!(res.unwrap().last().unwrap().as_span().end(), $input.len());
}; };
} }
#[test]
fn lex_text() {
let inputs = vec!["Hello world", "HEllo \n world", "Hello 1 2 true false 'hey'"];
for i in inputs {
assert_lex_rule!(Rule::text, i);
}
}
#[test]
fn lex_inline_shortcode() {
let inputs = vec![
"{{ youtube() }}",
"{{ youtube(id=1, autoplay=true, url='hey') }}",
"{{ youtube(id=1, \nautoplay=true, url='hey', array=[]) }}",
"{{ youtube(id=1, \nautoplay=true, url='hey', multi_aray=[[]]) }}",
];
for i in inputs {
assert_lex_rule!(Rule::inline_shortcode, i);
}
}
#[test]
fn lex_inline_ignored_shortcode() {
let inputs = vec![
"{{/* youtube() */}}",
"{{/* youtube(id=1, autoplay=true, url='hey') */}}",
"{{/* youtube(id=1, \nautoplay=true, \nurl='hey') */}}",
];
for i in inputs {
assert_lex_rule!(Rule::ignored_inline_shortcode, i);
}
}
#[test]
fn lex_shortcode_with_body() {
let inputs = vec![
r#"{% youtube() %}
Some text
{% end %}"#,
r#"{% youtube(id=1,
autoplay=true, url='hey') %}
Some text
{% end %}"#,
];
for i in inputs {
assert_lex_rule!(Rule::shortcode_with_body, i);
}
}
#[test]
fn lex_ignored_shortcode_with_body() {
let inputs = vec![
r#"{%/* youtube() */%}
Some text
{%/* end */%}"#,
r#"{%/* youtube(id=1,
autoplay=true, url='hey') */%}
Some text
{%/* end */%}"#,
];
for i in inputs {
assert_lex_rule!(Rule::ignored_shortcode_with_body, i);
}
}
#[test]
fn lex_page() {
let inputs = vec![
"Some text and a shortcode `{{/* youtube() */}}`",
"{{ youtube(id=1, autoplay=true, url='hey') }}",
"{{ youtube(id=1, \nautoplay=true, url='hey') }} that's it",
r#"
This is a test
{% hello() %}
Body {{ var }}
{% end %}
"#,
];
for i in inputs {
assert_lex_rule!(Rule::page, i);
}
}
#[test] #[test]
fn can_update_ranges() { fn can_update_ranges() {
let mut sc = Shortcode { let mut sc = Shortcode {
@ -355,56 +381,24 @@ mod tests {
assert_eq!(sc.span, 7..17); assert_eq!(sc.span, 7..17);
} }
#[test]
fn can_parse_inner_shortcode() {
let input = vec![
// txt, name, args
("hello() }}", "hello", HashMap::new()),
("hello_lo1() }}", "hello_lo1", HashMap::new()),
(
" shortcode(name='bob', age=45) }}",
"shortcode",
hashmap!("name".to_owned() => Value::String("bob".to_owned()), "age".to_owned() => to_value(45).unwrap()),
),
(
" shortcode(admin=true, age=45.1) }}",
"shortcode",
hashmap!("admin".to_owned() => Value::Bool(true), "age".to_owned() => to_value(45.1).unwrap()),
),
(
"with_array(hello=['true', false]) }}",
"with_array",
hashmap!("hello".to_owned() => Value::Array(vec![Value::String("true".to_owned()), Value::Bool(false)])),
),
(
"with_array(hello=['true', [true, false]]) }}",
"with_array",
hashmap!("hello".to_owned() => Value::Array(vec![Value::String("true".to_owned()), Value::Array(vec![Value::Bool(true), Value::Bool(false)])])),
),
];
for (txt, expected_name, expected_args) in input {
let mut lexer = InnerShortcode::lexer(txt);
let (name, args) = InnerShortcodeParser::parse(txt, &mut lexer).unwrap();
assert_eq!(&name, expected_name);
assert_eq!(args, to_value(expected_args).unwrap());
}
}
#[test] #[test]
fn can_extract_basic_inline_shortcode_with_args() { fn can_extract_basic_inline_shortcode_with_args() {
let (out, shortcodes) = ShortcodeExtractor::parse( let (out, shortcodes) = parse_for_shortcodes(
"Inline shortcode: {{ hello(string='hey', int=1, float=2.1, bool=true) }} hey", "Inline shortcode: {{ hello(string='hey', int=1, float=2.1, bool=true, array=[true, false]) }} hey",
) )
.unwrap(); .unwrap();
assert_eq!(out, format!("Inline shortcode: {} hey", SHORTCODE_PLACEHOLDER)); assert_eq!(out, format!("Inline shortcode: {} hey", SHORTCODE_PLACEHOLDER));
assert_eq!(shortcodes.len(), 1); assert_eq!(shortcodes.len(), 1);
assert_eq!(shortcodes[0].name, "hello"); assert_eq!(shortcodes[0].name, "hello");
assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 4); assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 5);
assert_eq!(shortcodes[0].args["string"], Value::String("hey".to_string())); assert_eq!(shortcodes[0].args["string"], Value::String("hey".to_string()));
assert_eq!(shortcodes[0].args["bool"], Value::Bool(true)); assert_eq!(shortcodes[0].args["bool"], Value::Bool(true));
assert_eq!(shortcodes[0].args["int"], to_value(1).unwrap()); assert_eq!(shortcodes[0].args["int"], to_value(1).unwrap());
assert_eq!(shortcodes[0].args["float"], to_value(2.1).unwrap()); assert_eq!(shortcodes[0].args["float"], to_value(2.1).unwrap());
assert_eq!(
shortcodes[0].args["array"],
Value::Array(vec![Value::Bool(true), Value::Bool(false)])
);
assert_eq!(shortcodes[0].span, 18..(18 + SHORTCODE_PLACEHOLDER.len())); assert_eq!(shortcodes[0].span, 18..(18 + SHORTCODE_PLACEHOLDER.len()));
assert_eq!(shortcodes[0].nth, 1); assert_eq!(shortcodes[0].nth, 1);
} }
@ -412,22 +406,26 @@ mod tests {
#[test] #[test]
fn can_unignore_ignored_inline_shortcode() { fn can_unignore_ignored_inline_shortcode() {
let (out, shortcodes) = let (out, shortcodes) =
ShortcodeExtractor::parse("Hello World {{/* youtube() */}} hey").unwrap(); parse_for_shortcodes("Hello World {{/* youtube() */}} hey").unwrap();
assert_eq!(out, "Hello World {{ youtube() }} hey"); assert_eq!(out, "Hello World {{ youtube() }} hey");
assert_eq!(shortcodes.len(), 0); assert_eq!(shortcodes.len(), 0);
} }
#[test] #[test]
fn can_extract_shortcode_with_body() { fn can_extract_shortcode_with_body() {
let (out, shortcodes) = ShortcodeExtractor::parse( let (out, shortcodes) = parse_for_shortcodes(
"Body shortcode\n {% quote(author='Bobby') %}DROP TABLES;{% end %} \n hey", "Body shortcode\n {% quote(author='Bobby', array=[[true]]) %}DROP TABLES;{% end %} \n hey",
) )
.unwrap(); .unwrap();
assert_eq!(out, format!("Body shortcode\n {} \n hey", SHORTCODE_PLACEHOLDER)); assert_eq!(out, format!("Body shortcode\n {} \n hey", SHORTCODE_PLACEHOLDER));
assert_eq!(shortcodes.len(), 1); assert_eq!(shortcodes.len(), 1);
assert_eq!(shortcodes[0].name, "quote"); assert_eq!(shortcodes[0].name, "quote");
assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 1); assert_eq!(shortcodes[0].args.as_object().unwrap().len(), 2);
assert_eq!(shortcodes[0].args["author"], Value::String("Bobby".to_string())); assert_eq!(shortcodes[0].args["author"], Value::String("Bobby".to_string()));
assert_eq!(
shortcodes[0].args["array"],
Value::Array(vec![Value::Array(vec![Value::Bool(true)])])
);
assert_eq!(shortcodes[0].body, Some("DROP TABLES;".to_owned())); assert_eq!(shortcodes[0].body, Some("DROP TABLES;".to_owned()));
assert_eq!(shortcodes[0].span, 16..(16 + SHORTCODE_PLACEHOLDER.len())); assert_eq!(shortcodes[0].span, 16..(16 + SHORTCODE_PLACEHOLDER.len()));
assert_eq!(shortcodes[0].nth, 1); assert_eq!(shortcodes[0].nth, 1);
@ -436,7 +434,7 @@ mod tests {
#[test] #[test]
fn can_unignore_ignored_shortcode_with_body() { fn can_unignore_ignored_shortcode_with_body() {
let (out, shortcodes) = let (out, shortcodes) =
ShortcodeExtractor::parse("Hello World {%/* youtube() */%} Somebody {%/* end */%} hey") parse_for_shortcodes("Hello World {%/* youtube() */%} Somebody {%/* end */%} hey")
.unwrap(); .unwrap();
assert_eq!(out, "Hello World {% youtube() %} Somebody {% end %} hey"); assert_eq!(out, "Hello World {% youtube() %} Somebody {% end %} hey");
assert_eq!(shortcodes.len(), 0); assert_eq!(shortcodes.len(), 0);
@ -444,7 +442,7 @@ mod tests {
#[test] #[test]
fn can_extract_multiple_shortcodes_and_increment_nth() { fn can_extract_multiple_shortcodes_and_increment_nth() {
let (out, shortcodes) = ShortcodeExtractor::parse( let (out, shortcodes) = parse_for_shortcodes(
"Hello World {% youtube() %} Somebody {% end %} {{ hello() }}\n {{hello()}}", "Hello World {% youtube() %} Somebody {% end %} {{ hello() }}\n {{hello()}}",
) )
.unwrap(); .unwrap();
@ -463,7 +461,7 @@ mod tests {
#[test] #[test]
fn can_handle_multiple_shortcodes() { fn can_handle_multiple_shortcodes() {
let (_, shortcodes) = ShortcodeExtractor::parse( let (_, shortcodes) = parse_for_shortcodes(
r#" r#"
{{ youtube(id="ub36ffWAqgQ") }} {{ youtube(id="ub36ffWAqgQ") }}
{{ youtube(id="ub36ffWAqgQ", autoplay=true) }} {{ youtube(id="ub36ffWAqgQ", autoplay=true) }}
@ -474,44 +472,4 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(shortcodes.len(), 5); assert_eq!(shortcodes.len(), 5);
} }
#[test]
fn can_provide_good_error_messages() {
let tests = vec![
("{{ hey()", "Unexpected end of content while parsing shortcodes"),
("{% hey()", "Unexpected end of content while parsing shortcodes"),
("{{ hey(=", "Unexpected token `=` while parsing arguments of shortcode hey"),
("{{ hey(ho==", "Unexpected token `=` while parsing arguments of shortcode hey"),
("{{ hey(ho=1h", "Unexpected end of content while parsing shortcode hey"),
("{{ hey)", "Unexpected token `)` while looking for `(` of shortcode hey"),
("{{ hey(ho=(", "Unexpected token `(` while parsing arguments of shortcode hey"),
("hello }}", "Unexpected token `}}` while parsing shortcodes"),
("hello %}", "Unexpected token `%}` while parsing shortcodes"),
("hello {% end %}", "Unexpected token `{% end %}` while parsing shortcodes"),
];
for (t, expected) in tests {
println!("Testing: {}", t);
let res = ShortcodeExtractor::parse(t);
assert!(res.is_err());
let err = res.unwrap_err();
assert_eq!(expected, err.to_string());
}
}
#[test]
fn can_parse_ok_with_problematic_chars() {
let inputs = vec![
("* a\n* b", "* a\n* b"),
("a regex {a,b}", "a regex {a,b}"),
("a slash //", "a slash //"),
("a slash */", "a slash */"),
("%a percent%", "%a percent%"),
];
for (input, expected) in inputs {
let (out, _) = ShortcodeExtractor::parse(input).unwrap();
assert_eq!(out, expected);
}
}
} }

4
docs/content/documentation/deployment/github-pages.md
View file

@ -18,7 +18,7 @@ We can use any continuous integration (CI) server to build and deploy our site.
In either case, it seems to work best if you use `git submodule` to include your theme, e.g.: In either case, it seems to work best if you use `git submodule` to include your theme, e.g.:
```shell ```bash
git submodule add https://github.com/getzola/after-dark.git themes/after-dark git submodule add https://github.com/getzola/after-dark.git themes/after-dark
``` ```
@ -110,7 +110,7 @@ Depending on how you added your theme, Travis may not know how to access
it. The best way to ensure that it will have full access to the theme is to use git it. The best way to ensure that it will have full access to the theme is to use git
submodules. When doing this, ensure that you are using the `https` version of the URL. submodules. When doing this, ensure that you are using the `https` version of the URL.
```shell ```bash
$ git submodule add {THEME_URL} themes/{THEME_NAME} $ git submodule add {THEME_URL} themes/{THEME_NAME}
``` ```

4
docs/content/documentation/deployment/gitlab-pages.md
View file

@ -21,12 +21,12 @@ This guide assumes that your Zola project is located in the root of your reposit
Depending on how you added your theme, your repository may not contain it. The best way to ensure that the theme will Depending on how you added your theme, your repository may not contain it. The best way to ensure that the theme will
be added is to use submodules. When doing this, ensure that you are using the `https` version of the URL. be added is to use submodules. When doing this, ensure that you are using the `https` version of the URL.
```shell ```bash
$ git submodule add {THEME_URL} themes/{THEME_NAME} $ git submodule add {THEME_URL} themes/{THEME_NAME}
``` ```
For example, this could look like: For example, this could look like:
```shell ```bash
$ git submodule add https://github.com/getzola/hyde.git themes/hyde $ git submodule add https://github.com/getzola/hyde.git themes/hyde
``` ```

2
docs/content/documentation/getting-started/configuration.md
View file

@ -237,7 +237,7 @@ Zola uses the Sublime Text themes, making it very easy to add more.
If you want a theme not listed above, please open an issue or a pull request on the [Zola repo](https://github.com/getzola/zola). If you want a theme not listed above, please open an issue or a pull request on the [Zola repo](https://github.com/getzola/zola).
Alternatively you can use the `extra_syntaxes_and_themes` configuration option to load your own custom themes from a .tmTheme file. Alternatively you can use the `extra_syntaxes_and_themes` configuration option to load your own custom themes from a .tmTheme file.
See [Syntax Highlighting](@/syntax-highlighting.md) for more details. See [Syntax Highlighting](@/documentation/content/syntax-highlighting.md) for more details.
## Slugification strategies ## Slugification strategies

2
docs/content/documentation/themes/installing-and-using-themes.md
View file

@ -35,7 +35,7 @@ in the configuration file is `simple-blog`. Also make sure to place the variable
Any file from the theme can be overridden by creating a file with the same path and name in your `templates` or `static` Any file from the theme can be overridden by creating a file with the same path and name in your `templates` or `static`
directory. Here are a few examples of that, assuming that the theme name is `simple-blog`: directory. Here are a few examples of that, assuming that the theme name is `simple-blog`:
```plain ```
templates/pages/post.html -> replace themes/simple-blog/templates/pages/post.html templates/pages/post.html -> replace themes/simple-blog/templates/pages/post.html
templates/macros.html -> replace themes/simple-blog/templates/macros.html templates/macros.html -> replace themes/simple-blog/templates/macros.html
static/js/site.js -> replace themes/simple-blog/static/js/site.js static/js/site.js -> replace themes/simple-blog/static/js/site.js