Update to pulldown-cmark 0.5. (#898)

* Update to pulldown-cmark 0.4.1.

* Update to pulldown-cmark 0.5.2.

* Remove pulldown-cmark-to-cmark dependency.

Since it is not compatible with the new pulldown-cmark. This example isn't
directly usable, anyways, and I think the no-op example sufficiently shows how
to make a preprocessor.

* cargo fmt

* Fix example link.

Cargo.lock

@@ -87,11 +87,6 @@ name = "bit-vec"
 version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "bitflags"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "bitflags"
 version = "1.0.4"
@@ -646,8 +641,7 @@ dependencies = [
  "notify 4.0.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "open 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pulldown-cmark 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "pulldown-cmark-to-cmark 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "select 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.91 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -936,19 +930,13 @@ dependencies = [
 
 [[package]]
 name = "pulldown-cmark"
-version = "0.1.2"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "getopts 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)",
-]
+ "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-
+ "unicase 2.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-[[package]]
-name = "pulldown-cmark-to-cmark"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1479,6 +1467,14 @@ dependencies = [
  "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "unicase"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.4"
@@ -1659,7 +1655,6 @@ dependencies = [
 "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643"
 "checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c"
 "checksum bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f"
-"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
 "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
 "checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
 "checksum block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4dc3af3ee2e12f3e5d224e5e1e3d73668abbeb69e566d361f7d5563a4fdf09"
@@ -1755,8 +1750,7 @@ dependencies = [
 "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 "checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427"
 "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
-"checksum pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d6fdf85cda6cadfae5428a54661d431330b312bc767ddbc57adbedc24da66e32"
+"checksum pulldown-cmark 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "051e60ace841b3bfecd402fe5051c06cb3bec4a6e6fdd060a37aa8eb829a1db3"
-"checksum pulldown-cmark-to-cmark 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57efca5f52f63336ee3a49bceee1a1169f18ef01c75aa7e71949441b49bbe7e4"
 "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
 "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
 "checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db"
@@ -1821,6 +1815,7 @@ dependencies = [
 "checksum ucd-trie 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "71a9c5b1fe77426cf144cc30e49e955270f5086e31a6441dfa8b32efc09b9d77"
 "checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86"
 "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33"
+"checksum unicase 2.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a84e5511b2a947f3ae965dcb29b13b7b1691b6e7332cf5dbc1744138d5acb7f6"
 "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
 "checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426"
 "checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9"

Cargo.toml

@@ -26,7 +26,7 @@ lazy_static = "1.0"
 log = "0.4"
 memchr = "2.0"
 open = "1.1"
-pulldown-cmark = "0.1.2"
+pulldown-cmark = "0.5"
 regex = "1.0.0"
 serde = "1.0"
 serde_derive = "1.0"
@@ -52,8 +52,6 @@ ammonia = { version = "2.1", optional = true }
 select = "0.4"
 pretty_assertions = "0.6"
 walkdir = "2.0"
-# FIXME(issue #898)
-pulldown-cmark-to-cmark = "=1.1.0"
 
 [features]
 default = ["output", "watch", "serve", "search"]

book-example/src/for_developers/preprocessors.md

@@ -109,7 +109,7 @@ For everything else, have a look [at the complete example][example].
 [preprocessor-docs]: https://docs.rs/mdbook/latest/mdbook/preprocess/trait.Preprocessor.html
 [pc]: https://crates.io/crates/pulldown-cmark
 [pctc]: https://crates.io/crates/pulldown-cmark-to-cmark
-[example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/de-emphasize.rs
+[example]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/nop-preprocessor.rs
 [an example no-op preprocessor]: https://github.com/rust-lang-nursery/mdBook/blob/master/examples/nop-preprocessor.rs
 [`CmdPreprocessor::parse_input()`]: https://docs.rs/mdbook/latest/mdbook/preprocess/trait.Preprocessor.html#method.parse_input
 [`Book::for_each_mut()`]: https://docs.rs/mdbook/latest/mdbook/book/struct.Book.html#method.for_each_mut

examples/de-emphasize.rs

@@ -1,75 +0,0 @@
-//! An example preprocessor for removing all forms of emphasis from a markdown
-//! book.
-use mdbook::book::{Book, BookItem, Chapter};
-use mdbook::errors::{Error, Result};
-use mdbook::preprocess::{Preprocessor, PreprocessorContext};
-use pulldown_cmark::{Event, Parser, Tag};
-use pulldown_cmark_to_cmark::fmt::cmark;
-
-const NAME: &str = "md-links-to-html-links";
-
-fn main() {
-    panic!("This example is intended to be part of a library");
-}
-
-#[allow(dead_code)]
-struct Deemphasize;
-
-impl Preprocessor for Deemphasize {
-    fn name(&self) -> &str {
-        NAME
-    }
-
-    fn run(&self, _ctx: &PreprocessorContext, mut book: Book) -> Result<Book> {
-        eprintln!("Running '{}' preprocessor", self.name());
-        let mut num_removed_items = 0;
-
-        process(&mut book.sections, &mut num_removed_items)?;
-
-        eprintln!(
-            "{}: removed {} events from markdown stream.",
-            self.name(),
-            num_removed_items
-        );
-
-        Ok(book)
-    }
-}
-
-fn process<'a, I>(items: I, num_removed_items: &mut usize) -> Result<()>
-where
-    I: IntoIterator<Item = &'a mut BookItem> + 'a,
-{
-    for item in items {
-        if let BookItem::Chapter(ref mut chapter) = *item {
-            eprintln!("{}: processing chapter '{}'", NAME, chapter.name);
-
-            let md = remove_emphasis(num_removed_items, chapter)?;
-            chapter.content = md;
-        }
-    }
-
-    Ok(())
-}
-
-fn remove_emphasis(num_removed_items: &mut usize, chapter: &mut Chapter) -> Result<String> {
-    let mut buf = String::with_capacity(chapter.content.len());
-
-    let events = Parser::new(&chapter.content).filter(|e| {
-        let should_keep = match *e {
-            Event::Start(Tag::Emphasis)
-            | Event::Start(Tag::Strong)
-            | Event::End(Tag::Emphasis)
-            | Event::End(Tag::Strong) => false,
-            _ => true,
-        };
-        if !should_keep {
-            *num_removed_items += 1;
-        }
-        should_keep
-    });
-
-    cmark(events, &mut buf, None)
-        .map(|_| buf)
-        .map_err(|err| Error::from(format!("Markdown serialization failed: {}", err)))
-}

src/book/summary.rs

@@ -259,7 +259,7 @@ impl<'a> SummaryParser<'a> {
                         bail!(self.parse_error("Suffix chapters cannot be followed by a list"));
                     }
                 }
-                Some(Event::Start(Tag::Link(href, _))) => {
+                Some(Event::Start(Tag::Link(_type, href, _title))) => {
                     let link = self.parse_link(href.to_string())?;
                     items.push(SummaryItem::Link(link));
                 }
@@ -397,7 +397,7 @@ impl<'a> SummaryParser<'a> {
         loop {
             match self.next_event() {
                 Some(Event::Start(Tag::Paragraph)) => continue,
-                Some(Event::Start(Tag::Link(href, _))) => {
+                Some(Event::Start(Tag::Link(_type, href, _title))) => {
                     let mut link = self.parse_link(href.to_string())?;
 
                     let mut number = parent.clone();
@@ -475,7 +475,7 @@ fn stringify_events(events: Vec<Event<'_>>) -> String {
     events
         .into_iter()
         .filter_map(|t| match t {
-            Event::Text(text) => Some(text.into_owned()),
+            Event::Text(text) | Event::Code(text) => Some(text.into_string()),
             _ => None,
         })
         .collect()
@@ -629,7 +629,7 @@ mod tests {
         let _ = parser.stream.next(); // skip past start of paragraph
 
         let href = match parser.stream.next() {
-            Some(Event::Start(Tag::Link(href, _))) => href.to_string(),
+            Some(Event::Start(Tag::Link(_type, href, _title))) => href.to_string(),
             other => panic!("Unreachable, {:?}", other),
         };
 

src/renderer/html_handlebars/helpers/toc.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 use crate::utils;
 
 use handlebars::{Context, Handlebars, Helper, HelperDef, Output, RenderContext, RenderError};
-use pulldown_cmark::{html, Event, Parser, Tag};
+use pulldown_cmark::{html, Event, Parser};
 
 // Handlebars helper to construct TOC
 #[derive(Clone, Copy)]
@@ -117,10 +117,7 @@ impl HelperDef for RenderToc {
 
                 // filter all events that are not inline code blocks
                 let parser = Parser::new(name).filter(|event| match *event {
-                    Event::Start(Tag::Code)
+                    Event::Code(_) | Event::InlineHtml(_) | Event::Text(_) => true,
-                    | Event::End(Tag::Code)
-                    | Event::InlineHtml(_)
-                    | Event::Text(_) => true,
                     _ => false,
                 });
 

src/renderer/html_handlebars/search.rs

@@ -82,8 +82,8 @@ fn render_item(
     let anchor_base = utils::fs::normalize_path(filepath);
 
     let mut opts = Options::empty();
-    opts.insert(OPTION_ENABLE_TABLES);
+    opts.insert(Options::ENABLE_TABLES);
-    opts.insert(OPTION_ENABLE_FOOTNOTES);
+    opts.insert(Options::ENABLE_FOOTNOTES);
     let p = Parser::new_ext(&chapter.content, opts);
 
     let mut in_header = false;
@@ -91,6 +91,7 @@ fn render_item(
     let mut section_id = None;
     let mut heading = String::new();
     let mut body = String::new();
+    let mut html_block = String::new();
     let mut breadcrumbs = chapter.parent_names.clone();
     let mut footnote_numbers = HashMap::new();
 
@@ -124,6 +125,13 @@ fn render_item(
                 let number = footnote_numbers.len() + 1;
                 footnote_numbers.entry(name).or_insert(number);
             }
+            Event::Html(html) => {
+                html_block.push_str(&html);
+            }
+            Event::End(Tag::HtmlBlock) => {
+                body.push_str(&clean_html(&html_block));
+                html_block.clear();
+            }
             Event::Start(_) | Event::End(_) | Event::SoftBreak | Event::HardBreak => {
                 // Insert spaces where HTML output would usually seperate text
                 // to ensure words don't get merged together
@@ -133,14 +141,14 @@ fn render_item(
                     body.push(' ');
                 }
             }
-            Event::Text(text) => {
+            Event::Text(text) | Event::Code(text) => {
                 if in_header {
                     heading.push_str(&text);
                 } else {
                     body.push_str(&text);
                 }
             }
-            Event::Html(html) | Event::InlineHtml(html) => {
+            Event::InlineHtml(html) => {
                 body.push_str(&clean_html(&html));
             }
             Event::FootnoteReference(name) => {
@@ -148,6 +156,7 @@ fn render_item(
                 let number = footnote_numbers.entry(name).or_insert(len);
                 body.push_str(&format!(" [{}] ", number));
             }
+            Event::TaskListMarker(_checked) => {}
         }
     }
 

src/utils/mod.rs

@@ -5,9 +5,7 @@ mod string;
 use crate::errors::Error;
 use regex::Regex;
 
-use pulldown_cmark::{
+use pulldown_cmark::{html, CowStr, Event, Options, Parser, Tag};
-    html, Event, Options, Parser, Tag, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES,
-};
 
 use std::borrow::Cow;
 
@@ -73,7 +71,7 @@ fn adjust_links<'a>(event: Event<'a>, with_base: &str) -> Event<'a> {
         static ref MD_LINK: Regex = Regex::new(r"(?P<link>.*)\.md(?P<anchor>#.*)?").unwrap();
     }
 
-    fn fix<'a>(dest: Cow<'a, str>, base: &str) -> Cow<'a, str> {
+    fn fix<'a>(dest: CowStr<'a>, base: &str) -> CowStr<'a> {
         // Don't modify links with schemes like `https`.
         if !SCHEME_LINK.is_match(&dest) {
             // This is a relative link, adjust it as necessary.
@@ -92,17 +90,17 @@ fn adjust_links<'a>(event: Event<'a>, with_base: &str) -> Event<'a> {
             } else {
                 fixed_link.push_str(&dest);
             };
-            return Cow::from(fixed_link);
+            return CowStr::from(fixed_link);
         }
         dest
     }
 
     match event {
-        Event::Start(Tag::Link(dest, title)) => {
+        Event::Start(Tag::Link(link_type, dest, title)) => {
-            Event::Start(Tag::Link(fix(dest, with_base), title))
+            Event::Start(Tag::Link(link_type, fix(dest, with_base), title))
         }
-        Event::Start(Tag::Image(dest, title)) => {
+        Event::Start(Tag::Image(link_type, dest, title)) => {
-            Event::Start(Tag::Image(fix(dest, with_base), title))
+            Event::Start(Tag::Image(link_type, fix(dest, with_base), title))
         }
         _ => event,
     }
@@ -117,8 +115,8 @@ pub fn render_markdown_with_base(text: &str, curly_quotes: bool, base: &str) ->
     let mut s = String::with_capacity(text.len() * 3 / 2);
 
     let mut opts = Options::empty();
-    opts.insert(OPTION_ENABLE_TABLES);
+    opts.insert(Options::ENABLE_TABLES);
-    opts.insert(OPTION_ENABLE_FOOTNOTES);
+    opts.insert(Options::ENABLE_FOOTNOTES);
 
     let p = Parser::new_ext(text, opts);
     let mut converter = EventQuoteConverter::new(curly_quotes);
@@ -150,16 +148,16 @@ impl EventQuoteConverter {
         }
 
         match event {
-            Event::Start(Tag::CodeBlock(_)) | Event::Start(Tag::Code) => {
+            Event::Start(Tag::CodeBlock(_)) => {
                 self.convert_text = false;
                 event
             }
-            Event::End(Tag::CodeBlock(_)) | Event::End(Tag::Code) => {
+            Event::End(Tag::CodeBlock(_)) => {
                 self.convert_text = true;
                 event
             }
             Event::Text(ref text) if self.convert_text => {
-                Event::Text(Cow::from(convert_quotes_to_curly(text)))
+                Event::Text(CowStr::from(convert_quotes_to_curly(text)))
             }
             _ => event,
         }
@@ -171,7 +169,7 @@ fn clean_codeblock_headers(event: Event<'_>) -> Event<'_> {
         Event::Start(Tag::CodeBlock(ref info)) => {
             let info: String = info.chars().filter(|ch| !ch.is_whitespace()).collect();
 
-            Event::Start(Tag::CodeBlock(Cow::from(info)))
+            Event::Start(Tag::CodeBlock(CowStr::from(info)))
         }
         _ => event,
     }