feat(data-files): Add data file support

Add capability to read yaml, json and toml files from a configurable
folder within source (defaults to `_data`) and expose them to the
templates under the variable `site.data` for processing.

Fixes #256

Cargo.lock

@@ -23,9 +23,11 @@ dependencies = [
  "rss 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "sass-rs 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_yaml 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "syntect 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "walkdir 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 

Cargo.toml

@@ -41,6 +41,8 @@ itertools = "0.6"
 ignore = "0.2"
 serde = "1.0"
 serde_yaml = "0.7"
+serde_json = "1.0"
+toml = "0.4.0"
 
 [dependencies.sass-rs]
 version = "0.2.0"

src/cobalt.rs

@@ -1,12 +1,16 @@
 use std::fs::{self, File};
 use std::collections::HashMap;
 use std::io::Write;
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::ffi::OsStr;
 use liquid::Value;
 use rss;
 use jsonfeed::Feed;
 use jsonfeed;
+use serde_yaml;
+use serde_json;
+use toml;
 
 #[cfg(feature = "sass")]
 use sass_rs;
@@ -20,6 +24,46 @@ use config::SassOutputStyle;
 use files::FilesBuilder;
 use frontmatter;
 
+fn deep_insert(data_map: &mut HashMap<String, Value>,
+               file_path: &Path,
+               target_key: String,
+               data: Value)
+               -> Result<()> {
+    // now find the nested map it is supposed to be in
+    let target_map = if let Some(path) = file_path.parent() {
+        let mut map = data_map;
+        for part in path.iter() {
+            if let Some(key) = part.to_str() {
+                let cur_map = map;
+                map = if let Some(sub_map) = cur_map
+                       .entry(String::from(key))
+                       .or_insert_with(|| Value::Object(HashMap::new()))
+                       .as_object_mut() {
+                    sub_map
+                } else {
+                    bail!("Aborting: Dublicate in data tree. Would overwrite {} ",
+                          &key)
+                }
+            } else {
+                bail!("The data from {:?} can't be loaded as it contains non utf-8 characters",
+                      path);
+            }
+        }
+        map
+    } else {
+        data_map
+    };
+
+    match target_map.insert(target_key, data) {
+        None => Ok(()),
+        _ => {
+            Err(format!("The data from {:?} can't be loaded: the key already exists",
+                        file_path)
+                        .into())
+        }
+    }
+}
+
 /// The primary build function that transforms a directory into a site
 pub fn build(config: &Config) -> Result<()> {
     trace!("Build configuration: {:?}", config);
@@ -125,6 +169,47 @@ pub fn build(config: &Config) -> Result<()> {
         }
     }
 
+    // load data files
+    let mut data_map: HashMap<String, Value> = HashMap::new();
+    let data_root = source.join(&config.data);
+    let data_files_builder = FilesBuilder::new(data_root.as_path())?;
+    let data_files = data_files_builder.build()?;
+
+    for df in data_files.files() {
+
+        let ext = df.extension().unwrap_or(OsStr::new(""));
+        let file_stem = df.file_stem()
+            .expect("Files will always return with a stem");
+
+        let file_name = String::from(file_stem.to_str().unwrap());
+        let full_path = data_root.join(df.clone());
+        let data: Value;
+
+        if ext == OsStr::new("yml") || ext == OsStr::new("yaml") {
+            let reader = File::open(full_path)?;
+            data = serde_yaml::from_reader(reader)?;
+        } else if ext == OsStr::new("json") {
+            let reader = File::open(full_path)?;
+            data = serde_json::from_reader(reader)?;
+        } else if ext == OsStr::new("toml") {
+            let mut reader = File::open(full_path)?;
+            let mut text = String::new();
+            reader.read_to_string(&mut text)?;
+            data = toml::from_str(&text)?;
+        } else {
+            warn!("Skipping loading of data {:?}: unknown file type.",
+                  full_path);
+            warn!("Supported data files extensions are: yml, yaml, json and toml.");
+            continue;
+        }
+
+        deep_insert(&mut data_map, &df, file_name, data)?;
+    }
+
+    // now wrap it all into the global site-object
+    let mut site = HashMap::new();
+    site.insert("data".to_owned(), Value::Object(data_map));
+
     // January 1, 1970 0:00:00 UTC, the beginning of time
     let default_date = datetime::DateTime::default();
 
@@ -182,6 +267,7 @@ pub fn build(config: &Config) -> Result<()> {
         }
 
         let mut context = post.get_render_context(&simple_posts_data);
+        context.set_val("site", Value::Object(site.clone()));
 
         post.render_excerpt(&mut context, source, &config.syntax_highlight.theme)
             .chain_err(|| format!("Failed to render excerpt for {:?}", post.file_path))?;
@@ -230,6 +316,8 @@ pub fn build(config: &Config) -> Result<()> {
         }
 
         let mut context = doc.get_render_context(&posts_data);
+        context.set_val("site", Value::Object(site.clone()));
+
         let doc_html = doc.render(&mut context,
                                   source,
                                   &layouts,

src/config.rs

@@ -86,6 +86,7 @@ pub struct Config {
     pub dest: String,
     pub layouts: String,
     pub drafts: String,
+    pub data: String,
     pub include_drafts: bool,
     pub posts: String,
     pub post_path: Option<String>,
@@ -112,6 +113,7 @@ impl Default for Config {
             dest: "./".to_owned(),
             layouts: "_layouts".to_owned(),
             drafts: "_drafts".to_owned(),
+            data: "_data".to_owned(),
             include_drafts: false,
             posts: "posts".to_owned(),
             post_path: None,

src/error.rs

@@ -6,6 +6,8 @@ use walkdir;
 use liquid;
 use ignore;
 use serde_yaml;
+use serde_json;
+use toml;
 
 error_chain! {
 
@@ -17,6 +19,8 @@ error_chain! {
         Liquid(liquid::Error);
         WalkDir(walkdir::Error);
         SerdeYaml(serde_yaml::Error);
+        SerdeJson(serde_json::Error);
+        Toml(toml::de::Error);
         Ignore(ignore::Error);
     }
 

src/legacy/wildwest.rs

@@ -115,6 +115,7 @@ pub struct GlobalConfig {
     pub dest: String,
     pub layouts: String,
     pub drafts: String,
+    pub data: String,
     pub include_drafts: bool,
     pub posts: String,
     pub post_path: Option<String>,
@@ -138,6 +139,7 @@ impl Default for GlobalConfig {
             dest: "./".to_owned(),
             layouts: "_layouts".to_owned(),
             drafts: "_drafts".to_owned(),
+            data: "_data".to_owned(),
             include_drafts: false,
             posts: "posts".to_owned(),
             post_path: None,
@@ -163,6 +165,7 @@ impl From<GlobalConfig> for config::Config {
             dest,
             layouts,
             drafts,
+            data,
             include_drafts,
             posts,
             post_path,
@@ -200,6 +203,7 @@ impl From<GlobalConfig> for config::Config {
             dest: dest,
             layouts: layouts,
             drafts: drafts,
+            data: data,
             include_drafts: include_drafts,
             posts: posts,
             post_path: post_path,

src/lib.rs

@@ -56,6 +56,8 @@ extern crate rss;
 extern crate jsonfeed;
 extern crate walkdir;
 extern crate serde_yaml;
+extern crate serde_json;
+extern crate toml;
 
 #[cfg(feature = "sass")]
 extern crate sass_rs;

tests/fixtures/data_files/_data/movies.yaml

@@ -0,0 +1,27 @@
+- name: Avatar"
+  grossed: "$2,787,965,087"
+  year: "2009"
+- name: Titanic 
+  grossed: "$2,186,772,302" 
+  year: "1997"
+- name: "Star Wars: The Force Awakens"
+  grossed: "$2,068,223,624" 
+  year: "2015"
+- name: Jurassic World 
+  grossed: "$1,671,713,208" 
+  year: "2015"
+- name: The Avengers 
+  grossed: "$1,518,812,988" 
+  year: "2012"
+- name: Furious 7 
+  grossed: "$1,516,045,911" 
+  year: "2015"
+- name: "Avengers: Age of Ultron"
+  grossed: "$1,405,403,694" 
+  year: "2015"
+- name: "Harry Potter and the Deathly Hallows – Part 2"
+  grossed: "$1,341,511,219" 
+  year: "2011"
+- name: Frozen 
+  grossed: "$1,287,000,000" 
+  year: "2013"

tests/fixtures/data_files/index.liquid

@@ -2,8 +2,8 @@
 <h1>Cool Dinos:</h1>
 
 <ul>
- <li><strong>{{data.dinos.earl.name}}</strong>, {{data.dinos.earl.species}}{% if data.dinos.earl.job %}, {{data.dinos.earl.job}}{% endif %}</li> 
- <li><strong>{{data.dinos.frances.name}}</strong>, {{data.dinos.frances.species}}{% if data.dinos.frances.job %}, {{data.dinos.frances.job}}{% endif %}</li> 
- <li><strong>{{data.dinos.roy.name}}</strong>, {{data.dinos.roy.species}}{% if data.dinos.roy.job %}, {{data.dinos.roy.job}}{% endif %}</li> 
- <li><strong>{{data.dinos.charlene.name}}</strong>, {{data.dinos.charlene.species}}{% if data.dinos.charlene.job %}, {{data.dinos.charlene.job}}{% endif %}</li>
+ <li><strong>{{site.data.dinos.earl.name}}</strong>, {{site.data.dinos.earl.species}}{% if site.data.dinos.earl.job %}, {{site.data.dinos.earl.job}}{% endif %}</li> 
+ <li><strong>{{site.data.dinos.frances.name}}</strong>, {{site.data.dinos.frances.species}}{% if site.data.dinos.frances.job %}, {{site.data.dinos.frances.job}}{% endif %}</li> 
+ <li><strong>{{site.data.dinos.roy.name}}</strong>, {{site.data.dinos.roy.species}}{% if site.data.dinos.roy.job %}, {{site.data.dinos.roy.job}}{% endif %}</li> 
+ <li><strong>{{site.data.dinos.charlene.name}}</strong>, {{site.data.dinos.charlene.species}}{% if site.data.dinos.charlene.job %}, {{site.data.dinos.charlene.job}}{% endif %}</li>
 </ul>

tests/fixtures/data_files/movies.liquid

@@ -0,0 +1,9 @@
+---
+
+<h1>Highest Grossing Movies</h1>
+
+<ul>
+{% for movie in site.data.movies %}
+  <li>{{movie.name}} ({{movie.year}}): {{movie.grossed}}</li>
+{% endfor %}
+</ul>

tests/target/data_files/index.html

@@ -1,8 +1,8 @@
- <h1>Cool Dinos:</h1>
+<h1>Cool Dinos:</h1>
 
 <ul>
  <li><strong>Earl Sinclair</strong>, Megalosaurus, Tree Pusher</li> 
  <li><strong>Frances Sinclair</strong>, Allosaurus</li> 
  <li><strong>Roy Hess</strong>, Tyrannosaurus rex, Tree Pusher</li> 
  <li><strong>Charlene Sinclair</strong>, Protoceratops, Teenager</li>
-</ul> 
+</ul>

tests/target/data_files/movies.html

@@ -0,0 +1,23 @@
+<h1>Highest Grossing Movies</h1>
+
+<ul>
+
+  <li>Avatar" (2009): $2,787,965,087</li>
+
+  <li>Titanic (1997): $2,186,772,302</li>
+
+  <li>Star Wars: The Force Awakens (2015): $2,068,223,624</li>
+
+  <li>Jurassic World (2015): $1,671,713,208</li>
+
+  <li>The Avengers (2012): $1,518,812,988</li>
+
+  <li>Furious 7 (2015): $1,516,045,911</li>
+
+  <li>Avengers: Age of Ultron (2015): $1,405,403,694</li>
+
+  <li>Harry Potter and the Deathly Hallows – Part 2 (2011): $1,341,511,219</li>
+
+  <li>Frozen (2013): $1,287,000,000</li>
+
+</ul>