WIP: Add vcard/ical support (#1504)

* Initial from-ical implementation

* Initial from-vcard implementation

* Rename from-ics and from-vcf for autoconvert

* Remove redundant clones

* Add from-vcf and from-ics tests

Co-authored-by: Jonathan Turner <jonathandturner@users.noreply.github.com>

Cargo.lock

@@ -901,6 +901,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b"
 dependencies = [
  "backtrace",
+ "failure_derive",
+]
+
+[[package]]
+name = "failure_derive"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
 ]
 
 [[package]]
@@ -1487,6 +1500,15 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "ical"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0e9337a9d901ffb92cf655e854c51734dc004287b959b47830a3308201f18c0"
+dependencies = [
+ "failure",
+]
+
 [[package]]
 name = "ichwh"
 version = "0.3.1"
@@ -2227,6 +2249,7 @@ dependencies = [
  "glob",
  "hex 0.4.0",
  "htmlescape",
+ "ical",
  "ichwh",
  "indexmap",
  "itertools 0.9.0",
@@ -3751,6 +3774,18 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "unicode-xid",
+]
+
 [[package]]
 name = "syntect"
 version = "3.2.0"

crates/nu-cli/Cargo.toml

@@ -45,6 +45,7 @@ git2 = { version = "0.13.0", default_features = false }
 glob = "0.3.0"
 hex = "0.4"
 htmlescape = "0.3.1"
+ical = "0.6.*"
 ichwh = "0.3"
 indexmap = { version = "1.3.2", features = ["serde-1"] }
 itertools = "0.9.0"

crates/nu-cli/src/cli.rs

@@ -342,6 +342,8 @@ pub fn create_default_context(
             whole_stream_command(FromXML),
             whole_stream_command(FromYAML),
             whole_stream_command(FromYML),
+            whole_stream_command(FromIcs),
+            whole_stream_command(FromVcf),
         ]);
 
         cfg_if::cfg_if! {

crates/nu-cli/src/commands.rs

@@ -30,6 +30,7 @@ pub(crate) mod first;
 pub(crate) mod format;
 pub(crate) mod from_bson;
 pub(crate) mod from_csv;
+pub(crate) mod from_ics;
 pub(crate) mod from_ini;
 pub(crate) mod from_json;
 pub(crate) mod from_ods;
@@ -38,6 +39,7 @@ pub(crate) mod from_ssv;
 pub(crate) mod from_toml;
 pub(crate) mod from_tsv;
 pub(crate) mod from_url;
+pub(crate) mod from_vcf;
 pub(crate) mod from_xlsx;
 pub(crate) mod from_xml;
 pub(crate) mod from_yaml;
@@ -136,6 +138,7 @@ pub(crate) use first::First;
 pub(crate) use format::Format;
 pub(crate) use from_bson::FromBSON;
 pub(crate) use from_csv::FromCSV;
+pub(crate) use from_ics::FromIcs;
 pub(crate) use from_ini::FromINI;
 pub(crate) use from_json::FromJSON;
 pub(crate) use from_ods::FromODS;
@@ -145,6 +148,7 @@ pub(crate) use from_ssv::FromSSV;
 pub(crate) use from_toml::FromTOML;
 pub(crate) use from_tsv::FromTSV;
 pub(crate) use from_url::FromURL;
+pub(crate) use from_vcf::FromVcf;
 pub(crate) use from_xlsx::FromXLSX;
 pub(crate) use from_xml::FromXML;
 pub(crate) use from_yaml::FromYAML;

crates/nu-cli/src/commands/from_ics.rs

@@ -0,0 +1,240 @@
+extern crate ical;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use ical::parser::ical::component::*;
+use ical::property::Property;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
+use std::io::BufReader;
+
+pub struct FromIcs;
+
+impl WholeStreamCommand for FromIcs {
+    fn name(&self) -> &str {
+        "from-ics"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-ics")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .ics and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_ics(args, registry)
+    }
+}
+
+fn from_ics(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream! {
+        let input_string = input.collect_string(tag.clone()).await?.item;
+        let input_bytes = input_string.as_bytes();
+        let buf_reader = BufReader::new(input_bytes);
+        let parser = ical::IcalParser::new(buf_reader);
+
+        for calendar in parser {
+            match calendar {
+                Ok(c) => yield ReturnSuccess::value(calendar_to_value(c, tag.clone())),
+                Err(_) => yield Err(ShellError::labeled_error(
+                    "Could not parse as .ics",
+                    "input cannot be parsed as .ics",
+                    tag.clone()
+                )),
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+fn calendar_to_value(calendar: IcalCalendar, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag.clone());
+
+    row.insert_untagged(
+        "properties",
+        properties_to_value(calendar.properties, tag.clone()),
+    );
+    row.insert_untagged("events", events_to_value(calendar.events, tag.clone()));
+    row.insert_untagged("alarms", alarms_to_value(calendar.alarms, tag.clone()));
+    row.insert_untagged("to-Dos", todos_to_value(calendar.todos, tag.clone()));
+    row.insert_untagged(
+        "journals",
+        journals_to_value(calendar.journals, tag.clone()),
+    );
+    row.insert_untagged(
+        "free-busys",
+        free_busys_to_value(calendar.free_busys, tag.clone()),
+    );
+    row.insert_untagged("timezones", timezones_to_value(calendar.timezones, tag));
+
+    row.into_value()
+}
+
+fn events_to_value(events: Vec<IcalEvent>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &events
+            .into_iter()
+            .map(|event| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(event.properties, tag.clone()),
+                );
+                row.insert_untagged("alarms", alarms_to_value(event.alarms, tag.clone()));
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn alarms_to_value(alarms: Vec<IcalAlarm>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &alarms
+            .into_iter()
+            .map(|alarm| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(alarm.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn todos_to_value(todos: Vec<IcalTodo>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &todos
+            .into_iter()
+            .map(|todo| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(todo.properties, tag.clone()),
+                );
+                row.insert_untagged("alarms", alarms_to_value(todo.alarms, tag.clone()));
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn journals_to_value(journals: Vec<IcalJournal>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &journals
+            .into_iter()
+            .map(|journal| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(journal.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn free_busys_to_value(free_busys: Vec<IcalFreeBusy>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &free_busys
+            .into_iter()
+            .map(|free_busy| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(free_busy.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn timezones_to_value(timezones: Vec<IcalTimeZone>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &timezones
+            .into_iter()
+            .map(|timezone| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(timezone.properties, tag.clone()),
+                );
+                row.insert_untagged(
+                    "transitions",
+                    timezone_transitions_to_value(timezone.transitions, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn timezone_transitions_to_value(
+    transitions: Vec<IcalTimeZoneTransition>,
+    tag: Tag,
+) -> UntaggedValue {
+    UntaggedValue::table(
+        &transitions
+            .into_iter()
+            .map(|transition| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+                row.insert_untagged(
+                    "properties",
+                    properties_to_value(transition.properties, tag.clone()),
+                );
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn properties_to_value(properties: Vec<Property>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &properties
+            .into_iter()
+            .map(|prop| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+
+                let name = UntaggedValue::string(prop.name);
+                let value = match prop.value {
+                    Some(val) => UntaggedValue::string(val),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+                let params = match prop.params {
+                    Some(param_list) => params_to_value(param_list, tag.clone()).into(),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+
+                row.insert_untagged("name", name);
+                row.insert_untagged("value", value);
+                row.insert_untagged("params", params);
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn params_to_value(params: Vec<(String, Vec<String>)>, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag);
+
+    for (param_name, param_values) in params {
+        let values: Vec<Value> = param_values.into_iter().map(|val| val.into()).collect();
+        let values = UntaggedValue::table(&values);
+        row.insert_untagged(param_name, values);
+    }
+
+    row.into_value()
+}

crates/nu-cli/src/commands/from_vcf.rs

@@ -0,0 +1,102 @@
+extern crate ical;
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use ical::parser::vcard::component::*;
+use ical::property::Property;
+use nu_errors::ShellError;
+use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value};
+use std::io::BufReader;
+
+pub struct FromVcf;
+
+impl WholeStreamCommand for FromVcf {
+    fn name(&self) -> &str {
+        "from-vcf"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-vcf")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .vcf and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        from_vcf(args, registry)
+    }
+}
+
+fn from_vcf(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once(registry)?;
+    let tag = args.name_tag();
+    let input = args.input;
+
+    let stream = async_stream! {
+        let input_string = input.collect_string(tag.clone()).await?.item;
+        let input_bytes = input_string.as_bytes();
+        let buf_reader = BufReader::new(input_bytes);
+        let parser = ical::VcardParser::new(buf_reader);
+
+        for contact in parser {
+            match contact {
+                Ok(c) => yield ReturnSuccess::value(contact_to_value(c, tag.clone())),
+                Err(_) => yield Err(ShellError::labeled_error(
+                    "Could not parse as .vcf",
+                    "input cannot be parsed as .vcf",
+                    tag.clone()
+                )),
+            }
+        }
+    };
+
+    Ok(stream.to_output_stream())
+}
+
+fn contact_to_value(contact: VcardContact, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag.clone());
+    row.insert_untagged("properties", properties_to_value(contact.properties, tag));
+    row.into_value()
+}
+
+fn properties_to_value(properties: Vec<Property>, tag: Tag) -> UntaggedValue {
+    UntaggedValue::table(
+        &properties
+            .into_iter()
+            .map(|prop| {
+                let mut row = TaggedDictBuilder::new(tag.clone());
+
+                let name = UntaggedValue::string(prop.name);
+                let value = match prop.value {
+                    Some(val) => UntaggedValue::string(val),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+                let params = match prop.params {
+                    Some(param_list) => params_to_value(param_list, tag.clone()).into(),
+                    None => UntaggedValue::Primitive(Primitive::Nothing),
+                };
+
+                row.insert_untagged("name", name);
+                row.insert_untagged("value", value);
+                row.insert_untagged("params", params);
+                row.into_value()
+            })
+            .collect::<Vec<Value>>(),
+    )
+}
+
+fn params_to_value(params: Vec<(String, Vec<String>)>, tag: Tag) -> Value {
+    let mut row = TaggedDictBuilder::new(tag);
+
+    for (param_name, param_values) in params {
+        let values: Vec<Value> = param_values.into_iter().map(|val| val.into()).collect();
+        let values = UntaggedValue::table(&values);
+        row.insert_untagged(param_name, values);
+    }
+
+    row.into_value()
+}

crates/nu-cli/tests/format_conversions/ics.rs

@@ -0,0 +1,101 @@
+use nu_test_support::fs::Stub::FileWithContentToBeTrimmed;
+use nu_test_support::playground::Playground;
+use nu_test_support::{nu, pipeline};
+
+#[test]
+fn infers_types() {
+    Playground::setup("filter_from_ics_test_1", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "calendar.ics",
+            r#"
+                BEGIN:VCALENDAR
+                PRODID:-//Google Inc//Google Calendar 70.9054//EN
+                VERSION:2.0
+                BEGIN:VEVENT
+                DTSTART:20171007T200000Z
+                DTEND:20171007T233000Z
+                DTSTAMP:20200319T182138Z
+                UID:4l80f6dcovnriq38g57g07btid@google.com
+                CREATED:20170719T202915Z
+                DESCRIPTION:
+                LAST-MODIFIED:20170930T190808Z
+                LOCATION:
+                SEQUENCE:1
+                STATUS:CONFIRMED
+                SUMMARY:Maryland Game
+                TRANSP:TRANSPARENT
+                END:VEVENT
+                BEGIN:VEVENT
+                DTSTART:20171002T010000Z
+                DTEND:20171002T020000Z
+                DTSTAMP:20200319T182138Z
+                UID:2v61g7mij4s7ieoubm3sjpun5d@google.com
+                CREATED:20171001T180103Z
+                DESCRIPTION:
+                LAST-MODIFIED:20171001T180103Z
+                LOCATION:
+                SEQUENCE:0
+                STATUS:CONFIRMED
+                SUMMARY:Halloween Wars
+                TRANSP:OPAQUE
+                END:VEVENT
+                END:VCALENDAR
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open calendar.ics
+                | get events
+                | count
+                | echo $it
+            "#
+        ));
+
+        assert_eq!(actual, "2");
+    })
+}
+
+#[test]
+fn from_ics_text_to_table() {
+    Playground::setup("filter_from_ics_test_2", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "calendar.txt",
+            r#"
+                BEGIN:VCALENDAR
+                BEGIN:VEVENT
+                DTSTART:20171007T200000Z
+                DTEND:20171007T233000Z
+                DTSTAMP:20200319T182138Z
+                UID:4l80f6dcovnriq38g57g07btid@google.com
+                CREATED:20170719T202915Z
+                DESCRIPTION:
+                LAST-MODIFIED:20170930T190808Z
+                LOCATION:
+                SEQUENCE:1
+                STATUS:CONFIRMED
+                SUMMARY:Maryland Game
+                TRANSP:TRANSPARENT
+                END:VEVENT
+                END:VCALENDAR
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open calendar.txt
+                | from-ics
+                | get events
+                | get properties
+                | where name == "SUMMARY"
+                | first
+                | get value
+                | echo $it
+            "#
+        ));
+
+        assert_eq!(actual, "Maryland Game");
+    })
+}

crates/nu-cli/tests/format_conversions/mod.rs

@@ -1,6 +1,7 @@
 mod bson;
 mod csv;
 mod html;
+mod ics;
 mod json;
 mod markdown;
 mod ods;
@@ -9,5 +10,6 @@ mod ssv;
 mod toml;
 mod tsv;
 mod url;
+mod vcf;
 mod xlsx;
 mod yaml;

crates/nu-cli/tests/format_conversions/vcf.rs

@@ -0,0 +1,84 @@
+use nu_test_support::fs::Stub::FileWithContentToBeTrimmed;
+use nu_test_support::playground::Playground;
+use nu_test_support::{nu, pipeline};
+
+#[test]
+fn infers_types() {
+    Playground::setup("filter_from_vcf_test_1", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "contacts.vcf",
+            r#"
+                BEGIN:VCARD
+                VERSION:3.0
+                FN:John Doe
+                N:Doe;John;;;
+                EMAIL;TYPE=INTERNET:john.doe99@gmail.com
+                item1.ORG:'Alpine Ski Resort'
+                item1.X-ABLabel:Other
+                item2.TITLE:'Ski Instructor'
+                item2.X-ABLabel:Other
+                BDAY:19001106
+                NOTE:Facebook: john.doe.3\nWebsite: \nHometown: Cleveland\, Ohio
+                CATEGORIES:myContacts
+                END:VCARD
+                BEGIN:VCARD
+                VERSION:3.0
+                FN:Alex Smith
+                N:Smith;Alex;;;
+                TEL;TYPE=CELL:(890) 123-4567
+                CATEGORIES:Band,myContacts
+                END:VCARD
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open contacts.vcf
+                | count
+                | echo $it
+            "#
+        ));
+
+        assert_eq!(actual, "2");
+    })
+}
+
+#[test]
+fn from_vcf_text_to_table() {
+    Playground::setup("filter_from_vcf_test_2", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "contacts.txt",
+            r#"
+                BEGIN:VCARD
+                VERSION:3.0
+                FN:John Doe
+                N:Doe;John;;;
+                EMAIL;TYPE=INTERNET:john.doe99@gmail.com
+                item1.ORG:'Alpine Ski Resort'
+                item1.X-ABLabel:Other
+                item2.TITLE:'Ski Instructor'
+                item2.X-ABLabel:Other
+                BDAY:19001106
+                NOTE:Facebook: john.doe.3\nWebsite: \nHometown: Cleveland\, Ohio
+                CATEGORIES:myContacts
+                END:VCARD
+            "#,
+        )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                open contacts.txt
+                | from-vcf
+                | get properties
+                | where name == "EMAIL"
+                | first
+                | get value
+                | echo $it
+            "#
+        ));
+
+        assert_eq!(actual, "john.doe99@gmail.com");
+    })
+}