From ab5e24a0e745326736cd7588974f896bdfa33e5a Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Fri, 20 Mar 2020 15:35:09 -0400 Subject: [PATCH] WIP: Add vcard/ical support (#1504) * Initial from-ical implementation * Initial from-vcard implementation * Rename from-ics and from-vcf for autoconvert * Remove redundant clones * Add from-vcf and from-ics tests Co-authored-by: Jonathan Turner --- Cargo.lock | 35 +++ crates/nu-cli/Cargo.toml | 1 + crates/nu-cli/src/cli.rs | 2 + crates/nu-cli/src/commands.rs | 4 + crates/nu-cli/src/commands/from_ics.rs | 240 ++++++++++++++++++ crates/nu-cli/src/commands/from_vcf.rs | 102 ++++++++ crates/nu-cli/tests/format_conversions/ics.rs | 101 ++++++++ crates/nu-cli/tests/format_conversions/mod.rs | 2 + crates/nu-cli/tests/format_conversions/vcf.rs | 84 ++++++ 9 files changed, 571 insertions(+) create mode 100644 crates/nu-cli/src/commands/from_ics.rs create mode 100644 crates/nu-cli/src/commands/from_vcf.rs create mode 100644 crates/nu-cli/tests/format_conversions/ics.rs create mode 100644 crates/nu-cli/tests/format_conversions/vcf.rs diff --git a/Cargo.lock b/Cargo.lock index 0e8a128cc8..6e4a2fc494 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,6 +901,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" dependencies = [ "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", ] [[package]] @@ -1487,6 +1500,15 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ical" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0e9337a9d901ffb92cf655e854c51734dc004287b959b47830a3308201f18c0" +dependencies = [ + "failure", +] + [[package]] name = "ichwh" version = "0.3.1" @@ -2227,6 +2249,7 @@ dependencies = [ "glob", "hex 0.4.0", "htmlescape", + "ical", "ichwh", "indexmap", "itertools 0.9.0", @@ -3751,6 +3774,18 @@ dependencies = [ "syn", ] +[[package]] +name = "synstructure" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "syntect" version = "3.2.0" diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index a3ffd4db71..3b4baee3d4 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -45,6 +45,7 @@ git2 = { version = "0.13.0", default_features = false } glob = "0.3.0" hex = "0.4" htmlescape = "0.3.1" +ical = "0.6.*" ichwh = "0.3" indexmap = { version = "1.3.2", features = ["serde-1"] } itertools = "0.9.0" diff --git a/crates/nu-cli/src/cli.rs b/crates/nu-cli/src/cli.rs index e37c1894a4..ff8df1ea60 100644 --- a/crates/nu-cli/src/cli.rs +++ b/crates/nu-cli/src/cli.rs @@ -342,6 +342,8 @@ pub fn create_default_context( whole_stream_command(FromXML), whole_stream_command(FromYAML), whole_stream_command(FromYML), + whole_stream_command(FromIcs), + whole_stream_command(FromVcf), ]); cfg_if::cfg_if! { diff --git a/crates/nu-cli/src/commands.rs b/crates/nu-cli/src/commands.rs index b97e68a88e..9afd76f455 100644 --- a/crates/nu-cli/src/commands.rs +++ b/crates/nu-cli/src/commands.rs @@ -30,6 +30,7 @@ pub(crate) mod first; pub(crate) mod format; pub(crate) mod from_bson; pub(crate) mod from_csv; +pub(crate) mod from_ics; pub(crate) mod from_ini; pub(crate) mod from_json; pub(crate) mod from_ods; @@ -38,6 +39,7 @@ pub(crate) mod from_ssv; pub(crate) mod from_toml; pub(crate) mod from_tsv; pub(crate) mod from_url; +pub(crate) mod from_vcf; pub(crate) mod from_xlsx; pub(crate) mod from_xml; pub(crate) mod from_yaml; @@ -136,6 +138,7 @@ pub(crate) use first::First; pub(crate) use format::Format; pub(crate) use from_bson::FromBSON; pub(crate) use from_csv::FromCSV; +pub(crate) use from_ics::FromIcs; pub(crate) use from_ini::FromINI; pub(crate) use from_json::FromJSON; pub(crate) use from_ods::FromODS; @@ -145,6 +148,7 @@ pub(crate) use from_ssv::FromSSV; pub(crate) use from_toml::FromTOML; pub(crate) use from_tsv::FromTSV; pub(crate) use from_url::FromURL; +pub(crate) use from_vcf::FromVcf; pub(crate) use from_xlsx::FromXLSX; pub(crate) use from_xml::FromXML; pub(crate) use from_yaml::FromYAML; diff --git a/crates/nu-cli/src/commands/from_ics.rs b/crates/nu-cli/src/commands/from_ics.rs new file mode 100644 index 0000000000..99e3f685e2 --- /dev/null +++ b/crates/nu-cli/src/commands/from_ics.rs @@ -0,0 +1,240 @@ +extern crate ical; +use crate::commands::WholeStreamCommand; +use crate::prelude::*; +use ical::parser::ical::component::*; +use ical::property::Property; +use nu_errors::ShellError; +use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value}; +use std::io::BufReader; + +pub struct FromIcs; + +impl WholeStreamCommand for FromIcs { + fn name(&self) -> &str { + "from-ics" + } + + fn signature(&self) -> Signature { + Signature::build("from-ics") + } + + fn usage(&self) -> &str { + "Parse text as .ics and create table." + } + + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + from_ics(args, registry) + } +} + +fn from_ics(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let tag = args.name_tag(); + let input = args.input; + + let stream = async_stream! { + let input_string = input.collect_string(tag.clone()).await?.item; + let input_bytes = input_string.as_bytes(); + let buf_reader = BufReader::new(input_bytes); + let parser = ical::IcalParser::new(buf_reader); + + for calendar in parser { + match calendar { + Ok(c) => yield ReturnSuccess::value(calendar_to_value(c, tag.clone())), + Err(_) => yield Err(ShellError::labeled_error( + "Could not parse as .ics", + "input cannot be parsed as .ics", + tag.clone() + )), + } + } + }; + + Ok(stream.to_output_stream()) +} + +fn calendar_to_value(calendar: IcalCalendar, tag: Tag) -> Value { + let mut row = TaggedDictBuilder::new(tag.clone()); + + row.insert_untagged( + "properties", + properties_to_value(calendar.properties, tag.clone()), + ); + row.insert_untagged("events", events_to_value(calendar.events, tag.clone())); + row.insert_untagged("alarms", alarms_to_value(calendar.alarms, tag.clone())); + row.insert_untagged("to-Dos", todos_to_value(calendar.todos, tag.clone())); + row.insert_untagged( + "journals", + journals_to_value(calendar.journals, tag.clone()), + ); + row.insert_untagged( + "free-busys", + free_busys_to_value(calendar.free_busys, tag.clone()), + ); + row.insert_untagged("timezones", timezones_to_value(calendar.timezones, tag)); + + row.into_value() +} + +fn events_to_value(events: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &events + .into_iter() + .map(|event| { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged( + "properties", + properties_to_value(event.properties, tag.clone()), + ); + row.insert_untagged("alarms", alarms_to_value(event.alarms, tag.clone())); + row.into_value() + }) + .collect::>(), + ) +} + +fn alarms_to_value(alarms: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &alarms + .into_iter() + .map(|alarm| { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged( + "properties", + properties_to_value(alarm.properties, tag.clone()), + ); + row.into_value() + }) + .collect::>(), + ) +} + +fn todos_to_value(todos: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &todos + .into_iter() + .map(|todo| { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged( + "properties", + properties_to_value(todo.properties, tag.clone()), + ); + row.insert_untagged("alarms", alarms_to_value(todo.alarms, tag.clone())); + row.into_value() + }) + .collect::>(), + ) +} + +fn journals_to_value(journals: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &journals + .into_iter() + .map(|journal| { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged( + "properties", + properties_to_value(journal.properties, tag.clone()), + ); + row.into_value() + }) + .collect::>(), + ) +} + +fn free_busys_to_value(free_busys: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &free_busys + .into_iter() + .map(|free_busy| { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged( + "properties", + properties_to_value(free_busy.properties, tag.clone()), + ); + row.into_value() + }) + .collect::>(), + ) +} + +fn timezones_to_value(timezones: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &timezones + .into_iter() + .map(|timezone| { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged( + "properties", + properties_to_value(timezone.properties, tag.clone()), + ); + row.insert_untagged( + "transitions", + timezone_transitions_to_value(timezone.transitions, tag.clone()), + ); + row.into_value() + }) + .collect::>(), + ) +} + +fn timezone_transitions_to_value( + transitions: Vec, + tag: Tag, +) -> UntaggedValue { + UntaggedValue::table( + &transitions + .into_iter() + .map(|transition| { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged( + "properties", + properties_to_value(transition.properties, tag.clone()), + ); + row.into_value() + }) + .collect::>(), + ) +} + +fn properties_to_value(properties: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &properties + .into_iter() + .map(|prop| { + let mut row = TaggedDictBuilder::new(tag.clone()); + + let name = UntaggedValue::string(prop.name); + let value = match prop.value { + Some(val) => UntaggedValue::string(val), + None => UntaggedValue::Primitive(Primitive::Nothing), + }; + let params = match prop.params { + Some(param_list) => params_to_value(param_list, tag.clone()).into(), + None => UntaggedValue::Primitive(Primitive::Nothing), + }; + + row.insert_untagged("name", name); + row.insert_untagged("value", value); + row.insert_untagged("params", params); + row.into_value() + }) + .collect::>(), + ) +} + +fn params_to_value(params: Vec<(String, Vec)>, tag: Tag) -> Value { + let mut row = TaggedDictBuilder::new(tag); + + for (param_name, param_values) in params { + let values: Vec = param_values.into_iter().map(|val| val.into()).collect(); + let values = UntaggedValue::table(&values); + row.insert_untagged(param_name, values); + } + + row.into_value() +} diff --git a/crates/nu-cli/src/commands/from_vcf.rs b/crates/nu-cli/src/commands/from_vcf.rs new file mode 100644 index 0000000000..6d4dadfecd --- /dev/null +++ b/crates/nu-cli/src/commands/from_vcf.rs @@ -0,0 +1,102 @@ +extern crate ical; +use crate::commands::WholeStreamCommand; +use crate::prelude::*; +use ical::parser::vcard::component::*; +use ical::property::Property; +use nu_errors::ShellError; +use nu_protocol::{Primitive, ReturnSuccess, Signature, TaggedDictBuilder, UntaggedValue, Value}; +use std::io::BufReader; + +pub struct FromVcf; + +impl WholeStreamCommand for FromVcf { + fn name(&self) -> &str { + "from-vcf" + } + + fn signature(&self) -> Signature { + Signature::build("from-vcf") + } + + fn usage(&self) -> &str { + "Parse text as .vcf and create table." + } + + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + from_vcf(args, registry) + } +} + +fn from_vcf(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let tag = args.name_tag(); + let input = args.input; + + let stream = async_stream! { + let input_string = input.collect_string(tag.clone()).await?.item; + let input_bytes = input_string.as_bytes(); + let buf_reader = BufReader::new(input_bytes); + let parser = ical::VcardParser::new(buf_reader); + + for contact in parser { + match contact { + Ok(c) => yield ReturnSuccess::value(contact_to_value(c, tag.clone())), + Err(_) => yield Err(ShellError::labeled_error( + "Could not parse as .vcf", + "input cannot be parsed as .vcf", + tag.clone() + )), + } + } + }; + + Ok(stream.to_output_stream()) +} + +fn contact_to_value(contact: VcardContact, tag: Tag) -> Value { + let mut row = TaggedDictBuilder::new(tag.clone()); + row.insert_untagged("properties", properties_to_value(contact.properties, tag)); + row.into_value() +} + +fn properties_to_value(properties: Vec, tag: Tag) -> UntaggedValue { + UntaggedValue::table( + &properties + .into_iter() + .map(|prop| { + let mut row = TaggedDictBuilder::new(tag.clone()); + + let name = UntaggedValue::string(prop.name); + let value = match prop.value { + Some(val) => UntaggedValue::string(val), + None => UntaggedValue::Primitive(Primitive::Nothing), + }; + let params = match prop.params { + Some(param_list) => params_to_value(param_list, tag.clone()).into(), + None => UntaggedValue::Primitive(Primitive::Nothing), + }; + + row.insert_untagged("name", name); + row.insert_untagged("value", value); + row.insert_untagged("params", params); + row.into_value() + }) + .collect::>(), + ) +} + +fn params_to_value(params: Vec<(String, Vec)>, tag: Tag) -> Value { + let mut row = TaggedDictBuilder::new(tag); + + for (param_name, param_values) in params { + let values: Vec = param_values.into_iter().map(|val| val.into()).collect(); + let values = UntaggedValue::table(&values); + row.insert_untagged(param_name, values); + } + + row.into_value() +} diff --git a/crates/nu-cli/tests/format_conversions/ics.rs b/crates/nu-cli/tests/format_conversions/ics.rs new file mode 100644 index 0000000000..9503ab91c0 --- /dev/null +++ b/crates/nu-cli/tests/format_conversions/ics.rs @@ -0,0 +1,101 @@ +use nu_test_support::fs::Stub::FileWithContentToBeTrimmed; +use nu_test_support::playground::Playground; +use nu_test_support::{nu, pipeline}; + +#[test] +fn infers_types() { + Playground::setup("filter_from_ics_test_1", |dirs, sandbox| { + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "calendar.ics", + r#" + BEGIN:VCALENDAR + PRODID:-//Google Inc//Google Calendar 70.9054//EN + VERSION:2.0 + BEGIN:VEVENT + DTSTART:20171007T200000Z + DTEND:20171007T233000Z + DTSTAMP:20200319T182138Z + UID:4l80f6dcovnriq38g57g07btid@google.com + CREATED:20170719T202915Z + DESCRIPTION: + LAST-MODIFIED:20170930T190808Z + LOCATION: + SEQUENCE:1 + STATUS:CONFIRMED + SUMMARY:Maryland Game + TRANSP:TRANSPARENT + END:VEVENT + BEGIN:VEVENT + DTSTART:20171002T010000Z + DTEND:20171002T020000Z + DTSTAMP:20200319T182138Z + UID:2v61g7mij4s7ieoubm3sjpun5d@google.com + CREATED:20171001T180103Z + DESCRIPTION: + LAST-MODIFIED:20171001T180103Z + LOCATION: + SEQUENCE:0 + STATUS:CONFIRMED + SUMMARY:Halloween Wars + TRANSP:OPAQUE + END:VEVENT + END:VCALENDAR + "#, + )]); + + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + open calendar.ics + | get events + | count + | echo $it + "# + )); + + assert_eq!(actual, "2"); + }) +} + +#[test] +fn from_ics_text_to_table() { + Playground::setup("filter_from_ics_test_2", |dirs, sandbox| { + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "calendar.txt", + r#" + BEGIN:VCALENDAR + BEGIN:VEVENT + DTSTART:20171007T200000Z + DTEND:20171007T233000Z + DTSTAMP:20200319T182138Z + UID:4l80f6dcovnriq38g57g07btid@google.com + CREATED:20170719T202915Z + DESCRIPTION: + LAST-MODIFIED:20170930T190808Z + LOCATION: + SEQUENCE:1 + STATUS:CONFIRMED + SUMMARY:Maryland Game + TRANSP:TRANSPARENT + END:VEVENT + END:VCALENDAR + "#, + )]); + + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + open calendar.txt + | from-ics + | get events + | get properties + | where name == "SUMMARY" + | first + | get value + | echo $it + "# + )); + + assert_eq!(actual, "Maryland Game"); + }) +} diff --git a/crates/nu-cli/tests/format_conversions/mod.rs b/crates/nu-cli/tests/format_conversions/mod.rs index cfc042ed5b..c79e34a23b 100644 --- a/crates/nu-cli/tests/format_conversions/mod.rs +++ b/crates/nu-cli/tests/format_conversions/mod.rs @@ -1,6 +1,7 @@ mod bson; mod csv; mod html; +mod ics; mod json; mod markdown; mod ods; @@ -9,5 +10,6 @@ mod ssv; mod toml; mod tsv; mod url; +mod vcf; mod xlsx; mod yaml; diff --git a/crates/nu-cli/tests/format_conversions/vcf.rs b/crates/nu-cli/tests/format_conversions/vcf.rs new file mode 100644 index 0000000000..4d86224b23 --- /dev/null +++ b/crates/nu-cli/tests/format_conversions/vcf.rs @@ -0,0 +1,84 @@ +use nu_test_support::fs::Stub::FileWithContentToBeTrimmed; +use nu_test_support::playground::Playground; +use nu_test_support::{nu, pipeline}; + +#[test] +fn infers_types() { + Playground::setup("filter_from_vcf_test_1", |dirs, sandbox| { + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "contacts.vcf", + r#" + BEGIN:VCARD + VERSION:3.0 + FN:John Doe + N:Doe;John;;; + EMAIL;TYPE=INTERNET:john.doe99@gmail.com + item1.ORG:'Alpine Ski Resort' + item1.X-ABLabel:Other + item2.TITLE:'Ski Instructor' + item2.X-ABLabel:Other + BDAY:19001106 + NOTE:Facebook: john.doe.3\nWebsite: \nHometown: Cleveland\, Ohio + CATEGORIES:myContacts + END:VCARD + BEGIN:VCARD + VERSION:3.0 + FN:Alex Smith + N:Smith;Alex;;; + TEL;TYPE=CELL:(890) 123-4567 + CATEGORIES:Band,myContacts + END:VCARD + "#, + )]); + + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + open contacts.vcf + | count + | echo $it + "# + )); + + assert_eq!(actual, "2"); + }) +} + +#[test] +fn from_vcf_text_to_table() { + Playground::setup("filter_from_vcf_test_2", |dirs, sandbox| { + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "contacts.txt", + r#" + BEGIN:VCARD + VERSION:3.0 + FN:John Doe + N:Doe;John;;; + EMAIL;TYPE=INTERNET:john.doe99@gmail.com + item1.ORG:'Alpine Ski Resort' + item1.X-ABLabel:Other + item2.TITLE:'Ski Instructor' + item2.X-ABLabel:Other + BDAY:19001106 + NOTE:Facebook: john.doe.3\nWebsite: \nHometown: Cleveland\, Ohio + CATEGORIES:myContacts + END:VCARD + "#, + )]); + + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + open contacts.txt + | from-vcf + | get properties + | where name == "EMAIL" + | first + | get value + | echo $it + "# + )); + + assert_eq!(actual, "john.doe99@gmail.com"); + }) +}