Add from-eml command (#1656)

* from-eml initial ver

* Adding tests for `from-eml`

* Add eml to prepares_and_decorates_filesystem_source_files

* Sort the file order

Co-authored-by: Jonathan Turner <jonathandturner@users.noreply.github.com>

crates/nu-cli/Cargo.toml

@@ -18,6 +18,7 @@ nu-parser = { version = "0.13.0", path = "../nu-parser" }
 nu-value-ext = { version = "0.13.0", path = "../nu-value-ext" }
 nu-test-support = { version = "0.13.0", path = "../nu-test-support" }
 
+
 ansi_term = "0.12.1"
 app_dirs = "1.2.1"
 async-stream = "0.2"
@@ -35,6 +36,7 @@ ctrlc = "3.1.4"
 derive-new = "0.5.8"
 dirs = "2.0.2"
 dunce = "1.0.0"
+eml-parser = "0.1.0"
 filesize = "0.2.0"
 futures = { version = "0.3", features = ["compat", "io-compat"] }
 futures-util = "0.3.4"

crates/nu-cli/src/cli.rs

@@ -328,6 +328,7 @@ pub fn create_default_context(
             whole_stream_command(ToYAML),
             // File format input
             whole_stream_command(FromCSV),
+            whole_stream_command(FromEML),
             whole_stream_command(FromTSV),
             whole_stream_command(FromSSV),
             whole_stream_command(FromINI),

crates/nu-cli/src/commands.rs

@@ -32,6 +32,7 @@ pub(crate) mod first;
 pub(crate) mod format;
 pub(crate) mod from_bson;
 pub(crate) mod from_csv;
+pub(crate) mod from_eml;
 pub(crate) mod from_ics;
 pub(crate) mod from_ini;
 pub(crate) mod from_json;
@@ -145,6 +146,7 @@ pub(crate) use first::First;
 pub(crate) use format::Format;
 pub(crate) use from_bson::FromBSON;
 pub(crate) use from_csv::FromCSV;
+pub(crate) use from_eml::FromEML;
 pub(crate) use from_ics::FromIcs;
 pub(crate) use from_ini::FromINI;
 pub(crate) use from_json::FromJSON;

crates/nu-cli/src/commands/from_eml.rs

@@ -0,0 +1,122 @@
+use crate::commands::WholeStreamCommand;
+use crate::prelude::*;
+use ::eml_parser::eml::*;
+use ::eml_parser::EmlParser;
+use nu_errors::ShellError;
+use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue};
+use nu_source::Tagged;
+
+pub struct FromEML;
+
+const DEFAULT_BODY_PREVIEW: usize = 50;
+
+#[derive(Deserialize, Clone)]
+pub struct FromEMLArgs {
+    #[serde(rename(deserialize = "preview-body"))]
+    preview_body: Option<Tagged<usize>>,
+}
+
+impl WholeStreamCommand for FromEML {
+    fn name(&self) -> &str {
+        "from-eml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from-eml").named(
+            "preview-body",
+            SyntaxShape::Int,
+            "How many bytes of the body to preview",
+            Some('b'),
+        )
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .eml and create table."
+    }
+
+    fn run(
+        &self,
+        args: CommandArgs,
+        registry: &CommandRegistry,
+    ) -> Result<OutputStream, ShellError> {
+        args.process(registry, from_eml)?.run()
+    }
+}
+
+fn emailaddress_to_value(tag: &Tag, email_address: &EmailAddress) -> TaggedDictBuilder {
+    let mut dict = TaggedDictBuilder::with_capacity(tag, 2);
+    let (n, a) = match email_address {
+        EmailAddress::AddressOnly { address } => {
+            (UntaggedValue::nothing(), UntaggedValue::string(address))
+        }
+        EmailAddress::NameAndEmailAddress { name, address } => {
+            (UntaggedValue::string(name), UntaggedValue::string(address))
+        }
+    };
+
+    dict.insert_untagged("Name", n);
+    dict.insert_untagged("Address", a);
+
+    dict
+}
+
+fn headerfieldvalue_to_value(tag: &Tag, value: &HeaderFieldValue) -> UntaggedValue {
+    use HeaderFieldValue::*;
+
+    match value {
+        SingleEmailAddress(address) => emailaddress_to_value(tag, address).into_untagged_value(),
+        MultipleEmailAddresses(addresses) => UntaggedValue::Table(
+            addresses
+                .iter()
+                .map(|a| emailaddress_to_value(tag, a).into_value())
+                .collect(),
+        ),
+        Unstructured(s) => UntaggedValue::string(s),
+        Empty => UntaggedValue::nothing(),
+    }
+}
+
+fn from_eml(
+    eml_args: FromEMLArgs,
+    runnable_context: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    let input = runnable_context.input;
+    let tag = runnable_context.name;
+
+    let stream = async_stream! {
+        let value = input.collect_string(tag.clone()).await?;
+
+        let body_preview = eml_args.preview_body.map(|b| b.item).unwrap_or(DEFAULT_BODY_PREVIEW);
+
+        let eml = EmlParser::from_string(value.item)
+        .with_body_preview(body_preview)
+        .parse()
+        .map_err(|_| ShellError::labeled_error("Could not parse .eml file", "could not parse .eml file", &tag))?;
+
+        let mut dict = TaggedDictBuilder::new(&tag);
+
+        if let Some(subj) = eml.subject {
+           dict.insert_untagged("Subject", UntaggedValue::string(subj));
+        }
+
+        if let Some(from) = eml.from {
+           dict.insert_untagged("From", headerfieldvalue_to_value(&tag, &from));
+        }
+
+        if let Some(to) = eml.to {
+           dict.insert_untagged("To", headerfieldvalue_to_value(&tag, &to));
+        }
+
+        for HeaderField{ name, value } in eml.headers.iter() {
+           dict.insert_untagged(name, headerfieldvalue_to_value(&tag, &value));
+        }
+
+        if let Some(body) = eml.body {
+           dict.insert_untagged("Body", UntaggedValue::string(body));
+        }
+
+        yield ReturnSuccess::value(dict.into_value());
+    };
+
+    Ok(stream.to_output_stream())
+}

crates/nu-cli/src/utils.rs

@@ -301,6 +301,10 @@ mod tests {
                     loc: fixtures().join("sample.db"),
                     at: 0
                 },
+                Res {
+                    loc: fixtures().join("sample.eml"),
+                    at: 0
+                },
                 Res {
                     loc: fixtures().join("sample.ini"),
                     at: 0

crates/nu-cli/tests/format_conversions/eml.rs

@@ -0,0 +1,84 @@
+use nu_test_support::{nu, pipeline};
+
+const TEST_CWD: &str = "tests/fixtures/formats";
+
+// The To field in this email is just "username@domain.com", which gets parsed out as the Address. The Name is empty.
+#[test]
+fn from_eml_get_to_field() {
+    let actual = nu!(
+        cwd: TEST_CWD,
+        pipeline(
+            r#"
+            open sample.eml
+            | get To
+            | get Address
+            | echo $it
+        "#
+        )
+    );
+
+    assert_eq!(actual, "username@domain.com");
+
+    let actual = nu!(
+        cwd: TEST_CWD,
+        pipeline(
+            r#"
+            open sample.eml
+            | get To
+            | get Name
+            | echo $it
+        "#
+        )
+    );
+
+    assert_eq!(actual, "");
+}
+
+// The Reply-To field in this email is "aw-confirm@ebay.com" <aw-confirm@ebay.com>, meaning both the Name and Address values are identical.
+#[test]
+fn from_eml_get_replyto_field() {
+    let actual = nu!(
+        cwd: TEST_CWD,
+        pipeline(
+            r#"
+            open sample.eml
+            | get Reply-To
+            | get Address
+            | echo $it
+        "#
+        )
+    );
+
+    assert_eq!(actual, "aw-confirm@ebay.com");
+
+    let actual = nu!(
+        cwd: TEST_CWD,
+        pipeline(
+            r#"
+            open sample.eml
+            | get Reply-To
+            | get Name
+            | echo $it
+        "#
+        )
+    );
+
+    assert_eq!(actual, "aw-confirm@ebay.com");
+}
+
+// The Reply-To field in this email is "aw-confirm@ebay.com" <aw-confirm@ebay.com>, meaning both the Name and Address values are identical.
+#[test]
+fn from_eml_get_subject_field() {
+    let actual = nu!(
+        cwd: TEST_CWD,
+        pipeline(
+            r#"
+            open sample.eml
+            | get Subject
+            | echo $it
+        "#
+        )
+    );
+
+    assert_eq!(actual, "Billing Issues");
+}

crates/nu-cli/tests/format_conversions/mod.rs

@@ -1,5 +1,6 @@
 mod bson;
 mod csv;
+mod eml;
 mod html;
 mod ics;
 mod json;

tests/fixtures/formats/sample.eml

@@ -0,0 +1,97 @@
+Return-Path: <aw-confirm@ebay.com>
+X-Original-To: username@domain.com
+Delivered-To: username@domain.com
+Received: from 81.18.87.130 (unknown [81.18.87.190])
+ by spanky.domain.com (Postfix) with SMTP id CCC115378FC
+ for <username@domain.com>; Sat, 27 Nov 2004 15:33:24 -0500 (EST)
+Received: from 20.84.152.113 by 65.23.81.142; Sat, 27 Nov 2004 15:24:27 -0500
+Message-ID: <LCCOVYFMRBWQCHUDWYCOYUW@hotmail.com>
+From: "aw-confirm@ebay.com" <aw-confirm@ebay.com>
+Reply-To: "aw-confirm@ebay.com" <aw-confirm@ebay.com>
+To: username@domain.com
+Subject: Billing Issues
+Date: Sun, 28 Nov 2004 00:30:27 +0400
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+	boundary="--591699981497957"
+X-Priority: 3
+X-CS-IP: 224.248.218.116
+Status: O
+X-Status: 
+X-Keywords: 
+X-UID: 1
+
+----591699981497957
+Content-Type: text/html;
+Content-Transfer-Encoding: quoted-printable
+
+<html>
+<p>
+  <A target=3D"_blank"     
+href=3D"http://click3.ebay.com/99019653.50692.0.20283"  
+><IMG
+src=3D"http://emailpics3.ebay.com/627422250/images/logo-18.gif"
+   border=3D0></A></p>
+<p><br>
+<FONT  SIZE=3D2 PTSIZE=3D10 FAMILY=3D"SANSSERIF"
+FACE=3D"Arial" LANG=3D"0"><FONT  SIZE=3D2 PTSIZE=3D10
+FAMILY=3D"SANSSERIF" FACE=3D"Arial" LANG=3D"0">Dear valued
+eBay member:<BR>
+  <BR>
+We recently have determined that different computers
+have logged onto your eBay account, and multiple
+password failures were present before the logons. We
+now need you to re-confirm your account information to
+us. If this is not completed by <strong>November 30,
+2004</strong>, we will be forced to suspend your
+account indefinitely, as it may have been used for
+fraudulent purposes. We thank you for your cooperation
+in this manner.</FONT>
+<p>To confirm your eBay records click here: <br>
+  <a
+HREF=3Dhttp://140.130.108.11/account/aw-confirm/ebayDLLupdate/index.html
+target=3D"_self">http://cgi1.ebay.com/aw-cgi/ebayISAPI.dll?UPdate</a></p>
+<p><font size=3D"2" face=3D"Arial, Helvetica,
+sans-serif"><font color=3D"#000000">We appreciate your
+support and understanding, as we work together to keep
+eBay a safe place to trade.</font></font><font
+color=3D"#000000" size=3D"2" face=3D"Arial, Helvetica,
+sans-serif"><BR>
+Thank you for your patience in this matter.</font><br>
+  <br>
+</p>
+<p><font color=3D"#000000" size=3D"2" face=3D"Arial,
+Helvetica, sans-serif">Trust and Safety
+Department</font><br>
+  <font color=3D"#000000" size=3D"2" face=3D"Arial,
+Helvetica, sans-serif">eBay Inc.</font></p>
+<p><font size=3D"2" face=3D"Arial, Helvetica,
+sans-serif"><font color=3D"#999999" size=3D"1">Please do
+not reply to this e-mail as this is only a
+notification. Mail sent to this address cannot be
+answered. </font></font></p>
+<p><font color=3D"#000000" size=3D"1" face=3D"Arial,
+Helvetica, sans-serif"><font color=3D"black"><span
+style=3D"color: black;">Copyright 1995-2004 <a
+target=3D"_blank"
+href=3D"http://pages.ebay.com/community/aboutebay/index.html">eBay
+Inc.</a> All Rights Reserved. Designated trademarks
+and brands are the property of their respective
+owners. Use of this Web site constitutes acceptance of
+the eBay <a target=3D"_blank"
+href=3D"http://pages.ebay.com/help/community/png-user.html">User
+Agreement</a> and <a target=3D"_blank"
+href=3D"http://pages.ebay.com/help/community/png-priv.html">Privacy
+Policy</a>.</span> </font></font><font color=3D"#000000"
+size=3D"1" face=3D"Arial,
+Helvetica, sans-serif">Designated trademarks and
+brands are the property of their respective owners.
+eBay and the eBay logo are trademarks of eBay Inc.
+eBay is located at 2145 Hamilton Avenue, San Jose, CA
+95125. </font><br>
+</p>
+</html>
+
+
+
+----591699981497957--