diff --git a/Cargo.lock b/Cargo.lock index c23be283b8..0778faa5b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,25 @@ name = "block" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bson" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "md5 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "try_from 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bstr" version = "0.2.6" @@ -1214,6 +1233,15 @@ name = "hex" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "hostname" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.60 (registry+https://github.com/rust-lang/crates.io-index)", + "winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "http" version = "0.1.18" @@ -1510,6 +1538,11 @@ name = "matches" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "md5" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "memchr" version = "2.2.1" @@ -1650,6 +1683,7 @@ dependencies = [ "ansi_term 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "battery 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bson 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "byte-unit 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2562,6 +2596,7 @@ name = "serde_json" version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2900,6 +2935,11 @@ dependencies = [ "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "try_from" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "typenum" version = "1.10.0" @@ -3186,6 +3226,14 @@ dependencies = [ "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "winutil" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "x11" version = "2.18.1" @@ -3254,6 +3302,7 @@ dependencies = [ "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" "checksum blake2b_simd 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "461f4b879a8eb70c1debf7d0788a9a5ff15f1ea9d25925fea264ef4258bed6b2" "checksum block 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" +"checksum bson 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8984b7b33b1f8ac97468df3cefa76c7035abb0786473aa2a437dea0c72855702" "checksum bstr 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e0a692f1c740e7e821ca71a22cf99b9b2322dfa94d10f71443befb1797b3946a" "checksum bumpalo 2.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2cd43d82f27d68911e6ee11ee791fb248f138f5d69424dc02e098d4f152b0b05" "checksum byte-unit 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90139954ec9776c4832d44f212e558ccdacbe915a881bf3de3a1a487fa8d1e87" @@ -3364,6 +3413,7 @@ dependencies = [ "checksum heim-runtime 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b3304dc68b138eb6e9b6c79785dd911306a4bca66757a88373cb034a4dfe3a4d" "checksum heim-virt 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6f68f73f66e6f00404d7b8ed97b458778f6ccafe1ecd65af797ec36f2003bf90" "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" +"checksum hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "21ceb46a83a85e824ef93669c8b390009623863b5c195d1ba747292c0c72f94e" "checksum http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "372bcb56f939e449117fb0869c2e8fd8753a8223d92a172c6e808cf123a5b6e4" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" "checksum ident_case 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" @@ -3400,6 +3450,7 @@ dependencies = [ "checksum mach 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" "checksum malloc_buf 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +"checksum md5 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "79c56d6a0b07f9e19282511c83fc5b086364cbae4ba8c7d5f190c3d9b0425a48" "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" "checksum memoffset 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce6075db033bbbb7ee5a0bbd3a3186bbae616f57fb001c485c7ff77955f8177f" @@ -3548,6 +3599,7 @@ dependencies = [ "checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926" "checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" "checksum toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c7aabe75941d914b72bf3e5d3932ed92ce0664d49d8432305a8b547c37227724" +"checksum try_from 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "923a7ee3e97dbfe8685261beb4511cc9620a1252405d02693d43169729570111" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum unicase 2.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a84e5511b2a947f3ae965dcb29b13b7b1691b6e7332cf5dbc1744138d5acb7f6" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" @@ -3585,6 +3637,7 @@ dependencies = [ "checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" +"checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e" "checksum x11 2.18.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39697e3123f715483d311b5826e254b6f3cfebdd83cf7ef3358f579c3d68e235" "checksum x11-clipboard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "89bd49c06c9eb5d98e6ba6536cf64ac9f7ee3a009b2f53996d405b3944f6bcea" "checksum xcb 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5e917a3f24142e9ff8be2414e36c649d47d6cc2ba81f16201cdef96e533e02de" diff --git a/Cargo.toml b/Cargo.toml index 51dc6622b2..d1b760b9ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ bytes = "0.4.12" log = "0.4.8" pretty_env_logger = "0.3.1" serde = { version = "1.0.98", features = ["derive"] } +bson = "0.13.0" serde_json = "1.0.40" serde-hjson = "0.9.1" serde_yaml = "0.8" diff --git a/src/cli.rs b/src/cli.rs index a04a6211b9..03fb9ac24b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -173,6 +173,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(FromArray), whole_stream_command(FromCSV), whole_stream_command(FromINI), + whole_stream_command(FromBSON), whole_stream_command(FromJSON), whole_stream_command(FromTOML), whole_stream_command(FromXML), diff --git a/src/commands.rs b/src/commands.rs index 3728474f22..a57219a22b 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -15,6 +15,7 @@ crate mod enter; crate mod exit; crate mod first; crate mod from_array; +crate mod from_bson; crate mod from_csv; crate mod from_ini; crate mod from_json; @@ -70,6 +71,7 @@ crate use enter::Enter; crate use exit::Exit; crate use first::First; crate use from_array::FromArray; +crate use from_bson::FromBSON; crate use from_csv::FromCSV; crate use from_ini::FromINI; crate use from_json::FromJSON; diff --git a/src/commands/classified.rs b/src/commands/classified.rs index 4582d5bee3..28005ceb7a 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -176,7 +176,7 @@ impl InternalCommand { match contents { Value::Primitive(Primitive::String(string)) => { - let value = crate::commands::open::parse_as_value( + let value = crate::commands::open::parse_string_as_value( file_extension, string, contents_tag, diff --git a/src/commands/from_bson.rs b/src/commands/from_bson.rs new file mode 100644 index 0000000000..0f0b8d7cbb --- /dev/null +++ b/src/commands/from_bson.rs @@ -0,0 +1,204 @@ +use crate::commands::WholeStreamCommand; +use crate::object::base::OF64; +use crate::object::{Primitive, TaggedDictBuilder, Value}; +use crate::prelude::*; +use bson::{decode_document, Bson, spec::BinarySubtype}; + +pub struct FromBSON; + +impl WholeStreamCommand for FromBSON { + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + from_bson(args, registry) + } + + fn name(&self) -> &str { + "from-bson" + } + + fn signature(&self) -> Signature { + Signature::build("from-bson") + } +} + +fn convert_bson_value_to_nu_value(v: &Bson, tag: impl Into) -> Tagged { + let tag = tag.into(); + + match v { + Bson::FloatingPoint(n) => Value::Primitive(Primitive::Float(OF64::from(*n))).tagged(tag), + Bson::String(s) => Value::Primitive(Primitive::String(String::from(s))).tagged(tag), + Bson::Array(a) => Value::List( + a.iter() + .map(|x| convert_bson_value_to_nu_value(x, tag)) + .collect(), + ) + .tagged(tag), + Bson::Document(doc) => { + let mut collected = TaggedDictBuilder::new(tag); + for (k, v) in doc.iter() { + collected.insert_tagged(k.clone(), convert_bson_value_to_nu_value(v, tag)); + } + + collected.into_tagged_value() + } + Bson::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)).tagged(tag), + Bson::Null => Value::Primitive(Primitive::String(String::from(""))).tagged(tag), + Bson::RegExp(r, opts) => { + let mut collected = TaggedDictBuilder::new(tag); + collected.insert_tagged( + "$regex".to_string(), + Value::Primitive(Primitive::String(String::from(r))).tagged(tag), + ); + collected.insert_tagged( + "$options".to_string(), + Value::Primitive(Primitive::String(String::from(opts))).tagged(tag), + ); + collected.into_tagged_value() + } + Bson::I32(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag), + Bson::I64(n) => Value::Primitive(Primitive::Int(*n as i64)).tagged(tag), + Bson::JavaScriptCode(js) => { + let mut collected = TaggedDictBuilder::new(tag); + collected.insert_tagged( + "$javascript".to_string(), + Value::Primitive(Primitive::String(String::from(js))).tagged(tag), + ); + collected.into_tagged_value() + } + Bson::JavaScriptCodeWithScope(js, doc) => { + let mut collected = TaggedDictBuilder::new(tag); + collected.insert_tagged( + "$javascript".to_string(), + Value::Primitive(Primitive::String(String::from(js))).tagged(tag), + ); + collected.insert_tagged( + "$scope".to_string(), + convert_bson_value_to_nu_value(&Bson::Document(doc.to_owned()), tag), + ); + collected.into_tagged_value() + } + Bson::TimeStamp(ts) => { + let mut collected = TaggedDictBuilder::new(tag); + collected.insert_tagged( + "$timestamp".to_string(), + Value::Primitive(Primitive::Int(*ts as i64)).tagged(tag), + ); + collected.into_tagged_value() + } + Bson::Binary(bst, bytes) => { + let mut collected = TaggedDictBuilder::new(tag); + collected.insert_tagged( + "$binary_subtype".to_string(), + match bst { + BinarySubtype::UserDefined(u) => Value::Primitive(Primitive::Int(*u as i64)), + _ => Value::Primitive(Primitive::String(binary_subtype_to_string(*bst))), + }.tagged(tag) + ); + collected.insert_tagged( + "$binary".to_string(), + Value::Binary(bytes.to_owned()).tagged(tag), + ); + collected.into_tagged_value() + } + Bson::ObjectId(obj_id) => Value::Primitive(Primitive::String(obj_id.to_hex())).tagged(tag), + Bson::UtcDatetime(dt) => Value::Primitive(Primitive::Date(*dt)).tagged(tag), + Bson::Symbol(s) => { + let mut collected = TaggedDictBuilder::new(tag); + collected.insert_tagged( + "$symbol".to_string(), + Value::Primitive(Primitive::String(String::from(s))).tagged(tag), + ); + collected.into_tagged_value() + } + } +} + +fn binary_subtype_to_string(bst: BinarySubtype) -> String { + match bst { + BinarySubtype::Generic => "generic", + BinarySubtype::Function => "function", + BinarySubtype::BinaryOld => "binary_old", + BinarySubtype::UuidOld => "uuid_old", + BinarySubtype::Uuid => "uuid", + BinarySubtype::Md5 => "md5", + _ => unreachable!(), + }.to_string() +} + +#[derive(Debug)] +struct BytesReader { + pos: usize, + inner: Vec, +} + +impl BytesReader { + fn new(bytes: Vec) -> BytesReader { + BytesReader { + pos: 0, + inner: bytes, + } + } +} + +impl std::io::Read for BytesReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let src: &mut &[u8] = &mut self.inner[self.pos..].as_ref(); + let diff = src.read(buf)?; + self.pos += diff; + Ok(diff) + } +} + +pub fn from_bson_bytes_to_value( + bytes: Vec, + tag: impl Into, +) -> bson::DecoderResult> { + let mut docs = Vec::new(); + let mut b_reader = BytesReader::new(bytes); + while let Ok(v) = decode_document(&mut b_reader) { + docs.push(Bson::Document(v)); + } + Ok(convert_bson_value_to_nu_value(&Bson::Array(docs), tag)) +} + +fn from_bson(args: CommandArgs, registry: &CommandRegistry) -> Result { + let args = args.evaluate_once(registry)?; + let span = args.name_span(); + let input = args.input; + + let stream = async_stream_block! { + let values: Vec> = input.values.collect().await; + + for value in values { + let value_tag = value.tag(); + match value.item { + Value::Binary(vb) => + match from_bson_bytes_to_value(vb, span) { + Ok(x) => yield ReturnSuccess::value(x), + Err(_) => { + yield Err(ShellError::labeled_error_with_secondary( + "Could not parse as BSON", + "input cannot be parsed as BSON", + span, + "value originates from here", + value_tag.span, + )) + } + } + _ => yield Err(ShellError::labeled_error_with_secondary( + "Expected a string from pipeline", + "requires string input", + span, + "value originates from here", + value_tag.span, + )), + + } + } + }; + + Ok(stream.to_output_stream()) +} diff --git a/src/commands/open.rs b/src/commands/open.rs index f3df998650..ffdf0bd68e 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -59,9 +59,12 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result Result { - let value = parse_as_value(file_extension, string, contents_tag, name_span).unwrap(); + let value = parse_string_as_value(file_extension, string, contents_tag, name_span).unwrap(); match value { Tagged { @@ -86,7 +89,21 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result yield ReturnSuccess::value(x), } } + Value::Binary(binary) => { + let value = parse_binary_as_value(file_extension, binary, contents_tag, name_span).unwrap(); + match value { + Tagged { + item: Value::List(list), + .. + } => { + for elem in list { + yield ReturnSuccess::value(elem); + } + } + x => yield ReturnSuccess::value(x), + } + } other => yield ReturnSuccess::value(other.tagged(contents_tag)), }; }; @@ -402,7 +419,7 @@ fn read_be_u16(input: &[u8]) -> Option> { } } -pub fn parse_as_value( +pub fn parse_string_as_value( extension: Option, contents: String, contents_tag: Tag, @@ -477,3 +494,25 @@ pub fn parse_as_value( _ => Ok(Value::string(contents).tagged(contents_tag)), } } + +pub fn parse_binary_as_value( + extension: Option, + contents: Vec, + contents_tag: Tag, + name_span: Span, +) -> Result, ShellError> { + match extension { + Some(x) if x == "bson" => { + crate::commands::from_bson::from_bson_bytes_to_value(contents, contents_tag).map_err( + move |_| { + ShellError::labeled_error( + "Could not open as BSON", + "could not open as BSON", + name_span, + ) + }, + ) + } + _ => Ok(Value::Binary(contents).tagged(contents_tag)), + } +} diff --git a/src/utils.rs b/src/utils.rs index 1e75898515..99a59e850e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -196,6 +196,10 @@ mod tests { loc: fixtures().join("jonathan.xml"), at: 0 }, + Res { + loc: fixtures().join("sample.bson"), + at: 0 + }, Res { loc: fixtures().join("sample.ini"), at: 0 diff --git a/tests/commands_test.rs b/tests/commands_test.rs index faced2631f..4e916dc903 100644 --- a/tests/commands_test.rs +++ b/tests/commands_test.rs @@ -24,6 +24,28 @@ fn open_can_parse_csv() { assert_eq!(output, "SPAIN"); } +#[test] +fn open_can_parse_bson_1() { + nu!( + output, + cwd("tests/fixtures/formats"), + "open sample.bson | nth 0 | get b | echo $it" + ); + + assert_eq!(output, "hello"); +} + +#[test] +fn open_can_parse_bson_2() { + nu!( + output, + cwd("tests/fixtures/formats"), + "open sample.bson | nth 6 | get b | get '$binary_subtype' | echo $it " + ); + + assert_eq!(output, "function"); +} + #[test] fn open_can_parse_toml() { nu!( diff --git a/tests/fixtures/formats/sample.bson b/tests/fixtures/formats/sample.bson new file mode 100644 index 0000000000..95c98eb4e1 Binary files /dev/null and b/tests/fixtures/formats/sample.bson differ