diff --git a/Cargo.lock b/Cargo.lock index d43c7a403c..efb00d9bae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2051,6 +2051,12 @@ dependencies = [ "url", ] +[[package]] +name = "gjson" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864178e25a00c41404f1728997c9a21a7b746be1faefe6ce4dc41eb48bb4234f" + [[package]] name = "glob" version = "0.3.0" @@ -3110,6 +3116,7 @@ dependencies = [ "nu_plugin_match", "nu_plugin_post", "nu_plugin_ps", + "nu_plugin_query_json", "nu_plugin_s3", "nu_plugin_selector", "nu_plugin_start", @@ -3704,6 +3711,18 @@ dependencies = [ "sysinfo", ] +[[package]] +name = "nu_plugin_query_json" +version = "0.29.2" +dependencies = [ + "gjson", + "nu-errors", + "nu-plugin", + "nu-protocol", + "nu-source", + "nu-test-support", +] + [[package]] name = "nu_plugin_s3" version = "0.29.2" diff --git a/Cargo.toml b/Cargo.toml index 85a38c0e7a..a56618513c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ nu_plugin_inc = { version = "0.29.2", path = "./crates/nu_plugin_inc", optional nu_plugin_match = { version = "0.29.2", path = "./crates/nu_plugin_match", optional = true } nu_plugin_post = { version = "0.29.2", path = "./crates/nu_plugin_post", optional = true } nu_plugin_ps = { version = "0.29.2", path = "./crates/nu_plugin_ps", optional = true } +nu_plugin_query_json = { version = "0.29.2", path = "./crates/nu_plugin_query_json", optional = true } nu_plugin_s3 = { version = "0.29.2", path = "./crates/nu_plugin_s3", optional = true } nu_plugin_selector = { version = "0.29.2", path = "./crates/nu_plugin_selector", optional = true } nu_plugin_start = { version = "0.29.2", path = "./crates/nu_plugin_start", optional = true } @@ -121,6 +122,7 @@ extra = [ "chart", "xpath", "selector", + "query-json", ] wasi = ["inc", "match", "ptree-support", "match", "tree", "rustyline-support"] @@ -141,6 +143,7 @@ binaryview = ["nu_plugin_binaryview"] bson = ["nu_plugin_from_bson", "nu_plugin_to_bson"] chart = ["nu_plugin_chart"] clipboard-cli = ["nu-cli/clipboard-cli", "nu-command/clipboard-cli"] +query-json = ["nu_plugin_query_json"] s3 = ["nu_plugin_s3"] selector = ["nu_plugin_selector"] sqlite = ["nu_plugin_from_sqlite", "nu_plugin_to_sqlite"] @@ -214,6 +217,11 @@ name = "nu_plugin_extra_tree" path = "src/plugins/nu_plugin_extra_tree.rs" required-features = ["tree"] +[[bin]] +name = "nu_plugin_extra_query_json" +path = "src/plugins/nu_plugin_extra_query_json.rs" +required-features = ["query-json"] + [[bin]] name = "nu_plugin_extra_start" path = "src/plugins/nu_plugin_extra_start.rs" diff --git a/crates/nu_plugin_query_json/Cargo.toml b/crates/nu_plugin_query_json/Cargo.toml new file mode 100644 index 0000000000..30fcbb889d --- /dev/null +++ b/crates/nu_plugin_query_json/Cargo.toml @@ -0,0 +1,20 @@ +[package] +authors = ["The Nu Project Contributors"] +description = "query json files with gjson" +edition = "2018" +license = "MIT" +name = "nu_plugin_query_json" +version = "0.29.2" + +[lib] +doctest = false + +[dependencies] +gjson = "0.7.1" +nu-errors = { version = "0.29.2", path = "../nu-errors" } +nu-plugin = { version = "0.29.2", path = "../nu-plugin" } +nu-protocol = { version = "0.29.2", path = "../nu-protocol" } +nu-source = { version = "0.29.2", path = "../nu-source" } + +[dev-dependencies] +nu-test-support = { path = "../nu-test-support", version = "0.29.2" } diff --git a/crates/nu_plugin_query_json/src/lib.rs b/crates/nu_plugin_query_json/src/lib.rs new file mode 100644 index 0000000000..3e18566bab --- /dev/null +++ b/crates/nu_plugin_query_json/src/lib.rs @@ -0,0 +1,4 @@ +mod nu; +mod query_json; + +pub use query_json::QueryJson; diff --git a/crates/nu_plugin_query_json/src/main.rs b/crates/nu_plugin_query_json/src/main.rs new file mode 100644 index 0000000000..19fecc26b7 --- /dev/null +++ b/crates/nu_plugin_query_json/src/main.rs @@ -0,0 +1,6 @@ +use nu_plugin::serve_plugin; +use nu_plugin_query_json::QueryJson; + +fn main() { + serve_plugin(&mut QueryJson::new()); +} diff --git a/crates/nu_plugin_query_json/src/nu/mod.rs b/crates/nu_plugin_query_json/src/nu/mod.rs new file mode 100644 index 0000000000..d92d372d2d --- /dev/null +++ b/crates/nu_plugin_query_json/src/nu/mod.rs @@ -0,0 +1,48 @@ +use nu_errors::ShellError; +use nu_plugin::Plugin; +use nu_protocol::{ + CallInfo, Primitive, ReturnSuccess, ReturnValue, Signature, SyntaxShape, UntaggedValue, Value, +}; +use nu_source::TaggedItem; + +use crate::{query_json::begin_json_query, QueryJson}; + +impl Plugin for QueryJson { + fn config(&mut self) -> Result { + Ok(Signature::build("query json") + .desc("execute json query on json file (open --raw | query json 'query string')\nsee https://gjson.dev/ for more info.") + .required("query", SyntaxShape::String, "json query") + .filter()) + } + + fn begin_filter(&mut self, call_info: CallInfo) -> Result, ShellError> { + let tag = call_info.name_tag; + let query = call_info.args.nth(0).ok_or_else(|| { + ShellError::labeled_error("json query not passed", "json query not passed", &tag) + })?; + + self.query = query.as_string()?; + self.tag = tag; + + Ok(vec![]) + } + + fn filter(&mut self, input: Value) -> Result, ShellError> { + match input { + Value { + value: UntaggedValue::Primitive(Primitive::String(s)), + .. + } => match begin_json_query(s, (*self.query).tagged(&self.tag)) { + Ok(result) => Ok(result.into_iter().map(ReturnSuccess::value).collect()), + Err(err) => Err(err), + }, + Value { tag, .. } => Err(ShellError::labeled_error_with_secondary( + "Expected text from pipeline", + "requires text input", + &self.tag, + "value originates from here", + tag, + )), + } + } +} diff --git a/crates/nu_plugin_query_json/src/query_json.rs b/crates/nu_plugin_query_json/src/query_json.rs new file mode 100644 index 0000000000..967faa2679 --- /dev/null +++ b/crates/nu_plugin_query_json/src/query_json.rs @@ -0,0 +1,162 @@ +use gjson::Value as gjValue; +use nu_errors::ShellError; +use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value}; +use nu_source::{Tag, Tagged}; + +pub struct QueryJson { + pub query: String, + pub tag: Tag, +} + +impl QueryJson { + pub fn new() -> QueryJson { + QueryJson { + query: String::new(), + tag: Tag::unknown(), + } + } +} + +impl Default for QueryJson { + fn default() -> Self { + Self::new() + } +} + +pub fn begin_json_query(input: String, query: Tagged<&str>) -> Result, ShellError> { + execute_json_query(input, query.item.to_string(), query.tag()) +} + +fn execute_json_query( + input_string: String, + query_string: String, + tag: impl Into, +) -> Result, ShellError> { + let tag = tag.into(); + + // Validate the json before trying to query it + let is_valid_json = gjson::valid(input_string.as_str()); + if !is_valid_json { + return Err(ShellError::labeled_error( + "invalid json", + "invalid json", + tag, + )); + } + + let mut ret: Vec = vec![]; + let val: gjValue = gjson::get(input_string.as_str(), &query_string); + + if query_contains_modifiers(query_string.as_str()) { + let json_str = val.json(); + let json_val = Value::from(json_str); + ret.push(json_val); + } else { + let gjv = convert_gjson_value_to_nu_value(&val, &tag); + + match gjv.value { + UntaggedValue::Primitive(_) => ret.push(gjv), + UntaggedValue::Row(_) => ret.push(gjv), + UntaggedValue::Table(t) => { + // Unravel the table so it's not a table inside of a table in the output + for v in t.iter() { + let c = v.clone(); + ret.push(c) + } + } + _ => (), + } + } + + Ok(ret) +} +fn query_contains_modifiers(query: &str) -> bool { + // https://github.com/tidwall/gjson.rs documents 7 modifiers as of 4/19/21 + // Some of these modifiers mean we really need to output the data as a string + // instead of tabular data. Others don't matter. + + // Output as String + // @ugly: Remove all whitespace from a json document. + // @pretty: Make the json document more human readable. + query.contains("@ugly") || query.contains("@pretty") + + // Output as Tablular + // Since it's output as tabular, which is our default, we can just ignore these + // @reverse: Reverse an array or the members of an object. + // @this: Returns the current element. It can be used to retrieve the root element. + // @valid: Ensure the json document is valid. + // @flatten: Flattens an array. + // @join: Joins multiple objects into a single object. +} + +fn convert_gjson_value_to_nu_value(v: &gjValue, tag: impl Into) -> Value { + let tag = tag.into(); + + match v.kind() { + gjson::Kind::Array => { + let mut values = vec![]; + v.each(|_k, v| { + values.push(convert_gjson_value_to_nu_value(&v, &tag)); + true + }); + + UntaggedValue::Table(values).into_value(&tag) + } + gjson::Kind::Null => UntaggedValue::nothing().into_value(&tag), + gjson::Kind::False => UntaggedValue::boolean(false).into_value(&tag), + gjson::Kind::Number => UntaggedValue::int(v.i64()).into_value(&tag), + gjson::Kind::String => UntaggedValue::string(v.str()).into_value(&tag), + gjson::Kind::True => UntaggedValue::boolean(true).into_value(&tag), + // I'm not sure how to test this, so it may not work + gjson::Kind::Object => { + // eprint!("Object: "); + let mut collected = TaggedDictBuilder::new(&tag); + v.each(|k, v| { + // eprintln!("k:{} v:{}", k.str(), v.str()); + collected.insert_value(k.str(), convert_gjson_value_to_nu_value(&v, &tag)); + true + }); + collected.into_value() + } + } +} + +#[cfg(test)] +mod tests { + use gjson::{valid, Value as gjValue}; + + #[test] + fn validate_string() { + let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#; + let val = valid(json); + assert_eq!(val, true); + } + + #[test] + fn answer_from_get_age() { + let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#; + let val: gjValue = gjson::get(json, "age"); + assert_eq!(val.str(), "37"); + } + + #[test] + fn answer_from_get_children() { + let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#; + let val: gjValue = gjson::get(json, "children"); + assert_eq!(val.str(), r#"["Sara", "Alex", "Jack"]"#); + } + + #[test] + fn answer_from_get_children_count() { + let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#; + let val: gjValue = gjson::get(json, "children.#"); + assert_eq!(val.str(), "3"); + } + + #[test] + fn answer_from_get_friends_first_name() { + let json = r#"{ "name": { "first": "Tom", "last": "Anderson" }, "age": 37, "children": ["Sara", "Alex", "Jack"], "friends": [ { "first": "James", "last": "Murphy" }, { "first": "Roger", "last": "Craig" } ] }"#; + let val: gjValue = gjson::get(json, "friends.#.first"); + assert_eq!(val.str(), r#"["James","Roger"]"#); + } +} diff --git a/src/plugins/nu_plugin_extra_query_json.rs b/src/plugins/nu_plugin_extra_query_json.rs new file mode 100644 index 0000000000..19fecc26b7 --- /dev/null +++ b/src/plugins/nu_plugin_extra_query_json.rs @@ -0,0 +1,6 @@ +use nu_plugin::serve_plugin; +use nu_plugin_query_json::QueryJson; + +fn main() { + serve_plugin(&mut QueryJson::new()); +} diff --git a/wix/main.wxs b/wix/main.wxs index 988f457401..321f1f990e 100644 --- a/wix/main.wxs +++ b/wix/main.wxs @@ -280,6 +280,14 @@ Source='target\$(var.Profile)\nu_plugin_selector.exe' KeyPath='yes'/> + + + @@ -324,6 +332,7 @@ +