to json -r not removing whitespaces fix (#11948)

fixes #11900 # Description Use `serde_json` instead. # User-Facing Changes The problem described in the issue now no longer persists. No whitespace in the output of `to json --raw` Output of unicode escape changed to consistent `\uffff` # Tests + Formatting I corrected all Tests that were affected by this change.
2024-11-10 07:04:13 +00:00 · 2024-03-20 22:14:31 +01:00 · 2024-03-20 22:14:31 +01:00 · 8237d15683
commit 8237d15683
parent fdf7f28d07
13 changed files with 45 additions and 34 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -2994,6 +2994,7 @@ dependencies = [
 "linked-hash-map",
 "num-traits",
 "serde",
+ "serde_json",
 ]

 [[package]]
@ -4946,9 +4947,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.112"
+version = "1.0.114"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d1bd37ce2324cf3bf85e5a25f96eb4baf0d5aa6eba43e7ae8958870c4ec48ed"
+checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
 dependencies = [
 "indexmap",
 "itoa",
--- a/crates/nu-command/tests/commands/cal.rs
+++ b/crates/nu-command/tests/commands/cal.rs
@ -5,7 +5,7 @@ fn cal_full_year() {
    let actual = nu!("cal -y --full-year 2010 | first | to json -r");

    let first_week_2010_json =
-        r#"{"year": 2010,"su": null,"mo": null,"tu": null,"we": null,"th": null,"fr": 1,"sa": 2}"#;
+        r#"{"year":2010,"su":null,"mo":null,"tu":null,"we":null,"th":null,"fr":1,"sa":2}"#;

    assert_eq!(actual.out, first_week_2010_json);
 }
@ -18,7 +18,7 @@ fn cal_february_2020_leap_year() {
        "#
    ));

-    let cal_february_json = r#"[{"year": 2020,"month": "february","su": null,"mo": null,"tu": null,"we": null,"th": null,"fr": null,"sa": 1},{"year": 2020,"month": "february","su": 2,"mo": 3,"tu": 4,"we": 5,"th": 6,"fr": 7,"sa": 8},{"year": 2020,"month": "february","su": 9,"mo": 10,"tu": 11,"we": 12,"th": 13,"fr": 14,"sa": 15},{"year": 2020,"month": "february","su": 16,"mo": 17,"tu": 18,"we": 19,"th": 20,"fr": 21,"sa": 22},{"year": 2020,"month": "february","su": 23,"mo": 24,"tu": 25,"we": 26,"th": 27,"fr": 28,"sa": 29}]"#;
+    let cal_february_json = r#"[{"year":2020,"month":"february","su":null,"mo":null,"tu":null,"we":null,"th":null,"fr":null,"sa":1},{"year":2020,"month":"february","su":2,"mo":3,"tu":4,"we":5,"th":6,"fr":7,"sa":8},{"year":2020,"month":"february","su":9,"mo":10,"tu":11,"we":12,"th":13,"fr":14,"sa":15},{"year":2020,"month":"february","su":16,"mo":17,"tu":18,"we":19,"th":20,"fr":21,"sa":22},{"year":2020,"month":"february","su":23,"mo":24,"tu":25,"we":26,"th":27,"fr":28,"sa":29}]"#;

    assert_eq!(actual.out, cal_february_json);
 }
@ -53,7 +53,7 @@ fn cal_week_day_start_mo() {
        "#
    ));

-    let cal_january_json = r#"[{"month": "january","mo": null,"tu": null,"we": 1,"th": 2,"fr": 3,"sa": 4,"su": 5},{"month": "january","mo": 6,"tu": 7,"we": 8,"th": 9,"fr": 10,"sa": 11,"su": 12},{"month": "january","mo": 13,"tu": 14,"we": 15,"th": 16,"fr": 17,"sa": 18,"su": 19},{"month": "january","mo": 20,"tu": 21,"we": 22,"th": 23,"fr": 24,"sa": 25,"su": 26},{"month": "january","mo": 27,"tu": 28,"we": 29,"th": 30,"fr": 31,"sa": null,"su": null}]"#;
+    let cal_january_json = r#"[{"month":"january","mo":null,"tu":null,"we":1,"th":2,"fr":3,"sa":4,"su":5},{"month":"january","mo":6,"tu":7,"we":8,"th":9,"fr":10,"sa":11,"su":12},{"month":"january","mo":13,"tu":14,"we":15,"th":16,"fr":17,"sa":18,"su":19},{"month":"january","mo":20,"tu":21,"we":22,"th":23,"fr":24,"sa":25,"su":26},{"month":"january","mo":27,"tu":28,"we":29,"th":30,"fr":31,"sa":null,"su":null}]"#;

    assert_eq!(actual.out, cal_january_json);
 }
--- a/crates/nu-command/tests/commands/each.rs
+++ b/crates/nu-command/tests/commands/each.rs
@ -32,7 +32,7 @@ fn each_window_stride() {
 fn each_no_args_in_block() {
    let actual = nu!("echo [[foo bar]; [a b] [c d] [e f]] | each {|i| $i | to json -r } | get 1");

-    assert_eq!(actual.out, r#"{"foo": "c","bar": "d"}"#);
+    assert_eq!(actual.out, r#"{"foo":"c","bar":"d"}"#);
 }

 #[test]
--- a/crates/nu-command/tests/commands/find.rs
+++ b/crates/nu-command/tests/commands/find.rs
@ -14,7 +14,7 @@ fn find_with_list_search_with_string() {
 fn find_with_list_search_with_char() {
    let actual = nu!("[moe larry curly] | find l | to json -r");

-    assert_eq!(actual.out, "[\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37ml\u{1b}[0m\u{1b}[37marry\u{1b}[0m\",\"\u{1b}[37mcur\u{1b}[0m\u{1b}[41;37ml\u{1b}[0m\u{1b}[37my\u{1b}[0m\"]");
+    assert_eq!(actual.out, "[\"\\u001b[37m\\u001b[0m\\u001b[41;37ml\\u001b[0m\\u001b[37marry\\u001b[0m\",\"\\u001b[37mcur\\u001b[0m\\u001b[41;37ml\\u001b[0m\\u001b[37my\\u001b[0m\"]");
 }

 #[test]
@ -48,7 +48,7 @@ fn find_with_filepath_search_with_string() {

    assert_eq!(
        actual.out,
-        "[\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37marep\u{1b}[0m\u{1b}[37mas.clu\u{1b}[0m\"]"
+        "[\"\\u001b[37m\\u001b[0m\\u001b[41;37marep\\u001b[0m\\u001b[37mas.clu\\u001b[0m\"]"
    );
 }

@ -57,7 +57,7 @@ fn find_with_filepath_search_with_multiple_patterns() {
    let actual =
        nu!(r#"["amigos.txt","arepas.clu","los.txt","tres.txt"] | find arep ami | to json -r"#);

-    assert_eq!(actual.out, "[\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37mami\u{1b}[0m\u{1b}[37mgos.txt\u{1b}[0m\",\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37marep\u{1b}[0m\u{1b}[37mas.clu\u{1b}[0m\"]");
+    assert_eq!(actual.out, "[\"\\u001b[37m\\u001b[0m\\u001b[41;37mami\\u001b[0m\\u001b[37mgos.txt\\u001b[0m\",\"\\u001b[37m\\u001b[0m\\u001b[41;37marep\\u001b[0m\\u001b[37mas.clu\\u001b[0m\"]");
 }

 #[test]
--- a/crates/nu-command/tests/commands/math/mod.rs
+++ b/crates/nu-command/tests/commands/math/mod.rs
@ -420,7 +420,7 @@ fn compound_where() {
        "#
    ));

-    assert_eq!(actual.out, r#"[{"a": 2,"b": 1}]"#);
+    assert_eq!(actual.out, r#"[{"a":2,"b":1}]"#);
 }

 #[test]
@ -431,7 +431,7 @@ fn compound_where_paren() {
        "#
    ));

-    assert_eq!(actual.out, r#"[{"a": 2,"b": 1},{"a": 2,"b": 2}]"#);
+    assert_eq!(actual.out, r#"[{"a":2,"b":1},{"a":2,"b":2}]"#);
 }

 // TODO: these ++ tests are not really testing *math* functionality, maybe find another place for them
--- a/crates/nu-command/tests/commands/sort_by.rs
+++ b/crates/nu-command/tests/commands/sort_by.rs
@ -62,7 +62,7 @@ fn ls_sort_by_name_sensitive() {
        "
    ));

-    let json_output = r#"[{"name": "B.txt"},{"name": "C"},{"name": "a.txt"}]"#;
+    let json_output = r#"[{"name":"B.txt"},{"name":"C"},{"name":"a.txt"}]"#;

    assert_eq!(actual.out, json_output);
 }
@ -79,7 +79,7 @@ fn ls_sort_by_name_insensitive() {
        "
    ));

-    let json_output = r#"[{"name": "a.txt"},{"name": "B.txt"},{"name": "C"}]"#;
+    let json_output = r#"[{"name":"a.txt"},{"name":"B.txt"},{"name":"C"}]"#;
    assert_eq!(actual.out, json_output);
 }

@ -95,7 +95,7 @@ fn ls_sort_by_type_name_sensitive() {
        "
    ));

-    let json_output = r#"[{"name": "C","type": "Dir"},{"name": "B.txt","type": "File"},{"name": "a.txt","type": "File"}]"#;
+    let json_output = r#"[{"name":"C","type":"Dir"},{"name":"B.txt","type":"File"},{"name":"a.txt","type":"File"}]"#;
    assert_eq!(actual.out, json_output);
 }

@ -111,7 +111,7 @@ fn ls_sort_by_type_name_insensitive() {
        "
    ));

-    let json_output = r#"[{"name": "C","type": "Dir"},{"name": "a.txt","type": "File"},{"name": "B.txt","type": "File"}]"#;
+    let json_output = r#"[{"name":"C","type":"Dir"},{"name":"a.txt","type":"File"},{"name":"B.txt","type":"File"}]"#;
    assert_eq!(actual.out, json_output);
 }

--- a/crates/nu-json/Cargo.toml
+++ b/crates/nu-json/Cargo.toml
@ -20,6 +20,7 @@ default = ["preserve_order"]
 linked-hash-map = { version = "0.5", optional = true }
 num-traits = "0.2"
 serde = "1.0"
+serde_json = "1.0.114"

 [dev-dependencies]
 # nu-path = { path="../nu-path", version = "0.91.1" }
--- a/crates/nu-json/src/ser.rs
+++ b/crates/nu-json/src/ser.rs
@ -1032,8 +1032,9 @@ pub fn to_string_raw<T>(value: &T) -> Result<String>
 where
    T: ser::Serialize,
 {
-    let vec = to_vec(value)?;
-    let string = String::from_utf8(vec)?;
-    let output = string.lines().map(str::trim).collect();
-    Ok(output)
+    let result = serde_json::to_string(value);
+    match result {
+        Ok(result_string) => Ok(result_string),
+        Err(error) => Err(Error::Io(std::io::Error::from(error))),
+    }
 }
--- a/crates/nu-std/tests/test_formats.nu
+++ b/crates/nu-std/tests/test_formats.nu
@ -2,12 +2,12 @@ use std assert

 def test_data_multiline [] {
  let lines = [
-    "{\"a\": 1}",
-    "{\"a\": 2}",
-    "{\"a\": 3}",
-    "{\"a\": 4}",
-    "{\"a\": 5}",
-    "{\"a\": 6}",
+    "{\"a\":1}",
+    "{\"a\":2}",
+    "{\"a\":3}",
+    "{\"a\":4}",
+    "{\"a\":5}",
+    "{\"a\":6}",
  ]

  if $nu.os-info.name == "windows" {
@ -73,7 +73,7 @@ def to_ndjson_multiple_objects [] {
 def to_ndjson_single_object [] {
  use std formats *
  let result = [{a:1}] | to ndjson | str trim
-  let expect = "{\"a\": 1}"
+  let expect = "{\"a\":1}"
  assert equal $result $expect "could not convert to NDJSON"
 }

@ -89,6 +89,6 @@ def to_jsonl_multiple_objects [] {
 def to_jsonl_single_object [] {
  use std formats *
  let result = [{a:1}] | to jsonl | str trim
-  let expect = "{\"a\": 1}"
+  let expect = "{\"a\":1}"
  assert equal $result $expect "could not convert to JSONL"
 }
--- a/src/tests/test_converters.rs
+++ b/src/tests/test_converters.rs
@ -18,7 +18,7 @@ fn from_json_2() -> TestResult {
 fn to_json_raw_flag_1() -> TestResult {
    run_test(
        "[[a b]; [jim susie] [3 4]] | to json -r",
-        r#"[{"a": "jim","b": "susie"},{"a": 3,"b": 4}]"#,
+        r#"[{"a":"jim","b":"susie"},{"a":3,"b":4}]"#,
    )
 }

@ -26,7 +26,7 @@ fn to_json_raw_flag_1() -> TestResult {
 fn to_json_raw_flag_2() -> TestResult {
    run_test(
        "[[\"a b\" c]; [jim susie] [3 4]] | to json -r",
-        r#"[{"a b": "jim","c": "susie"},{"a b": 3,"c": 4}]"#,
+        r#"[{"a b":"jim","c":"susie"},{"a b":3,"c":4}]"#,
    )
 }

@ -34,7 +34,7 @@ fn to_json_raw_flag_2() -> TestResult {
 fn to_json_raw_flag_3() -> TestResult {
    run_test(
        "[[\"a b\" \"c d\"]; [\"jim smith\" \"susie roberts\"] [3 4]] | to json -r",
-        r#"[{"a b": "jim smith","c d": "susie roberts"},{"a b": 3,"c d": 4}]"#,
+        r#"[{"a b":"jim smith","c d":"susie roberts"},{"a b":3,"c d":4}]"#,
    )
 }

@ -42,6 +42,14 @@ fn to_json_raw_flag_3() -> TestResult {
 fn to_json_escaped() -> TestResult {
    run_test(
        r#"{foo: {bar: '[{"a":"b","c": 2}]'}} | to json --raw"#,
-        r#"{"foo":{"bar": "[{\"a\":\"b\",\"c\": 2}]"}}"#,
+        r#"{"foo":{"bar":"[{\"a\":\"b\",\"c\": 2}]"}}"#,
+    )
+}
+
+#[test]
+fn to_json_raw_backslash_in_quotes() -> TestResult {
+    run_test(
+        r#"{a: '\', b: 'some text'} | to json -r"#,
+        r#"{"a":"\\","b":"some text"}"#,
    )
 }
--- a/src/tests/test_strings.rs
+++ b/src/tests/test_strings.rs
@ -68,6 +68,6 @@ fn case_insensitive_sort() -> TestResult {
 fn case_insensitive_sort_columns() -> TestResult {
    run_test(
        r#"[[version, package]; ["two", "Abc"], ["three", "abc"], ["four", "abc"]] | sort-by -i package version | to json --raw"#,
-        r#"[{"version": "four","package": "abc"},{"version": "three","package": "abc"},{"version": "two","package": "Abc"}]"#,
+        r#"[{"version":"four","package":"abc"},{"version":"three","package":"abc"},{"version":"two","package":"Abc"}]"#,
    )
 }
--- a/src/tests/test_table_operations.rs
+++ b/src/tests/test_table_operations.rs
@ -132,7 +132,7 @@ fn command_filter_reject_3() -> TestResult {
 fn command_filter_reject_4() -> TestResult {
    run_test(
        "[[lang, gems, grade]; [nu, 100, a]] | reject gems | to json -r",
-        r#"[{"lang": "nu","grade": "a"}]"#,
+        r#"[{"lang":"nu","grade":"a"}]"#,
    )
 }

--- a/tests/shell/pipeline/commands/internal.rs
+++ b/tests/shell/pipeline/commands/internal.rs
@ -607,7 +607,7 @@ fn index_row() {
        let foo = [[name]; [joe] [bob]]; echo $foo.1 | to json --raw
        ");

-    assert_eq!(actual.out, r#"{"name": "bob"}"#);
+    assert_eq!(actual.out, r#"{"name":"bob"}"#);
 }

 #[test]