update ast to support output to json (#8962)

# Description
This PR changes the `ast` command to be able to output `--json` as well
as `nuon` (default) with "pretty" and "minified" output. I'm hoping this
functionality will be usable in the vscode extension for semantic
tokenization and highlighting.

# User-Facing Changes
There's a new `--json`/`-j` option. Prior version output of nuon is
maintained as default.

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A
clippy::needless_collect` to check that you're using the standard code
style
- `cargo test --workspace` to check that all tests pass
- `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the
standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

Cargo.lock

@@ -2905,6 +2905,7 @@ dependencies = [
  "rust-embed",
  "same-file",
  "serde",
+ "serde_json",
  "serde_urlencoded",
  "serde_yaml",
  "sha2",

crates/nu-command/Cargo.toml

@@ -80,6 +80,7 @@ roxmltree = "0.18.0"
 rust-embed = "6.6.0"
 same-file = "1.0.6"
 serde = { version = "1.0.123", features = ["derive"] }
+serde_json = "1.0"
 serde_urlencoded = "0.7.0"
 serde_yaml = "0.9.4"
 sha2 = "0.10.0"

crates/nu-command/src/debug/ast.rs

@@ -3,8 +3,8 @@ use nu_parser::parse;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack, StateWorkingSet},
-    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Spanned, SyntaxShape,
-    Type, Value,
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned,
+    SyntaxShape, Type, Value,
 };
 
 #[derive(Clone)]
@@ -27,6 +27,8 @@ impl Command for Ast {
                 SyntaxShape::String,
                 "the pipeline to print the ast for",
             )
+            .switch("json", "serialize to json", Some('j'))
+            .switch("minify", "minify the nuon or json output", Some('m'))
             .allow_variants_without_examples(true)
             .category(Category::Debug)
     }
@@ -39,26 +41,86 @@ impl Command for Ast {
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let pipeline: Spanned<String> = call.req(engine_state, stack, 0)?;
+        let to_json = call.has_flag("json");
+        let minify = call.has_flag("minify");
         let mut working_set = StateWorkingSet::new(engine_state);
-
         let block_output = parse(&mut working_set, None, pipeline.item.as_bytes(), false);
-
         let error_output = working_set.parse_errors.first();
+        let block_span = match &block_output.span {
+            Some(span) => span,
+            None => &pipeline.span,
+        };
+        if to_json {
+            // Get the block as json
+            let serde_block_str = if minify {
+                serde_json::to_string(&block_output)
+            } else {
+                serde_json::to_string_pretty(&block_output)
+            };
+            let block_json = match serde_block_str {
+                Ok(json) => json,
+                Err(e) => Err(ShellError::CantConvert {
+                    to_type: "string".to_string(),
+                    from_type: "block".to_string(),
+                    span: *block_span,
+                    help: Some(format!(
+                        "Error: {e}\nCan't convert {block_output:?} to string"
+                    )),
+                })?,
+            };
+            // Get the error as json
+            let serde_error_str = if minify {
+                serde_json::to_string(&error_output)
+            } else {
+                serde_json::to_string_pretty(&error_output)
+            };
 
-        let block_value = Value::String {
-            val: format!("{block_output:#?}"),
-            span: pipeline.span,
-        };
-        let error_value = Value::String {
-            val: format!("{error_output:#?}"),
-            span: pipeline.span,
-        };
-        let output_record = Value::Record {
-            cols: vec!["block".to_string(), "error".to_string()],
-            vals: vec![block_value, error_value],
-            span: pipeline.span,
-        };
-        Ok(output_record.into_pipeline_data())
+            let error_json = match serde_error_str {
+                Ok(json) => json,
+                Err(e) => Err(ShellError::CantConvert {
+                    to_type: "string".to_string(),
+                    from_type: "error".to_string(),
+                    span: *block_span,
+                    help: Some(format!(
+                        "Error: {e}\nCan't convert {error_output:?} to string"
+                    )),
+                })?,
+            };
+
+            // Create a new output record, merging the block and error
+            let output_record = Value::Record {
+                cols: vec!["block".to_string(), "error".to_string()],
+                vals: vec![
+                    Value::string(block_json, *block_span),
+                    Value::string(error_json, Span::test_data()),
+                ],
+                span: pipeline.span,
+            };
+            Ok(output_record.into_pipeline_data())
+        } else {
+            let block_value = Value::String {
+                val: if minify {
+                    format!("{block_output:?}")
+                } else {
+                    format!("{block_output:#?}")
+                },
+                span: pipeline.span,
+            };
+            let error_value = Value::String {
+                val: if minify {
+                    format!("{error_output:?}")
+                } else {
+                    format!("{error_output:#?}")
+                },
+                span: pipeline.span,
+            };
+            let output_record = Value::Record {
+                cols: vec!["block".to_string(), "error".to_string()],
+                vals: vec![block_value, error_value],
+                span: pipeline.span,
+            };
+            Ok(output_record.into_pipeline_data())
+        }
     }
 
     fn examples(&self) -> Vec<Example> {
@@ -78,6 +140,17 @@ impl Command for Ast {
                 example: "ast 'for x in 1..10 { echo $x '",
                 result: None,
             },
+            Example {
+                description:
+                    "Print the ast of a pipeline with an error, as json, in a nushell table",
+                example: "ast 'for x in 1..10 { echo $x ' --json | get block | from json",
+                result: None,
+            },
+            Example {
+                description: "Print the ast of a pipeline with an error, as json, minified",
+                example: "ast 'for x in 1..10 { echo $x ' -j -m",
+                result: None,
+            },
         ]
     }
 }

crates/nu-protocol/src/ast/block.rs

@@ -1,10 +1,9 @@
+use super::Pipeline;
+use crate::{Signature, Span, VarId};
+use serde::{Deserialize, Serialize};
 use std::ops::{Index, IndexMut};
 
-use crate::{Signature, Span, VarId};
-
-use super::Pipeline;
-
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Block {
     pub signature: Box<Signature>,
     pub pipelines: Vec<Pipeline>,

crates/nu-protocol/src/ast/pipeline.rs

@@ -1,8 +1,8 @@
+use crate::{ast::Expression, engine::StateWorkingSet, Span, VarId};
+use serde::{Deserialize, Serialize};
 use std::ops::{Index, IndexMut};
 
-use crate::{ast::Expression, engine::StateWorkingSet, Span, VarId};
-
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum Redirection {
     Stdout,
     Stderr,
@@ -10,7 +10,7 @@ pub enum Redirection {
 }
 
 // Note: Span in the below is for the span of the connector not the whole element
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum PipelineElement {
     Expression(Option<Span>, Expression),
     Redirection(Span, Redirection, Expression),
@@ -106,7 +106,7 @@ impl PipelineElement {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Pipeline {
     pub elements: Vec<PipelineElement>,
 }

crates/nu-protocol/src/parse_error.rs

@@ -1,8 +1,9 @@
 use crate::{Span, Type};
 use miette::Diagnostic;
+use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
-#[derive(Clone, Debug, Error, Diagnostic)]
+#[derive(Clone, Debug, Error, Diagnostic, Serialize, Deserialize)]
 pub enum ParseError {
     /// The parser encountered unexpected tokens, when the code should have
     /// finished. You should remove these or finish adding what you intended