Url split query (#14211)

Addresses the following points from #14162

> - There is no built-in counterpart to url build-query for splitting a
query string

There is `from url`, which, due to naming, is a little hard to discover
and suffers from the following point

> - url parse can create records with duplicate keys
> - url parse's params should either:
>   - ~group the same keys into a list.~
> - instead of a record, be a key-value table. (table<key: string,
value: string>)

# Description

## `url split-query`

Counterpart to `url build-query`, splits a url encoded query string to
key value pairs, represented as `table<key: string, value: string>`

```
> "a=one&a=two&b=three" | url split-query
╭───┬─────┬───────╮
│ # │ key │ value │
├───┼─────┼───────┤
│ 0 │ a   │ one   │
│ 1 │ a   │ two   │
│ 2 │ b   │ three │
╰───┴─────┴───────╯
```

## `url parse`

The output's `param` field is now a table as well, mirroring the new
`url split-query`

```
> 'http://localhost?a=one&a=two&b=three' | url parse
╭──────────┬─────────────────────╮
│ scheme   │ http                │
│ username │                     │
│ password │                     │
│ host     │ localhost           │
│ port     │                     │
│ path     │ /                   │
│ query    │ a=one&a=two&b=three │
│ fragment │                     │
│          │ ╭───┬─────┬───────╮ │
│ params   │ │ # │ key │ value │ │
│          │ ├───┼─────┼───────┤ │
│          │ │ 0 │ a   │ one   │ │
│          │ │ 1 │ a   │ two   │ │
│          │ │ 2 │ b   │ three │ │
│          │ ╰───┴─────┴───────╯ │
╰──────────┴─────────────────────╯
```

# User-Facing Changes

- `url parse`'s output has the mentioned change, which is backwards
incompatible.

crates/nu-command/src/default_context.rs

@@ -387,6 +387,7 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
             HttpOptions,
             Url,
             UrlBuildQuery,
+            UrlSplitQuery,
             UrlDecode,
             UrlEncode,
             UrlJoin,

crates/nu-command/src/network/url/mod.rs

@@ -4,6 +4,7 @@ mod encode;
 mod join;
 mod parse;
 mod query;
+mod split_query;
 mod url_;
 
 pub use self::parse::SubCommand as UrlParse;
@@ -11,4 +12,5 @@ pub use build_query::SubCommand as UrlBuildQuery;
 pub use decode::SubCommand as UrlDecode;
 pub use encode::SubCommand as UrlEncode;
 pub use join::SubCommand as UrlJoin;
+pub use split_query::SubCommand as UrlSplitQuery;
 pub use url_::Url;

crates/nu-command/src/network/url/parse.rs

@@ -2,6 +2,8 @@ use nu_engine::command_prelude::*;
 use nu_protocol::Config;
 use url::Url;
 
+use super::query::query_string_to_table;
+
 #[derive(Clone)]
 pub struct SubCommand;
 
@@ -53,7 +55,7 @@ impl Command for SubCommand {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Parses a url",
-            example: "'http://user123:pass567@www.example.com:8081/foo/bar?param1=section&p2=&f[name]=vldc#hello' | url parse",
+            example: "'http://user123:pass567@www.example.com:8081/foo/bar?param1=section&p2=&f[name]=vldc&f[no]=42#hello' | url parse",
             result: Some(Value::test_record(record! {
                     "scheme" =>   Value::test_string("http"),
                     "username" => Value::test_string("user123"),
@@ -61,13 +63,14 @@ impl Command for SubCommand {
                     "host" =>     Value::test_string("www.example.com"),
                     "port" =>     Value::test_string("8081"),
                     "path" =>     Value::test_string("/foo/bar"),
-                    "query" =>    Value::test_string("param1=section&p2=&f[name]=vldc"),
+                    "query" =>    Value::test_string("param1=section&p2=&f[name]=vldc&f[no]=42"),
                     "fragment" => Value::test_string("hello"),
-                    "params" =>   Value::test_record(record! {
+                    "params" =>   Value::test_list(vec![
-                        "param1" =>  Value::test_string("section"),
+                        Value::test_record(record! {"key" => Value::test_string("param1"), "value" => Value::test_string("section") }),
-                        "p2" =>      Value::test_string(""),
+                        Value::test_record(record! {"key" => Value::test_string("p2"), "value" => Value::test_string("") }),
-                        "f[name]" => Value::test_string("vldc"),
+                        Value::test_record(record! {"key" => Value::test_string("f[name]"), "value" => Value::test_string("vldc") }),
-                    }),
+                        Value::test_record(record! {"key" => Value::test_string("f[no]"), "value" => Value::test_string("42") }),
+                    ]),
             })),
         }]
     }
@@ -80,54 +83,41 @@ fn get_url_string(value: &Value, config: &Config) -> String {
 fn parse(value: Value, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
     let url_string = get_url_string(&value, config);
 
-    let result_url = Url::parse(url_string.as_str());
-
     // This is the span of the original string, not the call head.
     let span = value.span();
 
-    match result_url {
+    let url = Url::parse(url_string.as_str()).map_err(|_| ShellError::UnsupportedInput {
-        Ok(url) => {
+        msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com"
-            let params =
+            .to_string(),
-                serde_urlencoded::from_str::<Vec<(String, String)>>(url.query().unwrap_or(""));
+        input: "value originates from here".into(),
-            match params {
+        msg_span: head,
-                Ok(result) => {
+        input_span: span,
-                    let params = result
+    })?;
-                        .into_iter()
-                        .map(|(k, v)| (k, Value::string(v, head)))
-                        .collect();
 
-                    let port = url.port().map(|p| p.to_string()).unwrap_or_default();
+    let params = query_string_to_table(url.query().unwrap_or(""), head, span).map_err(|_| {
-
+        ShellError::UnsupportedInput {
-                    let record = record! {
+            msg: "String not compatible with url-encoding".to_string(),
-                        "scheme" => Value::string(url.scheme(), head),
-                        "username" => Value::string(url.username(), head),
-                        "password" => Value::string(url.password().unwrap_or(""), head),
-                        "host" => Value::string(url.host_str().unwrap_or(""), head),
-                        "port" => Value::string(port, head),
-                        "path" => Value::string(url.path(), head),
-                        "query" => Value::string(url.query().unwrap_or(""), head),
-                        "fragment" => Value::string(url.fragment().unwrap_or(""), head),
-                        "params" => Value::record(params, head),
-                    };
-
-                    Ok(PipelineData::Value(Value::record(record, head), None))
-                }
-                _ => Err(ShellError::UnsupportedInput {
-                    msg: "String not compatible with url-encoding".to_string(),
-                    input: "value originates from here".into(),
-                    msg_span: head,
-                    input_span: span,
-                }),
-            }
-        }
-        Err(_e) => Err(ShellError::UnsupportedInput {
-            msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com"
-                .to_string(),
             input: "value originates from here".into(),
             msg_span: head,
             input_span: span,
-        }),
+        }
-    }
+    })?;
+
+    let port = url.port().map(|p| p.to_string()).unwrap_or_default();
+
+    let record = record! {
+        "scheme" => Value::string(url.scheme(), head),
+        "username" => Value::string(url.username(), head),
+        "password" => Value::string(url.password().unwrap_or(""), head),
+        "host" => Value::string(url.host_str().unwrap_or(""), head),
+        "port" => Value::string(port, head),
+        "path" => Value::string(url.path(), head),
+        "query" => Value::string(url.query().unwrap_or(""), head),
+        "fragment" => Value::string(url.fragment().unwrap_or(""), head),
+        "params" => params,
+    };
+
+    Ok(PipelineData::Value(Value::record(record, head), None))
 }
 
 #[cfg(test)]

crates/nu-command/src/network/url/query.rs

@@ -1,4 +1,4 @@
-use nu_protocol::{Record, ShellError, Span, Type, Value};
+use nu_protocol::{IntoValue, Record, ShellError, Span, Type, Value};
 
 pub fn record_to_query_string(
     record: &Record,
@@ -42,3 +42,26 @@ pub fn record_to_query_string(
         help: None,
     })
 }
+
+pub fn query_string_to_table(query: &str, head: Span, span: Span) -> Result<Value, ShellError> {
+    let params = serde_urlencoded::from_str::<Vec<(String, String)>>(query)
+        .map_err(|_| ShellError::UnsupportedInput {
+            msg: "String not compatible with url-encoding".to_string(),
+            input: "value originates from here".into(),
+            msg_span: head,
+            input_span: span,
+        })?
+        .into_iter()
+        .map(|(key, value)| {
+            Value::record(
+                nu_protocol::record! {
+                    "key" => key.into_value(head),
+                    "value" => value.into_value(head)
+                },
+                head,
+            )
+        })
+        .collect::<Vec<_>>();
+
+    Ok(Value::list(params, head))
+}

crates/nu-command/src/network/url/split_query.rs

@@ -0,0 +1,106 @@
+use nu_engine::command_prelude::*;
+
+use super::query::query_string_to_table;
+
+#[derive(Clone)]
+pub struct SubCommand;
+
+impl Command for SubCommand {
+    fn name(&self) -> &str {
+        "url split-query"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("url split-query")
+            .input_output_types(vec![(
+                Type::String,
+                Type::Table([("key".into(), Type::String), ("value".into(), Type::String)].into()),
+            )])
+            .category(Category::Network)
+    }
+
+    fn description(&self) -> &str {
+        "Converts query string into table applying percent-decoding."
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["convert", "record", "table"]
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Outputs a table representing the contents of this query string",
+                example: r#""mode=normal&userid=31415" | url split-query"#,
+                result: Some(Value::test_list(vec![
+                    Value::test_record(record!{
+                        "key" => Value::test_string("mode"),
+                        "value" => Value::test_string("normal"),
+                    }),
+                    Value::test_record(record!{
+                        "key" => Value::test_string("userid"),
+                        "value" => Value::test_string("31415"),
+                    })
+                ])),
+            },
+            Example {
+                description: "Outputs a table representing the contents of this query string, url-decoding the values",
+                example: r#""a=AT%26T&b=AT+T" | url split-query"#,
+                result: Some(Value::test_list(vec![
+                    Value::test_record(record!{
+                        "key" => Value::test_string("a"),
+                        "value" => Value::test_string("AT&T"),
+                    }),
+                    Value::test_record(record!{
+                        "key" => Value::test_string("b"),
+                        "value" => Value::test_string("AT T"),
+                    }),
+                ])),
+            },
+            Example {
+                description: "Outputs a table representing the contents of this query string",
+                example: r#""a=one&a=two&b=three" | url split-query"#,
+                result: Some(Value::test_list(vec![
+                    Value::test_record(record!{
+                        "key" => Value::test_string("a"),
+                        "value" => Value::test_string("one"),
+                    }),
+                    Value::test_record(record!{
+                        "key" => Value::test_string("a"),
+                        "value" => Value::test_string("two"),
+                    }),
+                    Value::test_record(record!{
+                        "key" => Value::test_string("b"),
+                        "value" => Value::test_string("three"),
+                    }),
+                ])),
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let value = input.into_value(call.head)?;
+        let span = value.span();
+        let query = value.to_expanded_string("", &stack.get_config(engine_state));
+        let table = query_string_to_table(&query, call.head, span)?;
+        Ok(PipelineData::Value(table, None))
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(SubCommand {})
+    }
+}

crates/nu-command/tests/commands/url/parse.rs

@@ -15,7 +15,7 @@ fn url_parse_simple() {
                     path: '/',
                     query: '',
                     fragment: '',
-                    params: {}
+                    params: []
                 }
             "#
     ));
@@ -37,7 +37,7 @@ fn url_parse_with_port() {
                     path: '/',
                     query: '',
                     fragment: '',
-                    params: {}
+                    params: []
                 }
             "#
     ));
@@ -60,7 +60,7 @@ fn url_parse_with_path() {
                     path: '/def/ghj',
                     query: '',
                     fragment: '',
-                    params: {}
+                    params: []
                 }
             "#
     ));
@@ -83,7 +83,30 @@ fn url_parse_with_params() {
                     path: '/def/ghj',
                     query: 'param1=11&param2=',
                     fragment: '',
-                    params: {param1: '11', param2: ''}
+                    params: [[key, value]; ["param1", "11"], ["param2", ""]]
+                }
+            "#
+    ));
+
+    assert_eq!(actual.out, "true");
+}
+
+#[test]
+fn url_parse_with_duplicate_params() {
+    let actual = nu!(pipeline(
+        r#"
+                ("http://www.abc.com:8811/def/ghj?param1=11&param2=&param1=22"
+                | url parse)
+                == {
+                    scheme: 'http',
+                    username: '',
+                    password: '',
+                    host: 'www.abc.com',
+                    port: '8811',
+                    path: '/def/ghj',
+                    query: 'param1=11&param2=&param1=22',
+                    fragment: '',
+                    params: [[key, value]; ["param1", "11"], ["param2", ""], ["param1", "22"]]
                 }
             "#
     ));
@@ -106,7 +129,7 @@ fn url_parse_with_fragment() {
                     path: '/def/ghj',
                     query: 'param1=11&param2=',
                     fragment: 'hello-fragment',
-                    params: {param1: '11', param2: ''}
+                    params: [[key, value]; ["param1", "11"], ["param2", ""]]
                 }
             "#
     ));
@@ -129,7 +152,7 @@ fn url_parse_with_username_and_password() {
                     path: '/def/ghj',
                     query: 'param1=11&param2=',
                     fragment: 'hello-fragment',
-                    params: {param1: '11', param2: ''}
+                    params: [[key, value]; ["param1", "11"], ["param2", ""]]
                 }
             "#
     ));