LazyRecord (#7619)

This is an attempt to implement a new `Value::LazyRecord` variant for
performance reasons.

`LazyRecord` is like a regular `Record`, but it's possible to access
individual columns without evaluating other columns. I've implemented
`LazyRecord` for the special `$nu` variable; accessing `$nu` is
relatively slow because of all the information in `scope`, and [`$nu`
accounts for about 2/3 of Nu's startup time on
Linux](https://github.com/nushell/nushell/issues/6677#issuecomment-1364618122).

### Benchmarks

I ran some benchmarks on my desktop (Linux, 12900K) and the results are
very pleasing.

Nu's time to start up and run a command (`cargo build --release;
hyperfine 'target/release/nu -c "echo \"Hello, world!\""' --shell=none
--warmup 10`) goes from **8.8ms to 3.2ms, about 2.8x faster**.

Tests are also much faster! Running `cargo nextest` (with our very slow
`proptest` tests disabled) goes from **7.2s to 4.4s (1.6x faster)**,
because most tests involve launching a new instance of Nu.

### Design (updated)

I've added a new `LazyRecord` trait and added a `Value` variant wrapping
those trait objects, much like `CustomValue`. `LazyRecord`
implementations must implement these 2 functions:

```rust
// All column names
fn column_names(&self) -> Vec<&'static str>;

// Get 1 specific column value
fn get_column_value(&self, column: &str) -> Result<Value, ShellError>;
 ```

### Serializability

`Value` variants must implement `Serializable` and `Deserializable`, which poses some problems because I want to use unserializable things like `EngineState` in `LazyRecord`s. To work around this, I basically lie to the type system:

1. Add `#[typetag::serde(tag = "type")]` to `LazyRecord` to make it serializable
2. Any unserializable fields in `LazyRecord` implementations get marked with `#[serde(skip)]`
3. At the point where a `LazyRecord` normally would get serialized and sent to a plugin, I instead collect it into a regular `Value::Record` (which can be serialized)

Cargo.lock

@@ -2821,6 +2821,7 @@ dependencies = [
  "nu-path",
  "nu-protocol",
  "nu-utils",
+ "serde",
  "sysinfo",
 ]
 

crates/nu-cli/src/completions/variable_completions.rs

@@ -262,6 +262,20 @@ fn nested_suggestions(
 
             output
         }
+        Value::LazyRecord { val, .. } => {
+            // Add all the columns as completion
+            for column_name in val.column_names() {
+                output.push(Suggestion {
+                    value: column_name.to_string(),
+                    description: None,
+                    extra: None,
+                    span: current_span,
+                    append_whitespace: false,
+                });
+            }
+
+            output
+        }
 
         _ => output,
     }

crates/nu-command/src/database/commands/into_sqlite.rs

@@ -262,6 +262,10 @@ fn nu_value_to_string(value: Value, separator: &str) -> String {
             .map(|(x, y)| format!("{}: {}", x, nu_value_to_string(y.clone(), ", ")))
             .collect::<Vec<_>>()
             .join(separator),
+        Value::LazyRecord { val, .. } => match val.collect() {
+            Ok(val) => nu_value_to_string(val, separator),
+            Err(error) => format!("{:?}", error),
+        },
         Value::Block { val, .. } => format!("<Block {}>", val),
         Value::Closure { val, .. } => format!("<Closure {}>", val),
         Value::Nothing { .. } => String::new(),

crates/nu-command/src/filters/columns.rs

@@ -122,6 +122,15 @@ fn getcol(
                 .into_pipeline_data(ctrlc)
                 .set_metadata(metadata))
         }
+        PipelineData::Value(Value::LazyRecord { val, .. }, ..) => Ok(val
+            .column_names()
+            .into_iter()
+            .map(move |x| Value::String {
+                val: x.into(),
+                span: head,
+            })
+            .into_pipeline_data(ctrlc)
+            .set_metadata(metadata)),
         PipelineData::Value(Value::Record { cols, .. }, ..) => Ok(cols
             .into_iter()
             .map(move |x| Value::String { val: x, span: head })

crates/nu-command/src/filters/find.rs

@@ -380,6 +380,26 @@ fn find_with_rest_and_highlight(
                                     .map_or(false, |aval| aval.is_true())
                             }
                         }),
+                        Value::LazyRecord { val, .. } => match val.collect() {
+                            Ok(val) => match val {
+                                Value::Record { vals, .. } => vals.iter().any(|val| {
+                                    if let Ok(span) = val.span() {
+                                        let lower_val = Value::string(
+                                            val.into_string("", &filter_config).to_lowercase(),
+                                            Span::test_data(),
+                                        );
+
+                                        term.r#in(span, &lower_val, span)
+                                            .map_or(false, |aval| aval.is_true())
+                                    } else {
+                                        term.r#in(span, val, span)
+                                            .map_or(false, |aval| aval.is_true())
+                                    }
+                                }),
+                                _ => false,
+                            },
+                            Err(_) => false,
+                        },
                         Value::Binary { .. } => false,
                     }) != invert
                 },
@@ -440,6 +460,26 @@ fn find_with_rest_and_highlight(
                                     .map_or(false, |value| value.is_true())
                             }
                         }),
+                        Value::LazyRecord { val, .. } => match val.collect() {
+                            Ok(val) => match val {
+                                Value::Record { vals, .. } => vals.iter().any(|val| {
+                                    if let Ok(span) = val.span() {
+                                        let lower_val = Value::string(
+                                            val.into_string("", &filter_config).to_lowercase(),
+                                            Span::test_data(),
+                                        );
+
+                                        term.r#in(span, &lower_val, span)
+                                            .map_or(false, |value| value.is_true())
+                                    } else {
+                                        term.r#in(span, val, span)
+                                            .map_or(false, |value| value.is_true())
+                                    }
+                                }),
+                                _ => false,
+                            },
+                            Err(_) => false,
+                        },
                         Value::Binary { .. } => false,
                     }) != invert
                 }),

crates/nu-command/src/formats/to/json.rs

@@ -136,6 +136,10 @@ pub fn value_to_json_value(v: &Value) -> Result<nu_json::Value, ShellError> {
             }
             nu_json::Value::Object(m)
         }
+        Value::LazyRecord { val, .. } => {
+            let collected = val.collect()?;
+            value_to_json_value(&collected)?
+        }
         Value::CustomValue { val, .. } => val.to_json(),
     })
 }

crates/nu-command/src/formats/to/nuon.rs

@@ -185,6 +185,10 @@ pub fn value_to_string(v: &Value, span: Span) -> Result<String, ShellError> {
             }
             Ok(format!("{{{}}}", collection.join(", ")))
         }
+        Value::LazyRecord { val, .. } => {
+            let collected = val.collect()?;
+            value_to_string(&collected, span)
+        }
         // All strings outside data structures are quoted because they are in 'command position'
         // (could be mistaken for commands by the Nu parser)
         Value::String { val, .. } => Ok(escape_quote_string(val)),

crates/nu-command/src/formats/to/text.rs

@@ -141,6 +141,10 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String {
             .map(|(x, y)| format!("{}: {}", x, local_into_string(y.clone(), ", ", config)))
             .collect::<Vec<_>>()
             .join(separator),
+        Value::LazyRecord { val, .. } => match val.collect() {
+            Ok(val) => local_into_string(val, separator, config),
+            Err(error) => format!("{:?}", error),
+        },
         Value::Block { val, .. } => format!("<Block {}>", val),
         Value::Closure { val, .. } => format!("<Closure {}>", val),
         Value::Nothing { .. } => String::new(),

crates/nu-command/src/formats/to/toml.rs

@@ -61,6 +61,10 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result<toml::Value, ShellErr
             }
             toml::Value::Table(m)
         }
+        Value::LazyRecord { val, .. } => {
+            let collected = val.collect()?;
+            helper(engine_state, &collected)?
+        }
         Value::List { vals, .. } => toml::Value::Array(toml_list(engine_state, vals)?),
         Value::Block { span, .. } => {
             let code = engine_state.get_span_contents(span);

crates/nu-command/src/formats/to/yaml.rs

@@ -62,6 +62,10 @@ pub fn value_to_yaml_value(v: &Value) -> Result<serde_yaml::Value, ShellError> {
             }
             serde_yaml::Value::Mapping(m)
         }
+        Value::LazyRecord { val, .. } => {
+            let collected = val.collect()?;
+            value_to_yaml_value(&collected)?
+        }
         Value::List { vals, .. } => {
             let mut out = vec![];
 

crates/nu-command/src/viewers/table.rs

@@ -331,6 +331,18 @@ fn handle_table_command(
 
             Ok(val.into_pipeline_data())
         }
+        PipelineData::Value(Value::LazyRecord { val, .. }, ..) => {
+            let collected = val.collect()?.into_pipeline_data();
+            handle_table_command(
+                engine_state,
+                stack,
+                call,
+                collected,
+                row_offset,
+                table_view,
+                term_width,
+            )
+        }
         PipelineData::Value(Value::Error { error }, ..) => {
             // Propagate this error outward, so that it goes to stderr
             // instead of stdout.

crates/nu-engine/Cargo.toml

@@ -14,6 +14,7 @@ nu-glob = { path = "../nu-glob", version = "0.74.1" }
 nu-utils = { path = "../nu-utils", version = "0.74.1"  }
 
 chrono = { version="0.4.23", features = ["std"], default-features = false }
+serde = {version = "1.0.143", default-features = false }
 sysinfo ="0.26.2"
 
 [features]

crates/nu-engine/src/eval.rs

@@ -1,4 +1,4 @@
-use crate::{current_dir_str, get_full_help, scope::create_scope};
+use crate::{current_dir_str, get_full_help, nu_variable::NuVariable};
 use nu_path::expand_path_with;
 use nu_protocol::{
     ast::{
@@ -6,12 +6,11 @@ use nu_protocol::{
         Operator, PathMember, PipelineElement, Redirection,
     },
     engine::{EngineState, Stack},
-    Config, HistoryFileFormat, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData,
+    Config, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData, Range, ShellError, Span,
-    Range, ShellError, Span, Spanned, Unit, Value, VarId, ENV_VARIABLE_ID,
+    Spanned, Unit, Value, VarId, ENV_VARIABLE_ID,
 };
 use nu_utils::stdout_write_all_and_flush;
 use std::collections::HashMap;
-use sysinfo::SystemExt;
 
 pub fn eval_operator(op: &Expression) -> Result<Operator, ShellError> {
     match op {
@@ -1120,146 +1119,15 @@ pub fn eval_variable(
     span: Span,
 ) -> Result<Value, ShellError> {
     match var_id {
-        nu_protocol::NU_VARIABLE_ID => {
         // $nu
-            let mut output_cols = vec![];
+        nu_protocol::NU_VARIABLE_ID => Ok(Value::LazyRecord {
-            let mut output_vals = vec![];
+            val: Box::new(NuVariable {
-
+                engine_state: engine_state.clone(),
-            if let Some(path) = engine_state.get_config_path("config-path") {
+                stack: stack.clone(),
-                output_cols.push("config-path".into());
-                output_vals.push(Value::String {
-                    val: path.to_string_lossy().to_string(),
                 span,
-                });
+            }),
-            }
-
-            if let Some(path) = engine_state.get_config_path("env-path") {
-                output_cols.push("env-path".into());
-                output_vals.push(Value::String {
-                    val: path.to_string_lossy().to_string(),
             span,
-                });
+        }),
-            }
-
-            if let Some(mut config_path) = nu_path::config_dir() {
-                config_path.push("nushell");
-                let mut env_config_path = config_path.clone();
-                let mut loginshell_path = config_path.clone();
-
-                let mut history_path = config_path.clone();
-
-                match engine_state.config.history_file_format {
-                    HistoryFileFormat::Sqlite => {
-                        history_path.push("history.sqlite3");
-                    }
-                    HistoryFileFormat::PlainText => {
-                        history_path.push("history.txt");
-                    }
-                }
-                // let mut history_path = config_files::get_history_path(); // todo: this should use the get_history_path method but idk where to put that function
-
-                output_cols.push("history-path".into());
-                output_vals.push(Value::String {
-                    val: history_path.to_string_lossy().to_string(),
-                    span,
-                });
-
-                if engine_state.get_config_path("config-path").is_none() {
-                    config_path.push("config.nu");
-
-                    output_cols.push("config-path".into());
-                    output_vals.push(Value::String {
-                        val: config_path.to_string_lossy().to_string(),
-                        span,
-                    });
-                }
-
-                if engine_state.get_config_path("env-path").is_none() {
-                    env_config_path.push("env.nu");
-
-                    output_cols.push("env-path".into());
-                    output_vals.push(Value::String {
-                        val: env_config_path.to_string_lossy().to_string(),
-                        span,
-                    });
-                }
-
-                loginshell_path.push("login.nu");
-
-                output_cols.push("loginshell-path".into());
-                output_vals.push(Value::String {
-                    val: loginshell_path.to_string_lossy().to_string(),
-                    span,
-                });
-            }
-
-            #[cfg(feature = "plugin")]
-            if let Some(path) = &engine_state.plugin_signatures {
-                if let Some(path_str) = path.to_str() {
-                    output_cols.push("plugin-path".into());
-                    output_vals.push(Value::String {
-                        val: path_str.into(),
-                        span,
-                    });
-                }
-            }
-
-            output_cols.push("scope".into());
-            output_vals.push(create_scope(engine_state, stack, span)?);
-
-            if let Some(home_path) = nu_path::home_dir() {
-                if let Some(home_path_str) = home_path.to_str() {
-                    output_cols.push("home-path".into());
-                    output_vals.push(Value::String {
-                        val: home_path_str.into(),
-                        span,
-                    })
-                }
-            }
-
-            let temp = std::env::temp_dir();
-            if let Some(temp_path) = temp.to_str() {
-                output_cols.push("temp-path".into());
-                output_vals.push(Value::String {
-                    val: temp_path.into(),
-                    span,
-                })
-            }
-
-            let pid = std::process::id();
-            output_cols.push("pid".into());
-            output_vals.push(Value::int(pid as i64, span));
-
-            let sys = sysinfo::System::new();
-            let ver = match sys.kernel_version() {
-                Some(v) => v,
-                None => "unknown".into(),
-            };
-
-            let os_record = Value::Record {
-                cols: vec![
-                    "name".into(),
-                    "arch".into(),
-                    "family".into(),
-                    "kernel_version".into(),
-                ],
-                vals: vec![
-                    Value::string(std::env::consts::OS, span),
-                    Value::string(std::env::consts::ARCH, span),
-                    Value::string(std::env::consts::FAMILY, span),
-                    Value::string(ver, span),
-                ],
-                span,
-            };
-            output_cols.push("os-info".into());
-            output_vals.push(os_record);
-
-            Ok(Value::Record {
-                cols: output_cols,
-                vals: output_vals,
-                span,
-            })
-        }
         ENV_VARIABLE_ID => {
             let env_vars = stack.get_env_vars(engine_state);
             let env_columns = env_vars.keys();

crates/nu-engine/src/lib.rs

@@ -4,6 +4,7 @@ pub mod documentation;
 pub mod env;
 mod eval;
 mod glob_from;
+mod nu_variable;
 pub mod scope;
 
 pub use call_ext::CallExt;

crates/nu-engine/src/nu_variable.rs

@@ -0,0 +1,200 @@
+use crate::scope::create_scope;
+use core::fmt;
+use nu_protocol::{
+    engine::{EngineState, Stack},
+    LazyRecord, ShellError, Span, Value,
+};
+use serde::{Deserialize, Serialize};
+use sysinfo::SystemExt;
+
+// NuVariable: a LazyRecord for the special $nu variable
+// $nu used to be a plain old Record, but LazyRecord lets us load different fields/columns lazily. This is important for performance;
+// collecting all the information in $nu is expensive and unnecessary if you just want a subset of the data
+
+// Note: NuVariable is not meaningfully serializable, this #[derive] is a lie to satisfy the type checker.
+// Make sure to collect() the record before serializing it
+#[derive(Serialize, Deserialize)]
+pub struct NuVariable {
+    #[serde(skip)]
+    pub engine_state: EngineState,
+    #[serde(skip)]
+    pub stack: Stack,
+    pub span: Span,
+}
+
+impl LazyRecord for NuVariable {
+    fn column_names(&self) -> Vec<&'static str> {
+        let mut cols = vec!["config-path", "env-path", "history-path", "loginshell-path"];
+
+        #[cfg(feature = "plugin")]
+        if self.engine_state.plugin_signatures.is_some() {
+            cols.push("plugin-path");
+        }
+
+        cols.push("scope");
+        cols.push("home-path");
+        cols.push("temp-path");
+        cols.push("pid");
+        cols.push("os-info");
+
+        cols
+    }
+
+    fn get_column_value(&self, column: &str) -> Result<Value, ShellError> {
+        let err = |message: &str| -> Result<Value, ShellError> {
+            Err(ShellError::LazyRecordAccessFailed {
+                message: message.into(),
+                column_name: column.to_string(),
+                span: self.span,
+            })
+        };
+
+        match column {
+            "config-path" => {
+                if let Some(path) = self.engine_state.get_config_path("config-path") {
+                    Ok(Value::String {
+                        val: path.to_string_lossy().to_string(),
+                        span: self.span,
+                    })
+                } else if let Some(mut path) = nu_path::config_dir() {
+                    path.push("nushell/config.nu");
+                    Ok(Value::String {
+                        val: path.to_string_lossy().to_string(),
+                        span: self.span,
+                    })
+                } else {
+                    err("Could not get config directory")
+                }
+            }
+            "env-path" => {
+                if let Some(path) = self.engine_state.get_config_path("env-path") {
+                    Ok(Value::String {
+                        val: path.to_string_lossy().to_string(),
+                        span: self.span,
+                    })
+                } else if let Some(mut path) = nu_path::config_dir() {
+                    path.push("nushell/env.nu");
+                    Ok(Value::String {
+                        val: path.to_string_lossy().to_string(),
+                        span: self.span,
+                    })
+                } else {
+                    err("Could not get config directory")
+                }
+            }
+            "history-path" => {
+                if let Some(mut path) = nu_path::config_dir() {
+                    path.push("nushell");
+                    match self.engine_state.config.history_file_format {
+                        nu_protocol::HistoryFileFormat::Sqlite => {
+                            path.push("history.sqlite3");
+                        }
+                        nu_protocol::HistoryFileFormat::PlainText => {
+                            path.push("history.txt");
+                        }
+                    }
+                    Ok(Value::String {
+                        val: path.to_string_lossy().to_string(),
+                        span: self.span,
+                    })
+                } else {
+                    err("Could not get config directory")
+                }
+            }
+            "loginshell-path" => {
+                if let Some(mut path) = nu_path::config_dir() {
+                    path.push("nushell/login.nu");
+                    Ok(Value::String {
+                        val: path.to_string_lossy().to_string(),
+                        span: self.span,
+                    })
+                } else {
+                    err("Could not get config directory")
+                }
+            }
+            "plugin-path" => {
+                #[cfg(feature = "plugin")]
+                {
+                    if let Some(path) = &self.engine_state.plugin_signatures {
+                        Ok(Value::String {
+                            val: path.to_string_lossy().to_string(),
+                            span: self.span,
+                        })
+                    } else {
+                        err("Could not get plugin signature location")
+                    }
+                }
+
+                #[cfg(not(feature = "plugin"))]
+                {
+                    err("Plugin feature not enabled")
+                }
+            }
+            "scope" => Ok(create_scope(&self.engine_state, &self.stack, self.span())?),
+            "home-path" => {
+                if let Some(home_path) = nu_path::home_dir() {
+                    Ok(Value::String {
+                        val: home_path.to_string_lossy().into(),
+                        span: self.span(),
+                    })
+                } else {
+                    err("Could not get home path")
+                }
+            }
+            "temp-path" => {
+                let temp_path = std::env::temp_dir();
+                Ok(Value::String {
+                    val: temp_path.to_string_lossy().into(),
+                    span: self.span(),
+                })
+            }
+            "pid" => Ok(Value::int(std::process::id().into(), self.span())),
+            "os-info" => {
+                let sys = sysinfo::System::new();
+                let ver = match sys.kernel_version() {
+                    Some(v) => v,
+                    None => "unknown".into(),
+                };
+
+                let os_record = Value::Record {
+                    cols: vec![
+                        "name".into(),
+                        "arch".into(),
+                        "family".into(),
+                        "kernel_version".into(),
+                    ],
+                    vals: vec![
+                        Value::string(std::env::consts::OS, self.span()),
+                        Value::string(std::env::consts::ARCH, self.span()),
+                        Value::string(std::env::consts::FAMILY, self.span()),
+                        Value::string(ver, self.span()),
+                    ],
+                    span: self.span(),
+                };
+
+                Ok(os_record)
+            }
+            _ => err(&format!("Could not find column '{column}'")),
+        }
+    }
+
+    fn span(&self) -> Span {
+        self.span
+    }
+
+    fn typetag_name(&self) -> &'static str {
+        "nu_variable"
+    }
+
+    fn typetag_deserialize(&self) {
+        unimplemented!("typetag_deserialize")
+    }
+}
+
+// manually implemented so we can skip engine_state which doesn't implement Debug
+// FIXME: find a better way
+impl fmt::Debug for NuVariable {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("NuVariable").finish()
+    }
+}

crates/nu-plugin/src/plugin/declaration.rs

@@ -95,6 +95,7 @@ impl Command for PluginDeclaration {
                     }
                 }
             }
+            Value::LazyRecord { val, .. } => CallInput::Value(val.collect()?),
             value => CallInput::Value(value),
         };
 

crates/nu-protocol/src/shell_error.rs

@@ -916,6 +916,16 @@ Either make sure {0} is a string, or add a 'to_string' entry for it in ENV_CONVE
         #[label("This called itself too many times")]
         span: Option<Span>,
     },
+
+    /// An attempt to access a record column failed.
+    #[error("Access failure: {message}")]
+    #[diagnostic(code(nu::shell::lazy_record_access_failed), url(docsrs))]
+    LazyRecordAccessFailed {
+        message: String,
+        column_name: String,
+        #[label("Could not access '{column_name}' on this record")]
+        span: Span,
+    },
 }
 
 impl From<std::io::Error> for ShellError {
@@ -940,9 +950,8 @@ pub fn into_code(err: &ShellError) -> Option<String> {
     err.code().map(|code| code.to_string())
 }
 
-pub fn did_you_mean(possibilities: &[String], input: &str) -> Option<String> {
+pub fn did_you_mean<S: AsRef<str>>(possibilities: &[S], input: &str) -> Option<String> {
-    let possibilities: Vec<&str> = possibilities.iter().map(|s| s.as_str()).collect();
+    let possibilities: Vec<&str> = possibilities.iter().map(|s| s.as_ref()).collect();
-
     let suggestion =
         crate::lev_distance::find_best_match_for_name_with_substrings(&possibilities, input, None)
             .map(|s| s.to_string());
@@ -1018,7 +1027,6 @@ mod tests {
             ),
         ];
         for (possibilities, cases) in all_cases {
-            let possibilities: Vec<String> = possibilities.iter().map(|s| s.to_string()).collect();
             for (input, expected_suggestion, discussion) in cases {
                 let suggestion = did_you_mean(&possibilities, input);
                 assert_eq!(

crates/nu-protocol/src/value/lazy_record.rs

@@ -0,0 +1,34 @@
+use crate::{ShellError, Span, Value};
+use std::fmt;
+
+// Trait definition for a lazy record (where columns are evaluated on-demand)
+// typetag is needed to make this implement Serialize+Deserialize... even though we should never actually serialize a LazyRecord.
+// To serialize a LazyRecord, collect it into a Value::Record with collect() first.
+#[typetag::serde(tag = "type")]
+pub trait LazyRecord: fmt::Debug + Send + Sync {
+    // All column names
+    fn column_names(&self) -> Vec<&'static str>;
+
+    // Get 1 specific column value
+    fn get_column_value(&self, column: &str) -> Result<Value, ShellError>;
+
+    fn span(&self) -> Span;
+
+    // Convert the lazy record into a regular Value::Record by collecting all its columns
+    fn collect(&self) -> Result<Value, ShellError> {
+        let mut cols = vec![];
+        let mut vals = vec![];
+
+        for column in self.column_names() {
+            cols.push(column.into());
+            let val = self.get_column_value(column)?;
+            vals.push(val);
+        }
+
+        Ok(Value::Record {
+            cols,
+            vals,
+            span: self.span(),
+        })
+    }
+}

crates/nu-protocol/src/value/mod.rs

@@ -1,6 +1,7 @@
 mod custom_value;
 mod from;
 mod from_value;
+mod lazy_record;
 mod range;
 mod stream;
 mod unit;
@@ -17,6 +18,7 @@ pub use custom_value::CustomValue;
 use fancy_regex::Regex;
 pub use from_value::FromValue;
 use indexmap::map::IndexMap;
+pub use lazy_record::LazyRecord;
 use nu_utils::get_system_locale;
 use num_format::ToFormattedString;
 pub use range::*;
@@ -101,10 +103,16 @@ pub enum Value {
         val: CellPath,
         span: Span,
     },
+    #[serde(skip_serializing)]
     CustomValue {
         val: Box<dyn CustomValue>,
         span: Span,
     },
+    #[serde(skip_serializing)]
+    LazyRecord {
+        val: Box<dyn LazyRecord>,
+        span: Span,
+    },
 }
 
 impl Clone for Value {
@@ -138,6 +146,13 @@ impl Clone for Value {
                 vals: vals.clone(),
                 span: *span,
             },
+            Value::LazyRecord { val, .. } => {
+                match val.collect() {
+                    Ok(val) => val,
+                    // this is a bit weird, but because clone() is infallible...
+                    Err(error) => Value::Error { error },
+                }
+            }
             Value::List { vals, span } => Value::List {
                 vals: vals.clone(),
                 span: *span,
@@ -350,6 +365,7 @@ impl Value {
             Value::Binary { span, .. } => Ok(*span),
             Value::CellPath { span, .. } => Ok(*span),
             Value::CustomValue { span, .. } => Ok(*span),
+            Value::LazyRecord { span, .. } => Ok(*span),
         }
     }
 
@@ -372,6 +388,7 @@ impl Value {
             Value::Range { span, .. } => *span = new_span,
             Value::String { span, .. } => *span = new_span,
             Value::Record { span, .. } => *span = new_span,
+            Value::LazyRecord { span, .. } => *span = new_span,
             Value::List { span, .. } => *span = new_span,
             Value::Closure { span, .. } => *span = new_span,
             Value::Block { span, .. } => *span = new_span,
@@ -426,6 +443,10 @@ impl Value {
                     None => Type::List(Box::new(ty.unwrap_or(Type::Any))),
                 }
             }
+            Value::LazyRecord { val, .. } => match val.collect() {
+                Ok(val) => val.get_type(),
+                Err(..) => Type::Error,
+            },
             Value::Nothing { .. } => Type::Nothing,
             Value::Block { .. } => Type::Block,
             Value::Closure { .. } => Type::Closure,
@@ -512,6 +533,13 @@ impl Value {
                     .collect::<Vec<_>>()
                     .join(separator)
             ),
+            Value::LazyRecord { val, .. } => {
+                let collected = match val.collect() {
+                    Ok(val) => val,
+                    Err(error) => Value::Error { error },
+                };
+                collected.into_string(separator, config)
+            }
             Value::Block { val, .. } => format!("<Block {}>", val),
             Value::Closure { val, .. } => format!("<Closure {}>", val),
             Value::Nothing { .. } => String::new(),
@@ -556,6 +584,10 @@ impl Value {
                 cols.len(),
                 if cols.len() == 1 { "" } else { "s" }
             ),
+            Value::LazyRecord { val, .. } => match val.collect() {
+                Ok(val) => val.into_abbreviated_string(config),
+                Err(error) => format!("{:?}", error),
+            },
             Value::Block { val, .. } => format!("<Block {}>", val),
             Value::Closure { val, .. } => format!("<Closure {}>", val),
             Value::Nothing { .. } => String::new(),
@@ -603,6 +635,10 @@ impl Value {
                     .collect::<Vec<_>>()
                     .join(separator)
             ),
+            Value::LazyRecord { val, .. } => match val.collect() {
+                Ok(val) => val.debug_string(separator, config),
+                Err(error) => format!("{:?}", error),
+            },
             Value::Block { val, .. } => format!("<Block {}>", val),
             Value::Closure { val, .. } => format!("<Closure {}>", val),
             Value::Nothing { .. } => String::new(),
@@ -777,6 +813,30 @@ impl Value {
                             );
                         }
                     }
+                    Value::LazyRecord { val, span } => {
+                        let columns = val.column_names();
+
+                        if columns.contains(&column_name.as_str()) {
+                            current = val.get_column_value(column_name)?;
+                        } else {
+                            if from_user_input {
+                                if let Some(suggestion) = did_you_mean(&columns, column_name) {
+                                    err_or_null!(
+                                        ShellError::DidYouMean(suggestion, *origin_span),
+                                        *origin_span
+                                    );
+                                }
+                            }
+                            err_or_null!(
+                                ShellError::CantFindColumn(
+                                    column_name.to_string(),
+                                    *origin_span,
+                                    *span,
+                                ),
+                                *origin_span
+                            );
+                        }
+                    }
                     // String access of Lists always means Table access.
                     // Create a List which contains each matching value for contained
                     // records in the source list.
@@ -1567,6 +1627,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Less),
                 Value::String { .. } => Some(Ordering::Less),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1586,6 +1647,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Less),
                 Value::String { .. } => Some(Ordering::Less),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1605,6 +1667,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Less),
                 Value::String { .. } => Some(Ordering::Less),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1624,6 +1687,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Less),
                 Value::String { .. } => Some(Ordering::Less),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1643,6 +1707,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Less),
                 Value::String { .. } => Some(Ordering::Less),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1662,6 +1727,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Less),
                 Value::String { .. } => Some(Ordering::Less),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1681,6 +1747,7 @@ impl PartialOrd for Value {
                 Value::Range { val: rhs, .. } => lhs.partial_cmp(rhs),
                 Value::String { .. } => Some(Ordering::Less),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1700,6 +1767,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Greater),
                 Value::String { val: rhs, .. } => lhs.partial_cmp(rhs),
                 Value::Record { .. } => Some(Ordering::Less),
+                Value::LazyRecord { .. } => Some(Ordering::Less),
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1745,6 +1813,10 @@ impl PartialOrd for Value {
                         result
                     }
                 }
+                Value::LazyRecord { val, .. } => match val.collect() {
+                    Ok(rhs) => self.partial_cmp(&rhs),
+                    Err(_) => None,
+                },
                 Value::List { .. } => Some(Ordering::Less),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1764,6 +1836,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Greater),
                 Value::String { .. } => Some(Ordering::Greater),
                 Value::Record { .. } => Some(Ordering::Greater),
+                Value::LazyRecord { .. } => Some(Ordering::Greater),
                 Value::List { vals: rhs, .. } => lhs.partial_cmp(rhs),
                 Value::Block { .. } => Some(Ordering::Less),
                 Value::Closure { .. } => Some(Ordering::Less),
@@ -1784,6 +1857,7 @@ impl PartialOrd for Value {
                 Value::String { .. } => Some(Ordering::Greater),
                 Value::Record { .. } => Some(Ordering::Greater),
                 Value::List { .. } => Some(Ordering::Greater),
+                Value::LazyRecord { .. } => Some(Ordering::Greater),
                 Value::Block { val: rhs, .. } => lhs.partial_cmp(rhs),
                 Value::Closure { .. } => Some(Ordering::Less),
                 Value::Nothing { .. } => Some(Ordering::Less),
@@ -1802,6 +1876,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Greater),
                 Value::String { .. } => Some(Ordering::Greater),
                 Value::Record { .. } => Some(Ordering::Greater),
+                Value::LazyRecord { .. } => Some(Ordering::Greater),
                 Value::List { .. } => Some(Ordering::Greater),
                 Value::Block { .. } => Some(Ordering::Greater),
                 Value::Closure { val: rhs, .. } => lhs.partial_cmp(rhs),
@@ -1821,6 +1896,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Greater),
                 Value::String { .. } => Some(Ordering::Greater),
                 Value::Record { .. } => Some(Ordering::Greater),
+                Value::LazyRecord { .. } => Some(Ordering::Greater),
                 Value::List { .. } => Some(Ordering::Greater),
                 Value::Block { .. } => Some(Ordering::Greater),
                 Value::Closure { .. } => Some(Ordering::Greater),
@@ -1840,6 +1916,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Greater),
                 Value::String { .. } => Some(Ordering::Greater),
                 Value::Record { .. } => Some(Ordering::Greater),
+                Value::LazyRecord { .. } => Some(Ordering::Greater),
                 Value::List { .. } => Some(Ordering::Greater),
                 Value::Block { .. } => Some(Ordering::Greater),
                 Value::Closure { .. } => Some(Ordering::Greater),
@@ -1859,6 +1936,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Greater),
                 Value::String { .. } => Some(Ordering::Greater),
                 Value::Record { .. } => Some(Ordering::Greater),
+                Value::LazyRecord { .. } => Some(Ordering::Greater),
                 Value::List { .. } => Some(Ordering::Greater),
                 Value::Block { .. } => Some(Ordering::Greater),
                 Value::Closure { .. } => Some(Ordering::Greater),
@@ -1878,6 +1956,7 @@ impl PartialOrd for Value {
                 Value::Range { .. } => Some(Ordering::Greater),
                 Value::String { .. } => Some(Ordering::Greater),
                 Value::Record { .. } => Some(Ordering::Greater),
+                Value::LazyRecord { .. } => Some(Ordering::Greater),
                 Value::List { .. } => Some(Ordering::Greater),
                 Value::Block { .. } => Some(Ordering::Greater),
                 Value::Closure { .. } => Some(Ordering::Greater),
@@ -1888,6 +1967,10 @@ impl PartialOrd for Value {
                 Value::CustomValue { .. } => Some(Ordering::Less),
             },
             (Value::CustomValue { val: lhs, .. }, rhs) => lhs.partial_cmp(rhs),
+            (Value::LazyRecord { val, .. }, rhs) => match val.collect() {
+                Ok(val) => val.partial_cmp(rhs),
+                Err(_) => None,
+            },
         }
     }
 }