Add virtual path abstraction layer (#9245)

Cargo.lock

@@ -2947,6 +2947,7 @@ name = "nu-std"
 version = "0.80.1"
 dependencies = [
  "miette",
+ "nu-engine",
  "nu-parser",
  "nu-protocol",
 ]

crates/nu-parser/src/lib.rs

@@ -7,6 +7,7 @@ mod lite_parser;
 mod parse_keywords;
 mod parse_patterns;
 mod parser;
+mod parser_path;
 mod type_check;
 
 pub use deparse::{escape_for_script_arg, escape_quote_string};
@@ -17,6 +18,7 @@ pub use known_external::KnownExternal;
 pub use lex::{lex, lex_signature, Token, TokenContents};
 pub use lite_parser::{lite_parse, LiteBlock, LiteElement};
 pub use parse_keywords::*;
+pub use parser_path::*;
 
 pub use parser::{
     is_math_expression_like, parse, parse_block, parse_expression, parse_external_call,

crates/nu-parser/src/parse_keywords.rs

@@ -1,3 +1,4 @@
+use crate::parser_path::ParserPath;
 use itertools::Itertools;
 use log::trace;
 use nu_path::canonicalize_with;
@@ -1724,14 +1725,14 @@ pub fn parse_module_block(
 
 fn parse_module_file(
     working_set: &mut StateWorkingSet,
-    path: PathBuf,
+    path: ParserPath,
     path_span: Span,
     name_override: Option<String>,
 ) -> Option<ModuleId> {
     if let Some(i) = working_set
         .parsed_module_files
         .iter()
-        .rposition(|p| p == &path)
+        .rposition(|p| p == path.path())
     {
         let mut files: Vec<String> = working_set
             .parsed_module_files
@@ -1740,7 +1741,7 @@ fn parse_module_file(
             .map(|p| p.to_string_lossy().to_string())
             .collect();
 
-        files.push(path.to_string_lossy().to_string());
+        files.push(path.path().to_string_lossy().to_string());
 
         let msg = files.join("\nuses ");
 
@@ -1757,14 +1758,14 @@ fn parse_module_file(
         return None;
     };
 
-    let contents = if let Ok(contents) = std::fs::read(&path) {
+    let contents = if let Some(contents) = path.read(working_set) {
         contents
     } else {
         working_set.error(ParseError::ModuleNotFound(path_span));
         return None;
     };
 
-    let file_id = working_set.add_file(path.to_string_lossy().to_string(), &contents);
+    let file_id = working_set.add_file(path.path().to_string_lossy().to_string(), &contents);
     let new_span = working_set.get_span_for_file(file_id);
 
     if let Some(module_id) = working_set.find_module_by_span(new_span) {
@@ -1773,17 +1774,13 @@ fn parse_module_file(
 
     // Change the currently parsed directory
     let prev_currently_parsed_cwd = if let Some(parent) = path.parent() {
-        let prev = working_set.currently_parsed_cwd.clone();
+        working_set.currently_parsed_cwd.replace(parent.into())
-
-        working_set.currently_parsed_cwd = Some(parent.into());
-
-        prev
     } else {
         working_set.currently_parsed_cwd.clone()
     };
 
     // Add the file to the stack of parsed module files
-    working_set.parsed_module_files.push(path);
+    working_set.parsed_module_files.push(path.path_buf());
 
     // Parse the module
     let (block, module, module_comments) =
@@ -1824,91 +1821,87 @@ pub fn parse_module_file_or_dir(
         };
 
     if module_path.is_dir() {
-        if let Ok(dir_contents) = std::fs::read_dir(&module_path) {
+        let Some(dir_contents) = module_path.read_dir() else {
-            let module_name = if let Some(stem) = module_path.file_stem() {
+            working_set.error(ParseError::ModuleNotFound(path_span));
-                stem.to_string_lossy().to_string()
+            return None;
-            } else {
+        };
-                working_set.error(ParseError::ModuleNotFound(path_span));
-                return None;
-            };
 
-            let mod_nu_path = module_path.join("mod.nu");
+        let module_name = if let Some(stem) = module_path.file_stem() {
-
+            stem.to_string_lossy().to_string()
-            if !(mod_nu_path.exists() && mod_nu_path.is_file()) {
-                working_set.error(ParseError::ModuleMissingModNuFile(path_span));
-                return None;
-            }
-
-            let mut paths = vec![];
-
-            for entry in dir_contents.flatten() {
-                let entry_path = entry.path();
-
-                if (entry_path.is_file()
-                    && entry_path.extension() == Some(OsStr::new("nu"))
-                    && entry_path.file_stem() != Some(OsStr::new("mod")))
-                    || (entry_path.is_dir() && entry_path.join("mod.nu").exists())
-                {
-                    if entry_path.file_stem() == Some(OsStr::new(&module_name)) {
-                        working_set.error(ParseError::InvalidModuleFileName(
-                            module_path.to_string_lossy().to_string(),
-                            module_name,
-                            path_span,
-                        ));
-                        return None;
-                    }
-
-                    paths.push(entry_path);
-                }
-            }
-
-            paths.sort();
-
-            // working_set.enter_scope();
-
-            let mut submodules = vec![];
-
-            for p in paths {
-                if let Some(submodule_id) = parse_module_file_or_dir(
-                    working_set,
-                    p.to_string_lossy().as_bytes(),
-                    path_span,
-                    None,
-                ) {
-                    let submodule_name = working_set.get_module(submodule_id).name();
-                    submodules.push((submodule_name, submodule_id));
-                }
-            }
-
-            if let Some(module_id) = parse_module_file(
-                working_set,
-                mod_nu_path,
-                path_span,
-                name_override.or(Some(module_name)),
-            ) {
-                let mut module = working_set.get_module(module_id).clone();
-
-                for (submodule_name, submodule_id) in submodules {
-                    module.add_submodule(submodule_name, submodule_id);
-                }
-
-                let module_name = String::from_utf8_lossy(&module.name).to_string();
-
-                let module_comments =
-                    if let Some(comments) = working_set.get_module_comments(module_id) {
-                        comments.to_vec()
-                    } else {
-                        vec![]
-                    };
-
-                let new_module_id = working_set.add_module(&module_name, module, module_comments);
-
-                Some(new_module_id)
-            } else {
-                None
-            }
         } else {
             working_set.error(ParseError::ModuleNotFound(path_span));
+            return None;
+        };
+
+        let mod_nu_path = module_path.clone().join("mod.nu");
+
+        if !(mod_nu_path.exists() && mod_nu_path.is_file()) {
+            working_set.error(ParseError::ModuleMissingModNuFile(path_span));
+            return None;
+        }
+
+        let mut paths = vec![];
+
+        for entry_path in dir_contents {
+            if (entry_path.is_file()
+                && entry_path.extension() == Some(OsStr::new("nu"))
+                && entry_path.file_stem() != Some(OsStr::new("mod")))
+                || (entry_path.is_dir() && entry_path.clone().join("mod.nu").exists())
+            {
+                if entry_path.file_stem() == Some(OsStr::new(&module_name)) {
+                    working_set.error(ParseError::InvalidModuleFileName(
+                        module_path.path().to_string_lossy().to_string(),
+                        module_name,
+                        path_span,
+                    ));
+                    return None;
+                }
+
+                paths.push(entry_path);
+            }
+        }
+
+        paths.sort();
+
+        let mut submodules = vec![];
+
+        for p in paths {
+            if let Some(submodule_id) = parse_module_file_or_dir(
+                working_set,
+                p.path().to_string_lossy().as_bytes(),
+                path_span,
+                None,
+            ) {
+                let submodule_name = working_set.get_module(submodule_id).name();
+                submodules.push((submodule_name, submodule_id));
+            }
+        }
+
+        if let Some(module_id) = parse_module_file(
+            working_set,
+            mod_nu_path,
+            path_span,
+            name_override.or(Some(module_name)),
+        ) {
+            let mut module = working_set.get_module(module_id).clone();
+
+            for (submodule_name, submodule_id) in submodules {
+                module.add_submodule(submodule_name, submodule_id);
+            }
+
+            let module_name = String::from_utf8_lossy(&module.name).to_string();
+
+            let module_comments = if let Some(comments) = working_set.get_module_comments(module_id)
+            {
+                comments.to_vec()
+            } else {
+                vec![]
+            };
+
+            let new_module_id = working_set.add_module(&module_name, module, module_comments);
+
+            Some(new_module_id)
+        } else {
             None
         }
     } else if module_path.is_file() {
@@ -2017,19 +2010,13 @@ pub fn parse_module(
     }]);
 
     if spans.len() == split_id + 1 {
-        let cwd = working_set.get_cwd();
+        if let Some(module_id) = parse_module_file_or_dir(
-
+            working_set,
-        if let Some(module_path) =
+            module_name_or_path.as_bytes(),
-            find_in_dirs(&module_name_or_path, working_set, &cwd, LIB_DIRS_VAR)
+            module_name_or_path_span,
-        {
+            None,
-            let path_str = module_path.to_string_lossy().to_string();
+        ) {
-            let maybe_module_id = parse_module_file_or_dir(
+            return (pipeline, Some(module_id));
-                working_set,
-                path_str.as_bytes(),
-                module_name_or_path_span,
-                None,
-            );
-            return (pipeline, maybe_module_id);
         } else {
             working_set.error(ParseError::ModuleNotFound(module_name_or_path_span));
             return (pipeline, None);
@@ -3043,14 +3030,10 @@ pub fn parse_source(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeli
                 };
 
                 if let Some(path) = find_in_dirs(&filename, working_set, &cwd, LIB_DIRS_VAR) {
-                    if let Ok(contents) = std::fs::read(&path) {
+                    if let Some(contents) = path.read(working_set) {
                         // Change currently parsed directory
                         let prev_currently_parsed_cwd = if let Some(parent) = path.parent() {
-                            let prev = working_set.currently_parsed_cwd.clone();
+                            working_set.currently_parsed_cwd.replace(parent.into())
-
-                            working_set.currently_parsed_cwd = Some(parent.into());
-
-                            prev
                         } else {
                             working_set.currently_parsed_cwd.clone()
                         };
@@ -3059,7 +3042,7 @@ pub fn parse_source(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeli
                         // working set, if it was a successful parse.
                         let block = parse(
                             working_set,
-                            Some(&path.to_string_lossy()),
+                            Some(&path.path().to_string_lossy()),
                             &contents,
                             scoped,
                         );
@@ -3300,6 +3283,7 @@ pub fn parse_register(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipe
         nu_engine::env::env_to_strings(working_set.permanent_state, &stack).unwrap_or_default();
     let error = match signature {
         Some(signature) => arguments.and_then(|(path, path_span)| {
+            let path = path.path_buf();
             // restrict plugin file name starts with `nu_plugin_`
             let valid_plugin_name = path
                 .file_name()
@@ -3320,13 +3304,14 @@ pub fn parse_register(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipe
             }
         }),
         None => arguments.and_then(|(path, path_span)| {
+            let path = path.path_buf();
             // restrict plugin file name starts with `nu_plugin_`
             let valid_plugin_name = path
                 .file_name()
                 .map(|s| s.to_string_lossy().starts_with("nu_plugin_"));
 
             if let Some(true) = valid_plugin_name {
-                get_signature(path.as_path(), &shell, &current_envs)
+                get_signature(&path, &shell, &current_envs)
                     .map_err(|err| {
                         ParseError::LabeledError(
                             "Error getting signatures".into(),
@@ -3393,28 +3378,52 @@ pub fn find_in_dirs(
     working_set: &StateWorkingSet,
     cwd: &str,
     dirs_var_name: &str,
-) -> Option<PathBuf> {
+) -> Option<ParserPath> {
     pub fn find_in_dirs_with_id(
         filename: &str,
         working_set: &StateWorkingSet,
         cwd: &str,
         dirs_var_name: &str,
-    ) -> Option<PathBuf> {
+    ) -> Option<ParserPath> {
         // Choose whether to use file-relative or PWD-relative path
         let actual_cwd = if let Some(currently_parsed_cwd) = &working_set.currently_parsed_cwd {
             currently_parsed_cwd.as_path()
         } else {
             Path::new(cwd)
         };
-        if let Ok(p) = canonicalize_with(filename, actual_cwd) {
+
-            return Some(p);
+        // Try if we have an existing virtual path
+        if let Some(virtual_path) = working_set.find_virtual_path(filename) {
+            return Some(ParserPath::from_virtual_path(
+                working_set,
+                filename,
+                virtual_path,
+            ));
+        } else {
+            let abs_virtual_filename = actual_cwd.join(filename);
+            let abs_virtual_filename = abs_virtual_filename.to_string_lossy();
+
+            if let Some(virtual_path) = working_set.find_virtual_path(&abs_virtual_filename) {
+                return Some(ParserPath::from_virtual_path(
+                    working_set,
+                    &abs_virtual_filename,
+                    virtual_path,
+                ));
+            }
         }
 
+        // Try if we have an existing physical path
+        if let Ok(p) = canonicalize_with(filename, actual_cwd) {
+            return Some(ParserPath::RealPath(p));
+        }
+
+        // Early-exit if path is non-existent absolute path
         let path = Path::new(filename);
         if !path.is_relative() {
             return None;
         }
 
+        // Look up relative path from NU_LIB_DIRS
         working_set
             .find_constant(find_dirs_var(working_set, dirs_var_name)?)?
             .as_list()
@@ -3427,9 +3436,11 @@ pub fn find_in_dirs(
             })
             .find(Option::is_some)
             .flatten()
+            .map(ParserPath::RealPath)
     }
 
     // TODO: remove (see #8310)
+    // Same as find_in_dirs_with_id but using $env.NU_LIB_DIRS instead of constant
     pub fn find_in_dirs_old(
         filename: &str,
         working_set: &StateWorkingSet,
@@ -3475,8 +3486,9 @@ pub fn find_in_dirs(
         }
     }
 
-    find_in_dirs_with_id(filename, working_set, cwd, dirs_var_name)
+    find_in_dirs_with_id(filename, working_set, cwd, dirs_var_name).or_else(|| {
-        .or_else(|| find_in_dirs_old(filename, working_set, cwd, dirs_var_name))
+        find_in_dirs_old(filename, working_set, cwd, dirs_var_name).map(ParserPath::RealPath)
+    })
 }
 
 fn detect_params_in_name(

crates/nu-parser/src/parser_path.rs

@@ -0,0 +1,134 @@
+use nu_protocol::engine::{StateWorkingSet, VirtualPath};
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+
+/// An abstraction over a PathBuf that can have virtual paths (files and directories). Virtual
+/// paths always exist and represent a way to ship Nushell code inside the binary without requiring
+/// paths to be present in the file system.
+///
+/// Created from VirtualPath found in the engine state.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ParserPath {
+    RealPath(PathBuf),
+    VirtualFile(PathBuf, usize),
+    VirtualDir(PathBuf, Vec<ParserPath>),
+}
+
+impl ParserPath {
+    pub fn is_dir(&self) -> bool {
+        match self {
+            ParserPath::RealPath(p) => p.is_dir(),
+            ParserPath::VirtualFile(..) => false,
+            ParserPath::VirtualDir(..) => true,
+        }
+    }
+
+    pub fn is_file(&self) -> bool {
+        match self {
+            ParserPath::RealPath(p) => p.is_file(),
+            ParserPath::VirtualFile(..) => true,
+            ParserPath::VirtualDir(..) => false,
+        }
+    }
+
+    pub fn exists(&self) -> bool {
+        match self {
+            ParserPath::RealPath(p) => p.exists(),
+            ParserPath::VirtualFile(..) => true,
+            ParserPath::VirtualDir(..) => true,
+        }
+    }
+
+    pub fn path(&self) -> &Path {
+        match self {
+            ParserPath::RealPath(p) => p,
+            ParserPath::VirtualFile(p, _) => p,
+            ParserPath::VirtualDir(p, _) => p,
+        }
+    }
+
+    pub fn path_buf(self) -> PathBuf {
+        match self {
+            ParserPath::RealPath(p) => p,
+            ParserPath::VirtualFile(p, _) => p,
+            ParserPath::VirtualDir(p, _) => p,
+        }
+    }
+
+    pub fn parent(&self) -> Option<&Path> {
+        match self {
+            ParserPath::RealPath(p) => p.parent(),
+            ParserPath::VirtualFile(p, _) => p.parent(),
+            ParserPath::VirtualDir(p, _) => p.parent(),
+        }
+    }
+
+    pub fn read_dir(&self) -> Option<Vec<ParserPath>> {
+        match self {
+            ParserPath::RealPath(p) => p.read_dir().ok().map(|read_dir| {
+                read_dir
+                    .flatten()
+                    .map(|dir_entry| ParserPath::RealPath(dir_entry.path()))
+                    .collect()
+            }),
+            ParserPath::VirtualFile(..) => None,
+            ParserPath::VirtualDir(_, files) => Some(files.clone()),
+        }
+    }
+
+    pub fn file_stem(&self) -> Option<&OsStr> {
+        self.path().file_stem()
+    }
+
+    pub fn extension(&self) -> Option<&OsStr> {
+        self.path().extension()
+    }
+
+    pub fn join(self, path: impl AsRef<Path>) -> ParserPath {
+        match self {
+            ParserPath::RealPath(p) => ParserPath::RealPath(p.join(path)),
+            ParserPath::VirtualFile(p, file_id) => ParserPath::VirtualFile(p.join(path), file_id),
+            ParserPath::VirtualDir(p, entries) => {
+                let new_p = p.join(path);
+                let mut pp = ParserPath::RealPath(new_p.clone());
+                for entry in entries {
+                    if new_p == entry.path() {
+                        pp = entry.clone();
+                    }
+                }
+                pp
+            }
+        }
+    }
+
+    pub fn read<'a>(&'a self, working_set: &'a StateWorkingSet) -> Option<Vec<u8>> {
+        match self {
+            ParserPath::RealPath(p) => std::fs::read(p).ok(),
+            ParserPath::VirtualFile(_, file_id) => working_set
+                .get_contents_of_file(*file_id)
+                .map(|bytes| bytes.to_vec()),
+
+            ParserPath::VirtualDir(..) => None,
+        }
+    }
+
+    pub fn from_virtual_path(
+        working_set: &StateWorkingSet,
+        name: &str,
+        virtual_path: &VirtualPath,
+    ) -> Self {
+        match virtual_path {
+            VirtualPath::File(file_id) => ParserPath::VirtualFile(PathBuf::from(name), *file_id),
+            VirtualPath::Dir(entries) => ParserPath::VirtualDir(
+                PathBuf::from(name),
+                entries
+                    .iter()
+                    .map(|virtual_path_id| {
+                        let (virt_name, virt_path) = working_set.get_virtual_path(*virtual_path_id);
+                        ParserPath::from_virtual_path(working_set, virt_name, virt_path)
+                    })
+                    .collect(),
+            ),
+        }
+    }
+}

crates/nu-protocol/src/engine/engine_state.rs

@@ -3,8 +3,8 @@ use lru::LruCache;
 
 use super::{Command, EnvVars, OverlayFrame, ScopeFrame, Stack, Visibility, DEFAULT_OVERLAY_NAME};
 use crate::{
-    ast::Block, BlockId, Config, DeclId, Example, Module, ModuleId, OverlayId, ShellError,
+    ast::Block, BlockId, Config, DeclId, Example, FileId, Module, ModuleId, OverlayId, ShellError,
-    Signature, Span, Type, VarId, Variable,
+    Signature, Span, Type, VarId, Variable, VirtualPathId,
 };
 use crate::{ParseError, Value};
 use core::panic;
@@ -56,6 +56,12 @@ impl Default for Usage {
     }
 }
 
+#[derive(Clone, Debug)]
+pub enum VirtualPath {
+    File(FileId),
+    Dir(Vec<VirtualPathId>),
+}
+
 /// The core global engine state. This includes all global definitions as well as any global state that
 /// will persist for the whole session.
 ///
@@ -102,6 +108,7 @@ impl Default for Usage {
 pub struct EngineState {
     files: Vec<(String, usize, usize)>,
     file_contents: Vec<(Vec<u8>, usize, usize)>,
+    virtual_paths: Vec<(String, VirtualPath)>,
     vars: Vec<Variable>,
     decls: Vec<Box<dyn Command + 'static>>,
     blocks: Vec<Block>,
@@ -144,6 +151,7 @@ impl EngineState {
         Self {
             files: vec![],
             file_contents: vec![],
+            virtual_paths: vec![],
             vars: vec![
                 Variable::new(Span::new(0, 0), Type::Any, false),
                 Variable::new(Span::new(0, 0), Type::Any, false),
@@ -196,6 +204,7 @@ impl EngineState {
         // Take the mutable reference and extend the permanent state from the working set
         self.files.extend(delta.files);
         self.file_contents.extend(delta.file_contents);
+        self.virtual_paths.extend(delta.virtual_paths);
         self.decls.extend(delta.decls);
         self.vars.extend(delta.vars);
         self.blocks.extend(delta.blocks);
@@ -555,6 +564,10 @@ impl EngineState {
         self.files.len()
     }
 
+    pub fn num_virtual_paths(&self) -> usize {
+        self.virtual_paths.len()
+    }
+
     pub fn num_vars(&self) -> usize {
         self.vars.len()
     }
@@ -828,6 +841,12 @@ impl EngineState {
             .expect("internal error: missing module")
     }
 
+    pub fn get_virtual_path(&self, virtual_path_id: VirtualPathId) -> &(String, VirtualPath) {
+        self.virtual_paths
+            .get(virtual_path_id)
+            .expect("internal error: missing virtual path")
+    }
+
     pub fn next_span_start(&self) -> usize {
         if let Some((_, _, last)) = self.file_contents.last() {
             *last
@@ -840,29 +859,6 @@ impl EngineState {
         self.files.iter()
     }
 
-    pub fn get_filename(&self, file_id: usize) -> String {
-        for file in self.files.iter().enumerate() {
-            if file.0 == file_id {
-                return file.1 .0.clone();
-            }
-        }
-
-        "<unknown>".into()
-    }
-
-    pub fn get_file_source(&self, file_id: usize) -> String {
-        for file in self.files.iter().enumerate() {
-            if file.0 == file_id {
-                let contents = self.get_span_contents(&Span::new(file.1 .1, file.1 .2));
-                let output = String::from_utf8_lossy(contents).to_string();
-
-                return output;
-            }
-        }
-
-        "<unknown>".into()
-    }
-
     pub fn add_file(&mut self, filename: String, contents: Vec<u8>) -> usize {
         let next_span_start = self.next_span_start();
         let next_span_end = next_span_start + contents.len();
@@ -1011,6 +1007,7 @@ impl TypeScope {
 pub struct StateDelta {
     files: Vec<(String, usize, usize)>,
     pub(crate) file_contents: Vec<(Vec<u8>, usize, usize)>,
+    virtual_paths: Vec<(String, VirtualPath)>,
     vars: Vec<Variable>,          // indexed by VarId
     decls: Vec<Box<dyn Command>>, // indexed by DeclId
     pub blocks: Vec<Block>,       // indexed by BlockId
@@ -1033,6 +1030,7 @@ impl StateDelta {
         StateDelta {
             files: vec![],
             file_contents: vec![],
+            virtual_paths: vec![],
             vars: vec![],
             decls: vec![],
             blocks: vec![],
@@ -1048,6 +1046,10 @@ impl StateDelta {
         self.files.len()
     }
 
+    pub fn num_virtual_paths(&self) -> usize {
+        self.virtual_paths.len()
+    }
+
     pub fn num_decls(&self) -> usize {
         self.decls.len()
     }
@@ -1145,6 +1147,10 @@ impl<'a> StateWorkingSet<'a> {
         self.delta.num_files() + self.permanent_state.num_files()
     }
 
+    pub fn num_virtual_paths(&self) -> usize {
+        self.delta.num_virtual_paths() + self.permanent_state.num_virtual_paths()
+    }
+
     pub fn num_decls(&self) -> usize {
         self.delta.num_decls() + self.permanent_state.num_decls()
     }
@@ -1346,33 +1352,24 @@ impl<'a> StateWorkingSet<'a> {
         self.permanent_state.files().chain(self.delta.files.iter())
     }
 
-    pub fn get_filename(&self, file_id: usize) -> String {
+    pub fn get_contents_of_file(&self, file_id: usize) -> Option<&[u8]> {
-        for file in self.files().enumerate() {
+        for (id, (contents, _, _)) in self.delta.file_contents.iter().enumerate() {
-            if file.0 == file_id {
+            if self.permanent_state.num_files() + id == file_id {
-                return file.1 .0.clone();
+                return Some(contents);
             }
         }
 
-        "<unknown>".into()
+        for (id, (contents, _, _)) in self.permanent_state.file_contents.iter().enumerate() {
-    }
+            if id == file_id {
-
+                return Some(contents);
-    pub fn get_file_source(&self, file_id: usize) -> String {
-        for file in self.files().enumerate() {
-            if file.0 == file_id {
-                let output = String::from_utf8_lossy(
-                    self.get_span_contents(Span::new(file.1 .1, file.1 .2)),
-                )
-                .to_string();
-
-                return output;
             }
         }
 
-        "<unknown>".into()
+        None
     }
 
     #[must_use]
-    pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize {
+    pub fn add_file(&mut self, filename: String, contents: &[u8]) -> FileId {
         // First, look for the file to see if we already have it
         for (idx, (fname, file_start, file_end)) in self.files().enumerate() {
             if fname == &filename {
@@ -1397,6 +1394,13 @@ impl<'a> StateWorkingSet<'a> {
         self.num_files() - 1
     }
 
+    #[must_use]
+    pub fn add_virtual_path(&mut self, name: String, virtual_path: VirtualPath) -> VirtualPathId {
+        self.delta.virtual_paths.push((name, virtual_path));
+
+        self.num_virtual_paths() - 1
+    }
+
     pub fn get_span_for_file(&self, file_id: usize) -> Span {
         let result = self
             .files()
@@ -2011,6 +2015,34 @@ impl<'a> StateWorkingSet<'a> {
 
         None
     }
+
+    pub fn find_virtual_path(&self, name: &str) -> Option<&VirtualPath> {
+        for (virtual_name, virtual_path) in self.delta.virtual_paths.iter().rev() {
+            if virtual_name == name {
+                return Some(virtual_path);
+            }
+        }
+
+        for (virtual_name, virtual_path) in self.permanent_state.virtual_paths.iter().rev() {
+            if virtual_name == name {
+                return Some(virtual_path);
+            }
+        }
+
+        None
+    }
+
+    pub fn get_virtual_path(&self, virtual_path_id: VirtualPathId) -> &(String, VirtualPath) {
+        let num_permanent_virtual_paths = self.permanent_state.num_virtual_paths();
+        if virtual_path_id < num_permanent_virtual_paths {
+            self.permanent_state.get_virtual_path(virtual_path_id)
+        } else {
+            self.delta
+                .virtual_paths
+                .get(virtual_path_id - num_permanent_virtual_paths)
+                .expect("internal error: missing virtual path")
+        }
+    }
 }
 
 impl Default for EngineState {

crates/nu-protocol/src/id.rs

@@ -3,3 +3,5 @@ pub type DeclId = usize;
 pub type BlockId = usize;
 pub type ModuleId = usize;
 pub type OverlayId = usize;
+pub type FileId = usize;
+pub type VirtualPathId = usize;

crates/nu-std/Cargo.toml

@@ -11,3 +11,4 @@ version = "0.80.1"
 miette = { version = "5.6.0", features = ["fancy-no-backtrace"] }
 nu-parser = { version = "0.80.1", path = "../nu-parser" }
 nu-protocol = { version = "0.80.1", path = "../nu-protocol" }
+nu-engine = { version = "0.80.1", path = "../nu-engine" }

crates/nu-std/src/lib.rs

@@ -1,126 +1,136 @@
-use nu_parser::{parse, parse_module_block};
+use std::path::PathBuf;
-use nu_protocol::report_error;
-use nu_protocol::{engine::StateWorkingSet, Module, ShellError, Span};
 
-fn add_file(
+use nu_engine::{env::current_dir, eval_block};
-    working_set: &mut StateWorkingSet,
+use nu_parser::parse;
-    name: &String,
+use nu_protocol::engine::{Stack, StateWorkingSet, VirtualPath};
-    content: &[u8],
+use nu_protocol::{report_error, PipelineData};
-) -> (Module, Vec<Span>) {
-    let file_id = working_set.add_file(name.clone(), content);
-    let new_span = working_set.get_span_for_file(file_id);
 
-    let (_, module, comments) = parse_module_block(working_set, new_span, name.as_bytes());
+// Virtual std directory unlikely to appear in user's file system
-
+const NU_STDLIB_VIRTUAL_DIR: &str = "NU_STDLIB_VIRTUAL_DIR";
-    if let Some(err) = working_set.parse_errors.first() {
-        report_error(working_set, err);
-    }
-
-    parse(working_set, Some(name), content, true);
-
-    if let Some(err) = working_set.parse_errors.first() {
-        report_error(working_set, err);
-    }
-
-    (module, comments)
-}
-
-fn load_prelude(working_set: &mut StateWorkingSet, prelude: Vec<(&str, &str)>, module: &Module) {
-    let mut decls = Vec::new();
-    let mut errs = Vec::new();
-    for (name, search_name) in prelude {
-        if let Some(id) = module.decls.get(&search_name.as_bytes().to_vec()) {
-            let decl = (name.as_bytes().to_vec(), id.to_owned());
-            decls.push(decl);
-        } else {
-            errs.push(ShellError::GenericError(
-                format!("could not load `{}` from `std`.", search_name),
-                String::new(),
-                None,
-                None,
-                Vec::new(),
-            ));
-        }
-    }
-
-    if !errs.is_empty() {
-        report_error(
-            working_set,
-            &ShellError::GenericError(
-                "Unable to load the prelude of the standard library.".into(),
-                String::new(),
-                None,
-                Some("this is a bug: please file an issue in the [issue tracker](https://github.com/nushell/nushell/issues/new/choose)".to_string()),
-                errs,
-            ),
-        );
-    }
-
-    working_set.use_decls(decls);
-}
 
 pub fn load_standard_library(
     engine_state: &mut nu_protocol::engine::EngineState,
 ) -> Result<(), miette::ErrReport> {
-    let delta = {
+    let (block, delta) = {
-        let name = "std".to_string();
+        let mut std_files = vec![
-        let content = include_str!("../lib/mod.nu");
+            (
-
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
-        // these modules are loaded in the order they appear in this list
+                    .join("std")
-        #[rustfmt::skip]
+                    .join("mod.nu"),
-        let submodules = vec![
+                include_str!("../std/mod.nu"),
-            // helper modules that could be used in other parts of the library
+            ),
-            ("log", include_str!("../lib/log.nu")),
+            (
-
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
-            // the rest of the library
+                    .join("std")
-            ("dirs", include_str!("../lib/dirs.nu")),
+                    .join("dirs.nu"),
-            ("iter", include_str!("../lib/iter.nu")),
+                include_str!("../std/dirs.nu"),
-            ("help", include_str!("../lib/help.nu")),
+            ),
-            ("testing", include_str!("../lib/testing.nu")),
+            (
-            ("xml", include_str!("../lib/xml.nu")),
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
-            ("dt", include_str!("../lib/dt.nu")),
+                    .join("std")
-        ];
+                    .join("dt.nu"),
-
+                include_str!("../std/dt.nu"),
-        // Define commands to be preloaded into the default (top level, unprefixed) namespace.
+            ),
-        // User can invoke these without having to `use std` beforehand.
+            (
-        // Entries are: (name to add to default namespace, path under std to find implementation)
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
-        //
+                    .join("std")
-        // Conventionally, for a command implemented as `std foo`, the name added
+                    .join("help.nu"),
-        // is either `std foo` or bare `foo`, not some arbitrary rename.
+                include_str!("../std/help.nu"),
-
+            ),
-        #[rustfmt::skip]
+            (
-        let prelude = vec![
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
-            ("std help", "help"),
+                    .join("std")
-            ("std help commands", "help commands"),
+                    .join("iter.nu"),
-            ("std help aliases", "help aliases"),
+                include_str!("../std/iter.nu"),
-            ("std help modules", "help modules"),
+            ),
-            ("std help externs", "help externs"),
+            (
-            ("std help operators", "help operators"),
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
-
+                    .join("std")
-            ("enter", "dirs enter"),
+                    .join("log.nu"),
-            ("shells", "dirs shells"),
+                include_str!("../std/log.nu"),
-            ("g", "dirs g"),
+            ),
-            ("n", "dirs n"),
+            (
-            ("p", "dirs p"),
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
-            ("dexit", "dirs dexit"),
+                    .join("std")
+                    .join("testing.nu"),
+                include_str!("../std/testing.nu"),
+            ),
+            (
+                PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
+                    .join("std")
+                    .join("xml.nu"),
+                include_str!("../std/xml.nu"),
+            ),
         ];
 
         let mut working_set = StateWorkingSet::new(engine_state);
+        let mut std_virt_paths = vec![];
 
-        for (name, content) in submodules {
+        for (name, content) in std_files.drain(..) {
-            let (module, comments) =
+            let file_id =
-                add_file(&mut working_set, &name.to_string(), content.as_bytes());
+                working_set.add_file(name.to_string_lossy().to_string(), content.as_bytes());
-            working_set.add_module(name, module, comments);
+            let virtual_file_id = working_set.add_virtual_path(
+                name.to_string_lossy().to_string(),
+                VirtualPath::File(file_id),
+            );
+            std_virt_paths.push(virtual_file_id);
         }
 
-        let (module, comments) = add_file(&mut working_set, &name, content.as_bytes());
+        // Using full virtual path to avoid potential conflicts with user having 'std' directory
-        load_prelude(&mut working_set, prelude, &module);
+        // in their working directory.
-        working_set.add_module(&name, module, comments);
+        let std_dir = PathBuf::from(NU_STDLIB_VIRTUAL_DIR)
+            .join("std")
+            .to_string_lossy()
+            .to_string();
+        let source = format!(
+            r#"
+# Define the `std` module
+module {std_dir}
 
-        working_set.render()
+# Prelude
+use std dirs [ enter, shells, g, n, p, dexit ]
+"#
+        );
+
+        let _ = working_set.add_virtual_path(std_dir, VirtualPath::Dir(std_virt_paths));
+
+        // Change the currently parsed directory
+        let prev_currently_parsed_cwd = working_set.currently_parsed_cwd.clone();
+        working_set.currently_parsed_cwd = Some(PathBuf::from(NU_STDLIB_VIRTUAL_DIR));
+
+        let block = parse(
+            &mut working_set,
+            Some("loading stdlib"),
+            source.as_bytes(),
+            false,
+        );
+
+        if let Some(err) = working_set.parse_errors.first() {
+            report_error(&working_set, err);
+        }
+
+        // Restore the currently parsed directory back
+        working_set.currently_parsed_cwd = prev_currently_parsed_cwd;
+
+        (block, working_set.render())
     };
 
     engine_state.merge_delta(delta)?;
 
+    // We need to evaluate the module in order to run the `export-env` blocks.
+    let mut stack = Stack::new();
+    let pipeline_data = PipelineData::Empty;
+    eval_block(
+        engine_state,
+        &mut stack,
+        &block,
+        pipeline_data,
+        false,
+        false,
+    )?;
+
+    let cwd = current_dir(engine_state, &stack)?;
+    engine_state.merge_env(&mut stack, cwd)?;
+
     Ok(())
 }

crates/nu-std/std/mod.nu

@@ -1,15 +1,12 @@
 # std.nu, `used` to load all standard library components
 
-export use dirs
 export-env {
-    use dirs *
+    use dirs.nu []
 }
-export use help
+
-export use iter
+export use testing.nu *
-export use log
+
-export use testing *
+use dt.nu [datetime-diff, pretty-print-duration]
-export use xml
-use dt [datetime-diff, pretty-print-duration]
 
 # Add the given paths to the PATH.
 #
@@ -46,7 +43,7 @@ export def-env "path add" [
             | if $append { append $paths }
             else { prepend $paths }
     )
-    
+
     if $ret {
         $env | get $path_name
     }

crates/nu-std/std/testing.nu

@@ -5,7 +5,7 @@
 # Assert commands and test runner.
 #
 ##################################################################################
-use log
+export use log.nu
 
 # Universal assert command
 #
@@ -17,7 +17,7 @@ use log
 # >_ assert (3 == 3)
 # >_ assert (42 == 3)
 # Error:
-#   × Assertion failed: 
+#   × Assertion failed:
 #     ╭─[myscript.nu:11:1]
 #  11 │ assert (3 == 3)
 #  12 │ assert (42 == 3)
@@ -38,7 +38,7 @@ use log
 # }
 # ```
 export def assert [
-    condition: bool, # Condition, which should be true 
+    condition: bool, # Condition, which should be true
     message?: string, # Optional error message
     --error-label: record # Label for `error make` if you want to create a custom assert
 ] {
@@ -64,7 +64,7 @@ export def assert [
 # >_ assert (42 == 3)
 # >_ assert (3 == 3)
 # Error:
-#   × Assertion failed: 
+#   × Assertion failed:
 #     ╭─[myscript.nu:11:1]
 #  11 │ assert (42 == 3)
 #  12 │ assert (3 == 3)
@@ -73,7 +73,7 @@ export def assert [
 #  13 │
 #     ╰────
 #
-# 
+#
 # The --error-label flag can be used if you want to create a custom assert command:
 # ```
 # def "assert not even" [number: int] {
@@ -86,7 +86,7 @@ export def assert [
 # ```
 #
 export def "assert not" [
-    condition: bool, # Condition, which should be false 
+    condition: bool, # Condition, which should be false
     message?: string, # Optional error message
     --error-label: record # Label for `error make` if you want to create a custom assert
 ] {
@@ -106,7 +106,7 @@ export def "assert not" [
 # Assert that executing the code generates an error
 #
 # For more documentation see the assert command
-# 
+#
 # # Examples
 #
 # > assert error {|| missing_command} # passes
@@ -138,7 +138,7 @@ export def "assert skip" [] {
 # For more documentation see the assert command
 #
 # # Examples
-# 
+#
 # > assert equal 1 1 # passes
 # > assert equal (0.1 + 0.2) 0.3
 # > assert equal 1 2 # fails

crates/nu-std/tests/test_dirs.nu

@@ -3,8 +3,7 @@ use std "assert length"
 use std "assert equal"
 use std "assert not equal"
 use std "assert error"
-use std "log info"
+use std log
-use std "log debug"
 
 # A couple of nuances to understand when testing module that exports environment:
 # Each 'use' for that module in the test script will execute the export def-env block.
@@ -41,20 +40,15 @@ def cur_ring_check [expect_dir:string, expect_position: int scenario:string] {
 export def test_dirs_command [] {
     # careful with order of these statements!
     # must capture value of $in before executing `use`s
-    let $c = $in    
+    let $c = $in
 
     # must set PWD *before* doing `use` that will run the export def-env block in dirs module.
     cd $c.base_path
 
     # must execute these uses for the UOT commands *after* the test and *not* just put them at top of test module.
     # the export def-env gets messed up
-    use std "dirs next"
+    use std dirs
-    use std "dirs prev"
+
-    use std "dirs add"
-    use std "dirs drop"
-    use std "dirs show"
-    use std "dirs goto"
-    
     assert equal [$c.base_path] $env.DIRS_LIST "list is just pwd after initialization"
 
     dirs next
@@ -85,13 +79,12 @@ export def test_dirs_command [] {
 
 export def test_dirs_next [] {
     # must capture value of $in before executing `use`s
-    let $c = $in    
+    let $c = $in
     # must set PWD *before* doing `use` that will run the export def-env block in dirs module.
     cd $c.base_path
     assert equal $env.PWD $c.base_path "test setup"
 
-    use std "dirs next"
+    use std dirs
-    use std "dirs add"
     cur_dir_check $c.base_path "use module test setup"
 
     dirs add $c.path_a $c.path_b
@@ -107,15 +100,11 @@ export def test_dirs_next [] {
 
 export def test_dirs_cd [] {
     # must capture value of $in before executing `use`s
-    let $c = $in    
+    let $c = $in
     # must set PWD *before* doing `use` that will run the export def-env block in dirs module.
     cd $c.base_path
 
-    use std # necessary to define $env.config??
+    use std dirs
-
-    use std "dirs next"
-    use std "dirs add"
-    use std "dirs drop"
 
     cur_dir_check $c.base_path "use module test setup"
 

src/main.rs

@@ -152,6 +152,18 @@ fn main() -> Result<()> {
         engine_state.add_env_var("NU_LIB_DIRS".into(), Value::List { vals, span });
     }
 
+    start_time = std::time::Instant::now();
+    // First, set up env vars as strings only
+    gather_parent_env_vars(&mut engine_state, &init_cwd);
+    perf(
+        "gather env vars",
+        start_time,
+        file!(),
+        line!(),
+        column!(),
+        use_color,
+    );
+
     if parsed_nu_cli_args.no_std_lib.is_none() {
         load_standard_library(&mut engine_state)?;
     }
@@ -243,18 +255,6 @@ fn main() -> Result<()> {
         use_color,
     );
 
-    start_time = std::time::Instant::now();
-    // First, set up env vars as strings only
-    gather_parent_env_vars(&mut engine_state, &init_cwd);
-    perf(
-        "gather env vars",
-        start_time,
-        file!(),
-        line!(),
-        column!(),
-        use_color,
-    );
-
     if let Some(commands) = parsed_nu_cli_args.commands.clone() {
         run_commands(
             &mut engine_state,

src/tests/test_stdlib.rs

@@ -2,15 +2,12 @@ use crate::tests::{fail_test, run_test_std, TestResult};
 
 #[test]
 fn library_loaded() -> TestResult {
-    run_test_std(
+    run_test_std("$nu.scope.modules | where name == 'std' | length", "1")
-        "help std | lines | first 1 | to text",
-        "std.nu, `used` to load all standard library components",
-    )
 }
 
 #[test]
 fn prelude_loaded() -> TestResult {
-    run_test_std("std help commands | where name == open | length", "1")
+    run_test_std("shells | length", "1")
 }
 
 #[test]