From 74724dee809c039440ce299db65ee5ad5aa321ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C5=BD=C3=A1dn=C3=ADk?= Date: Tue, 23 May 2023 23:48:50 +0300 Subject: [PATCH] Add virtual path abstraction layer (#9245) --- Cargo.lock | 1 + crates/nu-parser/src/lib.rs | 2 + crates/nu-parser/src/parse_keywords.rs | 252 +++++++++--------- crates/nu-parser/src/parser_path.rs | 134 ++++++++++ crates/nu-protocol/src/engine/engine_state.rs | 118 +++++--- crates/nu-protocol/src/id.rs | 2 + crates/nu-std/Cargo.toml | 1 + crates/nu-std/src/lib.rs | 224 ++++++++-------- crates/nu-std/{lib => std}/dirs.nu | 0 crates/nu-std/{lib => std}/dt.nu | 0 crates/nu-std/{lib => std}/help.nu | 0 crates/nu-std/{lib => std}/iter.nu | 0 crates/nu-std/{lib => std}/log.nu | 0 crates/nu-std/{lib => std}/mod.nu | 15 +- crates/nu-std/{lib => std}/testing.nu | 16 +- crates/nu-std/{lib => std}/xml.nu | 0 crates/nu-std/tests/test_dirs.nu | 27 +- src/main.rs | 24 +- src/tests/test_stdlib.rs | 7 +- 19 files changed, 500 insertions(+), 323 deletions(-) create mode 100644 crates/nu-parser/src/parser_path.rs rename crates/nu-std/{lib => std}/dirs.nu (100%) rename crates/nu-std/{lib => std}/dt.nu (100%) rename crates/nu-std/{lib => std}/help.nu (100%) rename crates/nu-std/{lib => std}/iter.nu (100%) rename crates/nu-std/{lib => std}/log.nu (100%) rename crates/nu-std/{lib => std}/mod.nu (98%) rename crates/nu-std/{lib => std}/testing.nu (98%) rename crates/nu-std/{lib => std}/xml.nu (100%) diff --git a/Cargo.lock b/Cargo.lock index 5b2078b0ce..e800089d29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2947,6 +2947,7 @@ name = "nu-std" version = "0.80.1" dependencies = [ "miette", + "nu-engine", "nu-parser", "nu-protocol", ] diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index 2bd346c8fc..c8842ad0e7 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -7,6 +7,7 @@ mod lite_parser; mod parse_keywords; mod parse_patterns; mod parser; +mod parser_path; mod type_check; pub use deparse::{escape_for_script_arg, escape_quote_string}; @@ -17,6 +18,7 @@ pub use known_external::KnownExternal; pub use lex::{lex, lex_signature, Token, TokenContents}; pub use lite_parser::{lite_parse, LiteBlock, LiteElement}; pub use parse_keywords::*; +pub use parser_path::*; pub use parser::{ is_math_expression_like, parse, parse_block, parse_expression, parse_external_call, diff --git a/crates/nu-parser/src/parse_keywords.rs b/crates/nu-parser/src/parse_keywords.rs index a0527ccac4..fdaf1875c3 100644 --- a/crates/nu-parser/src/parse_keywords.rs +++ b/crates/nu-parser/src/parse_keywords.rs @@ -1,3 +1,4 @@ +use crate::parser_path::ParserPath; use itertools::Itertools; use log::trace; use nu_path::canonicalize_with; @@ -1724,14 +1725,14 @@ pub fn parse_module_block( fn parse_module_file( working_set: &mut StateWorkingSet, - path: PathBuf, + path: ParserPath, path_span: Span, name_override: Option, ) -> Option { if let Some(i) = working_set .parsed_module_files .iter() - .rposition(|p| p == &path) + .rposition(|p| p == path.path()) { let mut files: Vec = working_set .parsed_module_files @@ -1740,7 +1741,7 @@ fn parse_module_file( .map(|p| p.to_string_lossy().to_string()) .collect(); - files.push(path.to_string_lossy().to_string()); + files.push(path.path().to_string_lossy().to_string()); let msg = files.join("\nuses "); @@ -1757,14 +1758,14 @@ fn parse_module_file( return None; }; - let contents = if let Ok(contents) = std::fs::read(&path) { + let contents = if let Some(contents) = path.read(working_set) { contents } else { working_set.error(ParseError::ModuleNotFound(path_span)); return None; }; - let file_id = working_set.add_file(path.to_string_lossy().to_string(), &contents); + let file_id = working_set.add_file(path.path().to_string_lossy().to_string(), &contents); let new_span = working_set.get_span_for_file(file_id); if let Some(module_id) = working_set.find_module_by_span(new_span) { @@ -1773,17 +1774,13 @@ fn parse_module_file( // Change the currently parsed directory let prev_currently_parsed_cwd = if let Some(parent) = path.parent() { - let prev = working_set.currently_parsed_cwd.clone(); - - working_set.currently_parsed_cwd = Some(parent.into()); - - prev + working_set.currently_parsed_cwd.replace(parent.into()) } else { working_set.currently_parsed_cwd.clone() }; // Add the file to the stack of parsed module files - working_set.parsed_module_files.push(path); + working_set.parsed_module_files.push(path.path_buf()); // Parse the module let (block, module, module_comments) = @@ -1824,91 +1821,87 @@ pub fn parse_module_file_or_dir( }; if module_path.is_dir() { - if let Ok(dir_contents) = std::fs::read_dir(&module_path) { - let module_name = if let Some(stem) = module_path.file_stem() { - stem.to_string_lossy().to_string() - } else { - working_set.error(ParseError::ModuleNotFound(path_span)); - return None; - }; + let Some(dir_contents) = module_path.read_dir() else { + working_set.error(ParseError::ModuleNotFound(path_span)); + return None; + }; - let mod_nu_path = module_path.join("mod.nu"); - - if !(mod_nu_path.exists() && mod_nu_path.is_file()) { - working_set.error(ParseError::ModuleMissingModNuFile(path_span)); - return None; - } - - let mut paths = vec![]; - - for entry in dir_contents.flatten() { - let entry_path = entry.path(); - - if (entry_path.is_file() - && entry_path.extension() == Some(OsStr::new("nu")) - && entry_path.file_stem() != Some(OsStr::new("mod"))) - || (entry_path.is_dir() && entry_path.join("mod.nu").exists()) - { - if entry_path.file_stem() == Some(OsStr::new(&module_name)) { - working_set.error(ParseError::InvalidModuleFileName( - module_path.to_string_lossy().to_string(), - module_name, - path_span, - )); - return None; - } - - paths.push(entry_path); - } - } - - paths.sort(); - - // working_set.enter_scope(); - - let mut submodules = vec![]; - - for p in paths { - if let Some(submodule_id) = parse_module_file_or_dir( - working_set, - p.to_string_lossy().as_bytes(), - path_span, - None, - ) { - let submodule_name = working_set.get_module(submodule_id).name(); - submodules.push((submodule_name, submodule_id)); - } - } - - if let Some(module_id) = parse_module_file( - working_set, - mod_nu_path, - path_span, - name_override.or(Some(module_name)), - ) { - let mut module = working_set.get_module(module_id).clone(); - - for (submodule_name, submodule_id) in submodules { - module.add_submodule(submodule_name, submodule_id); - } - - let module_name = String::from_utf8_lossy(&module.name).to_string(); - - let module_comments = - if let Some(comments) = working_set.get_module_comments(module_id) { - comments.to_vec() - } else { - vec![] - }; - - let new_module_id = working_set.add_module(&module_name, module, module_comments); - - Some(new_module_id) - } else { - None - } + let module_name = if let Some(stem) = module_path.file_stem() { + stem.to_string_lossy().to_string() } else { working_set.error(ParseError::ModuleNotFound(path_span)); + return None; + }; + + let mod_nu_path = module_path.clone().join("mod.nu"); + + if !(mod_nu_path.exists() && mod_nu_path.is_file()) { + working_set.error(ParseError::ModuleMissingModNuFile(path_span)); + return None; + } + + let mut paths = vec![]; + + for entry_path in dir_contents { + if (entry_path.is_file() + && entry_path.extension() == Some(OsStr::new("nu")) + && entry_path.file_stem() != Some(OsStr::new("mod"))) + || (entry_path.is_dir() && entry_path.clone().join("mod.nu").exists()) + { + if entry_path.file_stem() == Some(OsStr::new(&module_name)) { + working_set.error(ParseError::InvalidModuleFileName( + module_path.path().to_string_lossy().to_string(), + module_name, + path_span, + )); + return None; + } + + paths.push(entry_path); + } + } + + paths.sort(); + + let mut submodules = vec![]; + + for p in paths { + if let Some(submodule_id) = parse_module_file_or_dir( + working_set, + p.path().to_string_lossy().as_bytes(), + path_span, + None, + ) { + let submodule_name = working_set.get_module(submodule_id).name(); + submodules.push((submodule_name, submodule_id)); + } + } + + if let Some(module_id) = parse_module_file( + working_set, + mod_nu_path, + path_span, + name_override.or(Some(module_name)), + ) { + let mut module = working_set.get_module(module_id).clone(); + + for (submodule_name, submodule_id) in submodules { + module.add_submodule(submodule_name, submodule_id); + } + + let module_name = String::from_utf8_lossy(&module.name).to_string(); + + let module_comments = if let Some(comments) = working_set.get_module_comments(module_id) + { + comments.to_vec() + } else { + vec![] + }; + + let new_module_id = working_set.add_module(&module_name, module, module_comments); + + Some(new_module_id) + } else { None } } else if module_path.is_file() { @@ -2017,19 +2010,13 @@ pub fn parse_module( }]); if spans.len() == split_id + 1 { - let cwd = working_set.get_cwd(); - - if let Some(module_path) = - find_in_dirs(&module_name_or_path, working_set, &cwd, LIB_DIRS_VAR) - { - let path_str = module_path.to_string_lossy().to_string(); - let maybe_module_id = parse_module_file_or_dir( - working_set, - path_str.as_bytes(), - module_name_or_path_span, - None, - ); - return (pipeline, maybe_module_id); + if let Some(module_id) = parse_module_file_or_dir( + working_set, + module_name_or_path.as_bytes(), + module_name_or_path_span, + None, + ) { + return (pipeline, Some(module_id)); } else { working_set.error(ParseError::ModuleNotFound(module_name_or_path_span)); return (pipeline, None); @@ -3043,14 +3030,10 @@ pub fn parse_source(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeli }; if let Some(path) = find_in_dirs(&filename, working_set, &cwd, LIB_DIRS_VAR) { - if let Ok(contents) = std::fs::read(&path) { + if let Some(contents) = path.read(working_set) { // Change currently parsed directory let prev_currently_parsed_cwd = if let Some(parent) = path.parent() { - let prev = working_set.currently_parsed_cwd.clone(); - - working_set.currently_parsed_cwd = Some(parent.into()); - - prev + working_set.currently_parsed_cwd.replace(parent.into()) } else { working_set.currently_parsed_cwd.clone() }; @@ -3059,7 +3042,7 @@ pub fn parse_source(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeli // working set, if it was a successful parse. let block = parse( working_set, - Some(&path.to_string_lossy()), + Some(&path.path().to_string_lossy()), &contents, scoped, ); @@ -3300,6 +3283,7 @@ pub fn parse_register(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipe nu_engine::env::env_to_strings(working_set.permanent_state, &stack).unwrap_or_default(); let error = match signature { Some(signature) => arguments.and_then(|(path, path_span)| { + let path = path.path_buf(); // restrict plugin file name starts with `nu_plugin_` let valid_plugin_name = path .file_name() @@ -3320,13 +3304,14 @@ pub fn parse_register(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipe } }), None => arguments.and_then(|(path, path_span)| { + let path = path.path_buf(); // restrict plugin file name starts with `nu_plugin_` let valid_plugin_name = path .file_name() .map(|s| s.to_string_lossy().starts_with("nu_plugin_")); if let Some(true) = valid_plugin_name { - get_signature(path.as_path(), &shell, ¤t_envs) + get_signature(&path, &shell, ¤t_envs) .map_err(|err| { ParseError::LabeledError( "Error getting signatures".into(), @@ -3393,28 +3378,52 @@ pub fn find_in_dirs( working_set: &StateWorkingSet, cwd: &str, dirs_var_name: &str, -) -> Option { +) -> Option { pub fn find_in_dirs_with_id( filename: &str, working_set: &StateWorkingSet, cwd: &str, dirs_var_name: &str, - ) -> Option { + ) -> Option { // Choose whether to use file-relative or PWD-relative path let actual_cwd = if let Some(currently_parsed_cwd) = &working_set.currently_parsed_cwd { currently_parsed_cwd.as_path() } else { Path::new(cwd) }; - if let Ok(p) = canonicalize_with(filename, actual_cwd) { - return Some(p); + + // Try if we have an existing virtual path + if let Some(virtual_path) = working_set.find_virtual_path(filename) { + return Some(ParserPath::from_virtual_path( + working_set, + filename, + virtual_path, + )); + } else { + let abs_virtual_filename = actual_cwd.join(filename); + let abs_virtual_filename = abs_virtual_filename.to_string_lossy(); + + if let Some(virtual_path) = working_set.find_virtual_path(&abs_virtual_filename) { + return Some(ParserPath::from_virtual_path( + working_set, + &abs_virtual_filename, + virtual_path, + )); + } } + // Try if we have an existing physical path + if let Ok(p) = canonicalize_with(filename, actual_cwd) { + return Some(ParserPath::RealPath(p)); + } + + // Early-exit if path is non-existent absolute path let path = Path::new(filename); if !path.is_relative() { return None; } + // Look up relative path from NU_LIB_DIRS working_set .find_constant(find_dirs_var(working_set, dirs_var_name)?)? .as_list() @@ -3427,9 +3436,11 @@ pub fn find_in_dirs( }) .find(Option::is_some) .flatten() + .map(ParserPath::RealPath) } // TODO: remove (see #8310) + // Same as find_in_dirs_with_id but using $env.NU_LIB_DIRS instead of constant pub fn find_in_dirs_old( filename: &str, working_set: &StateWorkingSet, @@ -3475,8 +3486,9 @@ pub fn find_in_dirs( } } - find_in_dirs_with_id(filename, working_set, cwd, dirs_var_name) - .or_else(|| find_in_dirs_old(filename, working_set, cwd, dirs_var_name)) + find_in_dirs_with_id(filename, working_set, cwd, dirs_var_name).or_else(|| { + find_in_dirs_old(filename, working_set, cwd, dirs_var_name).map(ParserPath::RealPath) + }) } fn detect_params_in_name( diff --git a/crates/nu-parser/src/parser_path.rs b/crates/nu-parser/src/parser_path.rs new file mode 100644 index 0000000000..fa60d1ecd2 --- /dev/null +++ b/crates/nu-parser/src/parser_path.rs @@ -0,0 +1,134 @@ +use nu_protocol::engine::{StateWorkingSet, VirtualPath}; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; + +/// An abstraction over a PathBuf that can have virtual paths (files and directories). Virtual +/// paths always exist and represent a way to ship Nushell code inside the binary without requiring +/// paths to be present in the file system. +/// +/// Created from VirtualPath found in the engine state. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum ParserPath { + RealPath(PathBuf), + VirtualFile(PathBuf, usize), + VirtualDir(PathBuf, Vec), +} + +impl ParserPath { + pub fn is_dir(&self) -> bool { + match self { + ParserPath::RealPath(p) => p.is_dir(), + ParserPath::VirtualFile(..) => false, + ParserPath::VirtualDir(..) => true, + } + } + + pub fn is_file(&self) -> bool { + match self { + ParserPath::RealPath(p) => p.is_file(), + ParserPath::VirtualFile(..) => true, + ParserPath::VirtualDir(..) => false, + } + } + + pub fn exists(&self) -> bool { + match self { + ParserPath::RealPath(p) => p.exists(), + ParserPath::VirtualFile(..) => true, + ParserPath::VirtualDir(..) => true, + } + } + + pub fn path(&self) -> &Path { + match self { + ParserPath::RealPath(p) => p, + ParserPath::VirtualFile(p, _) => p, + ParserPath::VirtualDir(p, _) => p, + } + } + + pub fn path_buf(self) -> PathBuf { + match self { + ParserPath::RealPath(p) => p, + ParserPath::VirtualFile(p, _) => p, + ParserPath::VirtualDir(p, _) => p, + } + } + + pub fn parent(&self) -> Option<&Path> { + match self { + ParserPath::RealPath(p) => p.parent(), + ParserPath::VirtualFile(p, _) => p.parent(), + ParserPath::VirtualDir(p, _) => p.parent(), + } + } + + pub fn read_dir(&self) -> Option> { + match self { + ParserPath::RealPath(p) => p.read_dir().ok().map(|read_dir| { + read_dir + .flatten() + .map(|dir_entry| ParserPath::RealPath(dir_entry.path())) + .collect() + }), + ParserPath::VirtualFile(..) => None, + ParserPath::VirtualDir(_, files) => Some(files.clone()), + } + } + + pub fn file_stem(&self) -> Option<&OsStr> { + self.path().file_stem() + } + + pub fn extension(&self) -> Option<&OsStr> { + self.path().extension() + } + + pub fn join(self, path: impl AsRef) -> ParserPath { + match self { + ParserPath::RealPath(p) => ParserPath::RealPath(p.join(path)), + ParserPath::VirtualFile(p, file_id) => ParserPath::VirtualFile(p.join(path), file_id), + ParserPath::VirtualDir(p, entries) => { + let new_p = p.join(path); + let mut pp = ParserPath::RealPath(new_p.clone()); + for entry in entries { + if new_p == entry.path() { + pp = entry.clone(); + } + } + pp + } + } + } + + pub fn read<'a>(&'a self, working_set: &'a StateWorkingSet) -> Option> { + match self { + ParserPath::RealPath(p) => std::fs::read(p).ok(), + ParserPath::VirtualFile(_, file_id) => working_set + .get_contents_of_file(*file_id) + .map(|bytes| bytes.to_vec()), + + ParserPath::VirtualDir(..) => None, + } + } + + pub fn from_virtual_path( + working_set: &StateWorkingSet, + name: &str, + virtual_path: &VirtualPath, + ) -> Self { + match virtual_path { + VirtualPath::File(file_id) => ParserPath::VirtualFile(PathBuf::from(name), *file_id), + VirtualPath::Dir(entries) => ParserPath::VirtualDir( + PathBuf::from(name), + entries + .iter() + .map(|virtual_path_id| { + let (virt_name, virt_path) = working_set.get_virtual_path(*virtual_path_id); + ParserPath::from_virtual_path(working_set, virt_name, virt_path) + }) + .collect(), + ), + } + } +} diff --git a/crates/nu-protocol/src/engine/engine_state.rs b/crates/nu-protocol/src/engine/engine_state.rs index f9dfb8198a..7a7c2f0ffd 100644 --- a/crates/nu-protocol/src/engine/engine_state.rs +++ b/crates/nu-protocol/src/engine/engine_state.rs @@ -3,8 +3,8 @@ use lru::LruCache; use super::{Command, EnvVars, OverlayFrame, ScopeFrame, Stack, Visibility, DEFAULT_OVERLAY_NAME}; use crate::{ - ast::Block, BlockId, Config, DeclId, Example, Module, ModuleId, OverlayId, ShellError, - Signature, Span, Type, VarId, Variable, + ast::Block, BlockId, Config, DeclId, Example, FileId, Module, ModuleId, OverlayId, ShellError, + Signature, Span, Type, VarId, Variable, VirtualPathId, }; use crate::{ParseError, Value}; use core::panic; @@ -56,6 +56,12 @@ impl Default for Usage { } } +#[derive(Clone, Debug)] +pub enum VirtualPath { + File(FileId), + Dir(Vec), +} + /// The core global engine state. This includes all global definitions as well as any global state that /// will persist for the whole session. /// @@ -102,6 +108,7 @@ impl Default for Usage { pub struct EngineState { files: Vec<(String, usize, usize)>, file_contents: Vec<(Vec, usize, usize)>, + virtual_paths: Vec<(String, VirtualPath)>, vars: Vec, decls: Vec>, blocks: Vec, @@ -144,6 +151,7 @@ impl EngineState { Self { files: vec![], file_contents: vec![], + virtual_paths: vec![], vars: vec![ Variable::new(Span::new(0, 0), Type::Any, false), Variable::new(Span::new(0, 0), Type::Any, false), @@ -196,6 +204,7 @@ impl EngineState { // Take the mutable reference and extend the permanent state from the working set self.files.extend(delta.files); self.file_contents.extend(delta.file_contents); + self.virtual_paths.extend(delta.virtual_paths); self.decls.extend(delta.decls); self.vars.extend(delta.vars); self.blocks.extend(delta.blocks); @@ -555,6 +564,10 @@ impl EngineState { self.files.len() } + pub fn num_virtual_paths(&self) -> usize { + self.virtual_paths.len() + } + pub fn num_vars(&self) -> usize { self.vars.len() } @@ -828,6 +841,12 @@ impl EngineState { .expect("internal error: missing module") } + pub fn get_virtual_path(&self, virtual_path_id: VirtualPathId) -> &(String, VirtualPath) { + self.virtual_paths + .get(virtual_path_id) + .expect("internal error: missing virtual path") + } + pub fn next_span_start(&self) -> usize { if let Some((_, _, last)) = self.file_contents.last() { *last @@ -840,29 +859,6 @@ impl EngineState { self.files.iter() } - pub fn get_filename(&self, file_id: usize) -> String { - for file in self.files.iter().enumerate() { - if file.0 == file_id { - return file.1 .0.clone(); - } - } - - "".into() - } - - pub fn get_file_source(&self, file_id: usize) -> String { - for file in self.files.iter().enumerate() { - if file.0 == file_id { - let contents = self.get_span_contents(&Span::new(file.1 .1, file.1 .2)); - let output = String::from_utf8_lossy(contents).to_string(); - - return output; - } - } - - "".into() - } - pub fn add_file(&mut self, filename: String, contents: Vec) -> usize { let next_span_start = self.next_span_start(); let next_span_end = next_span_start + contents.len(); @@ -1011,6 +1007,7 @@ impl TypeScope { pub struct StateDelta { files: Vec<(String, usize, usize)>, pub(crate) file_contents: Vec<(Vec, usize, usize)>, + virtual_paths: Vec<(String, VirtualPath)>, vars: Vec, // indexed by VarId decls: Vec>, // indexed by DeclId pub blocks: Vec, // indexed by BlockId @@ -1033,6 +1030,7 @@ impl StateDelta { StateDelta { files: vec![], file_contents: vec![], + virtual_paths: vec![], vars: vec![], decls: vec![], blocks: vec![], @@ -1048,6 +1046,10 @@ impl StateDelta { self.files.len() } + pub fn num_virtual_paths(&self) -> usize { + self.virtual_paths.len() + } + pub fn num_decls(&self) -> usize { self.decls.len() } @@ -1145,6 +1147,10 @@ impl<'a> StateWorkingSet<'a> { self.delta.num_files() + self.permanent_state.num_files() } + pub fn num_virtual_paths(&self) -> usize { + self.delta.num_virtual_paths() + self.permanent_state.num_virtual_paths() + } + pub fn num_decls(&self) -> usize { self.delta.num_decls() + self.permanent_state.num_decls() } @@ -1346,33 +1352,24 @@ impl<'a> StateWorkingSet<'a> { self.permanent_state.files().chain(self.delta.files.iter()) } - pub fn get_filename(&self, file_id: usize) -> String { - for file in self.files().enumerate() { - if file.0 == file_id { - return file.1 .0.clone(); + pub fn get_contents_of_file(&self, file_id: usize) -> Option<&[u8]> { + for (id, (contents, _, _)) in self.delta.file_contents.iter().enumerate() { + if self.permanent_state.num_files() + id == file_id { + return Some(contents); } } - "".into() - } - - pub fn get_file_source(&self, file_id: usize) -> String { - for file in self.files().enumerate() { - if file.0 == file_id { - let output = String::from_utf8_lossy( - self.get_span_contents(Span::new(file.1 .1, file.1 .2)), - ) - .to_string(); - - return output; + for (id, (contents, _, _)) in self.permanent_state.file_contents.iter().enumerate() { + if id == file_id { + return Some(contents); } } - "".into() + None } #[must_use] - pub fn add_file(&mut self, filename: String, contents: &[u8]) -> usize { + pub fn add_file(&mut self, filename: String, contents: &[u8]) -> FileId { // First, look for the file to see if we already have it for (idx, (fname, file_start, file_end)) in self.files().enumerate() { if fname == &filename { @@ -1397,6 +1394,13 @@ impl<'a> StateWorkingSet<'a> { self.num_files() - 1 } + #[must_use] + pub fn add_virtual_path(&mut self, name: String, virtual_path: VirtualPath) -> VirtualPathId { + self.delta.virtual_paths.push((name, virtual_path)); + + self.num_virtual_paths() - 1 + } + pub fn get_span_for_file(&self, file_id: usize) -> Span { let result = self .files() @@ -2011,6 +2015,34 @@ impl<'a> StateWorkingSet<'a> { None } + + pub fn find_virtual_path(&self, name: &str) -> Option<&VirtualPath> { + for (virtual_name, virtual_path) in self.delta.virtual_paths.iter().rev() { + if virtual_name == name { + return Some(virtual_path); + } + } + + for (virtual_name, virtual_path) in self.permanent_state.virtual_paths.iter().rev() { + if virtual_name == name { + return Some(virtual_path); + } + } + + None + } + + pub fn get_virtual_path(&self, virtual_path_id: VirtualPathId) -> &(String, VirtualPath) { + let num_permanent_virtual_paths = self.permanent_state.num_virtual_paths(); + if virtual_path_id < num_permanent_virtual_paths { + self.permanent_state.get_virtual_path(virtual_path_id) + } else { + self.delta + .virtual_paths + .get(virtual_path_id - num_permanent_virtual_paths) + .expect("internal error: missing virtual path") + } + } } impl Default for EngineState { diff --git a/crates/nu-protocol/src/id.rs b/crates/nu-protocol/src/id.rs index 6afb6386a5..dc9978b844 100644 --- a/crates/nu-protocol/src/id.rs +++ b/crates/nu-protocol/src/id.rs @@ -3,3 +3,5 @@ pub type DeclId = usize; pub type BlockId = usize; pub type ModuleId = usize; pub type OverlayId = usize; +pub type FileId = usize; +pub type VirtualPathId = usize; diff --git a/crates/nu-std/Cargo.toml b/crates/nu-std/Cargo.toml index 696790ec6f..0f68176ffb 100644 --- a/crates/nu-std/Cargo.toml +++ b/crates/nu-std/Cargo.toml @@ -11,3 +11,4 @@ version = "0.80.1" miette = { version = "5.6.0", features = ["fancy-no-backtrace"] } nu-parser = { version = "0.80.1", path = "../nu-parser" } nu-protocol = { version = "0.80.1", path = "../nu-protocol" } +nu-engine = { version = "0.80.1", path = "../nu-engine" } diff --git a/crates/nu-std/src/lib.rs b/crates/nu-std/src/lib.rs index 1aae505562..0e3f0b7362 100644 --- a/crates/nu-std/src/lib.rs +++ b/crates/nu-std/src/lib.rs @@ -1,126 +1,136 @@ -use nu_parser::{parse, parse_module_block}; -use nu_protocol::report_error; -use nu_protocol::{engine::StateWorkingSet, Module, ShellError, Span}; +use std::path::PathBuf; -fn add_file( - working_set: &mut StateWorkingSet, - name: &String, - content: &[u8], -) -> (Module, Vec) { - let file_id = working_set.add_file(name.clone(), content); - let new_span = working_set.get_span_for_file(file_id); +use nu_engine::{env::current_dir, eval_block}; +use nu_parser::parse; +use nu_protocol::engine::{Stack, StateWorkingSet, VirtualPath}; +use nu_protocol::{report_error, PipelineData}; - let (_, module, comments) = parse_module_block(working_set, new_span, name.as_bytes()); - - if let Some(err) = working_set.parse_errors.first() { - report_error(working_set, err); - } - - parse(working_set, Some(name), content, true); - - if let Some(err) = working_set.parse_errors.first() { - report_error(working_set, err); - } - - (module, comments) -} - -fn load_prelude(working_set: &mut StateWorkingSet, prelude: Vec<(&str, &str)>, module: &Module) { - let mut decls = Vec::new(); - let mut errs = Vec::new(); - for (name, search_name) in prelude { - if let Some(id) = module.decls.get(&search_name.as_bytes().to_vec()) { - let decl = (name.as_bytes().to_vec(), id.to_owned()); - decls.push(decl); - } else { - errs.push(ShellError::GenericError( - format!("could not load `{}` from `std`.", search_name), - String::new(), - None, - None, - Vec::new(), - )); - } - } - - if !errs.is_empty() { - report_error( - working_set, - &ShellError::GenericError( - "Unable to load the prelude of the standard library.".into(), - String::new(), - None, - Some("this is a bug: please file an issue in the [issue tracker](https://github.com/nushell/nushell/issues/new/choose)".to_string()), - errs, - ), - ); - } - - working_set.use_decls(decls); -} +// Virtual std directory unlikely to appear in user's file system +const NU_STDLIB_VIRTUAL_DIR: &str = "NU_STDLIB_VIRTUAL_DIR"; pub fn load_standard_library( engine_state: &mut nu_protocol::engine::EngineState, ) -> Result<(), miette::ErrReport> { - let delta = { - let name = "std".to_string(); - let content = include_str!("../lib/mod.nu"); - - // these modules are loaded in the order they appear in this list - #[rustfmt::skip] - let submodules = vec![ - // helper modules that could be used in other parts of the library - ("log", include_str!("../lib/log.nu")), - - // the rest of the library - ("dirs", include_str!("../lib/dirs.nu")), - ("iter", include_str!("../lib/iter.nu")), - ("help", include_str!("../lib/help.nu")), - ("testing", include_str!("../lib/testing.nu")), - ("xml", include_str!("../lib/xml.nu")), - ("dt", include_str!("../lib/dt.nu")), - ]; - - // Define commands to be preloaded into the default (top level, unprefixed) namespace. - // User can invoke these without having to `use std` beforehand. - // Entries are: (name to add to default namespace, path under std to find implementation) - // - // Conventionally, for a command implemented as `std foo`, the name added - // is either `std foo` or bare `foo`, not some arbitrary rename. - - #[rustfmt::skip] - let prelude = vec![ - ("std help", "help"), - ("std help commands", "help commands"), - ("std help aliases", "help aliases"), - ("std help modules", "help modules"), - ("std help externs", "help externs"), - ("std help operators", "help operators"), - - ("enter", "dirs enter"), - ("shells", "dirs shells"), - ("g", "dirs g"), - ("n", "dirs n"), - ("p", "dirs p"), - ("dexit", "dirs dexit"), + let (block, delta) = { + let mut std_files = vec![ + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("mod.nu"), + include_str!("../std/mod.nu"), + ), + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("dirs.nu"), + include_str!("../std/dirs.nu"), + ), + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("dt.nu"), + include_str!("../std/dt.nu"), + ), + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("help.nu"), + include_str!("../std/help.nu"), + ), + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("iter.nu"), + include_str!("../std/iter.nu"), + ), + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("log.nu"), + include_str!("../std/log.nu"), + ), + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("testing.nu"), + include_str!("../std/testing.nu"), + ), + ( + PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .join("xml.nu"), + include_str!("../std/xml.nu"), + ), ]; let mut working_set = StateWorkingSet::new(engine_state); + let mut std_virt_paths = vec![]; - for (name, content) in submodules { - let (module, comments) = - add_file(&mut working_set, &name.to_string(), content.as_bytes()); - working_set.add_module(name, module, comments); + for (name, content) in std_files.drain(..) { + let file_id = + working_set.add_file(name.to_string_lossy().to_string(), content.as_bytes()); + let virtual_file_id = working_set.add_virtual_path( + name.to_string_lossy().to_string(), + VirtualPath::File(file_id), + ); + std_virt_paths.push(virtual_file_id); } - let (module, comments) = add_file(&mut working_set, &name, content.as_bytes()); - load_prelude(&mut working_set, prelude, &module); - working_set.add_module(&name, module, comments); + // Using full virtual path to avoid potential conflicts with user having 'std' directory + // in their working directory. + let std_dir = PathBuf::from(NU_STDLIB_VIRTUAL_DIR) + .join("std") + .to_string_lossy() + .to_string(); + let source = format!( + r#" +# Define the `std` module +module {std_dir} - working_set.render() +# Prelude +use std dirs [ enter, shells, g, n, p, dexit ] +"# + ); + + let _ = working_set.add_virtual_path(std_dir, VirtualPath::Dir(std_virt_paths)); + + // Change the currently parsed directory + let prev_currently_parsed_cwd = working_set.currently_parsed_cwd.clone(); + working_set.currently_parsed_cwd = Some(PathBuf::from(NU_STDLIB_VIRTUAL_DIR)); + + let block = parse( + &mut working_set, + Some("loading stdlib"), + source.as_bytes(), + false, + ); + + if let Some(err) = working_set.parse_errors.first() { + report_error(&working_set, err); + } + + // Restore the currently parsed directory back + working_set.currently_parsed_cwd = prev_currently_parsed_cwd; + + (block, working_set.render()) }; engine_state.merge_delta(delta)?; + // We need to evaluate the module in order to run the `export-env` blocks. + let mut stack = Stack::new(); + let pipeline_data = PipelineData::Empty; + eval_block( + engine_state, + &mut stack, + &block, + pipeline_data, + false, + false, + )?; + + let cwd = current_dir(engine_state, &stack)?; + engine_state.merge_env(&mut stack, cwd)?; + Ok(()) } diff --git a/crates/nu-std/lib/dirs.nu b/crates/nu-std/std/dirs.nu similarity index 100% rename from crates/nu-std/lib/dirs.nu rename to crates/nu-std/std/dirs.nu diff --git a/crates/nu-std/lib/dt.nu b/crates/nu-std/std/dt.nu similarity index 100% rename from crates/nu-std/lib/dt.nu rename to crates/nu-std/std/dt.nu diff --git a/crates/nu-std/lib/help.nu b/crates/nu-std/std/help.nu similarity index 100% rename from crates/nu-std/lib/help.nu rename to crates/nu-std/std/help.nu diff --git a/crates/nu-std/lib/iter.nu b/crates/nu-std/std/iter.nu similarity index 100% rename from crates/nu-std/lib/iter.nu rename to crates/nu-std/std/iter.nu diff --git a/crates/nu-std/lib/log.nu b/crates/nu-std/std/log.nu similarity index 100% rename from crates/nu-std/lib/log.nu rename to crates/nu-std/std/log.nu diff --git a/crates/nu-std/lib/mod.nu b/crates/nu-std/std/mod.nu similarity index 98% rename from crates/nu-std/lib/mod.nu rename to crates/nu-std/std/mod.nu index 8b5c940de8..5d6c7fb147 100644 --- a/crates/nu-std/lib/mod.nu +++ b/crates/nu-std/std/mod.nu @@ -1,15 +1,12 @@ # std.nu, `used` to load all standard library components -export use dirs export-env { - use dirs * + use dirs.nu [] } -export use help -export use iter -export use log -export use testing * -export use xml -use dt [datetime-diff, pretty-print-duration] + +export use testing.nu * + +use dt.nu [datetime-diff, pretty-print-duration] # Add the given paths to the PATH. # @@ -46,7 +43,7 @@ export def-env "path add" [ | if $append { append $paths } else { prepend $paths } ) - + if $ret { $env | get $path_name } diff --git a/crates/nu-std/lib/testing.nu b/crates/nu-std/std/testing.nu similarity index 98% rename from crates/nu-std/lib/testing.nu rename to crates/nu-std/std/testing.nu index b2dc5f2fa1..764d3c0c3b 100644 --- a/crates/nu-std/lib/testing.nu +++ b/crates/nu-std/std/testing.nu @@ -5,7 +5,7 @@ # Assert commands and test runner. # ################################################################################## -use log +export use log.nu # Universal assert command # @@ -17,7 +17,7 @@ use log # >_ assert (3 == 3) # >_ assert (42 == 3) # Error: -# × Assertion failed: +# × Assertion failed: # ╭─[myscript.nu:11:1] # 11 │ assert (3 == 3) # 12 │ assert (42 == 3) @@ -38,7 +38,7 @@ use log # } # ``` export def assert [ - condition: bool, # Condition, which should be true + condition: bool, # Condition, which should be true message?: string, # Optional error message --error-label: record # Label for `error make` if you want to create a custom assert ] { @@ -64,7 +64,7 @@ export def assert [ # >_ assert (42 == 3) # >_ assert (3 == 3) # Error: -# × Assertion failed: +# × Assertion failed: # ╭─[myscript.nu:11:1] # 11 │ assert (42 == 3) # 12 │ assert (3 == 3) @@ -73,7 +73,7 @@ export def assert [ # 13 │ # ╰──── # -# +# # The --error-label flag can be used if you want to create a custom assert command: # ``` # def "assert not even" [number: int] { @@ -86,7 +86,7 @@ export def assert [ # ``` # export def "assert not" [ - condition: bool, # Condition, which should be false + condition: bool, # Condition, which should be false message?: string, # Optional error message --error-label: record # Label for `error make` if you want to create a custom assert ] { @@ -106,7 +106,7 @@ export def "assert not" [ # Assert that executing the code generates an error # # For more documentation see the assert command -# +# # # Examples # # > assert error {|| missing_command} # passes @@ -138,7 +138,7 @@ export def "assert skip" [] { # For more documentation see the assert command # # # Examples -# +# # > assert equal 1 1 # passes # > assert equal (0.1 + 0.2) 0.3 # > assert equal 1 2 # fails diff --git a/crates/nu-std/lib/xml.nu b/crates/nu-std/std/xml.nu similarity index 100% rename from crates/nu-std/lib/xml.nu rename to crates/nu-std/std/xml.nu diff --git a/crates/nu-std/tests/test_dirs.nu b/crates/nu-std/tests/test_dirs.nu index 612982f0ff..fbb5dcff8e 100644 --- a/crates/nu-std/tests/test_dirs.nu +++ b/crates/nu-std/tests/test_dirs.nu @@ -3,8 +3,7 @@ use std "assert length" use std "assert equal" use std "assert not equal" use std "assert error" -use std "log info" -use std "log debug" +use std log # A couple of nuances to understand when testing module that exports environment: # Each 'use' for that module in the test script will execute the export def-env block. @@ -41,20 +40,15 @@ def cur_ring_check [expect_dir:string, expect_position: int scenario:string] { export def test_dirs_command [] { # careful with order of these statements! # must capture value of $in before executing `use`s - let $c = $in + let $c = $in # must set PWD *before* doing `use` that will run the export def-env block in dirs module. cd $c.base_path # must execute these uses for the UOT commands *after* the test and *not* just put them at top of test module. # the export def-env gets messed up - use std "dirs next" - use std "dirs prev" - use std "dirs add" - use std "dirs drop" - use std "dirs show" - use std "dirs goto" - + use std dirs + assert equal [$c.base_path] $env.DIRS_LIST "list is just pwd after initialization" dirs next @@ -85,13 +79,12 @@ export def test_dirs_command [] { export def test_dirs_next [] { # must capture value of $in before executing `use`s - let $c = $in + let $c = $in # must set PWD *before* doing `use` that will run the export def-env block in dirs module. cd $c.base_path assert equal $env.PWD $c.base_path "test setup" - use std "dirs next" - use std "dirs add" + use std dirs cur_dir_check $c.base_path "use module test setup" dirs add $c.path_a $c.path_b @@ -107,15 +100,11 @@ export def test_dirs_next [] { export def test_dirs_cd [] { # must capture value of $in before executing `use`s - let $c = $in + let $c = $in # must set PWD *before* doing `use` that will run the export def-env block in dirs module. cd $c.base_path - use std # necessary to define $env.config?? - - use std "dirs next" - use std "dirs add" - use std "dirs drop" + use std dirs cur_dir_check $c.base_path "use module test setup" diff --git a/src/main.rs b/src/main.rs index 0e7f592ba3..88878fc776 100644 --- a/src/main.rs +++ b/src/main.rs @@ -152,6 +152,18 @@ fn main() -> Result<()> { engine_state.add_env_var("NU_LIB_DIRS".into(), Value::List { vals, span }); } + start_time = std::time::Instant::now(); + // First, set up env vars as strings only + gather_parent_env_vars(&mut engine_state, &init_cwd); + perf( + "gather env vars", + start_time, + file!(), + line!(), + column!(), + use_color, + ); + if parsed_nu_cli_args.no_std_lib.is_none() { load_standard_library(&mut engine_state)?; } @@ -243,18 +255,6 @@ fn main() -> Result<()> { use_color, ); - start_time = std::time::Instant::now(); - // First, set up env vars as strings only - gather_parent_env_vars(&mut engine_state, &init_cwd); - perf( - "gather env vars", - start_time, - file!(), - line!(), - column!(), - use_color, - ); - if let Some(commands) = parsed_nu_cli_args.commands.clone() { run_commands( &mut engine_state, diff --git a/src/tests/test_stdlib.rs b/src/tests/test_stdlib.rs index 0c08d42395..3a7a4b226b 100644 --- a/src/tests/test_stdlib.rs +++ b/src/tests/test_stdlib.rs @@ -2,15 +2,12 @@ use crate::tests::{fail_test, run_test_std, TestResult}; #[test] fn library_loaded() -> TestResult { - run_test_std( - "help std | lines | first 1 | to text", - "std.nu, `used` to load all standard library components", - ) + run_test_std("$nu.scope.modules | where name == 'std' | length", "1") } #[test] fn prelude_loaded() -> TestResult { - run_test_std("std help commands | where name == open | length", "1") + run_test_std("shells | length", "1") } #[test]