Move from source to source-env (#6277)

* start working on source-env

* WIP

* Get most tests working, still one to go

* Fix file-relative paths; Report parser error

* Fix merge conflicts; Restore source as deprecated

* Tests: Use source-env; Remove redundant tests

* Fmt

* Respect hidden env vars

* Fix file-relative eval for source-env

* Add file-relative eval to "overlay use"

* Use FILE_PWD only in source-env and "overlay use"

* Ignore new tests for now

This will be another issue

* Throw an error if setting FILE_PWD manually

* Fix source-related test failures

* Fix nu-check to respect FILE_PWD

* Fix corrupted spans in source-env shell errors

* Fix up some references to old source

* Remove deprecation message

* Re-introduce deleted tests

Co-authored-by: kubouch <kubouch@gmail.com>

crates/nu-cli/src/completions/completer.rs

@@ -235,7 +235,7 @@ impl NuCompleter {
                             );
                         }
 
-                        // Completions that depends on the previous expression (e.g: use, source)
+                        // Completions that depends on the previous expression (e.g: use, source-env)
                         if flat_idx > 0 {
                             if let Some(previous_expr) = flattened.get(flat_idx - 1) {
                                 // Read the content for the previous expression
@@ -243,7 +243,7 @@ impl NuCompleter {
                                     working_set.get_span_contents(previous_expr.0).to_vec();
 
                                 // Completion for .nu files
-                                if prev_expr_str == b"use" || prev_expr_str == b"source" {
+                                if prev_expr_str == b"use" || prev_expr_str == b"source-env" {
                                     let mut completer =
                                         DotNuCompletion::new(self.engine_state.clone());
 

crates/nu-cli/tests/completions.rs

@@ -79,7 +79,7 @@ fn dotnu_completions() {
     let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
 
     // Test source completion
-    let completion_str = "source ".to_string();
+    let completion_str = "source-env ".to_string();
     let suggestions = completer.complete(&completion_str, completion_str.len());
 
     assert_eq!(1, suggestions.len());

crates/nu-command/src/core_commands/mod.rs

@@ -25,7 +25,6 @@ mod let_;
 mod metadata;
 mod module;
 pub(crate) mod overlay;
-mod source;
 mod use_;
 mod version;
 
@@ -56,7 +55,6 @@ pub use let_::Let;
 pub use metadata::Metadata;
 pub use module::Module;
 pub use overlay::*;
-pub use source::Source;
 pub use use_::Use;
 pub use version::Version;
 #[cfg(feature = "plugin")]

crates/nu-command/src/core_commands/overlay/use_.rs

@@ -1,7 +1,9 @@
-use nu_engine::{eval_block, redirect_env, CallExt};
+use nu_engine::{eval_block, find_in_dirs_env, redirect_env, CallExt};
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
 
 use std::path::Path;
 
@@ -79,7 +81,7 @@ impl Command for OverlayUse {
             .find_overlay(name_arg.item.as_bytes())
             .is_some()
         {
-            (name_arg.item, name_arg.span)
+            (name_arg.item.clone(), name_arg.span)
         } else if let Some(os_str) = Path::new(&name_arg.item).file_stem() {
             if let Some(name) = os_str.to_str() {
                 (name.to_string(), name_arg.span)
@@ -131,6 +133,22 @@ impl Command for OverlayUse {
 
                     // Evaluate the export-env block (if any) and keep its environment
                     if let Some(block_id) = module.env_block {
+                        let maybe_path =
+                            find_in_dirs_env(&name_arg.item, engine_state, caller_stack)?;
+
+                        if let Some(path) = &maybe_path {
+                            // Set the currently evaluated directory, if the argument is a valid path
+                            let mut parent = path.clone();
+                            parent.pop();
+
+                            let file_pwd = Value::String {
+                                val: parent.to_string_lossy().to_string(),
+                                span: call.head,
+                            };
+
+                            caller_stack.add_env_var("FILE_PWD".to_string(), file_pwd);
+                        }
+
                         let block = engine_state.get_block(block_id);
                         let mut callee_stack = caller_stack.gather_captures(&block.captures);
 
@@ -143,7 +161,13 @@ impl Command for OverlayUse {
                             call.redirect_stderr,
                         );
 
+                        // Merge the block's environment to the current stack
                         redirect_env(engine_state, caller_stack, &callee_stack);
+
+                        if maybe_path.is_some() {
+                            // Remove the file-relative PWD, if the argument is a valid path
+                            caller_stack.remove_env_var(engine_state, "FILE_PWD");
+                        }
                     }
                 }
             }

crates/nu-command/src/default_context.rs

@@ -59,7 +59,6 @@ pub fn create_default_context() -> EngineState {
             Let,
             Metadata,
             Module,
-            Source,
             Use,
             Version,
         };
@@ -358,6 +357,7 @@ pub fn create_default_context() -> EngineState {
             ExportEnv,
             LetEnv,
             LoadEnv,
+            SourceEnv,
             WithEnv,
             ConfigNu,
             ConfigEnv,
@@ -432,6 +432,7 @@ pub fn create_default_context() -> EngineState {
         // Deprecated
         bind_command! {
             HashBase64,
+            Source,
             StrDatetimeDeprecated,
             StrDecimalDeprecated,
             StrIntDeprecated,

crates/nu-command/src/deprecated/mod.rs

@@ -1,5 +1,6 @@
 mod deprecated_commands;
 mod hash_base64;
+mod source;
 mod str_datetime;
 mod str_decimal;
 mod str_find_replace;
@@ -7,6 +8,7 @@ mod str_int;
 
 pub use deprecated_commands::*;
 pub use hash_base64::HashBase64;
+pub use source::Source;
 pub use str_datetime::StrDatetimeDeprecated;
 pub use str_decimal::StrDecimalDeprecated;
 pub use str_find_replace::StrFindReplaceDeprecated;

crates/nu-command/src/env/let_env.rs

@@ -1,7 +1,9 @@
 use nu_engine::{current_dir, eval_expression_with_input, CallExt};
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, Example, PipelineData, Signature, SyntaxShape, Value};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
 
 #[derive(Clone)]
 pub struct LetEnv;
@@ -34,7 +36,7 @@ impl Command for LetEnv {
         input: PipelineData,
     ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
         // TODO: find and require the crossplatform restrictions on environment names
-        let env_var = call.req(engine_state, stack, 0)?;
+        let env_var: Spanned<String> = call.req(engine_state, stack, 0)?;
 
         let keyword_expr = call
             .positional_nth(1)
@@ -47,19 +49,26 @@ impl Command for LetEnv {
                 .0
                 .into_value(call.head);
 
-        if env_var == "PWD" {
+        if env_var.item == "FILE_PWD" {
+            return Err(ShellError::AutomaticEnvVarSetManually(
+                env_var.item,
+                env_var.span,
+            ));
+        }
+
+        if env_var.item == "PWD" {
             let cwd = current_dir(engine_state, stack)?;
             let rhs = rhs.as_string()?;
             let rhs = nu_path::expand_path_with(rhs, cwd);
             stack.add_env_var(
-                env_var,
+                env_var.item,
                 Value::String {
                     val: rhs.to_string_lossy().to_string(),
                     span: call.head,
                 },
             );
         } else {
-            stack.add_env_var(env_var, rhs);
+            stack.add_env_var(env_var.item, rhs);
         }
         Ok(PipelineData::new(call.head))
     }

crates/nu-command/src/env/load_env.rs

@@ -38,6 +38,10 @@ impl Command for LoadEnv {
         match arg {
             Some((cols, vals)) => {
                 for (env_var, rhs) in cols.into_iter().zip(vals) {
+                    if env_var == "FILE_PWD" {
+                        return Err(ShellError::AutomaticEnvVarSetManually(env_var, call.head));
+                    }
+
                     if env_var == "PWD" {
                         let cwd = current_dir(engine_state, stack)?;
                         let rhs = rhs.as_string()?;
@@ -58,6 +62,10 @@ impl Command for LoadEnv {
             None => match input {
                 PipelineData::Value(Value::Record { cols, vals, .. }, ..) => {
                     for (env_var, rhs) in cols.into_iter().zip(vals) {
+                        if env_var == "FILE_PWD" {
+                            return Err(ShellError::AutomaticEnvVarSetManually(env_var, call.head));
+                        }
+
                         if env_var == "PWD" {
                             let cwd = current_dir(engine_state, stack)?;
                             let rhs = rhs.as_string()?;

crates/nu-command/src/env/mod.rs

@@ -3,6 +3,7 @@ mod env_command;
 mod export_env;
 mod let_env;
 mod load_env;
+mod source_env;
 mod with_env;
 
 pub use config::ConfigEnv;
@@ -13,4 +14,5 @@ pub use env_command::Env;
 pub use export_env::ExportEnv;
 pub use let_env::LetEnv;
 pub use load_env::LoadEnv;
+pub use source_env::SourceEnv;
 pub use with_env::WithEnv;

crates/nu-command/src/env/source_env.rs

@@ -0,0 +1,142 @@
+use std::path::PathBuf;
+
+use nu_engine::{eval_block, find_in_dirs_env, redirect_env, CallExt};
+use nu_parser::parse;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack, StateWorkingSet};
+use nu_protocol::{
+    Category, CliError, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
+
+/// Source a file for environment variables.
+#[derive(Clone)]
+pub struct SourceEnv;
+
+impl Command for SourceEnv {
+    fn name(&self) -> &str {
+        "source-env"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("source-env")
+            .required(
+                "filename",
+                SyntaxShape::String, // type is string to avoid automatically canonicalizing the path
+                "the filepath to the script file to source the environment from",
+            )
+            .category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "Source the environment from a source file into the current environment."
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        caller_stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let source_filename: Spanned<String> = call.req(engine_state, caller_stack, 0)?;
+
+        if let Some(path) = find_in_dirs_env(&source_filename.item, engine_state, caller_stack)? {
+            if let Ok(content) = std::fs::read_to_string(&path) {
+                let mut parent = PathBuf::from(&path);
+                parent.pop();
+
+                let mut new_engine_state = engine_state.clone();
+
+                let (block, delta) = {
+                    let mut working_set = StateWorkingSet::new(&new_engine_state);
+
+                    // Set the currently parsed directory
+                    working_set.currently_parsed_cwd = Some(parent.clone());
+
+                    let (block, err) = parse(&mut working_set, None, content.as_bytes(), true, &[]);
+
+                    if let Some(err) = err {
+                        // Because the error span points at new_engine_state, we must create the error message now
+                        let msg = format!(
+                            r#"Found this parser error: {:?}"#,
+                            CliError(&err, &working_set)
+                        );
+
+                        return Err(ShellError::GenericError(
+                            "Failed to parse content".to_string(),
+                            "cannot parse this file".to_string(),
+                            Some(source_filename.span),
+                            Some(msg),
+                            vec![],
+                        ));
+                    } else {
+                        (block, working_set.render())
+                    }
+                };
+
+                // Merge parser changes to a temporary engine state
+                new_engine_state.merge_delta(delta)?;
+
+                // Set the currently evaluated directory
+                let file_pwd = Value::String {
+                    val: parent.to_string_lossy().to_string(),
+                    span: call.head,
+                };
+
+                caller_stack.add_env_var("FILE_PWD".to_string(), file_pwd);
+
+                // Evaluate the parsed file's block
+                let mut callee_stack = caller_stack.gather_captures(&block.captures);
+
+                let result = eval_block(
+                    &new_engine_state,
+                    &mut callee_stack,
+                    &block,
+                    input,
+                    true,
+                    true,
+                );
+
+                let result = if let Err(err) = result {
+                    // Because the error span points at new_engine_state, we must create the error message now
+                    let working_set = StateWorkingSet::new(&new_engine_state);
+
+                    let msg = format!(
+                        r#"Found this shell error: {:?}"#,
+                        CliError(&err, &working_set)
+                    );
+
+                    Err(ShellError::GenericError(
+                        "Failed to evaluate content".to_string(),
+                        "cannot evaluate this file".to_string(),
+                        Some(source_filename.span),
+                        Some(msg),
+                        vec![],
+                    ))
+                } else {
+                    result
+                };
+
+                // Merge the block's environment to the current stack
+                redirect_env(engine_state, caller_stack, &callee_stack);
+
+                // Remove the file-relative PWD
+                caller_stack.remove_env_var(engine_state, "FILE_PWD");
+
+                result
+            } else {
+                Err(ShellError::FileNotFound(source_filename.span))
+            }
+        } else {
+            Err(ShellError::FileNotFound(source_filename.span))
+        }
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Sources the environment from foo.nu in the current context",
+            example: r#"source-env foo.nu"#,
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/nu_check.rs

@@ -1,4 +1,4 @@
-use nu_engine::{current_dir, CallExt};
+use nu_engine::{find_in_dirs_env, CallExt};
 use nu_parser::{parse, parse_module_block, unescape_unquote_string};
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack, StateWorkingSet};
@@ -6,8 +6,6 @@ use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value,
 };
 
-use std::path::Path;
-
 #[derive(Clone)]
 pub struct NuCheck;
 
@@ -18,7 +16,8 @@ impl Command for NuCheck {
 
     fn signature(&self) -> Signature {
         Signature::build("nu-check")
-            .optional("path", SyntaxShape::Filepath, "File path to parse")
+            // type is string to avoid automatically canonicalizing the path
+            .optional("path", SyntaxShape::String, "File path to parse")
             .switch("as-module", "Parse content as module", Some('m'))
             .switch("debug", "Show error messages", Some('d'))
             .switch("all", "Parse content as script first, returns result if success, otherwise, try with module", Some('a'))
@@ -102,13 +101,23 @@ impl Command for NuCheck {
                 }
             }
             _ => {
-                if path.is_some() {
+                if let Some(path_str) = path {
-                    let path = match find_path(path, engine_state, stack, call.head) {
+                    // look up the path as relative to FILE_PWD or inside NU_LIB_DIRS (same process as source-env)
-                        Ok(path) => path,
+                    let path = match find_in_dirs_env(&path_str.item, engine_state, stack) {
+                        Ok(path) => {
+                            if let Some(path) = path {
+                                path
+                            } else {
+                                return Err(ShellError::FileNotFound(path_str.span));
+                            }
+                        }
                         Err(error) => return Err(error),
                     };
 
-                    let ext: Vec<_> = path.rsplitn(2, '.').collect();
+                    // get the expanded path as a string
+                    let path_str = path.to_string_lossy().to_string();
+
+                    let ext: Vec<_> = path_str.rsplitn(2, '.').collect();
                     if ext[0] != "nu" {
                         return Err(ShellError::GenericError(
                             "Cannot parse input".to_string(),
@@ -120,8 +129,7 @@ impl Command for NuCheck {
                     }
 
                     // Change currently parsed directory
-                    let prev_currently_parsed_cwd = if let Some(parent) = Path::new(&path).parent()
+                    let prev_currently_parsed_cwd = if let Some(parent) = path.parent() {
-                    {
                         let prev = working_set.currently_parsed_cwd.clone();
 
                         working_set.currently_parsed_cwd = Some(parent.into());
@@ -132,11 +140,11 @@ impl Command for NuCheck {
                     };
 
                     let result = if is_all {
-                        heuristic_parse_file(path, &mut working_set, call, is_debug)
+                        heuristic_parse_file(path_str, &mut working_set, call, is_debug)
                     } else if is_module {
-                        parse_file_module(path, &mut working_set, call, is_debug)
+                        parse_file_module(path_str, &mut working_set, call, is_debug)
                     } else {
-                        parse_file_script(path, &mut working_set, call, is_debug)
+                        parse_file_script(path_str, &mut working_set, call, is_debug)
                     };
 
                     // Restore the currently parsed directory back
@@ -202,46 +210,6 @@ impl Command for NuCheck {
     }
 }
 
-fn find_path(
-    path: Option<Spanned<String>>,
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    span: Span,
-) -> Result<String, ShellError> {
-    let cwd = current_dir(engine_state, stack)?;
-
-    let path = match path {
-        Some(s) => {
-            let path_no_whitespace = &s.item.trim_end_matches(|x| matches!(x, '\x09'..='\x0d'));
-
-            let path = match nu_path::canonicalize_with(path_no_whitespace, &cwd) {
-                Ok(p) => {
-                    if !p.is_file() {
-                        return Err(ShellError::GenericError(
-                            "Cannot parse input".to_string(),
-                            "Path is not a file".to_string(),
-                            Some(s.span),
-                            None,
-                            Vec::new(),
-                        ));
-                    } else {
-                        p
-                    }
-                }
-
-                Err(_) => {
-                    return Err(ShellError::FileNotFound(s.span));
-                }
-            };
-            path.to_string_lossy().to_string()
-        }
-        None => {
-            return Err(ShellError::NotFound(span));
-        }
-    };
-    Ok(path)
-}
-
 fn heuristic_parse(
     working_set: &mut StateWorkingSet,
     filename: Option<&str>,

crates/nu-command/tests/commands/alias.rs

@@ -5,7 +5,9 @@ fn alias_simple() {
     let actual = nu!(
         cwd: "tests/fixtures/formats", pipeline(
         r#"
-        alias bar = source sample_def.nu; bar; greet
+            alias bar = use sample_def.nu greet;
+            bar;
+            greet
         "#
     ));
 
@@ -13,11 +15,11 @@ fn alias_simple() {
 }
 
 #[test]
-fn alias_hiding1() {
+fn alias_hiding_1() {
     let actual = nu!(
         cwd: "tests/fixtures/formats", pipeline(
         r#"
-        source ./activate-foo.nu;
+            overlay use ./activate-foo.nu;
             $nu.scope.aliases | find deactivate-foo | length
         "#
     ));
@@ -26,11 +28,11 @@ fn alias_hiding1() {
 }
 
 #[test]
-fn alias_hiding2() {
+fn alias_hiding_2() {
     let actual = nu!(
         cwd: "tests/fixtures/formats", pipeline(
         r#"
-        source ./activate-foo.nu;
+            overlay use ./activate-foo.nu;
             deactivate-foo;
             $nu.scope.aliases | find deactivate-foo | length
         "#

crates/nu-command/tests/commands/def.rs

@@ -7,12 +7,12 @@ fn def_with_comment() {
     Playground::setup("def_with_comment", |dirs, _| {
         let data = r#"
 #My echo
-def e [arg] {echo $arg}
+export def e [arg] {echo $arg}
             "#;
         fs::write(dirs.root().join("def_test"), data).expect("Unable to write file");
         let actual = nu!(
             cwd: dirs.root(),
-            "source def_test; help e | to json -r"
+            "use def_test e; help e | to json -r"
         );
 
         assert!(actual.out.contains("My echo\\n\\n"));

crates/nu-command/tests/commands/mod.rs

@@ -69,7 +69,7 @@ mod semicolon;
 mod shells;
 mod skip;
 mod sort_by;
-mod source;
+mod source_env;
 mod split_by;
 mod split_column;
 mod split_row;

crates/nu-command/tests/commands/nu_check.rs

@@ -216,7 +216,9 @@ fn parse_dir_failure() {
             "#
         ));
 
-        assert!(actual.err.contains("Path is not a file"));
+        assert!(actual
+            .err
+            .contains("File extension must be the type of .nu"));
     })
 }
 
@@ -733,7 +735,7 @@ fn parse_script_with_nested_scripts_success() {
             .with_files(vec![FileWithContentToBeTrimmed(
                 "lol/lol.nu",
                 r#"
-                    source ../foo.nu
+                    source-env ../foo.nu
                     use lol_shell.nu
                     overlay use ../lol/lol_shell.nu
                 "#,
@@ -761,3 +763,33 @@ fn parse_script_with_nested_scripts_success() {
         assert_eq!(actual.out, "true");
     })
 }
+
+#[test]
+fn nu_check_respects_file_pwd() {
+    Playground::setup("nu_check_test_25", |dirs, sandbox| {
+        sandbox
+            .mkdir("lol")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "lol/lol.nu",
+                r#"
+                    let-env RETURN = (nu-check ../foo.nu)
+                "#,
+            )])
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "foo.nu",
+                r#"
+                    echo 'foo'
+                "#,
+            )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                source-env lol/lol.nu;
+                $env.RETURN
+            "#
+        ));
+
+        assert_eq!(actual.out, "true");
+    })
+}

crates/nu-command/tests/commands/source_env.rs

@@ -25,22 +25,20 @@ fn sources_also_files_under_custom_lib_dirs_path() {
         nu.within("lib").with_files(vec![FileWithContent(
             "my_library.nu",
             r#"
-                source my_library/main.nu
+                source-env my_library/main.nu
             "#,
         )]);
         nu.within("lib/my_library").with_files(vec![FileWithContent(
             "main.nu",
             r#"
-            def hello [] {
+                let-env hello = "hello nu"
-                echo "hello nu"
-            }
             "#,
         )]);
 
         let actual = nu!(
             cwd: ".", pipeline(
             r#"
-                source my_library.nu ;
+                source-env my_library.nu ;
 
                 hello
         "#
@@ -59,7 +57,7 @@ fn try_source_foo_with_double_quotes_in(testdir: &str, playdir: &str) {
         sandbox.mkdir(&testdir);
         sandbox.with_files(vec![FileWithContent(&foo_file, "echo foo")]);
 
-        let cmd = String::from("source ") + r#"""# + foo_file.as_str() + r#"""#;
+        let cmd = String::from("source-env ") + r#"""# + foo_file.as_str() + r#"""#;
 
         let actual = nu!(cwd: dirs.test(), &cmd);
 
@@ -76,7 +74,7 @@ fn try_source_foo_with_single_quotes_in(testdir: &str, playdir: &str) {
         sandbox.mkdir(&testdir);
         sandbox.with_files(vec![FileWithContent(&foo_file, "echo foo")]);
 
-        let cmd = String::from("source ") + r#"'"# + foo_file.as_str() + r#"'"#;
+        let cmd = String::from("source-env ") + r#"'"# + foo_file.as_str() + r#"'"#;
 
         let actual = nu!(cwd: dirs.test(), &cmd);
 
@@ -93,7 +91,7 @@ fn try_source_foo_without_quotes_in(testdir: &str, playdir: &str) {
         sandbox.mkdir(&testdir);
         sandbox.with_files(vec![FileWithContent(&foo_file, "echo foo")]);
 
-        let cmd = String::from("source ") + foo_file.as_str();
+        let cmd = String::from("source-env ") + foo_file.as_str();
 
         let actual = nu!(cwd: dirs.test(), &cmd);
 

crates/nu-command/tests/commands/zip.rs

@@ -3,7 +3,7 @@ use nu_test_support::playground::Playground;
 use nu_test_support::{nu, pipeline};
 
 const ZIP_POWERED_TEST_ASSERTION_SCRIPT: &str = r#"
-def expect [
+export def expect [
     left,
     --to-eq,
     right
@@ -26,7 +26,7 @@ fn zips_two_tables() {
             cwd: ".", pipeline(
             &format!(
                 r#"
-                source {} ;
+                use {} expect ;
 
                 let contributors = ([
                   [name, commits];

crates/nu-engine/src/env.rs

@@ -5,6 +5,8 @@ use nu_protocol::ast::PathMember;
 use nu_protocol::engine::{EngineState, Stack};
 use nu_protocol::{PipelineData, ShellError, Span, Value};
 
+use nu_path::canonicalize_with;
+
 use crate::eval_block;
 
 #[cfg(windows)]
@@ -16,6 +18,8 @@ const ENV_PATH_NAME: &str = "PATH";
 
 const ENV_CONVERSIONS: &str = "ENV_CONVERSIONS";
 
+static LIB_DIRS_ENV: &str = "NU_LIB_DIRS";
+
 enum ConversionResult {
     Ok(Value),
     ConversionError(ShellError), // Failure during the conversion itself
@@ -226,6 +230,76 @@ pub fn path_str(
     env_to_string(pathname, &pathval, engine_state, stack)
 }
 
+/// This helper function is used to find files during eval
+///
+/// First, the actual current working directory is selected as
+///   a) the directory of a file currently being parsed
+///   b) current working directory (PWD)
+///
+/// Then, if the file is not found in the actual cwd, NU_LIB_DIRS is checked.
+/// If there is a relative path in NU_LIB_DIRS, it is assumed to be relative to the actual cwd
+/// determined in the first step.
+///
+/// Always returns an absolute path
+pub fn find_in_dirs_env(
+    filename: &str,
+    engine_state: &EngineState,
+    stack: &Stack,
+) -> Result<Option<PathBuf>, ShellError> {
+    // Choose whether to use file-relative or PWD-relative path
+    let cwd = if let Some(pwd) = stack.get_env_var(engine_state, "FILE_PWD") {
+        match env_to_string("FILE_PWD", &pwd, engine_state, stack) {
+            Ok(cwd) => {
+                if Path::new(&cwd).is_absolute() {
+                    cwd
+                } else {
+                    return Err(ShellError::GenericError(
+                            "Invalid current directory".to_string(),
+                            format!("The 'FILE_PWD' environment variable must be set to an absolute path. Found: '{}'", cwd),
+                            Some(pwd.span()?),
+                            None,
+                            Vec::new()
+                    ));
+                }
+            }
+            Err(e) => return Err(e),
+        }
+    } else {
+        current_dir_str(engine_state, stack)?
+    };
+
+    if let Ok(p) = canonicalize_with(filename, &cwd) {
+        Ok(Some(p))
+    } else {
+        let path = Path::new(filename);
+
+        if path.is_relative() {
+            if let Some(lib_dirs) = stack.get_env_var(engine_state, LIB_DIRS_ENV) {
+                if let Ok(dirs) = lib_dirs.as_list() {
+                    for lib_dir in dirs {
+                        if let Ok(dir) = lib_dir.as_path() {
+                            // make sure the dir is absolute path
+                            if let Ok(dir_abs) = canonicalize_with(&dir, &cwd) {
+                                if let Ok(path) = canonicalize_with(filename, dir_abs) {
+                                    return Ok(Some(path));
+                                }
+                            }
+                        }
+                    }
+
+                    Ok(None)
+                } else {
+                    Ok(None)
+                }
+            } else {
+                Ok(None)
+            }
+        } else {
+            Ok(None)
+        }
+    }
+}
+
 fn get_converted_value(
     engine_state: &EngineState,
     stack: &Stack,

crates/nu-engine/src/eval.rs

@@ -173,6 +173,7 @@ pub fn eval_call(
 
 /// Redirect the environment from callee to the caller.
 pub fn redirect_env(engine_state: &EngineState, caller_stack: &mut Stack, callee_stack: &Stack) {
+    // Grab all environment variables from the callee
     let caller_env_vars = caller_stack.get_env_var_names(engine_state);
 
     // remove env vars that are present in the caller but not in the callee

crates/nu-parser/src/parse_keywords.rs

@@ -3190,7 +3190,7 @@ pub fn parse_register(
 /// determined in the first step.
 ///
 /// Always returns an absolute path
-fn find_in_dirs(
+pub fn find_in_dirs(
     filename: &str,
     working_set: &StateWorkingSet,
     cwd: &str,

crates/nu-protocol/src/shell_error.rs

@@ -301,6 +301,21 @@ Either make sure {0} is a string, or add a 'to_string' entry for it in ENV_CONVE
     )]
     EnvVarNotAString(String, #[label("value not representable as a string")] Span),
 
+    /// This environment variable cannot be set manually.
+    ///
+    /// ## Resolution
+    ///
+    /// This environment variable is set automatically by Nushell and cannot not be set manually.
+    #[error("{0} cannot be set manually.")]
+    #[diagnostic(
+        code(nu::shell::automatic_env_var_set_manually),
+        url(docsrs),
+        help(
+            r#"The environment variable '{0}' is set automatically by Nushell and cannot not be set manually."#
+        )
+    )]
+    AutomaticEnvVarSetManually(String, #[label("cannot set '{0}' manually")] Span),
+
     /// Division by zero is not a thing.
     ///
     /// ## Resolution

tests/fixtures/formats/activate-foo.nu

@@ -1 +1 @@
-alias deactivate-foo = source deactivate-foo.nu
+export alias deactivate-foo = overlay hide activate-foo

tests/fixtures/formats/deactivate-foo.nu

@@ -1 +0,0 @@
-hide deactivate-foo

tests/fixtures/formats/sample_def.nu

@@ -1,3 +1,3 @@
-def greet [] {
+export def greet [] {
   "hello"
 }

tests/hooks/mod.rs

@@ -367,7 +367,7 @@ fn env_change_block_condition_pwd() {
         &env_change_hook_code_condition(
             "PWD",
             r#"{|before, after| ($after | path basename) == samples }"#,
-            r#"'source .nu-env'"#,
+            r#"'source-env .nu-env'"#,
         ),
         "cd samples",
         "$env.SPAM",

tests/modules/mod.rs

@@ -83,35 +83,6 @@ fn module_private_import_decl_not_public() {
     })
 }
 
-// TODO -- doesn't work because modules are never evaluated
-#[ignore]
-#[test]
-fn module_private_import_env() {
-    Playground::setup("module_private_import_env", |dirs, sandbox| {
-        sandbox
-            .with_files(vec![FileWithContentToBeTrimmed(
-                "main.nu",
-                r#"
-                    use spam.nu FOO_HELPER
-
-                    export def foo [] { $env.FOO_HELPER }
-                "#,
-            )])
-            .with_files(vec![FileWithContentToBeTrimmed(
-                "spam.nu",
-                r#"
-                    export env FOO_HELPER { "foo" }
-                "#,
-            )]);
-
-        let inp = &[r#"use main.nu foo"#, r#"foo"#];
-
-        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
-
-        assert_eq!(actual.out, "foo");
-    })
-}
-
 #[test]
 fn module_public_import_decl() {
     Playground::setup("module_public_import_decl", |dirs, sandbox| {
@@ -163,33 +134,6 @@ fn module_public_import_alias() {
     })
 }
 
-// TODO -- doesn't work because modules are never evaluated
-#[ignore]
-#[test]
-fn module_public_import_env() {
-    Playground::setup("module_public_import_decl", |dirs, sandbox| {
-        sandbox
-            .with_files(vec![FileWithContentToBeTrimmed(
-                "main.nu",
-                r#"
-                    export use spam.nu FOO
-                "#,
-            )])
-            .with_files(vec![FileWithContentToBeTrimmed(
-                "spam.nu",
-                r#"
-                    export env FOO { "foo" }
-                "#,
-            )]);
-
-        let inp = &[r#"use main.nu FOO"#, r#"$env.FOO"#];
-
-        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
-
-        assert_eq!(actual.out, "foo");
-    })
-}
-
 #[test]
 fn module_nested_imports() {
     Playground::setup("module_nested_imports", |dirs, sandbox| {
@@ -347,16 +291,50 @@ fn module_nested_imports_in_dirs_prefixed() {
 }
 
 #[test]
-fn module_eval_export_env() {
+fn module_import_env_1() {
-    Playground::setup("module_eval_export_env", |dirs, sandbox| {
+    Playground::setup("module_imprt_env_1", |dirs, sandbox| {
-        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+        sandbox
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "main.nu",
+                r#"
+                    export-env { source-env spam.nu }
+
+                    export def foo [] { $env.FOO_HELPER }
+                "#,
+            )])
+            .with_files(vec![FileWithContentToBeTrimmed(
                 "spam.nu",
                 r#"
-                export-env { let-env FOO = 'foo' }
+                    export-env { let-env FOO_HELPER = "foo" }
                 "#,
             )]);
 
-        let inp = &[r#"source spam.nu"#, r#"$env.FOO"#];
+        let inp = &[r#"source-env main.nu"#, r#"use main.nu foo"#, r#"foo"#];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "foo");
+    })
+}
+
+#[test]
+fn module_import_env_2() {
+    Playground::setup("module_import_env_2", |dirs, sandbox| {
+        sandbox
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "main.nu",
+                r#"
+                    export-env { source-env spam.nu }
+                "#,
+            )])
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "spam.nu",
+                r#"
+                    export-env { let-env FOO = "foo" }
+                "#,
+            )]);
+
+        let inp = &[r#"source-env main.nu"#, r#"$env.FOO"#];
 
         let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
 

tests/overlays/mod.rs

@@ -1,3 +1,5 @@
+use nu_test_support::fs::Stub::FileWithContentToBeTrimmed;
+use nu_test_support::playground::Playground;
 use nu_test_support::{nu, nu_repl_code, pipeline};
 
 #[test]
@@ -767,3 +769,97 @@ fn overlay_use_export_env_hide() {
     assert!(actual.err.contains("did you mean"));
     assert!(actual_repl.err.contains("did you mean"));
 }
+
+#[test]
+fn overlay_use_do_cd() {
+    Playground::setup("overlay_use_do_cd", |dirs, sandbox| {
+        sandbox
+            .mkdir("test1/test2")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "test1/test2/spam.nu",
+                r#"
+                    export-env { cd test1/test2 }
+                "#,
+            )]);
+
+        let inp = &[
+            r#"overlay use test1/test2/spam.nu"#,
+            r#"$env.PWD | path basename"#,
+        ];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "test2");
+    })
+}
+
+#[test]
+fn overlay_use_do_cd_file_relative() {
+    Playground::setup("overlay_use_do_cd_file_relative", |dirs, sandbox| {
+        sandbox
+            .mkdir("test1/test2")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "test1/test2/spam.nu",
+                r#"
+                    export-env { cd ($env.FILE_PWD | path join '..') }
+                "#,
+            )]);
+
+        let inp = &[
+            r#"overlay use test1/test2/spam.nu"#,
+            r#"$env.PWD | path basename"#,
+        ];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "test1");
+    })
+}
+
+#[test]
+fn overlay_use_dont_cd_overlay() {
+    Playground::setup("overlay_use_dont_cd_overlay", |dirs, sandbox| {
+        sandbox
+            .mkdir("test1/test2")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "test1/test2/spam.nu",
+                r#"
+                    export-env {
+                        overlay new spam
+                        cd test1/test2
+                        overlay hide spam
+                    }
+                "#,
+            )]);
+
+        let inp = &[
+            r#"source-env test1/test2/spam.nu"#,
+            r#"$env.PWD | path basename"#,
+        ];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "overlay_use_dont_cd_overlay");
+    })
+}
+
+#[ignore]
+#[test]
+fn overlay_use_find_module_scoped() {
+    Playground::setup("overlay_use_find_module_scoped", |dirs, _| {
+        let inp = &[r#"
+                do {
+                    module spam { export def foo [] { 'foo' } }
+
+                    overlay use spam
+                    foo
+                }
+            "#];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+        let actual_repl = nu!(cwd: "tests/overlays", nu_repl_code(inp));
+
+        assert_eq!(actual.out, "foo");
+        assert_eq!(actual_repl.out, "foo");
+    })
+}

tests/parsing/mod.rs

@@ -47,6 +47,40 @@ fn run_nu_script_multiline_end_pipe_win() {
     assert_eq!(actual.out, "3");
 }
 
+#[test]
+fn parse_file_relative_to_parsed_file_simple() {
+    Playground::setup("relative_files_simple", |dirs, sandbox| {
+        sandbox
+            .mkdir("lol")
+            .mkdir("lol/lol")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "lol/lol/lol.nu",
+                r#"
+                    use ../lol_shell.nu
+
+                    let-env LOL = (lol_shell ls)
+                "#,
+            )])
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "lol/lol_shell.nu",
+                r#"
+                    export def ls [] { "lol" }
+                "#,
+            )]);
+
+        let actual = nu!(
+            cwd: dirs.test(), pipeline(
+            r#"
+                source-env lol/lol/lol.nu;
+                $env.LOL
+            "#
+        ));
+
+        assert_eq!(actual.out, "lol");
+    })
+}
+
+#[ignore]
 #[test]
 fn parse_file_relative_to_parsed_file() {
     Playground::setup("relative_files", |dirs, sandbox| {
@@ -56,11 +90,11 @@ fn parse_file_relative_to_parsed_file() {
             .with_files(vec![FileWithContentToBeTrimmed(
                 "lol/lol/lol.nu",
                 r#"
-                    source ../../foo.nu
+                    source-env ../../foo.nu
                     use ../lol_shell.nu
                     overlay use ../../lol/lol_shell.nu
 
-                    $'($env.FOO) (lol_shell ls) (ls)'
+                    let-env LOL = $'($env.FOO) (lol_shell ls) (ls)'
                 "#,
             )])
             .with_files(vec![FileWithContentToBeTrimmed(
@@ -79,7 +113,8 @@ fn parse_file_relative_to_parsed_file() {
         let actual = nu!(
             cwd: dirs.test(), pipeline(
             r#"
-                source lol/lol/lol.nu
+                source-env lol/lol/lol.nu;
+                $env.LOL
             "#
         ));
 
@@ -95,7 +130,7 @@ fn parse_file_relative_to_parsed_file_dont_use_cwd_1() {
             .with_files(vec![FileWithContentToBeTrimmed(
                 "lol/lol.nu",
                 r#"
-                    source foo.nu
+                    source-env foo.nu
                 "#,
             )])
             .with_files(vec![FileWithContentToBeTrimmed(
@@ -114,7 +149,7 @@ fn parse_file_relative_to_parsed_file_dont_use_cwd_1() {
         let actual = nu!(
             cwd: dirs.test(), pipeline(
             r#"
-                source lol/lol.nu;
+                source-env lol/lol.nu;
                 $env.FOO
             "#
         ));
@@ -131,7 +166,7 @@ fn parse_file_relative_to_parsed_file_dont_use_cwd_2() {
             .with_files(vec![FileWithContentToBeTrimmed(
                 "lol/lol.nu",
                 r#"
-                    source foo.nu
+                    source-env foo.nu
                 "#,
             )])
             .with_files(vec![FileWithContentToBeTrimmed(
@@ -144,7 +179,7 @@ fn parse_file_relative_to_parsed_file_dont_use_cwd_2() {
         let actual = nu!(
             cwd: dirs.test(), pipeline(
             r#"
-                source lol/lol.nu
+                source-env lol/lol.nu
             "#
         ));
 

tests/shell/environment/env.rs

@@ -77,6 +77,20 @@ fn env_shorthand_multi() {
     assert_eq!(actual.out, "barbaz");
 }
 
+#[test]
+fn let_env_file_pwd_env_var_fails() {
+    let actual = nu!(cwd: ".", r#"let-env FILE_PWD = 'foo'"#);
+
+    assert!(actual.err.contains("automatic_env_var_set_manually"));
+}
+
+#[test]
+fn load_env_file_pwd_env_var_fails() {
+    let actual = nu!(cwd: ".", r#"load-env { FILE_PWD : 'foo' }"#);
+
+    assert!(actual.err.contains("automatic_env_var_set_manually"));
+}
+
 // FIXME: for some reason Nu is attempting to execute foo in `let-env FOO = foo`
 #[ignore]
 #[test]

tests/shell/environment/mod.rs

@@ -1,4 +1,5 @@
 mod env;
+mod source_env;
 
 // FIXME: nu_env tests depend on autoenv which hasn't been ported yet
 // mod nu_env;

tests/shell/environment/source_env.rs

@@ -0,0 +1,152 @@
+use nu_test_support::fs::Stub::FileWithContentToBeTrimmed;
+use nu_test_support::playground::Playground;
+use nu_test_support::{nu, pipeline};
+
+#[test]
+fn source_env_eval_export_env() {
+    Playground::setup("source_env_eval_export_env", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "spam.nu",
+            r#"
+                export-env { let-env FOO = 'foo' }
+            "#,
+        )]);
+
+        let inp = &[r#"source-env spam.nu"#, r#"$env.FOO"#];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "foo");
+    })
+}
+
+#[test]
+fn source_env_eval_export_env_hide() {
+    Playground::setup("source_env_eval_export_env", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "spam.nu",
+            r#"
+                export-env { hide-env FOO }
+            "#,
+        )]);
+
+        let inp = &[
+            r#"let-env FOO = 'foo'"#,
+            r#"source-env spam.nu"#,
+            r#"$env.FOO"#,
+        ];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert!(actual.err.contains("did you mean"));
+    })
+}
+
+#[test]
+fn source_env_do_cd() {
+    Playground::setup("source_env_do_cd", |dirs, sandbox| {
+        sandbox
+            .mkdir("test1/test2")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "test1/test2/spam.nu",
+                r#"
+                    cd test1/test2
+                "#,
+            )]);
+
+        let inp = &[
+            r#"source-env test1/test2/spam.nu"#,
+            r#"$env.PWD | path basename"#,
+        ];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "test2");
+    })
+}
+
+#[test]
+fn source_env_do_cd_file_relative() {
+    Playground::setup("source_env_do_cd_file_relative", |dirs, sandbox| {
+        sandbox
+            .mkdir("test1/test2")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "test1/test2/spam.nu",
+                r#"
+                    cd ($env.FILE_PWD | path join '..')
+                "#,
+            )]);
+
+        let inp = &[
+            r#"source-env test1/test2/spam.nu"#,
+            r#"$env.PWD | path basename"#,
+        ];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "test1");
+    })
+}
+
+#[test]
+fn source_env_dont_cd_overlay() {
+    Playground::setup("source_env_dont_cd_overlay", |dirs, sandbox| {
+        sandbox
+            .mkdir("test1/test2")
+            .with_files(vec![FileWithContentToBeTrimmed(
+                "test1/test2/spam.nu",
+                r#"
+                    overlay new spam
+                    cd test1/test2
+                    overlay hide spam
+                "#,
+            )]);
+
+        let inp = &[
+            r#"source-env test1/test2/spam.nu"#,
+            r#"$env.PWD | path basename"#,
+        ];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert_eq!(actual.out, "source_env_dont_cd_overlay");
+    })
+}
+
+#[test]
+fn source_env_nice_parse_error() {
+    Playground::setup("source_env_nice_parse_error", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "spam.nu",
+            r#"
+                let x
+            "#,
+        )]);
+
+        let inp = &[r#"source-env spam.nu"#];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert!(actual.err.contains("cannot parse this file"));
+        assert!(actual.err.contains("───"));
+    })
+}
+
+#[test]
+fn source_env_nice_shell_error() {
+    Playground::setup("source_env_nice_shell_error", |dirs, sandbox| {
+        sandbox.with_files(vec![FileWithContentToBeTrimmed(
+            "spam.nu",
+            r#"
+                let-env FILE_PWD = 'foo'
+            "#,
+        )]);
+
+        let inp = &[r#"source-env spam.nu"#];
+
+        let actual = nu!(cwd: dirs.test(), pipeline(&inp.join("; ")));
+
+        assert!(actual.err.contains("cannot evaluate this file"));
+        assert!(actual.err.contains("───"));
+    })
+}

tests/shell/mod.rs

@@ -73,7 +73,7 @@ fn nu_lib_dirs_repl() {
 
         let inp_lines = &[
             r#"let-env NU_LIB_DIRS = [ ('scripts' | path expand) ]"#,
-            r#"source foo.nu"#,
+            r#"source-env foo.nu"#,
             r#"$env.FOO"#,
         ];
 
@@ -98,13 +98,13 @@ fn nu_lib_dirs_script() {
             .with_files(vec![FileWithContentToBeTrimmed(
                 "main.nu",
                 r#"
-                    source foo.nu
+                    source-env foo.nu
                 "#,
             )]);
 
         let inp_lines = &[
             r#"let-env NU_LIB_DIRS = [ ('scripts' | path expand) ]"#,
-            r#"source main.nu"#,
+            r#"source-env main.nu"#,
             r#"$env.FOO"#,
         ];
 
@@ -129,7 +129,7 @@ fn nu_lib_dirs_relative_repl() {
 
         let inp_lines = &[
             r#"let-env NU_LIB_DIRS = [ 'scripts' ]"#,
-            r#"source foo.nu"#,
+            r#"source-env foo.nu"#,
             r#"$env.FOO"#,
         ];
 
@@ -148,7 +148,7 @@ fn nu_lib_dirs_relative_script() {
             .with_files(vec![FileWithContentToBeTrimmed(
                 "scripts/main.nu",
                 r#"
-                    source ../foo.nu
+                    source-env ../foo.nu
                 "#,
             )])
             .with_files(vec![FileWithContentToBeTrimmed(
@@ -160,7 +160,7 @@ fn nu_lib_dirs_relative_script() {
 
         let inp_lines = &[
             r#"let-env NU_LIB_DIRS = [ 'scripts' ]"#,
-            r#"source scripts/main.nu"#,
+            r#"source-env scripts/main.nu"#,
             r#"$env.FOO"#,
         ];