Break up interdependencies of command crates (#9429)

# Description
Make sure that our different crates that contain commands can be
compiled in parallel.
This can under certain circumstances accelerate the compilation with
sufficient multithreading available.

## Details
- Move `help` commands from `nu-cmd-lang` back to `nu-command`
- This also makes sense as the commands are implemented in an
ANSI-terminal specific way
- Make `nu-cmd-lang` only a dev dependency for `nu-command`
- Change context creation helpers for `nu-cmd-extra` and
`nu-cmd-dataframe` to have a consistent api used in
`src/main.rs`:`get_engine_state()`
- `nu-command` now indepedent from `nu-cmd-extra` and `nu-cmd-dataframe`
that are now dependencies of `nu` directly. (change to internal
features)
- Fix tests that previously used `nu-command::create_default_context()`
with replacement functions

## From scratch compilation times:

just debug (dev) build and default features
```
cargo clean --profile dev && cargo build --timings
```

### before

![grafik](https://github.com/nushell/nushell/assets/15833959/e49f1f42-2e53-4a6c-bc23-625b686af1bc)

### after

![grafik](https://github.com/nushell/nushell/assets/15833959/8dec4723-e625-4a86-b91e-e6e808f64726)

# User-Facing Changes
None direct, only change to compilation on multithreaded jobs expected.

# Tests + Formatting
Tests that previously chose to use `nu-command` for their scope will
still use `nu-cmd-lang` + `nu-command` (command list in the granularity
at the time)

Cargo.lock

@@ -2652,6 +2652,7 @@ dependencies = [
  "nu-ansi-term",
  "nu-cli",
  "nu-cmd-dataframe",
+ "nu-cmd-extra",
  "nu-cmd-lang",
  "nu-color-config",
  "nu-command",
@@ -2705,6 +2706,7 @@ dependencies = [
  "log",
  "miette",
  "nu-ansi-term",
+ "nu-cmd-lang",
  "nu-color-config",
  "nu-command",
  "nu-engine",
@@ -2825,8 +2827,6 @@ dependencies = [
  "native-tls",
  "notify",
  "nu-ansi-term",
- "nu-cmd-dataframe",
- "nu-cmd-extra",
  "nu-cmd-lang",
  "nu-color-config",
  "nu-engine",

Cargo.toml

@@ -49,6 +49,7 @@ nu-cli = { path = "./crates/nu-cli", version = "0.81.1" }
 nu-color-config = { path = "./crates/nu-color-config", version = "0.81.1" }
 nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.81.1" }
 nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.81.1", optional = true }
+nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.81.1", optional = true }
 nu-command = { path = "./crates/nu-command", version = "0.81.1" }
 nu-engine = { path = "./crates/nu-engine", version = "0.81.1" }
 nu-explore = { path = "./crates/nu-explore", version = "0.81.1" }
@@ -112,6 +113,7 @@ plugin = [
 default = ["plugin", "which-support", "trash-support", "sqlite"]
 stable = ["default"]
 wasi = ["nu-cmd-lang/wasi"]
+# NOTE: individual features are also passed to `nu-cmd-lang` that uses them to generate the feature matrix in the `version` command
 
 # Enable to statically link OpenSSL; otherwise the system version will be used. Not enabled by default because it takes a while to build
 static-link-openssl = ["dep:openssl", "nu-cmd-lang/static-link-openssl"]
@@ -121,10 +123,10 @@ which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
 trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
 
 # Extra feature for nushell
-extra = ["nu-command/extra", "nu-cmd-lang/extra"]
+extra = ["dep:nu-cmd-extra", "nu-cmd-lang/extra"]
 
 # Dataframe feature for nushell
-dataframe = ["nu-command/dataframe", "nu-cmd-lang/dataframe"]
+dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
 
 # SQLite commands for nushell
 sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]

crates/nu-cli/Cargo.toml

@@ -11,6 +11,7 @@ version = "0.81.1"
 bench = false
 
 [dev-dependencies]
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
 nu-test-support = { path = "../nu-test-support", version = "0.81.1" }
 rstest = { version = "0.17.0", default-features = false }
 

crates/nu-cli/src/commands/default_context.rs

@@ -26,7 +26,7 @@ pub fn add_cli_context(mut engine_state: EngineState) -> EngineState {
     };
 
     if let Err(err) = engine_state.merge_delta(delta) {
-        eprintln!("Error creating default context: {err:?}");
+        eprintln!("Error creating CLI command context: {err:?}");
     }
 
     engine_state

crates/nu-cli/src/completions/completer.rs

@@ -501,7 +501,8 @@ mod completer_tests {
 
     #[test]
     fn test_completion_helper() {
-        let mut engine_state = nu_command::create_default_context();
+        let mut engine_state =
+            nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
 
         // Custom additions
         let delta = {

crates/nu-cli/tests/support/completions_helpers.rs

@@ -1,6 +1,5 @@
 use std::path::PathBuf;
 
-use nu_command::create_default_context;
 use nu_engine::eval_block;
 use nu_parser::parse;
 use nu_protocol::{
@@ -11,6 +10,10 @@ use nu_test_support::fs;
 use reedline::Suggestion;
 const SEP: char = std::path::MAIN_SEPARATOR;
 
+fn create_default_context() -> EngineState {
+    nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
+}
+
 // creates a new engine with the current path into the completions fixtures folder
 pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
     // Target folder inside assets

crates/nu-cmd-dataframe/Cargo.toml

@@ -13,7 +13,6 @@ version = "0.81.1"
 bench = false
 
 [dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
 nu-engine = { path = "../nu-engine", version = "0.81.1" }
 nu-parser = { path = "../nu-parser", version = "0.81.1" }
 nu-protocol = { path = "../nu-protocol", version = "0.81.1" }
@@ -66,4 +65,5 @@ dataframe = ["default"]
 default = ["num", "polars", "sqlparser"]
 
 [dev-dependencies]
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
 nu-test-support = { path = "../nu-test-support", version = "0.81.1" }

crates/nu-cmd-dataframe/src/dataframe/mod.rs

@@ -10,13 +10,24 @@ pub use expressions::add_expressions;
 pub use lazy::add_lazy_decls;
 pub use series::add_series_decls;
 
-use nu_protocol::engine::StateWorkingSet;
+use nu_protocol::engine::{EngineState, StateWorkingSet};
 
-pub fn add_dataframe_decls(working_set: &mut StateWorkingSet) {
-    add_series_decls(working_set);
-    add_eager_decls(working_set);
-    add_expressions(working_set);
-    add_lazy_decls(working_set);
+pub fn add_dataframe_context(mut engine_state: EngineState) -> EngineState {
+    let delta = {
+        let mut working_set = StateWorkingSet::new(&engine_state);
+        add_series_decls(&mut working_set);
+        add_eager_decls(&mut working_set);
+        add_expressions(&mut working_set);
+        add_lazy_decls(&mut working_set);
+
+        working_set.render()
+    };
+
+    if let Err(err) = engine_state.merge_delta(delta) {
+        eprintln!("Error creating dataframe command context: {err:?}");
+    }
+
+    engine_state
 }
 
 #[cfg(test)]

crates/nu-cmd-extra/Cargo.toml

@@ -13,7 +13,6 @@ version = "0.81.1"
 bench = false
 
 [dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
 nu-engine = { path = "../nu-engine", version = "0.81.1" }
 nu-parser = { path = "../nu-parser", version = "0.81.1" }
 nu-protocol = { path = "../nu-protocol", version = "0.81.1" }
@@ -27,4 +26,5 @@ extra = ["default"]
 default = []
 
 [dev-dependencies]
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
 nu-test-support = { path = "../nu-test-support", version = "0.81.1" }

crates/nu-cmd-extra/src/extra/mod.rs

@@ -1,9 +1,12 @@
 mod bits;
 
-use nu_protocol::engine::StateWorkingSet;
+use nu_protocol::engine::{EngineState, StateWorkingSet};
 
-pub fn add_extra_decls(working_set: &mut StateWorkingSet) {
-    macro_rules! bind_command {
+pub fn add_extra_command_context(mut engine_state: EngineState) -> EngineState {
+    let delta = {
+        let mut working_set = StateWorkingSet::new(&engine_state);
+
+        macro_rules! bind_command {
             ( $command:expr ) => {
                 working_set.add_decl(Box::new($command));
             };
@@ -12,15 +15,23 @@ pub fn add_extra_decls(working_set: &mut StateWorkingSet) {
             };
         }
 
-    bind_command!(
-        bits::bits_::Bits,
-        bits::and::BitsAnd,
-        bits::not::BitsNot,
-        bits::or::BitsOr,
-        bits::xor::BitsXor,
-        bits::rotate_left::BitsRol,
-        bits::rotate_right::BitsRor,
-        bits::shift_left::BitsShl,
-        bits::shift_right::BitsShr
-    );
+        bind_command!(
+            bits::bits_::Bits,
+            bits::and::BitsAnd,
+            bits::not::BitsNot,
+            bits::or::BitsOr,
+            bits::xor::BitsXor,
+            bits::rotate_left::BitsRol,
+            bits::rotate_right::BitsRor,
+            bits::shift_left::BitsShl,
+            bits::shift_right::BitsShr
+        );
+        working_set.render()
+    };
+
+    if let Err(err) = engine_state.merge_delta(delta) {
+        eprintln!("Error creating extra command context: {err:?}");
+    }
+
+    engine_state
 }

crates/nu-cmd-lang/src/core_commands/mod.rs

@@ -18,12 +18,6 @@ mod export_module;
 mod export_use;
 mod extern_;
 mod for_;
-pub mod help;
-pub mod help_aliases;
-pub mod help_commands;
-pub mod help_externs;
-pub mod help_modules;
-mod help_operators;
 mod hide;
 mod hide_env;
 mod if_;
@@ -61,12 +55,6 @@ pub use export_module::ExportModule;
 pub use export_use::ExportUse;
 pub use extern_::Extern;
 pub use for_::For;
-pub use help::Help;
-pub use help_aliases::HelpAliases;
-pub use help_commands::HelpCommands;
-pub use help_externs::HelpExterns;
-pub use help_modules::HelpModules;
-pub use help_operators::HelpOperators;
 pub use hide::Hide;
 pub use hide_env::HideEnv;
 pub use if_::If;

crates/nu-cmd-lang/src/default_context.rs

@@ -36,12 +36,6 @@ pub fn create_default_context() -> EngineState {
             ExportModule,
             Extern,
             For,
-            Help,
-            HelpAliases,
-            HelpCommands,
-            HelpModules,
-            HelpExterns,
-            HelpOperators,
             Hide,
             HideEnv,
             If,

crates/nu-command/Cargo.toml

@@ -13,9 +13,6 @@ version = "0.81.1"
 bench = false
 
 [dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
-nu-cmd-dataframe = { path = "../nu-cmd-dataframe", version = "0.81.1", optional = true }
-nu-cmd-extra = { path = "../nu-cmd-extra", version = "0.81.1", optional = true }
 nu-color-config = { path = "../nu-color-config", version = "0.81.1" }
 nu-engine = { path = "../nu-engine", version = "0.81.1" }
 nu-glob = { path = "../nu-glob", version = "0.81.1" }
@@ -117,17 +114,15 @@ features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_System
 version = "0.48"
 
 [features]
-dataframe = ["dep:nu-cmd-dataframe"]
-extra = ["dep:nu-cmd-extra"]
 plugin = ["nu-parser/plugin"]
-sqlite = [
-	"rusqlite",
-] # TODO: given that rusqlite is included in reedline, should we just always include it?
+sqlite = ["rusqlite"]
 trash-support = ["trash"]
 which-support = ["which"]
 
 [dev-dependencies]
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.81.1" }
 nu-test-support = { path = "../nu-test-support", version = "0.81.1" }
+
 dirs-next = "2.0"
 mockito = "1.0"
 quickcheck = "1.0"

crates/nu-command/src/default_context.rs

@@ -1,15 +1,10 @@
 use nu_protocol::engine::{EngineState, StateWorkingSet};
 
-use crate::*;
-#[cfg(feature = "dataframe")]
-use nu_cmd_dataframe::*;
-
-#[cfg(feature = "extra")]
-use nu_cmd_extra::*;
-
-pub fn create_default_context() -> EngineState {
-    let mut engine_state = nu_cmd_lang::create_default_context();
-
+use crate::{
+    help::{HelpAliases, HelpCommands, HelpExterns, HelpModules, HelpOperators},
+    *,
+};
+pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
     let delta = {
         let mut working_set = StateWorkingSet::new(&engine_state);
 
@@ -24,12 +19,6 @@ pub fn create_default_context() -> EngineState {
         // them only accessible if the correct input value category is used with the
         // declaration
 
-        #[cfg(feature = "extra")]
-        add_extra_decls(&mut working_set);
-
-        #[cfg(feature = "dataframe")]
-        add_dataframe_decls(&mut working_set);
-
         // Database-related
         // Adds all related commands to query databases
         #[cfg(feature = "sqlite")]
@@ -138,6 +127,16 @@ pub fn create_default_context() -> EngineState {
             Sys,
         };
 
+        // Help
+        bind_command! {
+            Help,
+            HelpAliases,
+            HelpExterns,
+            HelpCommands,
+            HelpModules,
+            HelpOperators,
+        };
+
         // Debug
         bind_command! {
             Ast,

crates/nu-command/src/filters/find.rs

@@ -1,5 +1,5 @@
+use crate::help::highlight_search_string;
 use itertools::Itertools;
-use nu_cmd_lang::help::highlight_search_string;
 
 use fancy_regex::Regex;
 use nu_ansi_term::Style;

crates/nu-command/src/help/help_.rs

@@ -1,6 +1,6 @@
-use crate::help_aliases::help_aliases;
-use crate::help_commands::help_commands;
-use crate::help_modules::help_modules;
+use crate::help::help_aliases;
+use crate::help::help_commands;
+use crate::help::help_modules;
 use fancy_regex::Regex;
 use nu_ansi_term::Style;
 use nu_engine::CallExt;

crates/nu-command/src/help/mod.rs

@@ -0,0 +1,18 @@
+mod help_;
+mod help_aliases;
+mod help_commands;
+mod help_externs;
+mod help_modules;
+mod help_operators;
+
+pub use help_::Help;
+pub use help_aliases::HelpAliases;
+pub use help_commands::HelpCommands;
+pub use help_externs::HelpExterns;
+pub use help_modules::HelpModules;
+pub use help_operators::HelpOperators;
+
+pub(crate) use help_::{highlight_search_in_table, highlight_search_string};
+pub(crate) use help_aliases::help_aliases;
+pub(crate) use help_commands::help_commands;
+pub(crate) use help_modules::help_modules;

crates/nu-command/src/lib.rs

@@ -13,6 +13,7 @@ mod filters;
 mod formats;
 mod generators;
 mod hash;
+mod help;
 pub mod hook;
 mod input_handler;
 mod math;
@@ -45,6 +46,7 @@ pub use filters::*;
 pub use formats::*;
 pub use generators::*;
 pub use hash::*;
+pub use help::*;
 pub use hook::*;
 pub use math::*;
 pub use misc::*;

crates/nu-command/tests/main.rs

@@ -1,10 +1,16 @@
-use nu_command::create_default_context;
-use nu_protocol::{engine::StateWorkingSet, Category, Span};
+use nu_protocol::{
+    engine::{EngineState, StateWorkingSet},
+    Category, Span,
+};
 use quickcheck_macros::quickcheck;
 
 mod commands;
 mod format_conversions;
 
+fn create_default_context() -> EngineState {
+    nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
+}
+
 #[quickcheck]
 fn quickcheck_parse(data: String) -> bool {
     let (tokens, err) = nu_parser::lex(data.as_bytes(), 0, b"", b"", true);
@@ -24,7 +30,7 @@ fn quickcheck_parse(data: String) -> bool {
 
 #[test]
 fn signature_name_matches_command_name() {
-    let ctx = crate::create_default_context();
+    let ctx = create_default_context();
     let decls = ctx.get_decls_sorted(true);
     let mut failures = Vec::new();
 
@@ -50,7 +56,7 @@ fn signature_name_matches_command_name() {
 
 #[test]
 fn commands_declare_input_output_types() {
-    let ctx = crate::create_default_context();
+    let ctx = create_default_context();
     let decls = ctx.get_decls_sorted(true);
     let mut failures = Vec::new();
 

src/main.rs

@@ -19,8 +19,8 @@ use command::gather_commandline_args;
 use log::Level;
 use miette::Result;
 use nu_cli::gather_parent_env_vars;
-use nu_command::{create_default_context, get_init_cwd};
-use nu_protocol::{report_error_new, Value};
+use nu_command::get_init_cwd;
+use nu_protocol::{engine::EngineState, report_error_new, Value};
 use nu_protocol::{util::BufferedReader, PipelineData, RawStream};
 use nu_std::load_standard_library;
 use nu_utils::utils::perf;
@@ -32,6 +32,17 @@ use std::{
     sync::{atomic::AtomicBool, Arc},
 };
 
+fn get_engine_state() -> EngineState {
+    let engine_state = nu_cmd_lang::create_default_context();
+    let engine_state = nu_command::add_shell_command_context(engine_state);
+    #[cfg(feature = "extra")]
+    let engine_state = nu_cmd_extra::add_extra_command_context(engine_state);
+    #[cfg(feature = "dataframe")]
+    let engine_state = nu_cmd_dataframe::add_dataframe_context(engine_state);
+    let engine_state = nu_cli::add_cli_context(engine_state);
+    nu_explore::add_explore_context(engine_state)
+}
+
 fn main() -> Result<()> {
     let entire_start_time = std::time::Instant::now();
     let mut start_time = std::time::Instant::now();
@@ -43,8 +54,7 @@ fn main() -> Result<()> {
 
     // Get initial current working directory.
     let init_cwd = get_init_cwd();
-    let mut engine_state =
-        nu_explore::add_explore_context(nu_cli::add_cli_context(create_default_context()));
+    let mut engine_state = get_engine_state();
 
     // Custom additions
     let delta = {

src/test_bins.rs

@@ -1,6 +1,5 @@
 use std::io::{self, BufRead, Read, Write};
 
-use nu_command::create_default_context;
 use nu_command::hook::{eval_env_change_hook, eval_hook};
 use nu_engine::eval_block;
 use nu_parser::parse;
@@ -168,12 +167,19 @@ fn outcome_ok(msg: String) -> ! {
     std::process::exit(0);
 }
 
+/// Generate a minimal engine state with just `nu-cmd-lang`, `nu-command`, and `nu-cli` commands.
+fn get_engine_state() -> EngineState {
+    let engine_state = nu_cmd_lang::create_default_context();
+    let engine_state = nu_command::add_shell_command_context(engine_state);
+    nu_cli::add_cli_context(engine_state)
+}
+
 pub fn nu_repl() {
     //cwd: &str, source_lines: &[&str]) {
     let cwd = std::env::current_dir().expect("Could not get current working directory.");
     let source_lines = args();
 
-    let mut engine_state = nu_cli::add_cli_context(create_default_context());
+    let mut engine_state = get_engine_state();
     let mut stack = Stack::new();
 
     engine_state.add_env_var("PWD".into(), Value::test_string(cwd.to_string_lossy()));