Port execution

Drop support for history file version 1.

ParseExecutionContext no longer contains an OperationContext because in my
first implementation, ParseExecutionContext didn't have interior mutability.
We should probably try to add it back.

Add a few to-do style comments. Search for "todo!" and "PORTING".

Co-authored-by: Xiretza <xiretza@xiretza.xyz>
(complete, wildcard, expand, history, history/file)
Co-authored-by: Henrik Hørlück Berg <36937807+henrikhorluck@users.noreply.github.com>
(builtins/set)

CMakeLists.txt

@@ -99,63 +99,40 @@ endif()
 
 # List of sources for builtin functions.
 set(FISH_BUILTIN_SRCS
-    src/builtin.cpp
     src/builtins/bind.cpp
     src/builtins/commandline.cpp
-    src/builtins/complete.cpp
-    src/builtins/disown.cpp
-    src/builtins/eval.cpp
-    src/builtins/fg.cpp
-    src/builtins/history.cpp
-    src/builtins/jobs.cpp
     src/builtins/read.cpp
-    src/builtins/set.cpp
-    src/builtins/source.cpp
     src/builtins/ulimit.cpp
 )
-
 # List of other sources.
 set(FISH_SRCS
     src/ast.cpp
-    src/autoload.cpp
+    src/builtin.cpp
     src/color.cpp
     src/common.cpp
-    src/complete.cpp
     src/env.cpp
     src/env_universal_common.cpp
     src/event.cpp
-    src/exec.cpp
     src/expand.cpp
     src/fallback.cpp
     src/fds.cpp
     src/fish_indent_common.cpp
     src/fish_version.cpp
     src/flog.cpp
-    src/function.cpp
     src/highlight.cpp
-    src/history.cpp
-    src/history_file.cpp
     src/input_common.cpp
     src/input.cpp
-    src/io.cpp
     src/null_terminated_array.cpp
-    src/operation_context.cpp
     src/output.cpp
     src/pager.cpp
-    src/parse_execution.cpp
-    src/parser.cpp
-    src/parser_keywords.cpp
     src/parse_util.cpp
     src/path.cpp
-    src/proc.cpp
     src/reader.cpp
     src/rustffi.cpp
     src/screen.cpp
-    src/signals.cpp
     src/utf8.cpp
     src/wcstringutil.cpp
     src/wgetopt.cpp
-    src/wildcard.cpp
     src/wutil.cpp
 )
 

fish-rust/build.rs

@@ -20,6 +20,14 @@ fn main() {
         )
         .compile("libcompat.a");
 
+    if compiles("fish-rust/src/cfg/w_exitcode.cpp") {
+        println!("cargo:rustc-cfg=HAVE_WAITSTATUS_SIGNAL_RET");
+    }
+
+    if compiles("fish-rust/src/cfg/spawn.c") {
+        println!("cargo:rustc-cfg=FISH_USE_POSIX_SPAWN");
+    }
+
     let rust_dir = env!("CARGO_MANIFEST_DIR");
     let target_dir =
         std::env::var("FISH_RUST_TARGET_DIR").unwrap_or(format!("{}/{}", rust_dir, "target/"));
@@ -58,30 +66,38 @@ fn main() {
         "fish-rust/src/abbrs.rs",
         "fish-rust/src/ast.rs",
         "fish-rust/src/builtins/shared.rs",
-        "fish-rust/src/builtins/function.rs",
         "fish-rust/src/common.rs",
-        "fish-rust/src/env/env_ffi.rs",
+        "fish-rust/src/complete.rs",
         "fish-rust/src/env_dispatch.rs",
+        "fish-rust/src/env/env_ffi.rs",
+        "fish-rust/src/env_universal_common.rs",
         "fish-rust/src/event.rs",
+        "fish-rust/src/exec.rs",
+        "fish-rust/src/expand.rs",
         "fish-rust/src/fd_monitor.rs",
         "fish-rust/src/fd_readable_set.rs",
         "fish-rust/src/fds.rs",
         "fish-rust/src/ffi_init.rs",
         "fish-rust/src/ffi_tests.rs",
-        "fish-rust/src/fish.rs",
         "fish-rust/src/fish_indent.rs",
-        "fish-rust/src/fork_exec/spawn.rs",
+        "fish-rust/src/fish.rs",
         "fish-rust/src/function.rs",
         "fish-rust/src/future_feature_flags.rs",
         "fish-rust/src/highlight.rs",
+        "fish-rust/src/history.rs",
+        "fish-rust/src/io.rs",
         "fish-rust/src/job_group.rs",
         "fish-rust/src/kill.rs",
         "fish-rust/src/null_terminated_array.rs",
+        "fish-rust/src/operation_context.rs",
         "fish-rust/src/output.rs",
         "fish-rust/src/parse_constants.rs",
+        "fish-rust/src/parser.rs",
         "fish-rust/src/parse_tree.rs",
         "fish-rust/src/parse_util.rs",
         "fish-rust/src/print_help.rs",
+        "fish-rust/src/proc.rs",
+        "fish-rust/src/reader.rs",
         "fish-rust/src/redirection.rs",
         "fish-rust/src/signal.rs",
         "fish-rust/src/smoke.rs",
@@ -89,10 +105,8 @@ fn main() {
         "fish-rust/src/threads.rs",
         "fish-rust/src/timer.rs",
         "fish-rust/src/tokenizer.rs",
-        "fish-rust/src/topic_monitor.rs",
         "fish-rust/src/trace.rs",
         "fish-rust/src/util.rs",
-        "fish-rust/src/wait_handle.rs",
         "fish-rust/src/wildcard.rs",
     ];
     cxx_build::bridges(&source_files)
@@ -149,6 +163,16 @@ fn detect_features(target: Target) {
     }
 }
 
+fn compiles(file: &str) -> bool {
+    rsconf::rebuild_if_path_changed(file);
+    let mut command = cc::Build::new()
+        .flag("-fsyntax-only")
+        .get_compiler()
+        .to_command();
+    command.arg(file);
+    command.status().unwrap().success()
+}
+
 /// Detect if we're being compiled for a BSD-derived OS, allowing targeting code conditionally with
 /// `#[cfg(feature = "bsd")]`.
 ///

fish-rust/src/abbrs.rs

@@ -194,9 +194,11 @@ impl Abbreviation {
 }
 
 /// The result of an abbreviation expansion.
+#[derive(Debug, Eq, PartialEq)]
 pub struct Replacer {
     /// The string to use to replace the incoming token, either literal or as a function name.
-    replacement: WString,
+    /// Exposed for testing.
+    pub replacement: WString,
 
     /// If true, treat 'replacement' as the name of a function.
     is_function: bool,

fish-rust/src/ast.rs

@@ -148,12 +148,11 @@ pub trait Node: Acceptor + ConcreteNode + std::fmt::Debug {
     fn pointer_eq(&self, rhs: &dyn Node) -> bool {
         std::ptr::eq(self.as_ptr(), rhs.as_ptr())
     }
+    fn as_node(&self) -> &dyn Node;
 }
 
 /// NodeMut is a mutable node.
-trait NodeMut: Node + AcceptorMut + ConcreteNodeMut {
-    fn as_node(&self) -> &dyn Node;
-}
+trait NodeMut: Node + AcceptorMut + ConcreteNodeMut {}
 
 pub trait ConcreteNode {
     // Cast to any sub-trait.
@@ -401,7 +400,7 @@ pub trait Leaf: Node {
     fn has_source(&self) -> bool {
         self.range().is_some()
     }
-    fn leaf_as_node_ffi(&self) -> &dyn Node;
+    fn leaf_as_node(&self) -> &dyn Node;
 }
 
 // A token node is a node which contains a token, which must be one of a fixed set.
@@ -439,6 +438,13 @@ pub trait List: Node {
     fn is_empty(&self) -> bool {
         self.contents().is_empty()
     }
+    /// Iteration support.
+    fn iter(&self) -> std::slice::Iter<Box<Self::ContentsNode>> {
+        self.contents().iter()
+    }
+    fn get(&self, index: usize) -> Option<&Self::ContentsNode> {
+        self.contents().get(index).map(|b| &**b)
+    }
 }
 
 /// Implement the node trait.
@@ -476,12 +482,11 @@ macro_rules! implement_node {
             fn as_ptr(&self) -> *const () {
                 (self as *const $name).cast()
             }
-        }
-        impl NodeMut for $name {
             fn as_node(&self) -> &dyn Node {
                 self
             }
         }
+        impl NodeMut for $name {}
     };
 }
 
@@ -495,7 +500,7 @@ macro_rules! implement_leaf {
             fn range_mut(&mut self) -> &mut Option<SourceRange> {
                 &mut self.range
             }
-            fn leaf_as_node_ffi(&self) -> &dyn Node {
+            fn leaf_as_node(&self) -> &dyn Node {
                 self
             }
         }
@@ -621,13 +626,11 @@ macro_rules! define_list_node {
                 &mut self.list_contents
             }
         }
-        impl $name {
-            /// Iteration support.
-            pub fn iter(&self) -> impl Iterator<Item = &<$name as List>::ContentsNode> {
-                self.contents().iter().map(|b| &**b)
-            }
-            pub fn get(&self, index: usize) -> Option<&$contents> {
-                self.contents().get(index).map(|b| &**b)
+        impl<'a> IntoIterator for &'a $name {
+            type Item = &'a Box<$contents>;
+            type IntoIter = std::slice::Iter<'a, Box<$contents>>;
+            fn into_iter(self) -> Self::IntoIter {
+                self.contents().into_iter()
             }
         }
         impl Index<usize> for $name {
@@ -1948,6 +1951,20 @@ impl ArgumentOrRedirectionVariant {
     pub fn try_source_range(&self) -> Option<SourceRange> {
         self.embedded_node().try_source_range()
     }
+
+    fn as_argument(&self) -> Option<&Argument> {
+        match self {
+            ArgumentOrRedirectionVariant::Argument(node) => Some(node),
+            _ => None,
+        }
+    }
+    fn as_redirection(&self) -> Option<&Redirection> {
+        match self {
+            ArgumentOrRedirectionVariant::Redirection(redirection) => Some(redirection),
+            _ => None,
+        }
+    }
+
     fn embedded_node(&self) -> &dyn NodeMut {
         match self {
             ArgumentOrRedirectionVariant::Argument(node) => node,
@@ -2044,6 +2061,38 @@ impl StatementVariant {
     pub fn try_source_range(&self) -> Option<SourceRange> {
         self.embedded_node().try_source_range()
     }
+
+    pub fn as_not_statement(&self) -> Option<&NotStatement> {
+        match self {
+            StatementVariant::NotStatement(node) => Some(node),
+            _ => None,
+        }
+    }
+    pub fn as_block_statement(&self) -> Option<&BlockStatement> {
+        match self {
+            StatementVariant::BlockStatement(node) => Some(node),
+            _ => None,
+        }
+    }
+    pub fn as_if_statement(&self) -> Option<&IfStatement> {
+        match self {
+            StatementVariant::IfStatement(node) => Some(node),
+            _ => None,
+        }
+    }
+    pub fn as_switch_statement(&self) -> Option<&SwitchStatement> {
+        match self {
+            StatementVariant::SwitchStatement(node) => Some(node),
+            _ => None,
+        }
+    }
+    pub fn as_decorated_statement(&self) -> Option<&DecoratedStatement> {
+        match self {
+            StatementVariant::DecoratedStatement(node) => Some(node),
+            _ => None,
+        }
+    }
+
     fn embedded_node(&self) -> &dyn NodeMut {
         match self {
             StatementVariant::None => panic!("cannot visit null statement"),
@@ -2131,6 +2180,32 @@ impl BlockStatementHeaderVariant {
     pub fn try_source_range(&self) -> Option<SourceRange> {
         self.embedded_node().try_source_range()
     }
+
+    pub fn as_for_header(&self) -> Option<&ForHeader> {
+        match self {
+            BlockStatementHeaderVariant::ForHeader(node) => Some(node),
+            _ => None,
+        }
+    }
+    pub fn as_while_header(&self) -> Option<&WhileHeader> {
+        match self {
+            BlockStatementHeaderVariant::WhileHeader(node) => Some(node),
+            _ => None,
+        }
+    }
+    pub fn as_function_header(&self) -> Option<&FunctionHeader> {
+        match self {
+            BlockStatementHeaderVariant::FunctionHeader(node) => Some(node),
+            _ => None,
+        }
+    }
+    pub fn as_begin_header(&self) -> Option<&BeginHeader> {
+        match self {
+            BlockStatementHeaderVariant::BeginHeader(node) => Some(node),
+            _ => None,
+        }
+    }
+
     fn embedded_node(&self) -> &dyn NodeMut {
         match self {
             BlockStatementHeaderVariant::None => panic!("cannot visit null block header"),
@@ -2325,7 +2400,7 @@ impl Ast {
 
     /// \return a textual representation of the tree.
     /// Pass the original source as \p orig.
-    fn dump(&self, orig: &wstr) -> WString {
+    pub fn dump(&self, orig: &wstr) -> WString {
         let mut result = WString::new();
 
         for node in self.walk() {
@@ -4417,6 +4492,11 @@ unsafe impl ExternType for Ast {
     type Kind = cxx::kind::Opaque;
 }
 
+unsafe impl ExternType for DecoratedStatement {
+    type Id = type_id!("DecoratedStatement");
+    type Kind = cxx::kind::Opaque;
+}
+
 impl Ast {
     fn top_ffi(&self) -> Box<NodeFfi> {
         Box::new(NodeFfi::new(self.top.as_node()))

fish-rust/src/autoload.rs

@@ -0,0 +1,390 @@
+//! The classes responsible for autoloading functions and completions.
+
+use crate::common::{escape, ScopeGuard};
+use crate::env::Environment;
+use crate::io::IoChain;
+use crate::parser::Parser;
+use crate::wchar::{wstr, WString, L};
+use crate::wutil::{file_id_for_path, FileId, INVALID_FILE_ID};
+use lru::LruCache;
+use std::collections::{HashMap, HashSet};
+use std::num::NonZeroUsize;
+use std::time;
+
+/// autoload_t is a class that knows how to autoload .fish files from a list of directories. This
+/// is used by autoloading functions and completions. It maintains a file cache, which is
+/// responsible for potentially cached accesses of files, and then a list of files that have
+/// actually been autoloaded. A client may request a file to autoload given a command name, and may
+/// be returned a path which it is expected to source.
+/// autoload_t does not have any internal locks; it is the responsibility of the caller to lock
+/// it.
+#[derive(Default)]
+pub struct Autoload {
+    /// The environment variable whose paths we observe.
+    env_var_name: &'static wstr,
+
+    /// A map from command to the files we have autoloaded.
+    autoloaded_files: HashMap<WString, FileId>,
+
+    /// The list of commands that we are currently autoloading.
+    current_autoloading: HashSet<WString>,
+
+    /// The autoload cache.
+    /// This is a unique_ptr because want to change it if the value of our environment variable
+    /// changes. This is never null (but it may be a cache with no paths).
+    cache: Box<AutoloadFileCache>,
+}
+
+impl Autoload {
+    /// Construct an autoloader that loads from the paths given by \p env_var_name.
+    pub fn new(env_var_name: &'static wstr) -> Self {
+        Self {
+            env_var_name,
+            ..Default::default()
+        }
+    }
+
+    /// Given a command, get a path to autoload.
+    /// For example, if the environment variable is 'fish_function_path' and the command is 'foo',
+    /// this will look for a file 'foo.fish' in one of the directories given by fish_function_path.
+    /// If there is no such file, OR if the file has been previously resolved and is now unchanged,
+    /// this will return none. But if the file is either new or changed, this will return the path.
+    /// After returning a path, the command is marked in-progress until the caller calls
+    /// mark_autoload_finished() with the same command. Note this does not actually execute any
+    /// code; it is the caller's responsibility to load the file.
+    pub fn resolve_command(&mut self, cmd: &wstr, env: &dyn Environment) -> Option<WString> {
+        if let Some(var) = env.get(self.env_var_name) {
+            self.resolve_command_impl(cmd, var.as_list())
+        } else {
+            self.resolve_command_impl(cmd, &[])
+        }
+    }
+
+    /// Helper to actually perform an autoload.
+    /// This is a static function because it executes fish script, and so must be called without
+    /// holding any particular locks.
+    pub fn perform_autoload(path: &wstr, parser: &Parser) {
+        // We do the useful part of what exec_subshell does ourselves
+        // - we source the file.
+        // We don't create a buffer or check ifs or create a read_limit
+
+        let script_source = L!("source ").to_owned() + &escape(path)[..];
+        let prev_statuses = parser.get_last_statuses();
+        let _put_back = ScopeGuard::new((), |()| parser.set_last_statuses(prev_statuses));
+        parser.eval(&script_source, &IoChain::new());
+    }
+
+    /// Mark that a command previously returned from path_to_autoload is finished autoloading.
+    pub fn mark_autoload_finished(&mut self, cmd: &wstr) {
+        let removed = self.current_autoloading.remove(cmd);
+        assert!(removed, "cmd was not being autoloaded");
+    }
+
+    /// \return whether a command is currently being autoloaded.
+    pub fn autoload_in_progress(&self, cmd: &wstr) -> bool {
+        self.current_autoloading.contains(cmd)
+    }
+
+    /// \return whether a command could potentially be autoloaded.
+    /// This does not actually mark the command as being autoloaded.
+    pub fn can_autoload(&mut self, cmd: &wstr) -> bool {
+        self.cache.check(cmd, true /* allow stale */).is_some()
+    }
+
+    /// \return whether autoloading has been attempted for a command.
+    pub fn has_attempted_autoload(&self, cmd: &wstr) -> bool {
+        self.cache.is_cached(cmd)
+    }
+
+    /// \return the names of all commands that have been autoloaded. Note this includes "in-flight"
+    /// commands.
+    pub fn get_autoloaded_commands(&self) -> Vec<WString> {
+        let mut result = Vec::with_capacity(self.autoloaded_files.len());
+        for k in self.autoloaded_files.keys() {
+            result.push(k.to_owned());
+        }
+        // Sort the output to make it easier to test.
+        result.sort();
+        result
+    }
+
+    /// Mark that all autoloaded files have been forgotten.
+    /// Future calls to path_to_autoload() will return previously-returned paths.
+    pub fn clear(&mut self) {
+        // Note there is no reason to invalidate the cache here.
+        self.autoloaded_files.clear();
+    }
+
+    /// Invalidate any underlying cache.
+    /// This is exposed for testing.
+    fn invalidate_cache(&mut self) {
+        self.cache = Box::new(AutoloadFileCache::with_dirs(self.cache.dirs().to_owned()));
+    }
+
+    /// Like resolve_autoload(), but accepts the paths directly.
+    /// This is exposed for testing.
+    fn resolve_command_impl(&mut self, cmd: &wstr, paths: &[WString]) -> Option<WString> {
+        // Are we currently in the process of autoloading this?
+        if self.current_autoloading.contains(cmd) {
+            return None;
+        }
+
+        // Check to see if our paths have changed. If so, replace our cache.
+        // Note we don't have to modify autoloadable_files_. We'll naturally detect if those have
+        // changed when we query the cache.
+        if paths != self.cache.dirs() {
+            self.cache = Box::new(AutoloadFileCache::with_dirs(paths.to_owned()));
+        }
+
+        // Do we have an entry to load?
+        let Some(file) = self.cache.check(cmd, false) else {
+            return None;
+        };
+
+        // Is this file the same as what we previously autoloaded?
+        if let Some(loaded_file) = self.autoloaded_files.get(cmd) {
+            if *loaded_file == file.file_id {
+                // The file has been autoloaded and is unchanged.
+                return None;
+            }
+        }
+
+        // We're going to (tell our caller to) autoload this command.
+        self.current_autoloading.insert(cmd.to_owned());
+        self.autoloaded_files.insert(cmd.to_owned(), file.file_id);
+        Some(file.path)
+    }
+}
+
+/// The time before we'll recheck an autoloaded file.
+const AUTOLOAD_STALENESS_INTERVALL: u64 = 15;
+
+/// Represents a file that we might want to autoload.
+#[derive(Clone)]
+struct AutoloadableFile {
+    /// The path to the file.
+    path: WString,
+    /// The metadata for the file.
+    file_id: FileId,
+}
+
+// A timestamp is a monotonic point in time.
+type Timestamp = time::Instant;
+type MissesLruCache = LruCache<WString, Timestamp>;
+
+struct KnownFile {
+    file: AutoloadableFile,
+    last_checked: Timestamp,
+}
+
+/// Class representing a cache of files that may be autoloaded.
+/// This is responsible for performing cached accesses to a set of paths.
+struct AutoloadFileCache {
+    /// The directories from which to load.
+    dirs: Vec<WString>,
+
+    /// Our LRU cache of checks that were misses.
+    /// The key is the command, the  value is the time of the check.
+    misses_cache: MissesLruCache,
+
+    /// The set of files that we have returned to the caller, along with the time of the check.
+    /// The key is the command (not the path).
+    known_files: HashMap<WString, KnownFile>,
+}
+
+impl Default for AutoloadFileCache {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AutoloadFileCache {
+    /// Initialize with a set of directories.
+    fn with_dirs(dirs: Vec<WString>) -> Self {
+        Self {
+            dirs,
+            misses_cache: MissesLruCache::new(NonZeroUsize::new(1024).unwrap()),
+            known_files: HashMap::new(),
+        }
+    }
+
+    /// Initialize with empty directories.
+    fn new() -> Self {
+        Self::with_dirs(vec![])
+    }
+
+    /// \return the directories.
+    fn dirs(&self) -> &[WString] {
+        &self.dirs
+    }
+
+    /// Check if a command \p cmd can be loaded.
+    /// If \p allow_stale is true, allow stale entries; otherwise discard them.
+    /// This returns an autoloadable file, or none() if there is no such file.
+    fn check(&mut self, cmd: &wstr, allow_stale: bool) -> Option<AutoloadableFile> {
+        // Check hits.
+        if let Some(value) = self.known_files.get(cmd) {
+            if allow_stale || Self::is_fresh(value.last_checked, Self::current_timestamp()) {
+                // Re-use this cached hit.
+                return Some(value.file.clone());
+            }
+            // The file is stale, remove it.
+            self.known_files.remove(cmd);
+        }
+
+        // Check misses.
+        if let Some(miss) = self.misses_cache.get(cmd) {
+            if allow_stale || Self::is_fresh(*miss, Self::current_timestamp()) {
+                // Re-use this cached miss.
+                return None;
+            }
+            // The miss is stale, remove it.
+            self.misses_cache.pop(cmd);
+        }
+
+        // We couldn't satisfy this request from the cache. Hit the disk.
+        let file = self.locate_file(cmd);
+        if let Some(file) = file.as_ref() {
+            let old_value = self.known_files.insert(
+                cmd.to_owned(),
+                KnownFile {
+                    file: file.clone(),
+                    last_checked: Self::current_timestamp(),
+                },
+            );
+            assert!(
+                old_value.is_none(),
+                "Known files cache should not have contained this cmd"
+            );
+        } else {
+            let old_value = self
+                .misses_cache
+                .put(cmd.to_owned(), Self::current_timestamp());
+            assert!(
+                old_value.is_none(),
+                "Misses cache should not have contained this cmd",
+            );
+        }
+        file
+    }
+
+    /// \return true if a command is cached (either as a hit or miss).
+    fn is_cached(&self, cmd: &wstr) -> bool {
+        self.known_files.contains_key(cmd) || self.misses_cache.contains(cmd)
+    }
+
+    /// \return the current timestamp.
+    fn current_timestamp() -> Timestamp {
+        Timestamp::now()
+    }
+
+    /// \return whether a timestamp is fresh enough to use.
+    fn is_fresh(then: Timestamp, now: Timestamp) -> bool {
+        let seconds = now.duration_since(then).as_secs();
+        seconds < AUTOLOAD_STALENESS_INTERVALL
+    }
+
+    /// Attempt to find an autoloadable file by searching our path list for a given command.
+    /// \return the file, or none() if none.
+    fn locate_file(&self, cmd: &wstr) -> Option<AutoloadableFile> {
+        // If the command is empty or starts with NULL (i.e. is empty as a path)
+        // we'd try to source the *directory*, which exists.
+        // So instead ignore these here.
+        if cmd.is_empty() {
+            return None;
+        }
+        if cmd.as_char_slice()[0] == '\0' {
+            return None;
+        }
+        // Re-use the storage for path.
+        let mut path;
+        for dir in self.dirs() {
+            // Construct the path as dir/cmd.fish
+            path = dir.to_owned();
+            path.push('/');
+            path.push_utfstr(cmd);
+            path.push_str(".fish");
+
+            let file_id = file_id_for_path(&path);
+            if file_id != INVALID_FILE_ID {
+                // Found it.
+                return Some(AutoloadableFile { path, file_id });
+            }
+        }
+        None
+    }
+}
+
+use crate::ffi_tests::add_test;
+#[widestring_suffix::widestrs]
+add_test!("test_autoload", || {
+    use crate::common::{charptr2wcstring, wcs2zstring, write_loop};
+    use crate::fds::wopen_cloexec;
+    use crate::wutil::sprintf;
+    use libc::{O_CREAT, O_RDWR};
+
+    macro_rules! run {
+        ( $fmt:expr $(, $arg:expr )* $(,)? ) => {
+             let cmd = wcs2zstring(&sprintf!($fmt $(, $arg)*));
+             let status = unsafe { libc::system(cmd.as_ptr()) };
+             assert!(status == 0);
+        };
+    }
+
+    fn touch_file(path: &wstr) {
+        let fd = wopen_cloexec(path, O_RDWR | O_CREAT, 0o666);
+        assert!(fd >= 0);
+        write_loop(&fd, "Hello".as_bytes()).unwrap();
+        unsafe { libc::close(fd) };
+    }
+
+    let mut t1 = "/tmp/fish_test_autoload.XXXXXX\0".as_bytes().to_vec();
+    let p1 = charptr2wcstring(unsafe { libc::mkdtemp(t1.as_mut_ptr().cast()) });
+    let mut t2 = "/tmp/fish_test_autoload.XXXXXX\0".as_bytes().to_vec();
+    let p2 = charptr2wcstring(unsafe { libc::mkdtemp(t2.as_mut_ptr().cast()) });
+
+    let paths = &[p1.clone(), p2.clone()];
+    let mut autoload = Autoload::new("test_var"L);
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_none());
+    assert!(autoload.resolve_command_impl("nothing"L, paths).is_none());
+    assert!(autoload.get_autoloaded_commands().is_empty());
+
+    run!("touch %ls/file1.fish", p1);
+    run!("touch %ls/file2.fish", p2);
+    autoload.invalidate_cache();
+
+    assert!(!autoload.autoload_in_progress("file1"L));
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_some());
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_none());
+    assert!(autoload.autoload_in_progress("file1"L));
+    assert!(autoload.get_autoloaded_commands() == vec!["file1"L]);
+    autoload.mark_autoload_finished("file1"L);
+    assert!(!autoload.autoload_in_progress("file1"L));
+    assert!(autoload.get_autoloaded_commands() == vec!["file1"L]);
+
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_none());
+    assert!(autoload.resolve_command_impl("nothing"L, paths).is_none());
+    assert!(autoload.resolve_command_impl("file2"L, paths).is_some());
+    assert!(autoload.resolve_command_impl("file2"L, paths).is_none());
+    autoload.mark_autoload_finished("file2"L);
+    assert!(autoload.resolve_command_impl("file2"L, paths).is_none());
+    assert!((autoload.get_autoloaded_commands() == vec!["file1"L, "file2"L]));
+
+    autoload.clear();
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_some());
+    autoload.mark_autoload_finished("file1"L);
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_none());
+    assert!(autoload.resolve_command_impl("nothing"L, paths).is_none());
+    assert!(autoload.resolve_command_impl("file2"L, paths).is_some());
+    assert!(autoload.resolve_command_impl("file2"L, paths).is_none());
+    autoload.mark_autoload_finished("file2"L);
+
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_none());
+    touch_file(&sprintf!("%ls/file1.fish", p1));
+    autoload.invalidate_cache();
+    assert!(autoload.resolve_command_impl("file1"L, paths).is_some());
+    autoload.mark_autoload_finished("file1"L);
+
+    run!("rm -Rf %ls"L, p1);
+    run!("rm -Rf %ls"L, p2);
+});

fish-rust/src/builtins/abbr.rs

@@ -1,8 +1,9 @@
 use super::prelude::*;
 use crate::abbrs::{self, Abbreviation, Position};
 use crate::common::{escape, escape_string, valid_func_name, EscapeStringStyle};
-use crate::env::status::{ENV_NOT_FOUND, ENV_OK};
-use crate::env::EnvMode;
+use crate::env::{EnvMode, EnvStackSetResult};
+use crate::io::IoStreams;
+use crate::parser::Parser;
 use crate::re::{regex_make_anchored, to_boxed_chars};
 use pcre2::utf32::{Regex, RegexBuilder};
 
@@ -391,7 +392,7 @@ fn abbr_add(opts: &Options, streams: &mut IoStreams) -> Option<c_int> {
 }
 
 // Erase the named abbreviations.
-fn abbr_erase(opts: &Options, parser: &mut Parser) -> Option<c_int> {
+fn abbr_erase(opts: &Options, parser: &Parser) -> Option<c_int> {
     if opts.args.is_empty() {
         // This has historically been a silent failure.
         return STATUS_CMD_ERROR;
@@ -402,15 +403,15 @@ fn abbr_erase(opts: &Options, parser: &mut Parser) -> Option<c_int> {
         let mut result = STATUS_CMD_OK;
         for arg in &opts.args {
             if !abbrs.erase(arg) {
-                result = Some(ENV_NOT_FOUND);
+                result = Some(EnvStackSetResult::ENV_NOT_FOUND.into());
             }
             // Erase the old uvar - this makes `abbr -e` work.
             let esc_src = escape(arg);
             if !esc_src.is_empty() {
                 let var_name = WString::from_str("_fish_abbr_") + esc_src.as_utfstr();
-                let ret = parser.remove_var(&var_name, EnvMode::UNIVERSAL.into());
+                let ret = parser.vars().remove(&var_name, EnvMode::UNIVERSAL);
 
-                if ret == autocxx::c_int(ENV_OK) {
+                if ret == EnvStackSetResult::ENV_OK {
                     result = STATUS_CMD_OK
                 };
             }
@@ -419,7 +420,7 @@ fn abbr_erase(opts: &Options, parser: &mut Parser) -> Option<c_int> {
     })
 }
 
-pub fn abbr(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn abbr(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let mut argv_read = Vec::with_capacity(argv.len());
     argv_read.extend_from_slice(argv);
 
@@ -539,7 +540,7 @@ pub fn abbr(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) ->
                     cmd,
                     argv_read[w.woptind - 1]
                 ));
-                builtin_print_error_trailer(parser, streams, cmd);
+                builtin_print_error_trailer(parser, streams.err, cmd);
             }
             'h' => {
                 builtin_print_help(parser, streams, cmd);

fish-rust/src/builtins/argparse.rs

@@ -3,6 +3,7 @@ use std::collections::HashMap;
 use super::prelude::*;
 
 use crate::env::{EnvMode, EnvStack};
+use crate::exec::exec_subshell;
 use crate::wcstringutil::split_string;
 use crate::wutil::fish_iswalnum;
 
@@ -81,29 +82,6 @@ const LONG_OPTIONS: &[woption] = &[
     wopt(L!("max-args"), woption_argument_t::required_argument, 'X'),
 ];
 
-fn exec_subshell(
-    cmd: &wstr,
-    parser: &mut Parser,
-    outputs: &mut Vec<WString>,
-    apply_exit_status: bool,
-) -> Option<c_int> {
-    use crate::ffi::exec_subshell_ffi;
-    use crate::wchar_ffi::wcstring_list_ffi_t;
-
-    let mut cmd_output: cxx::UniquePtr<wcstring_list_ffi_t> = wcstring_list_ffi_t::create();
-    let retval = Some(
-        exec_subshell_ffi(
-            cmd.to_ffi().as_ref().unwrap(),
-            parser.pin(),
-            cmd_output.pin_mut(),
-            apply_exit_status,
-        )
-        .into(),
-    );
-    *outputs = cmd_output.as_mut().unwrap().from_ffi();
-    retval
-}
-
 // Check if any pair of mutually exclusive options was seen. Note that since every option must have
 // a short name we only need to check those.
 fn check_for_mutually_exclusive_flags(
@@ -499,7 +477,7 @@ fn parse_cmd_opts<'args>(
     optind: &mut usize,
     argc: usize,
     args: &mut [&'args wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Option<c_int> {
     let cmd = args[0];
@@ -583,7 +561,7 @@ fn parse_cmd_opts<'args>(
         // If no name has been given, we default to the function name.
         // If any error happens, the backtrace will show which argparse it was.
         opts.name = parser
-            .get_func_name(1)
+            .get_function_name(1)
             .unwrap_or_else(|| L!("argparse").to_owned());
     }
 
@@ -624,7 +602,7 @@ fn populate_option_strings<'args>(
 }
 
 fn validate_arg<'opts>(
-    parser: &mut Parser,
+    parser: &Parser,
     opts_name: &wstr,
     opt_spec: &mut OptionSpec<'opts>,
     is_long_flag: bool,
@@ -636,7 +614,7 @@ fn validate_arg<'opts>(
         return STATUS_CMD_OK;
     }
 
-    let vars = parser.get_vars();
+    let vars = parser.vars();
     vars.push(true /* new_scope */);
 
     let env_mode = EnvMode::LOCAL | EnvMode::EXPORT;
@@ -659,13 +637,19 @@ fn validate_arg<'opts>(
 
     let mut cmd_output = Vec::new();
 
-    let retval = exec_subshell(opt_spec.validation_command, parser, &mut cmd_output, false);
+    let retval = exec_subshell(
+        opt_spec.validation_command,
+        parser,
+        Some(&mut cmd_output),
+        false,
+    );
 
     for output in cmd_output {
-        streams.err.appendln(output);
+        streams.err.append(output);
+        streams.err.append_char('\n');
     }
     vars.pop();
-    return retval;
+    Some(retval)
 }
 
 /// \return whether the option 'opt' is an implicit integer option.
@@ -681,7 +665,7 @@ fn is_implicit_int(opts: &ArgParseCmdOpts, val: &wstr) -> bool {
 
 // Store this value under the implicit int option.
 fn validate_and_store_implicit_int<'args>(
-    parser: &mut Parser,
+    parser: &Parser,
     opts: &mut ArgParseCmdOpts<'args>,
     val: &'args wstr,
     w: &mut wgetopter_t,
@@ -705,7 +689,7 @@ fn validate_and_store_implicit_int<'args>(
 }
 
 fn handle_flag<'args>(
-    parser: &mut Parser,
+    parser: &Parser,
     opts: &mut ArgParseCmdOpts<'args>,
     opt: char,
     is_long_flag: bool,
@@ -754,7 +738,7 @@ fn handle_flag<'args>(
 }
 
 fn argparse_parse_flags<'args>(
-    parser: &mut Parser,
+    parser: &Parser,
     opts: &mut ArgParseCmdOpts<'args>,
     argc: usize,
     args: &mut [&'args wstr],
@@ -855,7 +839,7 @@ fn argparse_parse_args<'args>(
     opts: &mut ArgParseCmdOpts<'args>,
     args: &mut [&'args wstr],
     argc: usize,
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Option<c_int> {
     if argc <= 1 {
@@ -943,7 +927,7 @@ fn set_argparse_result_vars(vars: &EnvStack, opts: &ArgParseCmdOpts) {
 /// an external command also means its output has to be in a form that can be eval'd. Because our
 /// version is a builtin it can directly set variables local to the current scope (e.g., a
 /// function). It doesn't need to write anything to stdout that then needs to be eval'd.
-pub fn argparse(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn argparse(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
     let argc = args.len();
 
@@ -954,7 +938,7 @@ pub fn argparse(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]
         // This is an error in argparse usage, so we append the error trailer with a stack trace.
         // The other errors are an error in using *the command* that is using argparse,
         // so our help doesn't apply.
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return retval;
     }
 
@@ -987,7 +971,7 @@ pub fn argparse(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]
         return retval;
     }
 
-    set_argparse_result_vars(&parser.get_vars(), &opts);
+    set_argparse_result_vars(parser.vars(), &opts);
 
     return retval;
 }

fish-rust/src/builtins/bg.rs

@@ -1,53 +1,52 @@
 // Implementation of the bg builtin.
 
-use std::pin::Pin;
+use libc::pid_t;
 
 use super::prelude::*;
 
 /// Helper function for builtin_bg().
 fn send_to_bg(
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
     cmd: &wstr,
     job_pos: usize,
 ) -> Option<c_int> {
-    let job = parser.get_jobs()[job_pos]
-        .as_ref()
-        .expect("job_pos must be valid");
-    if !job.wants_job_control() {
-        let err = wgettext_fmt!(
-            "%ls: Can't put job %d, '%ls' to background because it is not under job control\n",
-            cmd,
-            job.job_id().0,
-            job.command().from_ffi()
-        );
-        ffi::builtin_print_help(
-            parser.pin(),
-            streams.ffi_ref(),
-            c_str!(cmd),
-            err.to_ffi().as_ref()?,
-        );
-        return STATUS_CMD_ERROR;
+    {
+        let jobs = parser.jobs();
+        if !jobs[job_pos].wants_job_control() {
+            let err = {
+                let job = &jobs[job_pos];
+                wgettext_fmt!(
+                "%ls: Can't put job %s, '%ls' to background because it is not under job control\n",
+                cmd,
+                job.job_id().to_wstring(),
+                job.command()
+            )
+            };
+            builtin_print_help_error(parser, streams, cmd, &err);
+            return STATUS_CMD_ERROR;
+        }
+
+        let job = &jobs[job_pos];
+        streams.err.append(wgettext_fmt!(
+            "Send job %s '%ls' to background\n",
+            job.job_id().to_wstring(),
+            job.command()
+        ));
+
+        job.group().set_is_foreground(false);
+
+        if !job.resume() {
+            return STATUS_CMD_ERROR;
+        }
     }
-
-    streams.err.append(wgettext_fmt!(
-        "Send job %d '%ls' to background\n",
-        job.job_id().0,
-        job.command().from_ffi()
-    ));
-
-    job.get_job_group().set_is_foreground(false);
-
-    if !job.ffi_resume() {
-        return STATUS_CMD_ERROR;
-    }
-    parser.pin().job_promote_at(job_pos);
+    parser.job_promote_at(job_pos);
 
     return STATUS_CMD_OK;
 }
 
 /// Builtin for putting a job in the background.
-pub fn bg(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn bg(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let opts = match HelpOnlyCmdOpts::parse(args, parser, streams) {
         Ok(opts) => opts,
         Err(err @ Some(_)) if err != STATUS_CMD_OK => return err,
@@ -62,14 +61,11 @@ pub fn bg(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
 
     if opts.optind == args.len() {
         // No jobs were specified so use the most recent (i.e., last) job.
-        let jobs = parser.get_jobs();
-        let job_pos = jobs.iter().position(|job| {
-            if let Some(job) = job.as_ref() {
-                return job.is_stopped() && job.wants_job_control() && !job.is_completed();
-            }
-
-            false
-        });
+        let job_pos = {
+            let jobs = parser.jobs();
+            jobs.iter()
+                .position(|job| job.is_stopped() && job.wants_job_control() && !job.is_completed())
+        };
 
         let Some(job_pos) = job_pos else {
             streams
@@ -82,25 +78,23 @@ pub fn bg(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
     }
 
     // The user specified at least one job to be backgrounded.
-    let mut pids: Vec<libc::pid_t> = Vec::new();
-
     // If one argument is not a valid pid (i.e. integer >= 0), fail without backgrounding anything,
     // but still print errors for all of them.
     let mut retval: Option<i32> = STATUS_CMD_OK;
-    for arg in &args[opts.optind..] {
-        let pid = fish_wcstoi(arg);
-        #[allow(clippy::unnecessary_unwrap)]
-        if pid.is_err() || pid.unwrap() < 0 {
-            streams.err.append(wgettext_fmt!(
-                "%ls: '%ls' is not a valid job specifier\n",
-                cmd,
-                arg
-            ));
-            retval = STATUS_INVALID_ARGS;
-        } else {
-            pids.push(pid.unwrap());
-        }
-    }
+    let pids: Vec<pid_t> = args[opts.optind..]
+        .iter()
+        .map(|arg| {
+            fish_wcstoi(arg).unwrap_or_else(|_| {
+                streams.err.append(wgettext_fmt!(
+                    "%ls: '%ls' is not a valid job specifier\n",
+                    cmd,
+                    arg
+                ));
+                retval = STATUS_INVALID_ARGS;
+                0
+            })
+        })
+        .collect();
 
     if retval != STATUS_CMD_OK {
         return retval;
@@ -109,14 +103,7 @@ pub fn bg(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
     // Background all existing jobs that match the pids.
     // Non-existent jobs aren't an error, but information about them is useful.
     for pid in pids {
-        let mut job_pos = 0;
-        let job = unsafe {
-            parser
-                .job_get_from_pid1(autocxx::c_int(pid), Pin::new(&mut job_pos))
-                .as_ref()
-        };
-
-        if job.is_some() {
+        if let Some((job_pos, _job)) = parser.job_get_with_index_from_pid(pid) {
             send_to_bg(parser, streams, cmd, job_pos);
         } else {
             streams

fish-rust/src/builtins/bind.rs

@@ -0,0 +1,28 @@
+//! Implementation of the bind builtin.
+
+use super::prelude::*;
+
+const BIND_INSERT: c_int = 0;
+const BIND_ERASE: c_int = 1;
+const BIND_KEY_NAMES: c_int = 2;
+const BIND_FUNCTION_NAMES: c_int = 3;
+
+struct Options {
+    all: bool,
+    bind_mode_given: bool,
+    list_modes: bool,
+    print_help: bool,
+    silent: bool,
+    use_terminfo: bool,
+    have_user: bool,
+    user: bool,
+    have_preset: bool,
+    preset: bool,
+    mode: c_int,
+    bind_mode: &'static wstr,
+    sets_bind_mode: &'static wstr,
+}
+
+pub fn bind(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    run_builtin_ffi(crate::ffi::builtin_bind, parser, streams, args)
+}

fish-rust/src/builtins/block.rs

@@ -1,3 +1,5 @@
+use std::sync::atomic::Ordering;
+
 // Implementation of the block builtin.
 use super::prelude::*;
 
@@ -23,7 +25,7 @@ struct Options {
 
 fn parse_options(
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Result<(Options, usize), Option<c_int>> {
     let cmd = args[0];
@@ -71,7 +73,7 @@ fn parse_options(
 }
 
 /// The block builtin, used for temporarily blocking events.
-pub fn block(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn block(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
 
     let opts = match parse_options(args, parser, streams) {
@@ -94,49 +96,52 @@ pub fn block(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -
             return STATUS_INVALID_ARGS;
         }
 
-        if parser.ffi_global_event_blocks() == 0 {
+        if parser.global_event_blocks.load(Ordering::Relaxed) == 0 {
             streams
                 .err
                 .append(wgettext_fmt!("%ls: No blocks defined\n", cmd));
             return STATUS_CMD_ERROR;
         }
-        parser.pin().ffi_decr_global_event_blocks();
+        parser.global_event_blocks.fetch_sub(1, Ordering::Relaxed);
         return STATUS_CMD_OK;
     }
 
     let mut block_idx = 0;
-    let mut block = unsafe { parser.pin().block_at_index1(block_idx).as_mut() };
+    let have_block = {
+        let mut block = parser.block_at_index(block_idx);
 
-    match opts.scope {
-        Scope::Local => {
-            // If this is the outermost block, then we're global
-            if block_idx + 1 >= parser.ffi_blocks_size() {
+        match opts.scope {
+            Scope::Local => {
+                // If this is the outermost block, then we're global
+                if block_idx + 1 >= parser.blocks_size() {
+                    block = None;
+                }
+            }
+            Scope::Global => {
                 block = None;
             }
-        }
-        Scope::Global => {
-            block = None;
-        }
-        Scope::Unset => {
-            loop {
-                block = if let Some(block) = block.as_mut() {
-                    if !block.is_function_call() {
+            Scope::Unset => {
+                loop {
+                    block = if let Some(block) = block.as_mut() {
+                        if !block.is_function_call() {
+                            break;
+                        }
+                        // Set it in function scope
+                        block_idx += 1;
+                        parser.block_at_index(block_idx)
+                    } else {
                         break;
                     }
-                    // Set it in function scope
-                    block_idx += 1;
-                    unsafe { parser.pin().block_at_index1(block_idx).as_mut() }
-                } else {
-                    break;
                 }
             }
         }
-    }
+        block.is_some()
+    };
 
-    if let Some(block) = block.as_mut() {
-        block.pin().ffi_incr_event_blocks();
+    if have_block {
+        parser.block_at_index_mut(block_idx).unwrap().event_blocks += 1;
     } else {
-        parser.pin().ffi_incr_global_event_blocks();
+        parser.global_event_blocks.fetch_add(1, Ordering::Relaxed);
     }
 
     return STATUS_CMD_OK;

fish-rust/src/builtins/builtin.rs

@@ -1,5 +1,5 @@
 use super::prelude::*;
-use crate::ffi::{builtin_exists, builtin_get_names_ffi};
+use crate::builtins::shared::{builtin_exists, builtin_get_names};
 
 #[derive(Default)]
 struct builtin_cmd_opts_t {
@@ -7,11 +7,7 @@ struct builtin_cmd_opts_t {
     list_names: bool,
 }
 
-pub fn r#builtin(
-    parser: &mut Parser,
-    streams: &mut IoStreams,
-    argv: &mut [&wstr],
-) -> Option<c_int> {
+pub fn r#builtin(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
     let argc = argv.len();
     let print_hints = false;
@@ -67,7 +63,7 @@ pub fn r#builtin(
     if opts.query {
         let optind = w.woptind;
         for arg in argv.iter().take(argc).skip(optind) {
-            if builtin_exists(&arg.to_ffi()) {
+            if builtin_exists(arg) {
                 return STATUS_CMD_OK;
             }
         }
@@ -76,7 +72,7 @@ pub fn r#builtin(
 
     if opts.list_names {
         // List is guaranteed to be sorted by name.
-        let names: Vec<WString> = builtin_get_names_ffi().from_ffi();
+        let names = builtin_get_names();
         for name in names {
             streams.out.appendln(name);
         }

fish-rust/src/builtins/cd.rs

@@ -9,10 +9,11 @@ use crate::{
 };
 use errno::{self, Errno};
 use libc::{fchdir, EACCES, ELOOP, ENOENT, ENOTDIR, EPERM, O_RDONLY};
+use std::sync::Arc;
 
 // The cd builtin. Changes the current directory to the one specified or to $HOME if none is
 // specified. The directory can be relative to any directory in the CDPATH variable.
-pub fn cd(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn cd(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
 
     let opts = match HelpOnlyCmdOpts::parse(args, parser, streams) {
@@ -26,7 +27,7 @@ pub fn cd(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
         return STATUS_CMD_OK;
     }
 
-    let vars = parser.get_vars();
+    let vars = parser.vars();
     let tmpstr;
 
     let dir_in: &wstr = if args.len() > opts.optind {
@@ -54,14 +55,14 @@ pub fn cd(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
             dir_in
         ));
         if !parser.is_interactive() {
-            streams.err.append(parser.pin().current_line().from_ffi());
+            streams.err.append(parser.current_line());
         };
         return STATUS_CMD_ERROR;
     }
 
     let pwd = vars.get_pwd_slash();
 
-    let dirs = path_apply_cdpath(dir_in, &pwd, vars.as_ref());
+    let dirs = path_apply_cdpath(dir_in, &pwd, vars);
     if dirs.is_empty() {
         streams.err.append(wgettext_fmt!(
             "%ls: The directory '%ls' does not exist\n",
@@ -70,7 +71,7 @@ pub fn cd(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
         ));
 
         if !parser.is_interactive() {
-            streams.err.append(parser.pin().current_line().from_ffi());
+            streams.err.append(parser.current_line());
         }
 
         return STATUS_CMD_ERROR;
@@ -86,7 +87,7 @@ pub fn cd(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
         errno::set_errno(Errno(0));
 
         // We need to keep around the fd for this directory, in the parser.
-        let mut dir_fd = AutoCloseFd::new(wopen_cloexec(&norm_dir, O_RDONLY, 0));
+        let dir_fd = Arc::new(AutoCloseFd::new(wopen_cloexec(&norm_dir, O_RDONLY, 0)));
 
         if !(dir_fd.is_valid() && unsafe { fchdir(dir_fd.fd()) } == 0) {
             // Some errors we skip and only report if nothing worked.
@@ -113,13 +114,9 @@ pub fn cd(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
         }
 
         // Stash the fd for the cwd in the parser.
-        parser.pin().set_cwd_fd(autocxx::c_int(dir_fd.acquire()));
+        parser.libdata_mut().cwd_fd = Some(dir_fd);
 
-        parser.pin().set_var_and_fire(
-            &L!("PWD").to_ffi(),
-            EnvMode::EXPORT.bits() | EnvMode::GLOBAL.bits(),
-            norm_dir,
-        );
+        parser.set_var_and_fire(L!("PWD"), EnvMode::EXPORT | EnvMode::GLOBAL, vec![norm_dir]);
         return STATUS_CMD_OK;
     }
 
@@ -165,7 +162,7 @@ pub fn cd(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> O
     }
 
     if !parser.is_interactive() {
-        streams.err.append(parser.pin().current_line().from_ffi());
+        streams.err.append(parser.current_line());
     }
 
     return STATUS_CMD_ERROR;

fish-rust/src/builtins/command.rs

@@ -8,11 +8,7 @@ struct command_cmd_opts_t {
     find_path: bool,
 }
 
-pub fn r#command(
-    parser: &mut Parser,
-    streams: &mut IoStreams,
-    argv: &mut [&wstr],
-) -> Option<c_int> {
+pub fn r#command(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
     let argc = argv.len();
     let print_hints = false;
@@ -63,14 +59,13 @@ pub fn r#command(
     let optind = w.woptind;
     for arg in argv.iter().take(argc).skip(optind) {
         let paths = if opts.all {
-            path_get_paths(arg, &*parser.get_vars())
+            path_get_paths(arg, parser.vars())
         } else {
-            match path_get_path(arg, &*parser.get_vars()) {
+            match path_get_path(arg, parser.vars()) {
                 Some(p) => vec![p],
                 None => vec![],
             }
         };
-
         for path in paths.iter() {
             res = true;
             if opts.quiet {

fish-rust/src/builtins/commandline.rs

@@ -0,0 +1,5 @@
+use super::prelude::*;
+
+pub fn commandline(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    run_builtin_ffi(crate::ffi::builtin_commandline, parser, streams, args)
+}

fish-rust/src/builtins/complete.rs

@@ -0,0 +1,644 @@
+use super::prelude::*;
+use crate::common::{
+    unescape_string, unescape_string_in_place, ScopeGuard, UnescapeFlags, UnescapeStringStyle,
+};
+use crate::complete::{complete_add_wrapper, complete_remove_wrapper, CompletionRequestOptions};
+use crate::ffi;
+use crate::highlight::colorize;
+use crate::highlight::highlight_shell;
+use crate::parse_constants::ParseErrorList;
+use crate::parse_util::parse_util_detect_errors_in_argument_list;
+use crate::parse_util::{parse_util_detect_errors, parse_util_token_extent};
+use crate::wcstringutil::string_suffixes_string;
+use crate::{
+    common::str2wcstring,
+    complete::{
+        complete_add, complete_print, complete_remove, complete_remove_all, CompleteFlags,
+        CompleteOptionType, CompletionMode,
+    },
+};
+use libc::STDOUT_FILENO;
+
+// builtin_complete_* are a set of rather silly looping functions that make sure that all the proper
+// combinations of complete_add or complete_remove get called. This is needed since complete allows
+// you to specify multiple switches on a single commandline, like 'complete -s a -s b -s c', but the
+// complete_add function only accepts one short switch and one long switch.
+
+/// Silly function.
+fn builtin_complete_add2(
+    cmd: &wstr,
+    cmd_is_path: bool,
+    short_opt: &wstr,
+    gnu_opts: &[WString],
+    old_opts: &[WString],
+    result_mode: CompletionMode,
+    condition: &[WString],
+    comp: &wstr,
+    desc: &wstr,
+    flags: CompleteFlags,
+) {
+    for short_opt in short_opt.chars() {
+        complete_add(
+            cmd.to_owned(),
+            cmd_is_path,
+            WString::from(&[short_opt][..]),
+            CompleteOptionType::Short,
+            result_mode,
+            condition.to_vec(),
+            comp.to_owned(),
+            desc.to_owned(),
+            flags,
+        );
+    }
+
+    for gnu_opt in gnu_opts {
+        complete_add(
+            cmd.to_owned(),
+            cmd_is_path,
+            gnu_opt.to_owned(),
+            CompleteOptionType::DoubleLong,
+            result_mode,
+            condition.to_vec(),
+            comp.to_owned(),
+            desc.to_owned(),
+            flags,
+        );
+    }
+
+    for old_opt in old_opts {
+        complete_add(
+            cmd.to_owned(),
+            cmd_is_path,
+            old_opt.to_owned(),
+            CompleteOptionType::SingleLong,
+            result_mode,
+            condition.to_vec(),
+            comp.to_owned(),
+            desc.to_owned(),
+            flags,
+        );
+    }
+
+    if old_opts.is_empty() && gnu_opts.is_empty() && short_opt.is_empty() {
+        complete_add(
+            cmd.to_owned(),
+            cmd_is_path,
+            WString::new(),
+            CompleteOptionType::ArgsOnly,
+            result_mode,
+            condition.to_vec(),
+            comp.to_owned(),
+            desc.to_owned(),
+            flags,
+        );
+    }
+}
+
+/// Sily function.
+fn builtin_complete_add(
+    cmds: &[WString],
+    paths: &[WString],
+    short_opt: &wstr,
+    gnu_opt: &[WString],
+    old_opt: &[WString],
+    result_mode: CompletionMode,
+    condition: &[WString],
+    comp: &wstr,
+    desc: &wstr,
+    flags: CompleteFlags,
+) {
+    for cmd in cmds {
+        builtin_complete_add2(
+            cmd,
+            false, /* not path */
+            short_opt,
+            gnu_opt,
+            old_opt,
+            result_mode,
+            condition,
+            comp,
+            desc,
+            flags,
+        );
+    }
+    for path in paths {
+        builtin_complete_add2(
+            path,
+            true, /* is path */
+            short_opt,
+            gnu_opt,
+            old_opt,
+            result_mode,
+            condition,
+            comp,
+            desc,
+            flags,
+        );
+    }
+}
+
+fn builtin_complete_remove_cmd(
+    cmd: &WString,
+    cmd_is_path: bool,
+    short_opt: &wstr,
+    gnu_opt: &[WString],
+    old_opt: &[WString],
+) {
+    let mut removed = false;
+    for s in short_opt.chars() {
+        complete_remove(
+            cmd.to_owned(),
+            cmd_is_path,
+            wstr::from_char_slice(&[s]),
+            CompleteOptionType::Short,
+        );
+        removed = true;
+    }
+
+    for opt in old_opt {
+        complete_remove(
+            cmd.to_owned(),
+            cmd_is_path,
+            opt,
+            CompleteOptionType::SingleLong,
+        );
+        removed = true;
+    }
+
+    for opt in gnu_opt {
+        complete_remove(
+            cmd.to_owned(),
+            cmd_is_path,
+            opt,
+            CompleteOptionType::DoubleLong,
+        );
+        removed = true;
+    }
+
+    if !removed {
+        // This means that all loops were empty.
+        complete_remove_all(cmd.to_owned(), cmd_is_path);
+    }
+}
+
+fn builtin_complete_remove(
+    cmds: &[WString],
+    paths: &[WString],
+    short_opt: &wstr,
+    gnu_opt: &[WString],
+    old_opt: &[WString],
+) {
+    for cmd in cmds {
+        builtin_complete_remove_cmd(cmd, false /* not path */, short_opt, gnu_opt, old_opt);
+    }
+
+    for path in paths {
+        builtin_complete_remove_cmd(path, true /* is path */, short_opt, gnu_opt, old_opt);
+    }
+}
+
+fn builtin_complete_print(cmd: &wstr, streams: &mut IoStreams, parser: &Parser) {
+    let repr = complete_print(cmd);
+
+    // colorize if interactive
+    if !streams.out_is_redirected && unsafe { libc::isatty(STDOUT_FILENO) } != 0 {
+        let mut colors = vec![];
+        highlight_shell(&repr, &mut colors, &parser.context(), false, None);
+        streams
+            .out
+            .append(str2wcstring(&colorize(&repr, &colors, parser.vars())));
+    } else {
+        streams.out.append(repr);
+    }
+}
+
+/// Values used for long-only options.
+const OPT_ESCAPE: char = '\x01';
+
+/// The complete builtin. Used for specifying programmable tab-completions. Calls the functions in
+/// complete.cpp for any heavy lifting.
+pub fn complete(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+    let cmd = argv[0];
+    let argc = argv.len();
+    let mut result_mode = CompletionMode::default();
+    let mut remove = false;
+    let mut short_opt = WString::new();
+    // todo!("these whould be Vec<&wstr>");
+    let mut gnu_opt = vec![];
+    let mut old_opt = vec![];
+    let mut subcommand = vec![];
+    let mut comp = WString::new();
+    let mut desc = WString::new();
+    let mut condition = vec![];
+    let mut do_complete = false;
+    let mut do_complete_param = None;
+    let mut cmd_to_complete = vec![];
+    let mut path = vec![];
+    let mut wrap_targets = vec![];
+    let mut preserve_order = false;
+    let mut unescape_output = true;
+
+    const short_options: &wstr = L!(":a:c:p:s:l:o:d:fFrxeuAn:C::w:hk");
+    const long_options: &[woption] = &[
+        wopt(L!("exclusive"), woption_argument_t::no_argument, 'x'),
+        wopt(L!("no-files"), woption_argument_t::no_argument, 'f'),
+        wopt(L!("force-files"), woption_argument_t::no_argument, 'F'),
+        wopt(
+            L!("require-parameter"),
+            woption_argument_t::no_argument,
+            'r',
+        ),
+        wopt(L!("path"), woption_argument_t::required_argument, 'p'),
+        wopt(L!("command"), woption_argument_t::required_argument, 'c'),
+        wopt(
+            L!("short-option"),
+            woption_argument_t::required_argument,
+            's',
+        ),
+        wopt(
+            L!("long-option"),
+            woption_argument_t::required_argument,
+            'l',
+        ),
+        wopt(L!("old-option"), woption_argument_t::required_argument, 'o'),
+        wopt(L!("subcommand"), woption_argument_t::required_argument, 'S'),
+        wopt(
+            L!("description"),
+            woption_argument_t::required_argument,
+            'd',
+        ),
+        wopt(L!("arguments"), woption_argument_t::required_argument, 'a'),
+        wopt(L!("erase"), woption_argument_t::no_argument, 'e'),
+        wopt(L!("unauthoritative"), woption_argument_t::no_argument, 'u'),
+        wopt(L!("authoritative"), woption_argument_t::no_argument, 'A'),
+        wopt(L!("condition"), woption_argument_t::required_argument, 'n'),
+        wopt(L!("wraps"), woption_argument_t::required_argument, 'w'),
+        wopt(
+            L!("do-complete"),
+            woption_argument_t::optional_argument,
+            'C',
+        ),
+        wopt(L!("help"), woption_argument_t::no_argument, 'h'),
+        wopt(L!("keep-order"), woption_argument_t::no_argument, 'k'),
+        wopt(L!("escape"), woption_argument_t::no_argument, OPT_ESCAPE),
+    ];
+
+    let mut have_x = false;
+
+    let mut w = wgetopter_t::new(short_options, long_options, argv);
+    while let Some(opt) = w.wgetopt_long() {
+        match opt {
+            'x' => {
+                result_mode.no_files = true;
+                result_mode.requires_param = true;
+                // Needed to print an error later;
+                have_x = true;
+            }
+            'f' => {
+                result_mode.no_files = true;
+            }
+            'F' => {
+                result_mode.force_files = true;
+            }
+            'r' => {
+                result_mode.requires_param = true;
+            }
+            'k' => {
+                preserve_order = true;
+            }
+            'p' | 'c' => {
+                if let Some(tmp) = unescape_string(
+                    w.woptarg.unwrap(),
+                    UnescapeStringStyle::Script(UnescapeFlags::SPECIAL),
+                ) {
+                    if opt == 'p' {
+                        path.push(tmp);
+                    } else {
+                        cmd_to_complete.push(tmp);
+                    }
+                } else {
+                    streams.err.append(wgettext_fmt!(
+                        "%ls: Invalid token '%ls'\n",
+                        cmd,
+                        w.woptarg.unwrap()
+                    ));
+                    return STATUS_INVALID_ARGS;
+                }
+            }
+            'd' => {
+                desc = w.woptarg.unwrap().to_owned();
+            }
+            'u' => {
+                // This option was removed in commit 1911298 and is now a no-op.
+            }
+            'A' => {
+                // This option was removed in commit 1911298 and is now a no-op.
+            }
+            's' => {
+                let arg = w.woptarg.unwrap();
+                short_opt.extend(arg.chars());
+                if arg.is_empty() {
+                    streams
+                        .err
+                        .append(wgettext_fmt!("%ls: -s requires a non-empty string\n", cmd,));
+                    return STATUS_INVALID_ARGS;
+                }
+            }
+            'l' => {
+                let arg = w.woptarg.unwrap();
+                gnu_opt.push(arg.to_owned());
+                if arg.is_empty() {
+                    streams
+                        .err
+                        .append(wgettext_fmt!("%ls: -l requires a non-empty string\n", cmd,));
+                    return STATUS_INVALID_ARGS;
+                }
+            }
+            'o' => {
+                let arg = w.woptarg.unwrap();
+                old_opt.push(arg.to_owned());
+                if arg.is_empty() {
+                    streams
+                        .err
+                        .append(wgettext_fmt!("%ls: -o requires a non-empty string\n", cmd,));
+                    return STATUS_INVALID_ARGS;
+                }
+            }
+            'S' => {
+                let arg = w.woptarg.unwrap();
+                subcommand.push(arg.to_owned());
+                if arg.is_empty() {
+                    streams
+                        .err
+                        .append(wgettext_fmt!("%ls: -S requires a non-empty string\n", cmd,));
+                    return STATUS_INVALID_ARGS;
+                }
+            }
+            'a' => {
+                comp = w.woptarg.unwrap().to_owned();
+            }
+            'e' => remove = true,
+            'n' => {
+                condition.push(w.woptarg.unwrap().to_owned());
+            }
+            'w' => {
+                wrap_targets.push(w.woptarg.unwrap().to_owned());
+            }
+            'C' => {
+                do_complete = true;
+                if let Some(s) = w.woptarg {
+                    do_complete_param = Some(s.to_owned());
+                }
+            }
+            OPT_ESCAPE => {
+                unescape_output = false;
+            }
+            'h' => {
+                builtin_print_help(parser, streams, cmd);
+                return STATUS_CMD_OK;
+            }
+            ':' => {
+                builtin_missing_argument(parser, streams, cmd, argv[w.woptind - 1], true);
+                return STATUS_INVALID_ARGS;
+            }
+            '?' => {
+                builtin_unknown_option(parser, streams, cmd, argv[w.woptind - 1], true);
+                return STATUS_INVALID_ARGS;
+            }
+            _ => panic!("unexpected retval from wgetopt_long"),
+        }
+    }
+
+    if result_mode.no_files && result_mode.force_files {
+        if !have_x {
+            streams.err.append(wgettext_fmt!(
+                BUILTIN_ERR_COMBO2,
+                "complete",
+                "'--no-files' and '--force-files'"
+            ));
+        } else {
+            // The reason for us not wanting files is `-x`,
+            // which is short for `-rf`.
+            streams.err.append(wgettext_fmt!(
+                BUILTIN_ERR_COMBO2,
+                "complete",
+                "'--exclusive' and '--force-files'"
+            ));
+        }
+        return STATUS_INVALID_ARGS;
+    }
+
+    if w.woptind != argc {
+        // Use one left-over arg as the do-complete argument
+        // to enable `complete -C "git check"`.
+        if do_complete && do_complete_param.is_none() && argc == w.woptind + 1 {
+            do_complete_param = Some(argv[argc - 1].to_owned());
+        } else if !do_complete && cmd_to_complete.is_empty() && argc == w.woptind + 1 {
+            // Or use one left-over arg as the command to complete
+            cmd_to_complete.push(argv[argc - 1].to_owned());
+        } else {
+            streams
+                .err
+                .append(wgettext_fmt!(BUILTIN_ERR_TOO_MANY_ARGUMENTS, cmd));
+            builtin_print_error_trailer(parser, streams.err, cmd);
+            return STATUS_INVALID_ARGS;
+        }
+    }
+
+    for condition_string in &condition {
+        let mut errors = ParseErrorList::new();
+        if parse_util_detect_errors(condition_string, Some(&mut errors), false).is_err() {
+            for error in errors {
+                let prefix = cmd.to_owned() + L!(": -n '") + &condition_string[..] + L!("': ");
+                streams.err.append(error.describe_with_prefix(
+                    condition_string,
+                    &prefix,
+                    parser.is_interactive(),
+                    false,
+                ));
+                streams.err.push('\n');
+            }
+            return STATUS_CMD_ERROR;
+        }
+    }
+
+    if !comp.is_empty() {
+        let mut prefix = WString::new();
+        prefix.push_utfstr(cmd);
+        prefix.push_str(": ");
+
+        if let Err(err_text) = parse_util_detect_errors_in_argument_list(&comp, &prefix) {
+            streams.err.append(wgettext_fmt!(
+                "%ls: %ls: contains a syntax error\n",
+                cmd,
+                comp
+            ));
+            streams.err.append(err_text);
+            streams.err.push('\n');
+            return STATUS_CMD_ERROR;
+        }
+    }
+
+    if do_complete {
+        let have_do_complete_param = do_complete_param.is_some();
+        let do_complete_param = match do_complete_param {
+            None => {
+                // No argument given, try to use the current commandline.
+                if !ffi::commandline_get_state_initialized_ffi() {
+                    // This corresponds to using 'complete -C' in non-interactive mode.
+                    // See #2361    .
+                    builtin_missing_argument(parser, streams, cmd, L!("-C"), true);
+                    return STATUS_INVALID_ARGS;
+                }
+                ffi::commandline_get_state_text_ffi().from_ffi()
+            }
+            Some(param) => param,
+        };
+
+        let mut token = 0..0;
+        parse_util_token_extent(
+            &do_complete_param,
+            do_complete_param.len(),
+            &mut token,
+            None,
+        );
+
+        // Create a scoped transient command line, so that builtin_commandline will see our
+        // argument, not the reader buffer.
+        parser
+            .libdata_mut()
+            .transient_commandlines
+            .push(do_complete_param.clone());
+        let _remove_transient = ScopeGuard::new((), |()| {
+            parser.libdata_mut().transient_commandlines.pop();
+        });
+
+        // Prevent accidental recursion (see #6171).
+        if !parser.libdata().pods.builtin_complete_current_commandline {
+            if !have_do_complete_param {
+                parser
+                    .libdata_mut()
+                    .pods
+                    .builtin_complete_current_commandline = true;
+            }
+
+            let (mut comp, _needs_load) = crate::complete::complete(
+                &do_complete_param,
+                CompletionRequestOptions::normal(),
+                &parser.context(),
+            );
+
+            // Apply the same sort and deduplication treatment as pager completions
+            crate::complete::sort_and_prioritize(&mut comp, CompletionRequestOptions::default());
+
+            for next in comp {
+                // Make a fake commandline, and then apply the completion to it.
+                let faux_cmdline = &do_complete_param[token.clone()];
+                let mut tmp_cursor = faux_cmdline.len();
+                let mut faux_cmdline_with_completion = ffi::completion_apply_to_command_line(
+                    &next.completion.to_ffi(),
+                    unsafe { std::mem::transmute(next.flags) },
+                    &faux_cmdline.to_ffi(),
+                    &mut tmp_cursor,
+                    false,
+                )
+                .from_ffi();
+
+                // completion_apply_to_command_line will append a space unless COMPLETE_NO_SPACE
+                // is set. We don't want to set COMPLETE_NO_SPACE because that won't close
+                // quotes. What we want is to close the quote, but not append the space. So we
+                // just look for the space and clear it.
+                if !next.flags.contains(CompleteFlags::NO_SPACE)
+                    && string_suffixes_string(L!(" "), &faux_cmdline_with_completion)
+                {
+                    faux_cmdline_with_completion.truncate(faux_cmdline_with_completion.len() - 1);
+                }
+
+                if unescape_output {
+                    // The input data is meant to be something like you would have on the command
+                    // line, e.g. includes backslashes. The output should be raw, i.e. unescaped. So
+                    // we need to unescape the command line. See #1127.
+                    unescape_string_in_place(
+                        &mut faux_cmdline_with_completion,
+                        UnescapeStringStyle::Script(UnescapeFlags::default()),
+                    );
+                }
+
+                // Append any description.
+                if !next.description.is_empty() {
+                    faux_cmdline_with_completion
+                        .reserve(faux_cmdline_with_completion.len() + 2 + next.description.len());
+                    faux_cmdline_with_completion.push('\t');
+                    faux_cmdline_with_completion.push_utfstr(&next.description);
+                }
+                faux_cmdline_with_completion.push('\n');
+                streams.out.append(faux_cmdline_with_completion);
+            }
+
+            parser
+                .libdata_mut()
+                .pods
+                .builtin_complete_current_commandline = false;
+        }
+    } else if path.is_empty()
+        && gnu_opt.is_empty()
+        && short_opt.is_empty()
+        && old_opt.is_empty()
+        && !remove
+        && comp.is_empty()
+        && desc.is_empty()
+        && condition.is_empty()
+        && wrap_targets.is_empty()
+        && !result_mode.no_files
+        && !result_mode.force_files
+        && !result_mode.requires_param
+    {
+        // No arguments that would add or remove anything specified, so we print the definitions of
+        // all matching completions.
+        if cmd_to_complete.is_empty() {
+            builtin_complete_print(L!(""), streams, parser);
+        } else {
+            for cmd in cmd_to_complete {
+                builtin_complete_print(&cmd, streams, parser);
+            }
+        }
+    } else {
+        let mut flags = CompleteFlags::AUTO_SPACE;
+        // HACK: Don't escape tildes because at the beginning of a token they probably mean
+        // $HOME, for example as produced by a recursive call to "complete -C".
+        flags |= CompleteFlags::DONT_ESCAPE_TILDES;
+        if preserve_order {
+            flags |= CompleteFlags::DONT_SORT;
+        }
+
+        if remove {
+            builtin_complete_remove(&cmd_to_complete, &path, &short_opt, &gnu_opt, &old_opt);
+        } else {
+            builtin_complete_add(
+                &cmd_to_complete,
+                &path,
+                &short_opt,
+                &gnu_opt,
+                &old_opt,
+                result_mode,
+                &condition,
+                &comp,
+                &desc,
+                flags,
+            );
+        }
+
+        // Handle wrap targets (probably empty). We only wrap commands, not paths.
+        for wrap_target in wrap_targets {
+            for i in &cmd_to_complete {
+                if remove {
+                    complete_remove_wrapper(i.clone(), &wrap_target);
+                } else {
+                    complete_add_wrapper(i.clone(), wrap_target.clone());
+                }
+            }
+        }
+    }
+
+    STATUS_CMD_OK
+}

fish-rust/src/builtins/contains.rs

@@ -9,7 +9,7 @@ struct Options {
 
 fn parse_options(
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Result<(Options, usize), Option<c_int>> {
     let cmd = args[0];
@@ -46,7 +46,7 @@ fn parse_options(
 
 /// Implementation of the builtin contains command, used to check if a specified string is part of
 /// a list.
-pub fn contains(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn contains(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
 
     let (opts, optind) = match parse_options(args, parser, streams) {

fish-rust/src/builtins/count.rs

@@ -5,7 +5,7 @@ use super::prelude::*;
 const COUNT_CHUNK_SIZE: usize = 512 * 256;
 
 /// Implementation of the builtin count command, used to count the number of arguments sent to it.
-pub fn count(_parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn count(_parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     // Always add the size of argv (minus 0, which is "count").
     // That means if you call `something | count a b c`, you'll get the count of something _plus 3_.
     let mut numargs = argv.len() - 1;

fish-rust/src/builtins/disown.rs

@@ -0,0 +1,126 @@
+// Implementation of the disown builtin.
+
+use super::shared::{builtin_print_help, STATUS_CMD_ERROR, STATUS_INVALID_ARGS};
+use crate::io::IoStreams;
+use crate::parser::Parser;
+use crate::proc::{add_disowned_job, Job};
+use crate::{
+    builtins::shared::{HelpOnlyCmdOpts, STATUS_CMD_OK},
+    wchar::wstr,
+    wutil::{fish_wcstoi, wgettext_fmt},
+};
+use libc::c_int;
+use libc::{self, SIGCONT};
+
+/// Helper for builtin_disown.
+fn disown_job(cmd: &wstr, streams: &mut IoStreams, j: &Job) {
+    // Nothing to do if already disowned.
+    if j.flags().disown_requested {
+        return;
+    }
+
+    // Stopped disowned jobs must be manually signaled; explain how to do so.
+    let pgid = j.get_pgid();
+    if j.is_stopped() {
+        if let Some(pgid) = pgid {
+            unsafe {
+                libc::killpg(pgid, SIGCONT);
+            }
+        }
+        streams.err.append(wgettext_fmt!(
+            "%ls: job %d ('%ls') was stopped and has been signalled to continue.\n",
+            cmd,
+            j.job_id(),
+            j.command()
+        ));
+    }
+
+    // We cannot directly remove the job from the jobs() list as `disown` might be called
+    // within the context of a subjob which will cause the parent job to crash in exec_job().
+    // Instead, we set a flag and the parser removes the job from the jobs list later.
+    j.mut_flags().disown_requested = true;
+    add_disowned_job(j);
+}
+
+/// Builtin for removing jobs from the job list.
+pub fn disown(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    let opts = match HelpOnlyCmdOpts::parse(args, parser, streams) {
+        Ok(opts) => opts,
+        Err(err @ Some(_)) if err != STATUS_CMD_OK => return err,
+        Err(err) => panic!("Illogical exit code from parse_options(): {err:?}"),
+    };
+
+    let cmd = args[0];
+    if opts.print_help {
+        builtin_print_help(parser, streams, cmd);
+        return STATUS_CMD_OK;
+    }
+
+    let mut retval;
+    if opts.optind == args.len() {
+        // Select last constructed job (ie first job in the job queue) that is possible to disown.
+        // Stopped jobs can be disowned (they will be continued).
+        // Foreground jobs can be disowned.
+        // Even jobs that aren't under job control can be disowned!
+        let mut job = None;
+        for j in &parser.jobs()[..] {
+            if j.is_constructed() && !j.is_completed() {
+                job = Some(j.clone());
+                break;
+            }
+        }
+
+        if let Some(job) = job {
+            disown_job(cmd, streams, &job);
+            retval = STATUS_CMD_OK;
+        } else {
+            streams
+                .err
+                .append(wgettext_fmt!("%ls: There are no suitable jobs\n", cmd));
+            retval = STATUS_CMD_ERROR;
+        }
+    } else {
+        let mut jobs = vec![];
+
+        retval = STATUS_CMD_OK;
+
+        // If one argument is not a valid pid (i.e. integer >= 0), fail without disowning anything,
+        // but still print errors for all of them.
+        // Non-existent jobs aren't an error, but information about them is useful.
+        // Multiple PIDs may refer to the same job; include the job only once by using a set.
+        for arg in &args[1..] {
+            match fish_wcstoi(arg)
+                .ok()
+                .and_then(|pid| if pid < 0 { None } else { Some(pid) })
+            {
+                None => {
+                    streams.err.append(wgettext_fmt!(
+                        "%ls: '%ls' is not a valid job specifier\n",
+                        cmd,
+                        arg
+                    ));
+                    retval = STATUS_INVALID_ARGS;
+                }
+                Some(pid) => {
+                    if let Some(j) = parser.job_get_from_pid(pid) {
+                        jobs.push(j);
+                    } else {
+                        streams
+                            .err
+                            .append(wgettext_fmt!("%ls: Could not find job '%d'\n"));
+                    }
+                }
+            }
+        }
+        if retval != STATUS_CMD_OK {
+            return retval;
+        }
+
+        // Disown all target jobs.
+        for j in jobs {
+            disown_job(cmd, streams, &j);
+        }
+    }
+
+    retval
+}

fish-rust/src/builtins/echo.rs

@@ -22,7 +22,7 @@ impl Default for Options {
 
 fn parse_options(
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Result<(Options, usize), Option<c_int>> {
     let cmd = args[0];
@@ -135,7 +135,7 @@ where
 ///
 /// Bash only respects `-n` if it's the first argument. We'll do the same. We also support a new,
 /// fish specific, option `-s` to mean "no spaces".
-pub fn echo(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn echo(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let (opts, optind) = match parse_options(args, parser, streams) {
         Ok((opts, optind)) => (opts, optind),
         Err(err @ Some(_)) if err != STATUS_CMD_OK => return err,

fish-rust/src/builtins/emit.rs

@@ -2,7 +2,7 @@ use super::prelude::*;
 use crate::event;
 
 #[widestrs]
-pub fn emit(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn emit(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
 
     let opts = match HelpOnlyCmdOpts::parse(argv, parser, streams) {

fish-rust/src/builtins/eval.rs

@@ -0,0 +1,79 @@
+//! The eval builtin.
+
+use super::prelude::*;
+use crate::io::IoBufferfill;
+use crate::parser::BlockType;
+use crate::wcstringutil::join_strings;
+use libc::{STDERR_FILENO, STDOUT_FILENO};
+
+pub fn eval(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    let argc = args.len();
+    if argc <= 1 {
+        return STATUS_CMD_OK;
+    }
+
+    let new_cmd = join_strings(&args[1..], ' ');
+
+    // Copy the full io chain; we may append bufferfills.
+    let mut ios = unsafe { &*streams.io_chain }.clone();
+
+    // If stdout is piped, then its output must go to the streams, not to the io_chain in our
+    // streams, because the pipe may be intended to be consumed by a process which
+    // is not yet launched (#6806). If stdout is NOT redirected, it must see the tty (#6955). So
+    // create a bufferfill for stdout if and only if stdout is piped.
+    // Note do not do this if stdout is merely redirected (say, to a file); we don't want to
+    // buffer in that case.
+    let mut stdout_fill = None;
+    if streams.out_is_piped {
+        match IoBufferfill::create_opts(parser.libdata().pods.read_limit, STDOUT_FILENO) {
+            None => {
+                // We were unable to create a pipe, probably fd exhaustion.
+                return STATUS_CMD_ERROR;
+            }
+            Some(buffer) => {
+                stdout_fill = Some(buffer.clone());
+                ios.push(buffer);
+            }
+        }
+    }
+
+    // Of course the same applies to stderr.
+    let mut stderr_fill = None;
+    if streams.err_is_piped {
+        match IoBufferfill::create_opts(parser.libdata().pods.read_limit, STDERR_FILENO) {
+            None => {
+                // We were unable to create a pipe, probably fd exhaustion.
+                return STATUS_CMD_ERROR;
+            }
+            Some(buffer) => {
+                stderr_fill = Some(buffer.clone());
+                ios.push(buffer);
+            }
+        }
+    }
+
+    let res = parser.eval_with(&new_cmd, &ios, streams.job_group.as_ref(), BlockType::top);
+    let status = if res.was_empty {
+        // Issue #5692, in particular, to catch `eval ""`, `eval "begin; end;"`, etc.
+        // where we have an argument but nothing is executed.
+        STATUS_CMD_OK
+    } else {
+        Some(res.status.status_value())
+    };
+
+    // Finish the bufferfills - exhaust and close our pipes.
+    // Copy the output from the bufferfill back to the streams.
+    // Note it is important that we hold no other references to the bufferfills here - they need to
+    // deallocate to close.
+    ios.clear();
+    if let Some(stdout) = stdout_fill {
+        let output = IoBufferfill::finish(stdout);
+        streams.out.append_narrow_buffer(&output);
+    }
+    if let Some(stderr) = stderr_fill {
+        let errput = IoBufferfill::finish(stderr);
+        streams.err.append_narrow_buffer(&errput);
+    }
+
+    status
+}

fish-rust/src/builtins/exit.rs

@@ -2,7 +2,7 @@ use super::prelude::*;
 use super::r#return::parse_return_value;
 
 /// Function for handling the exit builtin.
-pub fn exit(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn exit(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let retval = match parse_return_value(args, parser, streams) {
         Ok(v) => v,
         Err(e) => return e,
@@ -12,7 +12,7 @@ pub fn exit(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) ->
     // TODO: in concurrent mode this won't successfully exit a pipeline, as there are other parsers
     // involved. That is, `exit | sleep 1000` may not exit as hoped. Need to rationalize what
     // behavior we want here.
-    parser.libdata_pod().exit_current_script = true;
+    parser.libdata_mut().pods.exit_current_script = true;
 
     return Some(retval);
 }

fish-rust/src/builtins/fg.rs

@@ -0,0 +1,177 @@
+//! Implementation of the fg builtin.
+
+use crate::fds::make_fd_blocking;
+use crate::reader::reader_write_title;
+use crate::tokenizer::tok_command;
+use crate::wutil::perror;
+use crate::{env::EnvMode, proc::TtyTransfer};
+use libc::{STDERR_FILENO, STDIN_FILENO, TCSADRAIN};
+
+use super::prelude::*;
+
+/// Builtin for putting a job in the foreground.
+pub fn fg(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+    let opts = match HelpOnlyCmdOpts::parse(argv, parser, streams) {
+        Ok(opts) => opts,
+        Err(err @ Some(_)) if err != STATUS_CMD_OK => return err,
+        Err(err) => panic!("Illogical exit code from parse_options(): {err:?}"),
+    };
+
+    let cmd = argv[0];
+    if opts.print_help {
+        builtin_print_help(parser, streams, cmd);
+        return STATUS_CMD_OK;
+    }
+
+    let job;
+    let job_pos;
+    let optind = opts.optind;
+    if optind == argv.len() {
+        // Select last constructed job (i.e. first job in the job queue) that can be brought
+        // to the foreground.
+        let jobs = parser.jobs();
+        match jobs.iter().enumerate().find(|(_pos, job)| {
+            job.is_constructed()
+                && !job.is_completed()
+                && ((job.is_stopped() || !job.is_foreground()) && job.wants_job_control())
+        }) {
+            None => {
+                streams
+                    .err
+                    .append(wgettext_fmt!("%ls: There are no suitable jobs\n", cmd));
+                return STATUS_INVALID_ARGS;
+            }
+            Some((pos, j)) => {
+                job_pos = Some(pos);
+                job = Some(j.clone());
+            }
+        }
+    } else if optind + 1 < argv.len() {
+        // Specifying more than one job to put to the foreground is a syntax error, we still
+        // try to locate the job $argv[1], since we need to determine which error message to
+        // emit (ambigous job specification vs malformed job id).
+        let mut found_job = false;
+        match fish_wcstoi(argv[optind]) {
+            Ok(pid) if pid > 0 => found_job = parser.job_get_from_pid(pid).is_some(),
+            _ => (),
+        };
+
+        if found_job {
+            streams
+                .err
+                .append(wgettext_fmt!("%ls: Ambiguous job\n", cmd));
+        } else {
+            streams.err.append(wgettext_fmt!(
+                "%ls: '%ls' is not a job\n",
+                cmd,
+                argv[optind]
+            ));
+        }
+
+        builtin_print_error_trailer(parser, streams.err, cmd);
+        job_pos = None;
+        job = None;
+    } else {
+        match fish_wcstoi(argv[optind]) {
+            Err(_) => {
+                streams
+                    .err
+                    .append(wgettext_fmt!(BUILTIN_ERR_NOT_NUMBER, cmd, argv[optind]));
+                job_pos = None;
+                job = None;
+                builtin_print_error_trailer(parser, streams.err, cmd);
+            }
+            Ok(pid) => {
+                let pid = pid.abs();
+                let j = parser.job_get_with_index_from_pid(pid);
+                if j.as_ref()
+                    .map_or(true, |(_pos, j)| !j.is_constructed() || j.is_completed())
+                {
+                    streams
+                        .err
+                        .append(wgettext_fmt!("%ls: No suitable job: %d\n", cmd, pid));
+                    job_pos = None;
+                    job = None
+                } else {
+                    let (pos, j) = j.unwrap();
+                    job_pos = Some(pos);
+                    job = if !j.wants_job_control() {
+                        streams.err.append(wgettext_fmt!(
+                            concat!(
+                                "%ls: Can't put job %d, '%ls' to foreground because ",
+                                "it is not under job control\n"
+                            ),
+                            cmd,
+                            pid,
+                            j.command()
+                        ));
+                        None
+                    } else {
+                        Some(j)
+                    };
+                }
+            }
+        }
+    };
+
+    let Some(job) = job else {
+        return STATUS_INVALID_ARGS;
+    };
+    let job_pos = job_pos.unwrap();
+
+    if streams.err_is_redirected {
+        streams
+            .err
+            .append(wgettext_fmt!(FG_MSG, job.job_id(), job.command()));
+    } else {
+        // If we aren't redirecting, send output to real stderr, since stuff in sb_err won't get
+        // printed until the command finishes.
+        fwprintf!(
+            STDERR_FILENO,
+            "%s",
+            wgettext_fmt!(FG_MSG, job.job_id(), job.command())
+        );
+    }
+
+    let ft = tok_command(job.command());
+    if !ft.is_empty() {
+        // Provide value for `status current-command`
+        parser.libdata_mut().status_vars.command = ft.clone();
+        // Also provide a value for the deprecated fish 2.0 $_ variable
+        parser.set_var_and_fire(L!("_"), EnvMode::EXPORT, vec![ft]);
+        // Provide value for `status current-commandline`
+        parser.libdata_mut().status_vars.commandline = job.command().to_owned();
+    }
+    reader_write_title(job.command(), parser, true);
+
+    // Note if tty transfer fails, we still try running the job.
+    parser.job_promote_at(job_pos);
+    let _ = make_fd_blocking(STDIN_FILENO);
+    {
+        let job_group = job.group();
+        job_group.set_is_foreground(true);
+        let tmodes = job_group.tmodes.borrow();
+        if job_group.wants_terminal() && tmodes.is_some() {
+            let termios = tmodes.as_ref().unwrap();
+            let res = unsafe { libc::tcsetattr(STDIN_FILENO, TCSADRAIN, termios) };
+            if res < 0 {
+                perror("tcsetattr");
+            }
+        }
+    }
+    let mut transfer = TtyTransfer::new();
+    transfer.to_job_group(job.group.as_ref().unwrap());
+    let resumed = job.resume();
+    if resumed {
+        job.continue_job(parser);
+    }
+    if job.is_stopped() {
+        transfer.save_tty_modes();
+    }
+    transfer.reclaim();
+    if resumed {
+        STATUS_CMD_OK
+    } else {
+        STATUS_CMD_ERROR
+    }
+}

fish-rust/src/builtins/function.rs

@@ -1,17 +1,16 @@
 use super::prelude::*;
 use crate::ast::BlockStatement;
 use crate::common::{valid_func_name, valid_var_name};
+use crate::complete::complete_add_wrapper;
 use crate::env::environment::Environment;
 use crate::event::{self, EventDescription, EventHandler};
-use crate::ffi::IoStreams as io_streams_ffi_t;
 use crate::function;
 use crate::global_safety::RelaxedAtomicBool;
+use crate::io::IoStreams;
 use crate::parse_tree::NodeRef;
-use crate::parse_tree::ParsedSourceRefFFI;
+use crate::parser::Parser;
 use crate::parser_keywords::parser_keywords_is_reserved;
 use crate::signal::Signal;
-use crate::wchar_ffi::{wcstring_list_ffi_t, WCharFromFFI, WCharToFFI};
-use std::pin::Pin;
 use std::sync::Arc;
 
 struct FunctionCmdOpts {
@@ -60,7 +59,7 @@ const LONG_OPTIONS: &[woption] = &[
 /// This looks through both active and finished jobs.
 fn job_id_for_pid(pid: i32, parser: &Parser) -> Option<u64> {
     if let Some(job) = parser.job_get_from_pid(pid) {
-        Some(job.get_internal_job_id())
+        Some(job.internal_job_id)
     } else {
         parser
             .get_wait_handles()
@@ -75,7 +74,7 @@ fn parse_cmd_opts(
     opts: &mut FunctionCmdOpts,
     optind: &mut usize,
     argv: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Option<c_int> {
     let cmd = L!("function");
@@ -134,9 +133,9 @@ fn parse_cmd_opts(
                 let woptarg = w.woptarg.unwrap();
                 let e: EventDescription;
                 if opt == 'j' && woptarg == "caller" {
-                    let libdata = parser.ffi_libdata_pod_const();
-                    let caller_id = if libdata.is_subshell {
-                        libdata.caller_id
+                    let libdata = parser.libdata();
+                    let caller_id = if libdata.pods.is_subshell {
+                        libdata.pods.caller_id
                     } else {
                         0
                     };
@@ -252,7 +251,7 @@ fn validate_function_name(
 /// function. Note this isn't strictly a "builtin": it is called directly from parse_execution.
 /// That is why its signature is different from the other builtins.
 pub fn function(
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
     c_args: &mut [&wstr],
     func_node: NodeRef<BlockStatement>,
@@ -281,7 +280,7 @@ pub fn function(
     }
 
     if opts.print_help {
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return STATUS_CMD_OK;
     }
 
@@ -308,8 +307,7 @@ pub fn function(
     }
 
     // Extract the current filename.
-    let definition_file = unsafe { parser.pin().libdata().get_current_filename().as_ref() }
-        .map(|s| Arc::new(s.from_ffi()));
+    let definition_file = parser.libdata().current_filename.clone();
 
     // Ensure inherit_vars is unique and then populate it.
     opts.inherit_vars.sort_unstable();
@@ -319,7 +317,7 @@ pub fn function(
         .inherit_vars
         .into_iter()
         .filter_map(|name| {
-            let vals = parser.get_vars().get(&name)?.as_list().to_vec();
+            let vals = parser.vars().get(&name)?.as_list().to_vec();
             Some((name, vals))
         })
         .collect();
@@ -343,7 +341,7 @@ pub fn function(
 
     // Handle wrap targets by creating the appropriate completions.
     for wt in opts.wrap_targets.into_iter() {
-        ffi::complete_add_wrapper(&function_name.to_ffi(), &wt.to_ffi());
+        complete_add_wrapper(function_name.clone(), wt.clone());
     }
 
     // Add any event handlers.
@@ -375,56 +373,3 @@ pub fn function(
 
     STATUS_CMD_OK
 }
-
-fn builtin_function_ffi(
-    parser: Pin<&mut Parser>,
-    streams: Pin<&mut io_streams_ffi_t>,
-    c_args: &wcstring_list_ffi_t,
-    source_u8: *const u8, // unowned ParsedSourceRefFFI
-    func_node: &BlockStatement,
-) -> i32 {
-    let storage = c_args.from_ffi();
-    let mut args = truncate_args_on_nul(&storage);
-    let node = unsafe {
-        let source_ref: &ParsedSourceRefFFI = &*(source_u8.cast());
-        NodeRef::from_parts(
-            source_ref
-                .0
-                .as_ref()
-                .expect("Should have parsed source")
-                .clone(),
-            func_node,
-        )
-    };
-    function(
-        parser.unpin(),
-        &mut IoStreams::new(streams),
-        args.as_mut_slice(),
-        node,
-    )
-    .expect("function builtin should always return a non-None status")
-}
-
-#[cxx::bridge]
-mod builtin_function {
-    extern "C++" {
-        include!("ast.h");
-        include!("parser.h");
-        include!("io.h");
-        type Parser = crate::ffi::Parser;
-        type IoStreams = crate::ffi::IoStreams;
-        type wcstring_list_ffi_t = crate::ffi::wcstring_list_ffi_t;
-
-        type BlockStatement = crate::ast::BlockStatement;
-    }
-
-    extern "Rust" {
-        fn builtin_function_ffi(
-            parser: Pin<&mut Parser>,
-            streams: Pin<&mut IoStreams>,
-            c_args: &wcstring_list_ffi_t,
-            source: *const u8, // unowned ParsedSourceRefFFI
-            func_node: &BlockStatement,
-        ) -> i32;
-    }
-}

fish-rust/src/builtins/functions.rs

@@ -1,11 +1,14 @@
 use super::prelude::*;
 use crate::common::escape_string;
 use crate::common::reformat_for_screen;
+use crate::common::str2wcstring;
 use crate::common::valid_func_name;
 use crate::common::{EscapeFlags, EscapeStringStyle};
 use crate::event::{self};
-use crate::ffi::colorize_shell;
 use crate::function;
+use crate::highlight::colorize;
+use crate::highlight::highlight_shell;
+use crate::parser::Parser;
 use crate::parser_keywords::parser_keywords_is_reserved;
 use crate::termsize::termsize_last;
 
@@ -68,7 +71,7 @@ fn parse_cmd_opts<'args>(
     opts: &mut FunctionsCmdOpts<'args>,
     optind: &mut usize,
     argv: &mut [&'args wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Option<c_int> {
     let cmd = L!("functions");
@@ -111,11 +114,7 @@ fn parse_cmd_opts<'args>(
     STATUS_CMD_OK
 }
 
-pub fn functions(
-    parser: &mut Parser,
-    streams: &mut IoStreams,
-    args: &mut [&wstr],
-) -> Option<c_int> {
+pub fn functions(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
 
     let mut opts = FunctionsCmdOpts::default();
@@ -128,7 +127,7 @@ pub fn functions(
     let args = &args[optind..];
 
     if opts.print_help {
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return STATUS_CMD_OK;
     }
 
@@ -140,13 +139,13 @@ pub fn functions(
         > 1
     {
         streams.err.append(wgettext_fmt!(BUILTIN_ERR_COMBO, cmd));
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return STATUS_INVALID_ARGS;
     }
 
     if opts.report_metadata && opts.no_metadata {
         streams.err.append(wgettext_fmt!(BUILTIN_ERR_COMBO, cmd));
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return STATUS_INVALID_ARGS;
     }
 
@@ -164,7 +163,7 @@ pub fn functions(
                 "%ls: Expected exactly one function name\n",
                 cmd
             ));
-            builtin_print_error_trailer(parser, streams, cmd);
+            builtin_print_error_trailer(parser, streams.err, cmd);
             return STATUS_INVALID_ARGS;
         }
         let current_func = args[0];
@@ -175,7 +174,7 @@ pub fn functions(
                 cmd,
                 current_func
             ));
-            builtin_print_error_trailer(parser, streams, cmd);
+            builtin_print_error_trailer(parser, streams.err, cmd);
             return STATUS_CMD_ERROR;
         }
 
@@ -304,7 +303,7 @@ pub fn functions(
                 "%ls: Expected exactly two names (current function name, and new function name)\n",
                 cmd
             ));
-            builtin_print_error_trailer(parser, streams, cmd);
+            builtin_print_error_trailer(parser, streams.err, cmd);
             return STATUS_INVALID_ARGS;
         }
         let current_func = args[0];
@@ -316,7 +315,7 @@ pub fn functions(
                 cmd,
                 current_func
             ));
-            builtin_print_error_trailer(parser, streams, cmd);
+            builtin_print_error_trailer(parser, streams.err, cmd);
             return STATUS_CMD_ERROR;
         }
 
@@ -326,7 +325,7 @@ pub fn functions(
                 cmd,
                 new_func
             ));
-            builtin_print_error_trailer(parser, streams, cmd);
+            builtin_print_error_trailer(parser, streams.err, cmd);
             return STATUS_INVALID_ARGS;
         }
 
@@ -337,7 +336,7 @@ pub fn functions(
                 new_func,
                 current_func
             ));
-            builtin_print_error_trailer(parser, streams, cmd);
+            builtin_print_error_trailer(parser, streams.err, cmd);
             return STATUS_CMD_ERROR;
         }
         if function::copy(current_func, new_func.into(), parser) {
@@ -410,8 +409,11 @@ pub fn functions(
         }
 
         if streams.out_is_terminal() {
-            let col = colorize_shell(&def.to_ffi(), parser.pin()).from_ffi();
-            streams.out.append(col);
+            let mut colors = vec![];
+            highlight_shell(&def, &mut colors, &parser.context(), false, None);
+            streams
+                .out
+                .append(str2wcstring(&colorize(&def, &colors, parser.vars())));
         } else {
             streams.out.append(def);
         }

fish-rust/src/builtins/history.rs

@@ -0,0 +1,362 @@
+//! Implementation of the history builtin.
+
+use crate::ffi::{self};
+use crate::history::{self, history_session_id, History};
+use crate::history::{in_private_mode, HistorySharedPtr};
+
+use super::prelude::*;
+
+#[derive(Default, Eq, PartialEq)]
+enum HistCmd {
+    HIST_SEARCH = 1,
+    HIST_DELETE,
+    HIST_CLEAR,
+    HIST_MERGE,
+    HIST_SAVE,
+    #[default]
+    HIST_UNDEF,
+    HIST_CLEAR_SESSION,
+}
+
+impl HistCmd {
+    fn to_wstr(&self) -> &'static wstr {
+        match self {
+            HistCmd::HIST_SEARCH => L!("search"),
+            HistCmd::HIST_DELETE => L!("delete"),
+            HistCmd::HIST_CLEAR => L!("clear"),
+            HistCmd::HIST_MERGE => L!("merge"),
+            HistCmd::HIST_SAVE => L!("save"),
+            HistCmd::HIST_UNDEF => panic!(),
+            HistCmd::HIST_CLEAR_SESSION => L!("clear-session"),
+        }
+    }
+}
+
+impl TryFrom<&wstr> for HistCmd {
+    type Error = ();
+    fn try_from(val: &wstr) -> Result<Self, ()> {
+        match val {
+            _ if val == "search" => Ok(HistCmd::HIST_SEARCH),
+            _ if val == "delete" => Ok(HistCmd::HIST_DELETE),
+            _ if val == "clear" => Ok(HistCmd::HIST_CLEAR),
+            _ if val == "merge" => Ok(HistCmd::HIST_MERGE),
+            _ if val == "save" => Ok(HistCmd::HIST_SAVE),
+            _ if val == "clear-session" => Ok(HistCmd::HIST_CLEAR_SESSION),
+            _ => Err(()),
+        }
+    }
+}
+
+#[derive(Default)]
+struct HistoryCmdOpts {
+    hist_cmd: HistCmd,
+    search_type: Option<history::SearchType>,
+    show_time_format: Option<String>,
+    max_items: Option<usize>,
+    print_help: bool,
+    case_sensitive: bool,
+    null_terminate: bool,
+    reverse: bool,
+}
+
+/// Note: Do not add new flags that represent subcommands. We're encouraging people to switch to
+/// the non-flag subcommand form. While many of these flags are deprecated they must be
+/// supported at least until fish 3.0 and possibly longer to avoid breaking everyones
+/// config.fish and other scripts.
+const short_options: &wstr = L!(":CRcehmn:pt::z");
+const longopts: &[woption] = &[
+    wopt(L!("prefix"), woption_argument_t::no_argument, 'p'),
+    wopt(L!("contains"), woption_argument_t::no_argument, 'c'),
+    wopt(L!("help"), woption_argument_t::no_argument, 'h'),
+    wopt(L!("show-time"), woption_argument_t::optional_argument, 't'),
+    wopt(L!("exact"), woption_argument_t::no_argument, 'e'),
+    wopt(L!("max"), woption_argument_t::required_argument, 'n'),
+    wopt(L!("null"), woption_argument_t::no_argument, 'z'),
+    wopt(L!("case-sensitive"), woption_argument_t::no_argument, 'C'),
+    wopt(L!("delete"), woption_argument_t::no_argument, '\x01'),
+    wopt(L!("search"), woption_argument_t::no_argument, '\x02'),
+    wopt(L!("save"), woption_argument_t::no_argument, '\x03'),
+    wopt(L!("clear"), woption_argument_t::no_argument, '\x04'),
+    wopt(L!("merge"), woption_argument_t::no_argument, '\x05'),
+    wopt(L!("reverse"), woption_argument_t::no_argument, 'R'),
+];
+
+/// Remember the history subcommand and disallow selecting more than one history subcommand.
+fn set_hist_cmd(
+    cmd: &wstr,
+    hist_cmd: &mut HistCmd,
+    sub_cmd: HistCmd,
+    streams: &mut IoStreams,
+) -> bool {
+    if *hist_cmd != HistCmd::HIST_UNDEF {
+        streams.err.append(wgettext_fmt!(
+            BUILTIN_ERR_COMBO2_EXCLUSIVE,
+            cmd,
+            hist_cmd.to_wstr(),
+            sub_cmd.to_wstr()
+        ));
+        return false;
+    }
+    *hist_cmd = sub_cmd;
+    true
+}
+
+fn check_for_unexpected_hist_args(
+    opts: &HistoryCmdOpts,
+    cmd: &wstr,
+    args: &[&wstr],
+    streams: &mut IoStreams,
+) -> bool {
+    if opts.search_type.is_some() || opts.show_time_format.is_some() || opts.null_terminate {
+        let subcmd_str = opts.hist_cmd.to_wstr();
+        streams.err.append(wgettext_fmt!(
+            "%ls: %ls: subcommand takes no options\n",
+            cmd,
+            subcmd_str
+        ));
+        return true;
+    }
+    if !args.is_empty() {
+        let subcmd_str = opts.hist_cmd.to_wstr();
+        streams.err.append(wgettext_fmt!(
+            BUILTIN_ERR_ARG_COUNT2,
+            cmd,
+            subcmd_str,
+            0,
+            args.len()
+        ));
+        return true;
+    }
+    false
+}
+
+fn parse_cmd_opts(
+    opts: &mut HistoryCmdOpts,
+    optind: &mut usize,
+    argv: &mut [&wstr],
+    parser: &Parser,
+    streams: &mut IoStreams,
+) -> Option<c_int> {
+    let cmd = argv[0];
+    let mut w = wgetopter_t::new(short_options, longopts, argv);
+    while let Some(opt) = w.wgetopt_long() {
+        match opt {
+            '\x01' => {
+                if !set_hist_cmd(cmd, &mut opts.hist_cmd, HistCmd::HIST_DELETE, streams) {
+                    return STATUS_CMD_ERROR;
+                }
+            }
+            '\x02' => {
+                if !set_hist_cmd(cmd, &mut opts.hist_cmd, HistCmd::HIST_SEARCH, streams) {
+                    return STATUS_CMD_ERROR;
+                }
+            }
+            '\x03' => {
+                if !set_hist_cmd(cmd, &mut opts.hist_cmd, HistCmd::HIST_SAVE, streams) {
+                    return STATUS_CMD_ERROR;
+                }
+            }
+            '\x04' => {
+                if !set_hist_cmd(cmd, &mut opts.hist_cmd, HistCmd::HIST_CLEAR, streams) {
+                    return STATUS_CMD_ERROR;
+                }
+            }
+            '\x05' => {
+                if !set_hist_cmd(cmd, &mut opts.hist_cmd, HistCmd::HIST_MERGE, streams) {
+                    return STATUS_CMD_ERROR;
+                }
+            }
+            'C' => {
+                opts.case_sensitive = true;
+            }
+            'R' => {
+                opts.reverse = true;
+            }
+            'p' => {
+                opts.search_type = Some(history::SearchType::PrefixGlob);
+            }
+            'c' => {
+                opts.search_type = Some(history::SearchType::ContainsGlob);
+            }
+            'e' => {
+                opts.search_type = Some(history::SearchType::Exact);
+            }
+            't' => {
+                opts.show_time_format = Some(w.woptarg.unwrap_or(L!("# %c%n")).to_string());
+            }
+            'n' => match fish_wcstol(w.woptarg.unwrap()) {
+                Ok(x) => opts.max_items = Some(x as _), // todo!("historical behavior is to cast")
+                Err(_) => {
+                    streams.err.append(wgettext_fmt!(
+                        BUILTIN_ERR_NOT_NUMBER,
+                        cmd,
+                        w.woptarg.unwrap()
+                    ));
+                    return STATUS_INVALID_ARGS;
+                }
+            },
+            'z' => {
+                opts.null_terminate = true;
+            }
+            'h' => {
+                opts.print_help = true;
+            }
+            ':' => {
+                builtin_missing_argument(parser, streams, cmd, argv[w.woptind - 1], true);
+                return STATUS_INVALID_ARGS;
+            }
+            '?' => {
+                // Try to parse it as a number; e.g., "-123".
+                match fish_wcstol(&w.argv[w.woptind - 1][1..]) {
+                    Ok(x) => opts.max_items = Some(x as _), // todo!("historical behavior is to cast")
+                    Err(_) => {
+                        builtin_unknown_option(parser, streams, cmd, argv[w.woptind - 1], true);
+                        return STATUS_INVALID_ARGS;
+                    }
+                }
+                w.nextchar = L!("");
+            }
+            _ => {
+                panic!("unexpected retval from wgetopt_long");
+            }
+        }
+    }
+
+    *optind = w.woptind;
+    STATUS_CMD_OK
+}
+
+/// Manipulate history of interactive commands executed by the user.
+pub fn history(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    let mut opts = HistoryCmdOpts::default();
+    let mut optind = 0;
+    let retval = parse_cmd_opts(&mut opts, &mut optind, args, parser, streams);
+    if retval != STATUS_CMD_OK {
+        return retval;
+    }
+    let cmd = &args[0];
+
+    if opts.print_help {
+        builtin_print_help(parser, streams, cmd);
+        return STATUS_CMD_OK;
+    }
+
+    // Use the default history if we have none (which happens if invoked non-interactively, e.g.
+    // from webconfig.py.
+    let history = ffi::commandline_get_state_history_ffi();
+    let history = if history.is_null() {
+        History::with_name(&history_session_id(parser.vars()))
+    } else {
+        {
+            *unsafe {
+                Box::from_raw(ffi::commandline_get_state_history_ffi() as *mut HistorySharedPtr)
+            }
+        }
+        .0
+    };
+
+    // If a history command hasn't already been specified via a flag check the first word.
+    // Note that this can be simplified after we eliminate allowing subcommands as flags.
+    // See the TODO above regarding the `long_options` array.
+    if optind < args.len() {
+        if let Ok(subcmd) = HistCmd::try_from(args[optind]) {
+            if !set_hist_cmd(cmd, &mut opts.hist_cmd, subcmd, streams) {
+                return STATUS_INVALID_ARGS;
+            }
+            optind += 1;
+        }
+    }
+
+    // Every argument that we haven't consumed already is an argument for a subcommand (e.g., a
+    // search term).
+    let args = &args[optind..];
+
+    // Establish appropriate defaults.
+    if opts.hist_cmd == HistCmd::HIST_UNDEF {
+        opts.hist_cmd = HistCmd::HIST_SEARCH;
+    }
+    if opts.search_type.is_none() {
+        if opts.hist_cmd == HistCmd::HIST_SEARCH {
+            opts.search_type = Some(history::SearchType::ContainsGlob);
+        }
+        if opts.hist_cmd == HistCmd::HIST_DELETE {
+            opts.search_type = Some(history::SearchType::Exact);
+        }
+    }
+
+    let mut status = STATUS_CMD_OK;
+    match opts.hist_cmd {
+        HistCmd::HIST_SEARCH => {
+            if !history.search(
+                opts.search_type.unwrap(),
+                args,
+                opts.show_time_format.as_deref(),
+                opts.max_items.unwrap_or(usize::MAX),
+                opts.case_sensitive,
+                opts.null_terminate,
+                opts.reverse,
+                &parser.context().cancel_checker,
+                streams,
+            ) {
+                status = STATUS_CMD_ERROR;
+            }
+        }
+        HistCmd::HIST_DELETE => {
+            // TODO: Move this code to the history module and support the other search types
+            // including case-insensitive matches. At this time we expect the non-exact deletions to
+            // be handled only by the history function's interactive delete feature.
+            if opts.search_type.unwrap() != history::SearchType::Exact {
+                streams
+                    .err
+                    .append(wgettext!("builtin history delete only supports --exact\n"));
+                return STATUS_INVALID_ARGS;
+            }
+            if !opts.case_sensitive {
+                streams.err.append(wgettext!(
+                    "builtin history delete --exact requires --case-sensitive\n"
+                ));
+                return STATUS_INVALID_ARGS;
+            }
+            for delete_string in args.iter().copied() {
+                history.remove(delete_string.to_owned());
+            }
+        }
+        HistCmd::HIST_CLEAR => {
+            if check_for_unexpected_hist_args(&opts, cmd, args, streams) {
+                return STATUS_INVALID_ARGS;
+            }
+            history.clear();
+            history.save();
+        }
+        HistCmd::HIST_CLEAR_SESSION => {
+            if check_for_unexpected_hist_args(&opts, cmd, args, streams) {
+                return STATUS_INVALID_ARGS;
+            }
+            history.clear_session();
+            history.save();
+        }
+        HistCmd::HIST_MERGE => {
+            if check_for_unexpected_hist_args(&opts, cmd, args, streams) {
+                return STATUS_INVALID_ARGS;
+            }
+
+            if in_private_mode(parser.vars()) {
+                streams.err.append(wgettext_fmt!(
+                    "%ls: can't merge history in private mode\n",
+                    cmd
+                ));
+                return STATUS_INVALID_ARGS;
+            }
+            history.incorporate_external_changes();
+        }
+        HistCmd::HIST_SAVE => {
+            if check_for_unexpected_hist_args(&opts, cmd, args, streams) {
+                return STATUS_INVALID_ARGS;
+            }
+            history.save();
+        }
+        HistCmd::HIST_UNDEF => panic!("Unexpected HIST_UNDEF seen"),
+    }
+
+    status
+}

fish-rust/src/builtins/jobs.rs

@@ -0,0 +1,267 @@
+// Functions for executing the jobs builtin.
+
+use super::shared::{
+    builtin_missing_argument, builtin_print_help, builtin_unknown_option, STATUS_CMD_ERROR,
+    STATUS_INVALID_ARGS,
+};
+use crate::common::{escape_string, timef, EscapeFlags, EscapeStringStyle};
+use crate::io::IoStreams;
+use crate::job_group::{JobId, MaybeJobId};
+use crate::parser::Parser;
+use crate::proc::{clock_ticks_to_seconds, have_proc_stat, proc_get_jiffies, Job, INVALID_PID};
+use crate::wchar_ext::WExt;
+use crate::wgetopt::{wgetopter_t, wopt, woption, woption_argument_t};
+use crate::wutil::wgettext;
+use crate::{
+    builtins::shared::STATUS_CMD_OK,
+    wchar::{wstr, WString, L},
+    wutil::{fish_wcstoi, wgettext_fmt},
+};
+use libc::c_int;
+use libc::{self};
+use printf_compat::sprintf;
+use std::num::NonZeroU32;
+use std::sync::atomic::Ordering;
+
+/// Print modes for the jobs builtin.
+
+#[derive(Clone, Copy, Eq, PartialEq)]
+enum JobsPrintMode {
+    Default,      // print lots of general info
+    PrintPid,     // print pid of each process in job
+    PrintCommand, // print command name of each process in job
+    PrintGroup,   // print group id of job
+    PrintNothing, // print nothing (exit status only)
+}
+
+/// Calculates the cpu usage (as a fraction of 1) of the specified job.
+/// This may exceed 1 if there are multiple CPUs!
+fn cpu_use(j: &Job) -> f64 {
+    let mut u = 0.0;
+    for p in j.processes() {
+        let now = timef();
+        let jiffies = proc_get_jiffies(p.pid.load(Ordering::Relaxed));
+        let last_jiffies = p.last_times.get().jiffies;
+        let since = now - last_jiffies as f64;
+        if since > 0.0 && jiffies > last_jiffies {
+            u += clock_ticks_to_seconds(jiffies - last_jiffies) / since;
+        }
+    }
+    u
+}
+
+/// Print information about the specified job.
+fn builtin_jobs_print(j: &Job, mode: JobsPrintMode, header: bool, streams: &mut IoStreams) {
+    let mut pgid = INVALID_PID;
+    {
+        if let Some(job_pgid) = j.get_pgid() {
+            pgid = job_pgid;
+        }
+    }
+
+    let mut out = WString::new();
+    match mode {
+        JobsPrintMode::PrintNothing => (),
+        JobsPrintMode::Default => {
+            if header {
+                // Print table header before first job.
+                out += wgettext!("Job\tGroup\t");
+                if have_proc_stat() {
+                    out += wgettext!("CPU\t");
+                }
+                out += wgettext!("State\tCommand\n");
+            }
+
+            sprintf!(=> &mut out, "%d\t%d\t", j.job_id(), pgid);
+
+            if have_proc_stat() {
+                sprintf!(=> &mut out, "%.0f%%\t", 100.0 * cpu_use(j));
+            }
+
+            out += if j.is_stopped() {
+                wgettext!("stopped")
+            } else {
+                wgettext!("running")
+            };
+            out += "\t";
+
+            let cmd = escape_string(
+                j.command(),
+                EscapeStringStyle::Script(EscapeFlags::NO_PRINTABLES),
+            );
+            out += &cmd[..];
+
+            out += "\n";
+            streams.out.append(out);
+        }
+        JobsPrintMode::PrintGroup => {
+            if header {
+                // Print table header before first job.
+                out += &wgettext!("Group\n")[..];
+            }
+            out += &sprintf!("%d\n", pgid)[..];
+            streams.out.append(out);
+        }
+        JobsPrintMode::PrintPid => {
+            if header {
+                // Print table header before first job.
+                out += &wgettext!("Process\n")[..];
+            }
+
+            for p in j.processes() {
+                out += &sprintf!("%d\n", p.pid.load(Ordering::Relaxed))[..];
+            }
+            streams.out.append(out);
+        }
+        JobsPrintMode::PrintCommand => {
+            if header {
+                // Print table header before first job.
+                out += &wgettext!("Command\n")[..];
+            }
+
+            for p in j.processes() {
+                out += &sprintf!("%ls\n", p.argv0().unwrap())[..];
+            }
+            streams.out.append(out);
+        }
+    };
+}
+
+const SHORT_OPTIONS: &wstr = L!(":cghlpq");
+const LONG_OPTIONS: &[woption] = &[
+    wopt(L!("command"), woption_argument_t::no_argument, 'c'),
+    wopt(L!("group"), woption_argument_t::no_argument, 'g'),
+    wopt(L!("help"), woption_argument_t::no_argument, 'h'),
+    wopt(L!("last"), woption_argument_t::no_argument, 'l'),
+    wopt(L!("pid"), woption_argument_t::no_argument, 'p'),
+    wopt(L!("quiet"), woption_argument_t::no_argument, 'q'),
+    wopt(L!("query"), woption_argument_t::no_argument, 'q'),
+];
+
+/// The jobs builtin. Used for printing running jobs. Defined in builtin_jobs.c.
+pub fn jobs(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+    let cmd = argv[0];
+    let argc = argv.len();
+    let mut found = false;
+    let mut mode = JobsPrintMode::Default;
+    let mut print_last = false;
+
+    let mut w = wgetopter_t::new(SHORT_OPTIONS, LONG_OPTIONS, argv);
+    while let Some(c) = w.wgetopt_long() {
+        match c {
+            'p' => {
+                mode = JobsPrintMode::PrintPid;
+            }
+            'q' => {
+                mode = JobsPrintMode::PrintNothing;
+            }
+            'c' => {
+                mode = JobsPrintMode::PrintCommand;
+            }
+            'g' => {
+                mode = JobsPrintMode::PrintGroup;
+            }
+            'l' => {
+                print_last = true;
+            }
+            'h' => {
+                builtin_print_help(parser, streams, cmd);
+                return STATUS_CMD_OK;
+            }
+            ':' => {
+                builtin_missing_argument(parser, streams, cmd, argv[w.woptind - 1], true);
+                return STATUS_INVALID_ARGS;
+            }
+            '?' => {
+                builtin_unknown_option(parser, streams, cmd, argv[w.woptind - 1], true);
+                return STATUS_INVALID_ARGS;
+            }
+            _ => panic!("unexpected retval from wgetopt_long"),
+        }
+    }
+
+    if print_last {
+        // Ignore unconstructed jobs, i.e. ourself.
+        for j in &parser.jobs()[..] {
+            if j.is_visible() {
+                builtin_jobs_print(j, mode, !streams.out_is_redirected, streams);
+                return STATUS_CMD_OK;
+            }
+        }
+        return STATUS_CMD_ERROR;
+    }
+
+    if w.woptind < argc {
+        for arg in &w.argv[w.woptind..] {
+            let j;
+            if arg.char_at(0) == '%' {
+                match fish_wcstoi(&arg[1..]).ok().filter(|&job_id| job_id >= 0) {
+                    None => {
+                        streams.err.append(wgettext_fmt!(
+                            "%ls: '%ls' is not a valid job id\n",
+                            cmd,
+                            arg
+                        ));
+                        return STATUS_INVALID_ARGS;
+                    }
+                    Some(job_id) => {
+                        let job_id = if job_id == 0 {
+                            JobId::NONE
+                        } else {
+                            let job_id = u32::try_from(job_id).unwrap();
+                            let job_id = NonZeroU32::try_from(job_id).unwrap();
+                            MaybeJobId(Some(JobId::new(job_id)))
+                        };
+                        j = parser.job_with_id(job_id);
+                    }
+                }
+            } else {
+                match fish_wcstoi(arg).ok().filter(|&pid| pid >= 0) {
+                    None => {
+                        streams.err.append(wgettext_fmt!(
+                            "%ls: '%ls' is not a valid process id\n",
+                            cmd,
+                            arg
+                        ));
+                        return STATUS_INVALID_ARGS;
+                    }
+                    Some(job_id) => {
+                        j = parser.job_get_from_pid(job_id);
+                    }
+                }
+            }
+
+            if let Some(j) = j.filter(|j| !j.is_completed() && j.is_constructed()) {
+                builtin_jobs_print(&j, mode, false, streams);
+                found = true;
+            } else {
+                if mode != JobsPrintMode::PrintNothing {
+                    streams
+                        .err
+                        .append(wgettext_fmt!("%ls: No suitable job: %ls\n", cmd, arg));
+                }
+                return STATUS_CMD_ERROR;
+            }
+        }
+    } else {
+        for j in &parser.jobs()[..] {
+            // Ignore unconstructed jobs, i.e. ourself.
+            if j.is_visible() {
+                builtin_jobs_print(j, mode, !found && !streams.out_is_redirected, streams);
+                found = true;
+            }
+        }
+    }
+
+    if !found {
+        // Do not babble if not interactive.
+        if !streams.out_is_redirected && mode != JobsPrintMode::PrintNothing {
+            streams
+                .out
+                .append(wgettext_fmt!("%ls: There are no jobs\n", argv[0]));
+        }
+        return STATUS_CMD_ERROR;
+    }
+
+    STATUS_CMD_OK
+}

fish-rust/src/builtins/math.rs

@@ -17,7 +17,7 @@ struct Options {
 #[widestrs]
 fn parse_cmd_opts(
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Result<(Options, usize), Option<c_int>> {
     const cmd: &wstr = "math"L;
@@ -219,7 +219,7 @@ const MATH_CHUNK_SIZE: usize = 1024;
 
 /// The math builtin evaluates math expressions.
 #[widestrs]
-pub fn math(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn math(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
 
     let (opts, mut optind) = match parse_cmd_opts(argv, parser, streams) {

fish-rust/src/builtins/mod.rs

@@ -3,29 +3,41 @@ pub mod shared;
 pub mod abbr;
 pub mod argparse;
 pub mod bg;
+pub mod bind;
 pub mod block;
 pub mod builtin;
 pub mod cd;
 pub mod command;
+pub mod commandline;
+pub mod complete;
 pub mod contains;
 pub mod count;
+pub mod disown;
 pub mod echo;
 pub mod emit;
+pub mod eval;
 pub mod exit;
+pub mod fg;
 pub mod function;
 pub mod functions;
+pub mod history;
+pub mod jobs;
 pub mod math;
 pub mod path;
 pub mod printf;
 pub mod pwd;
 pub mod random;
+pub mod read;
 pub mod realpath;
 pub mod r#return;
+pub mod set;
 pub mod set_color;
+pub mod source;
 pub mod status;
 pub mod string;
 pub mod test;
 pub mod r#type;
+pub mod ulimit;
 pub mod wait;
 
 // Note these tests will NOT run with cfg(test).
@@ -34,9 +46,13 @@ mod tests;
 mod prelude {
     pub use super::shared::*;
     pub use libc::c_int;
+    pub use std::borrow::Cow;
 
+    #[allow(unused_imports)]
     pub(crate) use crate::{
-        ffi::{self, separation_type_t, Parser, Repin},
+        flog::{FLOG, FLOGF},
+        io::{IoStreams, SeparationType},
+        parser::Parser,
         wchar::prelude::*,
         wchar_ffi::{c_str, AsWstr, WCharFromFFI, WCharToFFI},
         wgetopt::{

fish-rust/src/builtins/path.rs

@@ -28,9 +28,9 @@ macro_rules! path_error {
     };
 }
 
-fn path_unknown_option(parser: &mut Parser, streams: &mut IoStreams, subcmd: &wstr, opt: &wstr) {
+fn path_unknown_option(parser: &Parser, streams: &mut IoStreams, subcmd: &wstr, opt: &wstr) {
     path_error!(streams, BUILTIN_ERR_UNKNOWN, subcmd, opt);
-    builtin_print_error_trailer(parser, streams, L!("path"));
+    builtin_print_error_trailer(parser, streams.err, L!("path"));
 }
 
 // How many bytes we read() at once.
@@ -161,7 +161,7 @@ fn path_out(streams: &mut IoStreams, opts: &Options<'_>, s: impl AsRef<wstr>) {
         if !opts.null_out {
             streams
                 .out
-                .append_with_separation(s, separation_type_t::explicitly, true);
+                .append_with_separation(s, SeparationType::explicitly, true);
         } else {
             let mut output = WString::with_capacity(s.len() + 1);
             output.push_utfstr(s);
@@ -221,7 +221,7 @@ fn parse_opts<'args>(
     optind: &mut usize,
     n_req_args: usize,
     args: &mut [&'args wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Option<c_int> {
     let cmd = args[0];
@@ -352,7 +352,7 @@ fn parse_opts<'args>(
     }
 
     // At this point we should not have optional args and be reading args from stdin.
-    if streams.stdin_is_directly_redirected() && args.len() > *optind {
+    if streams.stdin_is_directly_redirected && args.len() > *optind {
         path_error!(streams, BUILTIN_ERR_TOO_MANY_ARGUMENTS, cmd);
         return STATUS_INVALID_ARGS;
     }
@@ -361,7 +361,7 @@ fn parse_opts<'args>(
 }
 
 fn path_transform(
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
     args: &mut [&wstr],
     func: impl Fn(&wstr) -> WString,
@@ -402,15 +402,11 @@ fn path_transform(
     }
 }
 
-fn path_basename(
-    parser: &mut Parser,
-    streams: &mut IoStreams,
-    args: &mut [&wstr],
-) -> Option<c_int> {
+fn path_basename(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     path_transform(parser, streams, args, |s| wbasename(s).to_owned())
 }
 
-fn path_dirname(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+fn path_dirname(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     path_transform(parser, streams, args, |s| wdirname(s).to_owned())
 }
 
@@ -422,15 +418,11 @@ fn normalize_help(path: &wstr) -> WString {
     np
 }
 
-fn path_normalize(
-    parser: &mut Parser,
-    streams: &mut IoStreams,
-    args: &mut [&wstr],
-) -> Option<c_int> {
+fn path_normalize(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     path_transform(parser, streams, args, normalize_help)
 }
 
-fn path_mtime(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+fn path_mtime(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let mut opts = Options::default();
     opts.relative_valid = true;
     let mut optind = 0;
@@ -514,11 +506,7 @@ fn test_find_extension() {
     }
 }
 
-fn path_extension(
-    parser: &mut Parser,
-    streams: &mut IoStreams,
-    args: &mut [&wstr],
-) -> Option<c_int> {
+fn path_extension(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let mut opts = Options::default();
     let mut optind = 0;
     let retval = parse_opts(&mut opts, &mut optind, 0, args, parser, streams);
@@ -556,7 +544,7 @@ fn path_extension(
 }
 
 fn path_change_extension(
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
     args: &mut [&wstr],
 ) -> Option<c_int> {
@@ -604,7 +592,7 @@ fn path_change_extension(
     }
 }
 
-fn path_resolve(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+fn path_resolve(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let mut opts = Options::default();
     let mut optind = 0;
     let retval = parse_opts(&mut opts, &mut optind, 0, args, parser, streams);
@@ -626,7 +614,7 @@ fn path_resolve(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]
                 let mut next = arg.into_owned();
                 // First add $PWD if we're relative
                 if !next.is_empty() && next.char_at(0) != '/' {
-                    next = path_apply_working_directory(&next, &parser.get_vars().get_pwd_slash());
+                    next = path_apply_working_directory(&next, &parser.vars().get_pwd_slash());
                 }
                 let mut rest = wbasename(&next).to_owned();
                 let mut real = None;
@@ -669,7 +657,7 @@ fn path_resolve(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]
     }
 }
 
-fn path_sort(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+fn path_sort(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let mut opts = Options::default();
     opts.reverse_valid = true;
     opts.unique_valid = true;
@@ -835,7 +823,7 @@ fn filter_path(opts: &Options, path: &wstr) -> bool {
 }
 
 fn path_filter_maybe_is(
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
     args: &mut [&wstr],
     is_is: bool,
@@ -894,16 +882,16 @@ fn path_filter_maybe_is(
     }
 }
 
-fn path_filter(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+fn path_filter(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     path_filter_maybe_is(parser, streams, args, false)
 }
 
-fn path_is(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+fn path_is(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     path_filter_maybe_is(parser, streams, args, true)
 }
 
 /// The path builtin, for handling paths.
-pub fn path(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn path(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
     let argc = args.len();
 
@@ -911,7 +899,7 @@ pub fn path(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) ->
         streams
             .err
             .append(wgettext_fmt!(BUILTIN_ERR_MISSING_SUBCMD, cmd));
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return STATUS_INVALID_ARGS;
     }
 
@@ -937,7 +925,7 @@ pub fn path(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) ->
             streams
                 .err
                 .append(wgettext_fmt!(BUILTIN_ERR_INVALID_SUBCMD, cmd, subcmd_name));
-            builtin_print_error_trailer(parser, streams, cmd);
+            builtin_print_error_trailer(parser, streams.err, cmd);
             return STATUS_INVALID_ARGS;
         }
     };

fish-rust/src/builtins/printf.rs

@@ -77,9 +77,9 @@ fn iswxdigit(c: char) -> bool {
     c.is_ascii_hexdigit()
 }
 
-struct builtin_printf_state_t<'a> {
+struct builtin_printf_state_t<'a, 'b> {
     // Out and err streams. Note this is a captured reference!
-    streams: &'a mut IoStreams,
+    streams: &'a mut IoStreams<'b>,
 
     // The status of the operation.
     exit_code: c_int,
@@ -203,7 +203,7 @@ fn modify_allowed_format_specifiers(ok: &mut [bool; 256], str: &str, flag: bool)
     }
 }
 
-impl<'a> builtin_printf_state_t<'a> {
+impl<'a, 'b> builtin_printf_state_t<'a, 'b> {
     #[allow(clippy::partialeq_to_none)]
     fn verify_numeric(&mut self, s: &wstr, end: &wstr, errcode: Option<Error>) {
         // This check matches the historic `errcode != EINVAL` check from C++.
@@ -579,7 +579,7 @@ impl<'a> builtin_printf_state_t<'a> {
 
         self.streams.err.append(errstr);
         if !errstr.ends_with('\n') {
-            self.streams.err.append1('\n');
+            self.streams.err.push('\n');
         }
 
         // We set the exit code to error, because one occurred,
@@ -603,7 +603,7 @@ impl<'a> builtin_printf_state_t<'a> {
 
         self.streams.err.append(errstr);
         if !errstr.ends_with('\n') {
-            self.streams.err.append1('\n');
+            self.streams.err.push('\n');
         }
 
         self.exit_code = STATUS_CMD_ERROR.unwrap();
@@ -763,7 +763,7 @@ impl<'a> builtin_printf_state_t<'a> {
 }
 
 /// The printf builtin.
-pub fn printf(_parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn printf(_parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let mut argc = argv.len();
 
     // Rebind argv as immutable slice (can't rearrange its elements), skipping the command name.

fish-rust/src/builtins/pwd.rs

@@ -2,7 +2,7 @@
 use errno::errno;
 
 use super::prelude::*;
-use crate::{env::EnvMode, wutil::wrealpath};
+use crate::{env::Environment, wutil::wrealpath};
 
 // The pwd builtin. Respect -P to resolve symbolic links. Respect -L to not do that (the default).
 const short_options: &wstr = L!("LPh");
@@ -12,7 +12,7 @@ const long_options: &[woption] = &[
     wopt(L!("physical"), no_argument, 'P'),
 ];
 
-pub fn pwd(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn pwd(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
     let argc = argv.len();
     let mut resolve_symlinks = false;
@@ -41,11 +41,8 @@ pub fn pwd(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) ->
     }
 
     let mut pwd = WString::new();
-    let tmp = parser
-        .vars1()
-        .get_or_null(&L!("PWD").to_ffi(), EnvMode::default().bits());
-    if !tmp.is_null() {
-        pwd = tmp.as_string().from_ffi();
+    if let Some(tmp) = parser.vars().get(L!("PWD")) {
+        pwd = tmp.as_string();
     }
     if resolve_symlinks {
         if let Some(real_pwd) = wrealpath(&pwd) {

fish-rust/src/builtins/random.rs

@@ -8,7 +8,7 @@ use std::sync::Mutex;
 
 static RNG: Lazy<Mutex<SmallRng>> = Lazy::new(|| Mutex::new(SmallRng::from_entropy()));
 
-pub fn random(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn random(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
     let argc = argv.len();
     let print_hints = false;
@@ -47,7 +47,7 @@ pub fn random(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
         if arg_count == 1 {
             streams
                 .err
-                .append(wgettext_fmt!("%ls: nothing to choose from\n", cmd,));
+                .append(wgettext_fmt!("%ls: nothing to choose from\n", cmd));
             return STATUS_INVALID_ARGS;
         }
 

fish-rust/src/builtins/read.rs

@@ -0,0 +1,7 @@
+//! Implementation of the read builtin.
+
+use super::prelude::*;
+
+pub fn read(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    run_builtin_ffi(crate::ffi::builtin_read, parser, streams, args)
+}

fish-rust/src/builtins/realpath.rs

@@ -3,6 +3,8 @@
 use errno::errno;
 
 use super::prelude::*;
+use crate::env::Environment;
+use crate::io::IoStreams;
 use crate::{
     path::path_apply_working_directory,
     wutil::{normalize_path, wrealpath},
@@ -22,7 +24,7 @@ const long_options: &[woption] = &[
 
 fn parse_options(
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Result<(Options, usize), Option<c_int>> {
     let cmd = args[0];
@@ -53,7 +55,7 @@ fn parse_options(
 /// An implementation of the external realpath command. Doesn't support any options.
 /// In general scripts shouldn't invoke this directly. They should just use `realpath` which
 /// will fallback to this builtin if an external command cannot be found.
-pub fn realpath(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn realpath(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
     let (opts, optind) = match parse_options(args, parser, streams) {
         Ok((opts, optind)) => (opts, optind),
@@ -105,7 +107,7 @@ pub fn realpath(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]
         }
     } else {
         // We need to get the *physical* pwd here.
-        let realpwd = wrealpath(parser.vars1().get_pwd_slash().as_wstr());
+        let realpwd = wrealpath(&parser.vars().get_pwd_slash());
 
         if let Some(realpwd) = realpwd {
             let absolute_arg = if arg.starts_with(L!("/")) {

fish-rust/src/builtins/return.rs

@@ -11,7 +11,7 @@ struct Options {
 
 fn parse_options(
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Result<(Options, usize), Option<c_int>> {
     let cmd = args[0];
@@ -46,13 +46,13 @@ fn parse_options(
 }
 
 /// Function for handling the return builtin.
-pub fn r#return(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn r#return(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let mut retval = match parse_return_value(args, parser, streams) {
         Ok(v) => v,
         Err(e) => return e,
     };
 
-    let has_function_block = parser.ffi_has_funtion_block();
+    let has_function_block = parser.blocks().iter().any(|b| b.is_function_call());
 
     // *nix does not support negative return values, but our `return` builtin happily accepts being
     // called with negative literals (e.g. `return -1`).
@@ -65,7 +65,7 @@ pub fn r#return(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]
 
     // If we're not in a function, exit the current script (but not an interactive shell).
     if !has_function_block {
-        let ld = parser.libdata_pod();
+        let ld = &mut parser.libdata_mut().pods;
         if !ld.is_interactive {
             ld.exit_current_script = true;
         }
@@ -73,14 +73,14 @@ pub fn r#return(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]
     }
 
     // Mark a return in the libdata.
-    parser.libdata_pod().returning = true;
+    parser.libdata_mut().pods.returning = true;
 
     return Some(retval);
 }
 
 pub fn parse_return_value(
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Result<i32, Option<c_int>> {
     let cmd = args[0];
@@ -97,11 +97,11 @@ pub fn parse_return_value(
         streams
             .err
             .append(wgettext_fmt!(BUILTIN_ERR_TOO_MANY_ARGUMENTS, cmd));
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return Err(STATUS_INVALID_ARGS);
     }
     if optind == args.len() {
-        Ok(parser.get_last_status().into())
+        Ok(parser.get_last_status())
     } else {
         match fish_wcstoi(args[optind]) {
             Ok(i) => Ok(i),
@@ -109,7 +109,7 @@ pub fn parse_return_value(
                 streams
                     .err
                     .append(wgettext_fmt!(BUILTIN_ERR_NOT_NUMBER, cmd, args[1]));
-                builtin_print_error_trailer(parser, streams, cmd);
+                builtin_print_error_trailer(parser, streams.err, cmd);
                 return Err(STATUS_INVALID_ARGS);
             }
         }

fish-rust/src/builtins/set_color.rs

@@ -116,11 +116,7 @@ const LONG_OPTIONS: &[woption] = &[
 ];
 
 /// set_color builtin.
-pub fn set_color(
-    parser: &mut Parser,
-    streams: &mut IoStreams,
-    argv: &mut [&wstr],
-) -> Option<c_int> {
+pub fn set_color(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     // Variables used for parsing the argument list.
     let argc = argv.len();
 

fish-rust/src/builtins/source.rs

@@ -0,0 +1,132 @@
+use crate::{
+    common::{escape, scoped_push_replacer, FilenameRef},
+    fds::{wopen_cloexec, AutoCloseFd},
+    ffi::reader_read_ffi,
+    io::IoChain,
+    parser::Block,
+};
+use libc::{c_int, O_RDONLY, S_IFMT, S_IFREG};
+
+use super::prelude::*;
+
+/// The  source builtin, sometimes called `.`. Evaluates the contents of a file in the current
+/// context.
+pub fn source(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    let argc = args.len();
+
+    let opts = match HelpOnlyCmdOpts::parse(args, parser, streams) {
+        Ok(opts) => opts,
+        Err(err) => return err,
+    };
+    let cmd = args[0];
+
+    if opts.print_help {
+        builtin_print_help(parser, streams, cmd);
+        return STATUS_CMD_OK;
+    }
+
+    // If we open a file, this ensures we close it.
+    let opened_fd;
+
+    // The fd that we read from, either from opened_fd or stdin.
+    let fd;
+    let func_filename;
+    let optind = opts.optind;
+
+    if argc == optind || args[optind] == L!("-") {
+        if streams.stdin_fd < 0 {
+            streams
+                .err
+                .append(wgettext_fmt!("%ls: stdin is closed\n", cmd));
+            return STATUS_CMD_ERROR;
+        }
+        // Either a bare `source` which means to implicitly read from stdin or an explicit `-`.
+        if argc == optind && unsafe { libc::isatty(streams.stdin_fd) } != 0 {
+            // Don't implicitly read from the terminal.
+            return STATUS_CMD_ERROR;
+        }
+        func_filename = FilenameRef::new(L!("-").to_owned());
+        fd = streams.stdin_fd;
+    } else {
+        opened_fd = AutoCloseFd::new(wopen_cloexec(args[optind], O_RDONLY, 0));
+        if !opened_fd.is_valid() {
+            let esc = escape(args[optind]);
+            streams.err.append(wgettext_fmt!(
+                "%ls: Error encountered while sourcing file '%ls':\n",
+                cmd,
+                &esc
+            ));
+            builtin_wperror(cmd, streams);
+            return STATUS_CMD_ERROR;
+        }
+
+        fd = opened_fd.fd();
+        let mut buf: libc::stat = unsafe { std::mem::zeroed() };
+        if unsafe { libc::fstat(fd, &mut buf) } == -1 {
+            let esc = escape(args[optind]);
+            streams.err.append(wgettext_fmt!(
+                "%ls: Error encountered while sourcing file '%ls':\n",
+                cmd,
+                &esc
+            ));
+            return STATUS_CMD_ERROR;
+        }
+
+        if buf.st_mode & S_IFMT != S_IFREG {
+            let esc = escape(args[optind]);
+            streams
+                .err
+                .append(wgettext_fmt!("%ls: '%ls' is not a file\n", cmd, esc));
+            return STATUS_CMD_ERROR;
+        }
+
+        func_filename = FilenameRef::new(args[optind].to_owned());
+    }
+
+    assert!(fd >= 0, "Should have a valid fd");
+
+    let sb = parser.push_block(Block::source_block(func_filename.clone()));
+    let _filename_push = scoped_push_replacer(
+        |new_value| std::mem::replace(&mut parser.libdata_mut().current_filename, new_value),
+        Some(func_filename.clone()),
+    );
+
+    // Construct argv from our null-terminated list.
+    // This is slightly subtle. If this is a bare `source` with no args then `argv + optind` already
+    // points to the end of argv. Otherwise we want to skip the file name to get to the args if any.
+    let mut argv_list: Vec<WString> = vec![];
+    let remaining_args = &args[optind + if argc == optind { 0 } else { 1 }..];
+    for arg in remaining_args.iter().copied() {
+        argv_list.push(arg.to_owned());
+    }
+    parser.vars().set_argv(argv_list);
+
+    let empty_io_chain = IoChain::new();
+    let retval = reader_read_ffi(
+        parser as *const Parser as *const autocxx::c_void,
+        unsafe { std::mem::transmute(fd) },
+        if !streams.io_chain.is_null() {
+            unsafe { &*streams.io_chain }
+        } else {
+            &empty_io_chain
+        } as *const _ as *const autocxx::c_void,
+    );
+    let mut retval: c_int = unsafe { std::mem::transmute(retval) };
+
+    parser.pop_block(sb);
+
+    if retval != STATUS_CMD_OK.unwrap() {
+        let esc = escape(&func_filename);
+        streams.err.append(wgettext_fmt!(
+            "%ls: Error while reading file '%ls'\n",
+            cmd,
+            if esc == "-" { L!("<stdin>") } else { &esc }
+        ));
+    } else {
+        retval = parser.get_last_status();
+    }
+
+    // Do not close fd after calling reader_read. reader_read automatically closes it before calling
+    // eval.
+    Some(retval)
+}

fish-rust/src/builtins/status.rs

@@ -2,10 +2,10 @@ use std::os::unix::prelude::*;
 
 use super::prelude::*;
 use crate::common::{get_executable_path, str2wcstring, PROGRAM_NAME};
-use crate::ffi::{
-    get_job_control_mode, get_login, is_interactive_session, job_control_t, set_job_control_mode,
-};
 use crate::future_feature_flags::{self as features, feature_test};
+use crate::proc::{
+    get_job_control_mode, get_login, is_interactive_session, set_job_control_mode, JobControl,
+};
 use crate::wutil::{waccess, wbasename, wdirname, wrealpath, Error};
 use libc::F_OK;
 use nix::errno::Errno;
@@ -114,7 +114,7 @@ enum TestFeatureRetVal {
 
 struct StatusCmdOpts {
     level: i32,
-    new_job_control_mode: Option<job_control_t>,
+    new_job_control_mode: Option<JobControl>,
     status_cmd: Option<StatusCmd>,
     print_help: bool,
 }
@@ -191,7 +191,7 @@ fn parse_cmd_opts(
     opts: &mut StatusCmdOpts,
     optind: &mut usize,
     args: &mut [&wstr],
-    parser: &mut Parser,
+    parser: &Parser,
     streams: &mut IoStreams,
 ) -> Option<c_int> {
     let cmd = args[0];
@@ -305,7 +305,7 @@ fn parse_cmd_opts(
     return STATUS_CMD_OK;
 }
 
-pub fn status(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn status(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
     let argc = args.len();
 
@@ -357,14 +357,14 @@ pub fn status(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
             streams.out.append(wgettext!("This is not a login shell\n"));
         }
         let job_control_mode = match get_job_control_mode() {
-            job_control_t::interactive => wgettext!("Only on interactive jobs"),
-            job_control_t::none => wgettext!("Never"),
-            job_control_t::all => wgettext!("Always"),
+            JobControl::interactive => wgettext!("Only on interactive jobs"),
+            JobControl::none => wgettext!("Never"),
+            JobControl::all => wgettext!("Always"),
         };
         streams
             .out
             .append(wgettext_fmt!("Job control: %ls\n", job_control_mode));
-        streams.out.append(parser.stack_trace().as_wstr());
+        streams.out.append(parser.stack_trace());
 
         return STATUS_CMD_OK;
     };
@@ -447,17 +447,18 @@ pub fn status(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
             }
             match s {
                 STATUS_BASENAME | STATUS_DIRNAME | STATUS_FILENAME => {
-                    let res = parser.current_filename_ffi().from_ffi();
-                    let f = match (res.is_empty(), s) {
-                        (false, STATUS_DIRNAME) => wdirname(&res),
-                        (false, STATUS_BASENAME) => wbasename(&res),
+                    let res = parser.current_filename();
+                    let function = res.unwrap_or_default();
+                    let f = match (function.is_empty(), s) {
+                        (false, STATUS_DIRNAME) => wdirname(&function),
+                        (false, STATUS_BASENAME) => wbasename(&function),
                         (true, _) => wgettext!("Standard input"),
-                        (false, _) => &res,
+                        (false, _) => &function,
                     };
                     streams.out.appendln(f);
                 }
                 STATUS_FUNCTION => {
-                    let f = match parser.get_func_name(opts.level) {
+                    let f = match parser.get_function_name(opts.level) {
                         Some(f) => f,
                         None => wgettext!("Not a function").to_owned(),
                     };
@@ -467,7 +468,9 @@ pub fn status(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
                     // TBD is how to interpret the level argument when fetching the line number.
                     // See issue #4161.
                     // streams.out.append_format(L"%d\n", parser.get_lineno(opts.level));
-                    streams.out.appendln(parser.get_lineno().0.to_wstring());
+                    streams
+                        .out
+                        .appendln(parser.get_lineno().unwrap_or(0).to_wstring());
                 }
                 STATUS_IS_INTERACTIVE => {
                     if is_interactive_session() {
@@ -477,7 +480,7 @@ pub fn status(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
                     }
                 }
                 STATUS_IS_COMMAND_SUB => {
-                    if parser.libdata_pod().is_subshell {
+                    if parser.libdata().pods.is_subshell {
                         return STATUS_CMD_OK;
                     } else {
                         return STATUS_CMD_ERROR;
@@ -505,44 +508,42 @@ pub fn status(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
                     }
                 }
                 STATUS_IS_FULL_JOB_CTRL => {
-                    if get_job_control_mode() == job_control_t::all {
+                    if get_job_control_mode() == JobControl::all {
                         return STATUS_CMD_OK;
                     } else {
                         return STATUS_CMD_ERROR;
                     }
                 }
                 STATUS_IS_INTERACTIVE_JOB_CTRL => {
-                    if get_job_control_mode() == job_control_t::interactive {
+                    if get_job_control_mode() == JobControl::interactive {
                         return STATUS_CMD_OK;
                     } else {
                         return STATUS_CMD_ERROR;
                     }
                 }
                 STATUS_IS_NO_JOB_CTRL => {
-                    if get_job_control_mode() == job_control_t::none {
+                    if get_job_control_mode() == JobControl::none {
                         return STATUS_CMD_OK;
                     } else {
                         return STATUS_CMD_ERROR;
                     }
                 }
                 STATUS_STACK_TRACE => {
-                    streams.out.append(parser.stack_trace().as_wstr());
+                    streams.out.append(parser.stack_trace());
                 }
                 STATUS_CURRENT_CMD => {
-                    let var = parser.pin().libdata().get_status_vars_command().from_ffi();
-                    if !var.is_empty() {
-                        streams.out.appendln(var);
+                    let command = &parser.libdata().status_vars.command;
+                    if !command.is_empty() {
+                        streams.out.append(command);
                     } else {
                         streams.out.appendln(*PROGRAM_NAME.get().unwrap());
                     }
+                    streams.out.append_char('\n');
                 }
                 STATUS_CURRENT_COMMANDLINE => {
-                    let var = parser
-                        .pin()
-                        .libdata()
-                        .get_status_vars_commandline()
-                        .from_ffi();
-                    streams.out.appendln(var);
+                    let commandline = &parser.libdata().status_vars.commandline;
+                    streams.out.append(commandline);
+                    streams.out.append_char('\n');
                 }
                 STATUS_FISH_PATH => {
                     let path = get_executable_path("fish");
@@ -563,7 +564,8 @@ pub fn status(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
                             _ => path,
                         };
 
-                        streams.out.appendln(real);
+                        streams.out.append(real);
+                        streams.out.append_char('\n');
                     } else {
                         // This is a relative path, we can't canonicalize it
                         let path = str2wcstring(path.as_os_str().as_bytes());

fish-rust/src/builtins/string.rs

@@ -1,7 +1,6 @@
 use crate::wcstringutil::fish_wcwidth_visible;
 // Forward some imports to make subcmd implementations easier
 use super::prelude::*;
-use crate::ffi::separation_type_t;
 
 mod collect;
 mod escape;
@@ -31,9 +30,9 @@ macro_rules! string_error {
 }
 use string_error;
 
-fn string_unknown_option(parser: &mut Parser, streams: &mut IoStreams, subcmd: &wstr, opt: &wstr) {
+fn string_unknown_option(parser: &Parser, streams: &mut IoStreams, subcmd: &wstr, opt: &wstr) {
     string_error!(streams, BUILTIN_ERR_UNKNOWN, subcmd, opt);
-    builtin_print_error_trailer(parser, streams, L!("string"));
+    builtin_print_error_trailer(parser, streams.err, L!("string"));
 }
 
 trait StringSubCommand<'args> {
@@ -51,7 +50,7 @@ trait StringSubCommand<'args> {
     fn parse_opts(
         &mut self,
         args: &mut [&'args wstr],
-        parser: &mut Parser,
+        parser: &Parser,
         streams: &mut IoStreams,
     ) -> Result<usize, Option<c_int>> {
         let cmd = args[0];
@@ -97,7 +96,7 @@ trait StringSubCommand<'args> {
     /// Perform the business logic of the command.
     fn handle(
         &mut self,
-        parser: &mut Parser,
+        parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&'args wstr],
@@ -105,7 +104,7 @@ trait StringSubCommand<'args> {
 
     fn run(
         &mut self,
-        parser: &mut Parser,
+        parser: &Parser,
         streams: &mut IoStreams,
         args: &mut [&'args wstr],
     ) -> Option<c_int> {
@@ -127,7 +126,7 @@ trait StringSubCommand<'args> {
             return retval;
         }
 
-        if streams.stdin_is_directly_redirected() && args.len() > optind {
+        if streams.stdin_is_directly_redirected && args.len() > optind {
             string_error!(streams, BUILTIN_ERR_TOO_MANY_ARGUMENTS, args[0]);
             return STATUS_INVALID_ARGS;
         }
@@ -199,7 +198,7 @@ impl StringError {
     fn print_error(
         &self,
         args: &[&wstr],
-        parser: &mut Parser,
+        parser: &Parser,
         streams: &mut IoStreams,
         optarg: Option<&wstr>,
         optind: usize,
@@ -291,7 +290,7 @@ fn arguments<'iter, 'args>(
 }
 
 /// The string builtin, for manipulating strings.
-pub fn string(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+pub fn string(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
     let cmd = args[0];
     let argc = args.len();
 
@@ -299,7 +298,7 @@ pub fn string(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
         streams
             .err
             .append(wgettext_fmt!(BUILTIN_ERR_MISSING_SUBCMD, cmd));
-        builtin_print_error_trailer(parser, streams, cmd);
+        builtin_print_error_trailer(parser, streams.err, cmd);
         return STATUS_INVALID_ARGS;
     }
 
@@ -320,13 +319,11 @@ pub fn string(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
             cmd.run(parser, streams, args)
         }
         "length" => length::Length::default().run(parser, streams, args),
-        "lower" => {
-            let mut cmd = transform::Transform {
-                quiet: false,
-                func: wstr::to_lowercase,
-            };
-            cmd.run(parser, streams, args)
+        "lower" => transform::Transform {
+            quiet: false,
+            func: wstr::to_lowercase,
         }
+        .run(parser, streams, args),
         "match" => r#match::Match::default().run(parser, streams, args),
         "pad" => pad::Pad::default().run(parser, streams, args),
         "repeat" => repeat::Repeat::default().run(parser, streams, args),
@@ -341,19 +338,17 @@ pub fn string(parser: &mut Parser, streams: &mut IoStreams, args: &mut [&wstr])
         "sub" => sub::Sub::default().run(parser, streams, args),
         "trim" => trim::Trim::default().run(parser, streams, args),
         "unescape" => unescape::Unescape::default().run(parser, streams, args),
-        "upper" => {
-            let mut cmd = transform::Transform {
-                quiet: false,
-                func: wstr::to_uppercase,
-            };
-            cmd.run(parser, streams, args)
+        "upper" => transform::Transform {
+            quiet: false,
+            func: wstr::to_uppercase,
         }
+        .run(parser, streams, args),
         _ => {
             streams
                 .err
-                .append(wgettext_fmt!(BUILTIN_ERR_INVALID_SUBCMD, cmd, subcmd_name));
-            builtin_print_error_trailer(parser, streams, cmd);
-            STATUS_INVALID_ARGS
+                .append(wgettext_fmt!(BUILTIN_ERR_INVALID_SUBCMD, cmd, args[0]));
+            builtin_print_error_trailer(parser, streams.err, cmd);
+            return STATUS_INVALID_ARGS;
         }
     }
 }

fish-rust/src/builtins/string/collect.rs

@@ -24,7 +24,7 @@ impl StringSubCommand<'_> for Collect {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],
@@ -43,7 +43,7 @@ impl StringSubCommand<'_> for Collect {
 
             streams
                 .out
-                .append_with_separation(arg, separation_type_t::explicitly, want_newline);
+                .append_with_separation(arg, SeparationType::explicitly, want_newline);
             appended += arg.len();
         }
 
@@ -54,7 +54,7 @@ impl StringSubCommand<'_> for Collect {
         if self.allow_empty && appended == 0 {
             streams.out.append_with_separation(
                 L!(""),
-                separation_type_t::explicitly,
+                SeparationType::explicitly,
                 true, /* historical behavior is to always print a newline */
             );
         }

fish-rust/src/builtins/string/escape.rs

@@ -30,7 +30,7 @@ impl StringSubCommand<'_> for Escape {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/string/join.rs

@@ -56,7 +56,7 @@ impl<'args> StringSubCommand<'args> for Join<'args> {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],
@@ -84,9 +84,9 @@ impl<'args> StringSubCommand<'args> for Join<'args> {
 
         if nargs > 0 && !self.quiet {
             if self.is_join0 {
-                streams.out.append1('\0');
+                streams.out.append_char('\0');
             } else if print_trailing_newline {
-                streams.out.append1('\n');
+                streams.out.append_char('\n');
             }
         }
 

fish-rust/src/builtins/string/length.rs

@@ -26,7 +26,7 @@ impl StringSubCommand<'_> for Length {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/string/match.rs

@@ -67,7 +67,7 @@ impl<'args> StringSubCommand<'args> for Match<'args> {
 
     fn handle(
         &mut self,
-        parser: &mut Parser,
+        parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],
@@ -125,7 +125,7 @@ impl<'args> StringSubCommand<'args> for Match<'args> {
             ..
         }) = matcher
         {
-            let vars = parser.get_vars();
+            let vars = parser.vars();
             for (name, vals) in first_match_captures.into_iter() {
                 vars.set(&WString::from(name), EnvMode::default(), vals);
             }

fish-rust/src/builtins/string/pad.rs

@@ -65,7 +65,7 @@ impl StringSubCommand<'_> for Pad {
 
     fn handle<'args>(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&'args wstr],

fish-rust/src/builtins/string/repeat.rs

@@ -39,7 +39,7 @@ impl StringSubCommand<'_> for Repeat {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/string/replace.rs

@@ -62,7 +62,7 @@ impl<'args> StringSubCommand<'args> for Replace<'args> {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/string/shorten.rs

@@ -64,7 +64,7 @@ impl<'args> StringSubCommand<'args> for Shorten<'args> {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],
@@ -100,7 +100,7 @@ impl<'args> StringSubCommand<'args> for Shorten<'args> {
                     Direction::Left => splits.last(),
                 }
                 .unwrap();
-                s.push_utfstr(self.ellipsis);
+                s.push_utfstr(&self.ellipsis);
                 let width = width_without_escapes(&s, 0);
 
                 if width > 0 && width < min_width {
@@ -125,7 +125,7 @@ impl<'args> StringSubCommand<'args> for Shorten<'args> {
             // truncating instead.
             (L!(""), 0)
         } else {
-            (self.ellipsis, self.ellipsis_width)
+            (&self.ellipsis[..], self.ellipsis_width)
         };
 
         let mut nsub = 0usize;

fish-rust/src/builtins/string/split.rs

@@ -1,4 +1,3 @@
-use std::borrow::Cow;
 use std::ops::Deref;
 
 use super::*;
@@ -162,7 +161,7 @@ impl<'args> StringSubCommand<'args> for Split<'args> {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&'args wstr],
@@ -259,18 +258,16 @@ impl<'args> StringSubCommand<'args> for Split<'args> {
                 }
                 for field in self.fields.iter() {
                     if let Some(val) = splits.get(*field) {
-                        streams.out.append_with_separation(
-                            val,
-                            separation_type_t::explicitly,
-                            true,
-                        );
+                        streams
+                            .out
+                            .append_with_separation(val, SeparationType::explicitly, true);
                     }
                 }
             } else {
-                for split in &splits {
+                for split in splits {
                     streams
                         .out
-                        .append_with_separation(split, separation_type_t::explicitly, true);
+                        .append_with_separation(&split, SeparationType::explicitly, true);
                 }
             }
         }

fish-rust/src/builtins/string/sub.rs

@@ -49,7 +49,7 @@ impl StringSubCommand<'_> for Sub {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/string/transform.rs

@@ -18,7 +18,7 @@ impl StringSubCommand<'_> for Transform {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/string/trim.rs

@@ -46,7 +46,7 @@ impl<'args> StringSubCommand<'args> for Trim<'args> {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/string/unescape.rs

@@ -32,7 +32,7 @@ impl StringSubCommand<'_> for Unescape {
 
     fn handle(
         &mut self,
-        _parser: &mut Parser,
+        _parser: &Parser,
         streams: &mut IoStreams,
         optind: &mut usize,
         args: &[&wstr],

fish-rust/src/builtins/test.rs

@@ -82,7 +82,7 @@ mod test_expressions {
         }
 
         // Return true if the number is a tty().
-        fn isatty(&self, streams: &IoStreams) -> bool {
+        fn isatty(&self, streams: &mut IoStreams) -> bool {
             fn istty(fd: libc::c_int) -> bool {
                 // Safety: isatty cannot crash.
                 unsafe { libc::isatty(fd) > 0 }
@@ -92,7 +92,10 @@ mod test_expressions {
             }
             let bint = self.base as i32;
             if bint == 0 {
-                streams.stdin_fd().map(istty).unwrap_or(false)
+                match streams.stdin_fd {
+                    -1 => false,
+                    fd => istty(fd),
+                }
             } else if bint == 1 {
                 !streams.out_is_redirected && istty(libc::STDOUT_FILENO)
             } else if bint == 2 {
@@ -1003,7 +1006,7 @@ mod test_expressions {
 /// Evaluate a conditional expression given the arguments. For POSIX conformance this
 /// supports a more limited range of functionality.
 /// Return status is the final shell status, i.e. 0 for true, 1 for false and 2 for error.
-pub fn test(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn test(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     // The first argument should be the name of the command ('test').
     if argv.is_empty() {
         return STATUS_INVALID_ARGS;
@@ -1025,7 +1028,7 @@ pub fn test(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) ->
             streams
                 .err
                 .appendln(wgettext!("[: the last argument must be ']'"));
-            builtin_print_error_trailer(parser, streams, program_name);
+            builtin_print_error_trailer(parser, streams.err, program_name);
             return STATUS_INVALID_ARGS;
         }
     }
@@ -1053,7 +1056,7 @@ pub fn test(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) ->
     let expr = test_expressions::TestParser::parse_args(args, &mut err, program_name);
     let Some(expr) = expr else {
         streams.err.append(err);
-        streams.err.append(parser.pin().current_line().as_wstr());
+        streams.err.append(parser.current_line());
         return STATUS_CMD_ERROR;
     };
 
@@ -1062,11 +1065,11 @@ pub fn test(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) ->
     if !eval_errors.is_empty() {
         if !common::should_suppress_stderr_for_tests() {
             for eval_error in eval_errors {
-                streams.err.appendln(eval_error);
+                streams.err.appendln(&eval_error);
             }
             // Add a backtrace but not the "see help" message
             // because this isn't about passing the wrong options.
-            streams.err.append(parser.pin().current_line().as_wstr());
+            streams.err.append(parser.current_line());
         }
         return STATUS_INVALID_ARGS;
     }

fish-rust/src/builtins/tests/string_tests.rs

@@ -1,11 +1,11 @@
+use super::super::prelude::*;
+use crate::common::escape;
 use crate::ffi_tests::add_test;
+use crate::io::{OutputStream, StringOutputStream};
 
 add_test! {"test_string", || {
-    use crate::ffi::Parser;
-    use crate::ffi;
+    use crate::parser::Parser;
     use crate::builtins::string::string;
-    use crate::wchar_ffi::WCharFromFFI;
-    use crate::common::{EscapeStringStyle, escape_string};
     use crate::wchar::wstr;
     use crate::wchar::L;
     use crate::builtins::shared::{STATUS_CMD_ERROR,STATUS_CMD_OK, STATUS_INVALID_ARGS};
@@ -20,17 +20,18 @@ add_test! {"test_string", || {
 
     // TODO: these should be individual tests, not all in one, port when we can run these with `cargo test`
     fn string_test(mut args: Vec<&wstr>, expected_rc: Option<i32>, expected_out: &wstr) {
-        let parser: &mut Parser = unsafe { &mut *Parser::principal_parser_ffi() };
-        let mut streams = ffi::make_test_io_streams_ffi();
-        let mut io = crate::builtins::shared::IoStreams::new(streams.pin_mut());
+        let parser: &Parser = Parser::principal_parser();
+        let mut outs = OutputStream::String(StringOutputStream::new());
+        let mut errs = OutputStream::Null;
+        let mut streams = IoStreams::new(&mut outs, &mut errs);
+        streams.stdin_is_directly_redirected = false; // read from argv instead of stdin
 
-        let rc = string(parser, &mut io, args.as_mut_slice()).expect("string failed");
+        let rc = string(parser, &mut streams, args.as_mut_slice()).expect("string failed");
 
         assert_eq!(expected_rc.unwrap(), rc, "string builtin returned unexpected return code");
 
-        let string_stream_contents = &ffi::get_test_output_ffi(&streams);
-        let actual = escape_string(&string_stream_contents.from_ffi(), EscapeStringStyle::default());
-        let expected = escape_string(expected_out, EscapeStringStyle::default());
+        let actual = escape(outs.contents());
+        let expected = escape(expected_out);
         assert_eq!(expected, actual, "string builtin returned unexpected output");
     }
 

fish-rust/src/builtins/tests/test_tests.rs

@@ -1,10 +1,9 @@
 use crate::builtins::prelude::*;
 use crate::builtins::test::test as builtin_test;
-
-use crate::ffi::make_null_io_streams_ffi;
+use crate::io::OutputStream;
 
 fn run_one_test_test_mbracket(expected: i32, lst: &[&str], bracket: bool) -> bool {
-    let parser: &mut Parser = unsafe { &mut *Parser::principal_parser_ffi() };
+    let parser = Parser::principal_parser();
     let mut argv = Vec::new();
     if bracket {
         argv.push(L!("[").to_owned());
@@ -20,9 +19,10 @@ fn run_one_test_test_mbracket(expected: i32, lst: &[&str], bracket: bool) -> boo
 
     // Convert to &[&wstr].
     let mut argv = argv.iter().map(|s| s.as_ref()).collect::<Vec<_>>();
+    let mut out = OutputStream::Null;
+    let mut err = OutputStream::Null;
+    let mut streams = IoStreams::new(&mut out, &mut err);
 
-    let mut streams_ffi = make_null_io_streams_ffi();
-    let mut streams = IoStreams::new(streams_ffi.as_mut().unwrap());
     let result: Option<i32> = builtin_test(parser, &mut streams, &mut argv);
 
     if result != Some(expected) {
@@ -48,9 +48,11 @@ fn run_test_test(expected: i32, lst: &[&str]) -> bool {
 #[widestrs]
 fn test_test_brackets() {
     // Ensure [ knows it needs a ].
-    let parser: &mut Parser = unsafe { &mut *Parser::principal_parser_ffi() };
-    let mut streams_ffi = make_null_io_streams_ffi();
-    let mut streams = IoStreams::new(streams_ffi.as_mut().unwrap());
+    let parser = Parser::principal_parser();
+
+    let mut out = OutputStream::Null;
+    let mut err = OutputStream::Null;
+    let mut streams = IoStreams::new(&mut out, &mut err);
 
     let args1 = &mut ["["L, "foo"L];
     assert_eq!(

fish-rust/src/builtins/type.rs

@@ -1,6 +1,8 @@
 use super::prelude::*;
-use crate::ffi::{builtin_exists, colorize_shell};
+use crate::builtins::shared::builtin_exists;
+use crate::common::str2wcstring;
 use crate::function;
+use crate::highlight::{colorize, highlight_shell};
 
 use crate::path::{path_get_path, path_get_paths};
 
@@ -16,7 +18,7 @@ struct type_cmd_opts_t {
     query: bool,
 }
 
-pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn r#type(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
     let argc = argv.len();
     let print_hints = false;
@@ -87,7 +89,7 @@ pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
 
                     if path.is_empty() {
                         comment.push_utfstr(&wgettext_fmt!("Defined interactively"));
-                    } else if path == "-" {
+                    } else if path == L!("-") {
                         comment.push_utfstr(&wgettext_fmt!("Defined via `source`"));
                     } else {
                         let lineno: i32 = props.definition_lineno();
@@ -102,10 +104,10 @@ pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
                         let path = props.copy_definition_file().unwrap_or(L!(""));
                         if path.is_empty() {
                             comment.push_utfstr(&wgettext_fmt!(", copied interactively"));
-                        } else if path == "-" {
+                        } else if path == L!("-") {
                             comment.push_utfstr(&wgettext_fmt!(", copied via `source`"));
                         } else {
-                            let lineno: i32 = props.copy_definition_lineno();
+                            let lineno = props.copy_definition_lineno();
                             comment.push_utfstr(&wgettext_fmt!(
                                 ", copied in %ls @ line %d",
                                 path,
@@ -130,7 +132,15 @@ pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
                         ));
 
                         if streams.out_is_terminal() {
-                            let col = colorize_shell(&def.to_ffi(), parser.pin()).from_ffi();
+                            let mut color = vec![];
+                            highlight_shell(
+                                &def,
+                                &mut color,
+                                &parser.context(),
+                                /*io_ok=*/ false,
+                                /*cursor=*/ None,
+                            );
+                            let col = str2wcstring(&colorize(&def, &color, parser.vars()));
                             streams.out.append(col);
                         } else {
                             streams.out.append(def);
@@ -140,7 +150,7 @@ pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
                         streams.out.append(wgettext_fmt!(" (%ls)\n", comment));
                     }
                 } else if opts.get_type {
-                    streams.out.appendln("function");
+                    streams.out.appendln(L!("function"));
                 }
                 if !opts.all {
                     continue;
@@ -148,7 +158,7 @@ pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
             }
         }
 
-        if !opts.force_path && builtin_exists(&arg.to_ffi()) {
+        if !opts.force_path && builtin_exists(arg) {
             found += 1;
             res = true;
             if opts.query {
@@ -165,9 +175,9 @@ pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
         }
 
         let paths = if opts.all {
-            path_get_paths(arg, &*parser.get_vars())
+            path_get_paths(arg, parser.vars())
         } else {
-            match path_get_path(arg, &*parser.get_vars()) {
+            match path_get_path(arg, parser.vars()) {
                 Some(p) => vec![p],
                 None => vec![],
             }
@@ -186,7 +196,7 @@ pub fn r#type(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr])
                     streams.out.append(wgettext_fmt!("%ls is %ls\n", arg, path));
                 }
             } else if opts.get_type {
-                streams.out.appendln("file");
+                streams.out.appendln(L!("file"));
                 break;
             }
             if !opts.all {

fish-rust/src/builtins/ulimit.rs

@@ -0,0 +1,16 @@
+use super::prelude::*;
+
+pub fn ulimit(parser: &Parser, streams: &mut IoStreams, args: &mut [&wstr]) -> Option<c_int> {
+    run_builtin_ffi(crate::ffi::builtin_ulimit, parser, streams, args)
+}
+
+/// Struct describing a resource limit.
+struct Resource {
+    resource: c_int,     // resource ID
+    desc: &'static wstr, // description of resource
+    switch_char: char,   // switch used on commandline to specify resource
+    multiplier: c_int,   // the implicit multiplier used when setting getting values
+}
+
+/// Array of resource_t structs, describing all known resource types.
+const resource_arr: &[Resource] = &[];

fish-rust/src/builtins/wait.rs

@@ -1,13 +1,13 @@
 use libc::pid_t;
 
 use super::prelude::*;
-use crate::ffi::{job_t, proc_wait_any, Parser};
+use crate::proc::{proc_wait_any, Job};
 use crate::signal::SigChecker;
 use crate::wait_handle::{WaitHandleRef, WaitHandleStore};
 use crate::wutil;
 
 /// \return true if we can wait on a job.
-fn can_wait_on_job(j: &cxx::SharedPtr<job_t>) -> bool {
+fn can_wait_on_job(j: &Job) -> bool {
     j.is_constructed() && !j.is_foreground() && !j.is_stopped()
 }
 
@@ -35,13 +35,13 @@ enum WaitHandleQuery<'a> {
 /// \return true if we found a matching job (even if not waitable), false if not.
 fn find_wait_handles(
     query: WaitHandleQuery<'_>,
-    parser: &mut Parser,
+    parser: &Parser,
     handles: &mut Vec<WaitHandleRef>,
 ) -> bool {
     // Has a job already completed?
     // TODO: we can avoid traversing this list if searching by pid.
     let mut matched = false;
-    let wait_handles: &mut WaitHandleStore = parser.get_wait_handles_mut();
+    let wait_handles: &mut WaitHandleStore = &mut parser.mut_wait_handles();
     for wh in wait_handles.iter() {
         if wait_handle_matches(query, wh) {
             handles.push(wh.clone());
@@ -50,15 +50,12 @@ fn find_wait_handles(
     }
 
     // Is there a running job match?
-    for j in parser.get_jobs() {
+    for j in &*parser.jobs() {
         // We want to set 'matched' to true if we could have matched, even if the job was stopped.
         let provide_handle = can_wait_on_job(j);
-        for proc in j.get_procs() {
-            let wh = proc
-                .pin_mut()
-                .unpin()
-                .make_wait_handle(j.get_internal_job_id());
-            let Some(wh) = wh else {
+        let internal_job_id = j.internal_job_id;
+        for proc in j.processes().iter() {
+            let Some(wh) = proc.make_wait_handle(internal_job_id) else {
                 continue;
             };
             if wait_handle_matches(query, &wh) {
@@ -77,13 +74,13 @@ fn get_all_wait_handles(parser: &Parser) -> Vec<WaitHandleRef> {
     let mut result = parser.get_wait_handles().get_list();
 
     // Get wait handles for running jobs.
-    for j in parser.get_jobs() {
+    for j in &*parser.jobs() {
         if !can_wait_on_job(j) {
             continue;
         }
-        for proc_ptr in j.get_procs().iter_mut() {
-            let proc = proc_ptr.pin_mut().unpin();
-            if let Some(wh) = proc.make_wait_handle(j.get_internal_job_id()) {
+        let internal_job_id = j.internal_job_id;
+        for proc in j.processes().iter() {
+            if let Some(wh) = proc.make_wait_handle(internal_job_id) {
                 result.push(wh);
             }
         }
@@ -98,11 +95,7 @@ fn is_completed(wh: &WaitHandleRef) -> bool {
 /// Wait for the given wait handles to be marked as completed.
 /// If \p any_flag is set, wait for the first one; otherwise wait for all.
 /// \return a status code.
-fn wait_for_completion(
-    parser: &mut Parser,
-    whs: &[WaitHandleRef],
-    any_flag: bool,
-) -> Option<c_int> {
+fn wait_for_completion(parser: &Parser, whs: &[WaitHandleRef], any_flag: bool) -> Option<c_int> {
     if whs.is_empty() {
         return Some(0);
     }
@@ -119,7 +112,7 @@ fn wait_for_completion(
             // Remove completed wait handles (at most 1 if any_flag is set).
             for wh in whs {
                 if is_completed(wh) {
-                    parser.get_wait_handles_mut().remove(wh);
+                    parser.mut_wait_handles().remove(wh);
                     if any_flag {
                         break;
                     }
@@ -130,12 +123,12 @@ fn wait_for_completion(
         if sigint.check() {
             return Some(128 + libc::SIGINT);
         }
-        proc_wait_any(parser.pin());
+        proc_wait_any(parser);
     }
 }
 
 #[widestrs]
-pub fn wait(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
+pub fn wait(parser: &Parser, streams: &mut IoStreams, argv: &mut [&wstr]) -> Option<c_int> {
     let cmd = argv[0];
     let argc = argv.len();
     let mut any_flag = false; // flag for -n option
@@ -158,11 +151,11 @@ pub fn wait(parser: &mut Parser, streams: &mut IoStreams, argv: &mut [&wstr]) ->
                 print_help = true;
             }
             ':' => {
-                builtin_missing_argument(parser, streams, cmd, argv[w.woptind - 1], print_hints);
+                builtin_missing_argument(parser, streams, cmd, &argv[w.woptind - 1], print_hints);
                 return STATUS_INVALID_ARGS;
             }
             '?' => {
-                builtin_unknown_option(parser, streams, cmd, argv[w.woptind - 1], print_hints);
+                builtin_unknown_option(parser, streams, cmd, &argv[w.woptind - 1], print_hints);
                 return STATUS_INVALID_ARGS;
             }
             _ => {

fish-rust/src/cfg/spawn.c

@@ -0,0 +1 @@
+#include <spawn.h>

fish-rust/src/cfg/w_exitcode.cpp

@@ -0,0 +1,2 @@
+#include <sys/wait.h>
+static_assert(WEXITSTATUS(0x007f) == 0x7f, "");

fish-rust/src/common.rs

@@ -6,7 +6,7 @@ use crate::expand::{
     PROCESS_EXPAND_SELF, PROCESS_EXPAND_SELF_STR, VARIABLE_EXPAND, VARIABLE_EXPAND_SINGLE,
 };
 use crate::fallback::fish_wcwidth;
-use crate::ffi::{self};
+use crate::ffi;
 use crate::flog::FLOG;
 use crate::future_feature_flags::{feature_test, FeatureFlag};
 use crate::global_safety::RelaxedAtomicBool;
@@ -1004,7 +1004,7 @@ fn debug_thread_error() {
 }
 
 /// Exits without invoking destructors (via _exit), useful for code after fork.
-pub fn exit_without_destructors(code: i32) -> ! {
+pub fn exit_without_destructors(code: libc::c_int) -> ! {
     unsafe { libc::_exit(code) };
 }
 
@@ -1074,7 +1074,8 @@ pub static EMPTY_STRING_LIST: Vec<WString> = vec![];
 
 /// A function type to check for cancellation.
 /// \return true if execution should cancel.
-pub type CancelChecker = dyn Fn() -> bool;
+/// todo!("Maybe remove the box? It is only needed for get_bg_context.")
+pub type CancelChecker = Box<dyn Fn() -> bool>;
 
 /// Converts the narrow character string \c in into its wide equivalent, and return it.
 ///
@@ -1200,6 +1201,7 @@ pub fn wcs2osstring(input: &wstr) -> OsString {
     OsString::from_vec(result)
 }
 
+/// Same as [`wcs2string`]. Meant to be used when we need a zero-terminated string to feed legacy APIs.
 pub fn wcs2zstring(input: &wstr) -> CString {
     if input.is_empty() {
         return CString::default();
@@ -1311,7 +1313,11 @@ pub fn format_size_safe(buff: &mut [u8; 128], mut sz: u64) {
 }
 
 /// Writes out a long safely.
-pub fn format_llong_safe<CharT: From<u8>>(buff: &mut [CharT; 64], val: i64) {
+pub fn format_llong_safe<CharT: From<u8>, I64>(buff: &mut [CharT; 64], val: I64)
+where
+    i64: From<I64>,
+{
+    let val = i64::from(val);
     let uval = val.unsigned_abs();
     if val >= 0 {
         format_safe_impl(buff, 64, uval);
@@ -1659,6 +1665,10 @@ fn slice_contains_slice<T: Eq>(a: &[T], b: &[T]) -> bool {
     a.windows(b.len()).any(|aw| aw == b)
 }
 
+pub fn subslice_position<T: Eq>(a: &[T], b: &[T]) -> Option<usize> {
+    a.windows(b.len()).position(|aw| aw == b)
+}
+
 /// Determines if we are running under Microsoft's Windows Subsystem for Linux to work around
 /// some known limitations and/or bugs.
 ///
@@ -1898,6 +1908,22 @@ where
     ScopeGuard::new(ctx, restore_saved)
 }
 
+/// Similar to scoped_push but takes a function like "std::mem::replace" instead of a function
+/// that returns a mutable reference.
+pub fn scoped_push_replacer<Replacer, T>(
+    replacer: Replacer,
+    new_value: T,
+) -> impl ScopeGuarding<Target = ()>
+where
+    Replacer: Fn(T) -> T,
+{
+    let saved = replacer(new_value);
+    let restore_saved = move |_ctx: &mut ()| {
+        replacer(saved);
+    };
+    ScopeGuard::new((), restore_saved)
+}
+
 pub const fn assert_send<T: Send>() {}
 pub const fn assert_sync<T: Sync>() {}
 
@@ -2119,11 +2145,10 @@ macro_rules! err {
     }
 }
 
-#[allow(unused_macros)]
 macro_rules! fwprintf {
-    ($fd:expr, $format:literal $(, $arg:expr)*) => {
+    ($fd:expr, $format:expr $(, $arg:expr)*) => {
         {
-            let wide = crate::wutil::sprintf!($format $(, $arg )*);
+            let wide = crate::wutil::sprintf!($format, $( $arg ),*);
             crate::wutil::wwrite_to_fd(&wide, $fd);
         }
     }
@@ -2141,7 +2166,7 @@ mod common_ffi {
         type escape_string_style_t = crate::ffi::escape_string_style_t;
     }
     extern "Rust" {
-        #[cxx_name = "rust_unescape_string"]
+        #[cxx_name = "unescape_string"]
         fn unescape_string_ffi(
             input: *const wchar_t,
             len: usize,
@@ -2149,17 +2174,17 @@ mod common_ffi {
             style: escape_string_style_t,
         ) -> UniquePtr<CxxWString>;
 
-        #[cxx_name = "rust_escape_string_script"]
+        #[cxx_name = "escape_string_script"]
         fn escape_string_script_ffi(
             input: *const wchar_t,
             len: usize,
             flags: u32,
         ) -> UniquePtr<CxxWString>;
 
-        #[cxx_name = "rust_escape_string_url"]
+        #[cxx_name = "escape_string_url"]
         fn escape_string_url_ffi(input: *const wchar_t, len: usize) -> UniquePtr<CxxWString>;
 
-        #[cxx_name = "rust_escape_string_var"]
+        #[cxx_name = "escape_string_var"]
         fn escape_string_var_ffi(input: *const wchar_t, len: usize) -> UniquePtr<CxxWString>;
 
     }

fish-rust/src/compat.c

@@ -1,7 +1,9 @@
 #include "config.h"
 
+#include <paths.h>
 #include <stdbool.h>
 #include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <term.h>
 #include <unistd.h>
@@ -51,6 +53,26 @@ uint64_t C_MNT_LOCAL() {
 #endif
 }
 
+const char* C_PATH_BSHELL() { return _PATH_BSHELL; }
+
+int C_PC_CASE_SENSITIVE() {
+#ifdef _PC_CASE_SENSITIVE
+    return _PC_CASE_SENSITIVE;
+#else
+    return 0;
+#endif
+}
+
+FILE* stdout_stream() { return stdout; }
+
+int C_O_EXLOCK() {
+#ifdef O_EXLOCK
+    return O_EXLOCK;
+#else
+    return 0;
+#endif
+}
+
 static const bool uvar_file_set_mtime_hack =
 #ifdef UVAR_FILE_SET_MTIME_HACK
     true;

fish-rust/src/compat.rs

@@ -1,3 +1,8 @@
+use std::sync::atomic::AtomicPtr;
+
+use libc::c_int;
+use once_cell::sync::Lazy;
+
 #[allow(non_snake_case)]
 pub fn MB_CUR_MAX() -> usize {
     unsafe { C_MB_CUR_MAX() }
@@ -22,6 +27,12 @@ pub fn _CS_PATH() -> i32 {
     unsafe { C_CS_PATH() }
 }
 
+#[allow(non_snake_case)]
+pub static _PATH_BSHELL: AtomicPtr<i8> = AtomicPtr::new(std::ptr::null_mut());
+
+#[allow(non_snake_case)]
+pub static _PC_CASE_SENSITIVE: Lazy<c_int> = Lazy::new(|| unsafe { C_PC_CASE_SENSITIVE() });
+
 extern "C" {
     fn C_MB_CUR_MAX() -> usize;
     fn has_cur_term() -> bool;
@@ -33,4 +44,9 @@ extern "C" {
         buf: *mut libc::c_char,
         len: libc::size_t,
     ) -> libc::size_t;
+    pub fn C_PATH_BSHELL() -> *const i8;
+    fn C_PC_CASE_SENSITIVE() -> c_int;
+    pub fn C_O_EXLOCK() -> c_int;
+    pub fn stdout_stream() -> *mut libc::FILE;
+    pub fn UVAR_FILE_SET_MTIME_HACK() -> bool;
 }

fish-rust/src/env/env_ffi.rs

@@ -1,22 +1,40 @@
-use super::environment::{self, EnvNull, EnvStack, EnvStackRef, Environment};
+use super::environment::{self, EnvDyn, EnvNull, EnvStack, EnvStackRef, Environment};
 use super::var::{ElectricVar, EnvVar, EnvVarFlags, Statuses};
 use crate::env::EnvMode;
-use crate::event::Event;
-use crate::ffi::{event_list_ffi_t, wchar_t, wcharz_t, wcstring_list_ffi_t};
-use crate::function::FunctionPropertiesRefFFI;
+use crate::ffi::{wchar_t, wcharz_t, wcstring_list_ffi_t};
 use crate::null_terminated_array::OwningNullTerminatedArrayRefFFI;
 use crate::signal::Signal;
 use crate::wchar_ffi::WCharToFFI;
 use crate::wchar_ffi::{AsWstr, WCharFromFFI};
+use core::ffi::c_char;
 use cxx::{CxxVector, CxxWString, UniquePtr};
+use lazy_static::lazy_static;
+use std::ffi::c_int;
 use std::pin::Pin;
 
+use crate::env::misc_init;
+
+impl From<EnvStackSetResult> for c_int {
+    fn from(r: EnvStackSetResult) -> Self {
+        match r {
+            EnvStackSetResult::ENV_OK => 0,
+            EnvStackSetResult::ENV_PERM => 1,
+            EnvStackSetResult::ENV_SCOPE => 2,
+            EnvStackSetResult::ENV_INVALID => 3,
+            EnvStackSetResult::ENV_NOT_FOUND => 4,
+            _ => panic!(),
+        }
+    }
+}
+
 #[allow(clippy::module_inception)]
 #[cxx::bridge]
 mod env_ffi {
+
     /// Return values for `EnvStack::set()`.
     #[repr(u8)]
     #[cxx_name = "env_stack_set_result_t"]
+    #[derive(Debug)]
     enum EnvStackSetResult {
         ENV_OK,
         ENV_PERM,
@@ -27,15 +45,9 @@ mod env_ffi {
 
     extern "C++" {
         include!("env.h");
-        include!("null_terminated_array.h");
         include!("wutil.h");
-        type event_list_ffi_t = super::event_list_ffi_t;
         type wcstring_list_ffi_t = super::wcstring_list_ffi_t;
         type wcharz_t = super::wcharz_t;
-        type function_properties_t = super::FunctionPropertiesRefFFI;
-
-        type OwningNullTerminatedArrayRefFFI =
-            crate::null_terminated_array::OwningNullTerminatedArrayRefFFI;
     }
 
     extern "Rust" {
@@ -92,6 +104,9 @@ mod env_ffi {
         #[cxx_name = "get_status"]
         fn get_status_ffi(&self) -> i32;
 
+        #[cxx_name = "statuses_just"]
+        fn statuses_just_ffi(s: i32) -> Box<Statuses>;
+
         #[cxx_name = "get_pipestatus"]
         fn get_pipestatus_ffi(&self) -> &Vec<i32>;
 
@@ -102,6 +117,7 @@ mod env_ffi {
     extern "Rust" {
         #[cxx_name = "EnvDyn"]
         type EnvDynFFI;
+        fn get(&self, name: &CxxWString) -> *mut EnvVar;
         fn getf(&self, name: &CxxWString, mode: u16) -> *mut EnvVar;
         fn get_names(&self, flags: u16, out: Pin<&mut wcstring_list_ffi_t>);
     }
@@ -109,7 +125,13 @@ mod env_ffi {
     extern "Rust" {
         #[cxx_name = "EnvStackRef"]
         type EnvStackRefFFI;
+        fn env_stack_globals() -> &'static EnvStackRefFFI;
+        fn env_stack_principal() -> &'static EnvStackRefFFI;
+        fn set_one(&self, name: &CxxWString, flags: u16, value: &CxxWString) -> EnvStackSetResult;
+        fn get(&self, name: &CxxWString) -> *mut EnvVar;
         fn getf(&self, name: &CxxWString, mode: u16) -> *mut EnvVar;
+        fn get_unless_empty(&self, name: &CxxWString) -> *mut EnvVar;
+        fn getf_unless_empty(&self, name: &CxxWString, flags: u16) -> *mut EnvVar;
         fn get_names(&self, flags: u16, out: Pin<&mut wcstring_list_ffi_t>);
         fn is_principal(&self) -> bool;
         fn get_last_statuses(&self) -> Box<Statuses>;
@@ -124,11 +146,12 @@ mod env_ffi {
         fn get_pwd_slash(&self) -> UniquePtr<CxxWString>;
         fn set_pwd_from_getcwd(&self);
 
-        fn push(&mut self, new_scope: bool);
-        fn pop(&mut self);
+        fn push(&self, new_scope: bool);
+        fn pop(&self);
 
-        // Returns a ``Box<OwningNullTerminatedArrayRefFFI>.into_raw()``.
-        fn export_array(&self) -> *mut OwningNullTerminatedArrayRefFFI;
+        // Returns a Box<OwningNullTerminatedArrayRefFFI>.into_raw() cast to a void*.
+        // This is because we can't use the same C++ bindings to a Rust type from two different bridges.
+        fn export_array(&self) -> *mut c_char;
 
         fn snapshot(&self) -> Box<EnvDynFFI>;
 
@@ -138,10 +161,6 @@ mod env_ffi {
 
         // Access the principal variable stack.
         fn env_get_principal_ffi() -> Box<EnvStackRefFFI>;
-
-        fn universal_sync(&self, always: bool, out_events: Pin<&mut event_list_ffi_t>);
-
-        fn apply_inherited_ffi(&self, props: &function_properties_t);
     }
 
     extern "Rust" {
@@ -151,6 +170,8 @@ mod env_ffi {
         #[cxx_name = "rust_env_init"]
         fn rust_env_init_ffi(do_uvars: bool);
 
+        fn misc_init();
+
         #[cxx_name = "env_flags_for"]
         fn env_flags_for_ffi(name: wcharz_t) -> u8;
     }
@@ -212,24 +233,64 @@ fn env_null_create_ffi() -> Box<EnvNull> {
     Box::new(EnvNull::new())
 }
 
-/// FFI wrapper around dyn Environment.
-pub struct EnvDynFFI(Box<dyn Environment>);
+/// FFI wrapper around EnvDyn
+pub struct EnvDynFFI(pub EnvDyn);
 impl EnvDynFFI {
+    fn get(&self, name: &CxxWString) -> *mut EnvVar {
+        EnvironmentFFI::getf_ffi(&self.0, name, 0)
+    }
     fn getf(&self, name: &CxxWString, mode: u16) -> *mut EnvVar {
-        EnvironmentFFI::getf_ffi(&*self.0, name, mode)
+        EnvironmentFFI::getf_ffi(&self.0, name, mode)
     }
     fn get_names(&self, flags: u16, out: Pin<&mut wcstring_list_ffi_t>) {
-        EnvironmentFFI::get_names_ffi(&*self.0, flags, out)
+        EnvironmentFFI::get_names_ffi(&self.0, flags, out)
     }
 }
+unsafe impl cxx::ExternType for EnvDynFFI {
+    type Id = cxx::type_id!("EnvDyn"); // CXX name!
+    type Kind = cxx::kind::Opaque;
+}
 
 /// FFI wrapper around EnvStackRef.
+#[derive(Clone)]
 pub struct EnvStackRefFFI(pub EnvStackRef);
 
+lazy_static! {
+    static ref GLOBALS: EnvStackRefFFI = EnvStackRefFFI(EnvStack::globals().clone());
+}
+lazy_static! {
+    static ref PRINCIPAL_STACK: EnvStackRefFFI = EnvStackRefFFI(EnvStack::principal().clone());
+}
+
+fn env_stack_globals() -> &'static EnvStackRefFFI {
+    &GLOBALS
+}
+fn env_stack_principal() -> &'static EnvStackRefFFI {
+    &PRINCIPAL_STACK
+}
+
 impl EnvStackRefFFI {
+    fn set_one(&self, name: &CxxWString, flags: u16, value: &CxxWString) -> EnvStackSetResult {
+        self.0.set_one(
+            name.as_wstr(),
+            EnvMode::from_bits(flags).unwrap(),
+            value.from_ffi(),
+        )
+    }
+
+    fn get(&self, name: &CxxWString) -> *mut EnvVar {
+        EnvironmentFFI::getf_ffi(&*self.0, name, 0)
+    }
     fn getf(&self, name: &CxxWString, mode: u16) -> *mut EnvVar {
         EnvironmentFFI::getf_ffi(&*self.0, name, mode)
     }
+    fn get_unless_empty(&self, name: &CxxWString) -> *mut EnvVar {
+        EnvironmentFFI::getf_unless_empty_ffi(&*self.0, name, 0)
+    }
+    fn getf_unless_empty(&self, name: &CxxWString, mode: u16) -> *mut EnvVar {
+        EnvironmentFFI::getf_unless_empty_ffi(&*self.0, name, mode)
+    }
+
     fn get_names(&self, flags: u16, out: Pin<&mut wcstring_list_ffi_t>) {
         EnvironmentFFI::get_names_ffi(&*self.0, flags, out)
     }
@@ -280,39 +341,29 @@ impl EnvStackRefFFI {
         self.0.remove(name.as_wstr(), mode)
     }
 
-    fn export_array(&self) -> *mut OwningNullTerminatedArrayRefFFI {
+    fn export_array(&self) -> *mut c_char {
         Box::into_raw(Box::new(OwningNullTerminatedArrayRefFFI(
             self.0.export_array(),
         )))
+        .cast()
     }
 
     fn snapshot(&self) -> Box<EnvDynFFI> {
         Box::new(EnvDynFFI(self.0.snapshot()))
     }
-
-    fn universal_sync(
-        self: &EnvStackRefFFI,
-        always: bool,
-        mut out_events: Pin<&mut event_list_ffi_t>,
-    ) {
-        let events: Vec<Box<Event>> = self.0.universal_sync(always);
-        for event in events {
-            out_events.as_mut().push(Box::into_raw(event).cast());
-        }
-    }
-
-    fn apply_inherited_ffi(&self, props: &FunctionPropertiesRefFFI) {
-        // Ported from C++:
-        // for (const auto &kv : props.inherit_vars) {
-        //     vars.set(kv.first, ENV_LOCAL | ENV_USER, kv.second);
-        // }
-        for (name, vals) in props.0.inherit_vars() {
-            self.0
-                .set(name, EnvMode::LOCAL | EnvMode::USER, vals.clone());
-        }
-    }
+}
+unsafe impl cxx::ExternType for EnvStackRefFFI {
+    type Id = cxx::type_id!("EnvStackRef"); // CXX name!
+    type Kind = cxx::kind::Opaque;
 }
 
+unsafe impl cxx::ExternType for Statuses {
+    type Id = cxx::type_id!("Statuses");
+    type Kind = cxx::kind::Opaque;
+}
+fn statuses_just_ffi(s: i32) -> Box<Statuses> {
+    Box::new(Statuses::just(s))
+}
 impl Statuses {
     fn get_status_ffi(&self) -> i32 {
         self.status
@@ -361,6 +412,21 @@ trait EnvironmentFFI: Environment {
             Some(var) => Box::into_raw(Box::new(var)),
         }
     }
+    fn getf_unless_empty_ffi(&self, name: &CxxWString, mode: u16) -> *mut EnvVar {
+        match self.getf(
+            name.as_wstr(),
+            EnvMode::from_bits(mode).expect("Invalid mode bits"),
+        ) {
+            None => std::ptr::null_mut(),
+            Some(var) => {
+                if var.is_empty() {
+                    std::ptr::null_mut()
+                } else {
+                    Box::into_raw(Box::new(var))
+                }
+            }
+        }
+    }
     fn get_names_ffi(&self, mode: u16, mut out: Pin<&mut wcstring_list_ffi_t>) {
         let names = self.get_names(EnvMode::from_bits(mode).expect("Invalid mode bits"));
         for name in names {
@@ -369,7 +435,7 @@ trait EnvironmentFFI: Environment {
     }
 }
 
-impl<T: Environment + ?Sized> EnvironmentFFI for T {}
+impl<T: Environment> EnvironmentFFI for T {}
 
 fn var_is_electric_ffi(name: &CxxWString) -> bool {
     ElectricVar::for_name(name.as_wstr()).is_some()

fish-rust/src/env/environment.rs

@@ -5,10 +5,12 @@ use super::environment_impl::{
 use super::{ConfigPaths, ElectricVar};
 use crate::abbrs::{abbrs_get_set, Abbreviation, Position};
 use crate::common::{str2wcstring, unescape_string, wcs2zstring, UnescapeStringStyle};
+use crate::compat::{stdout_stream, C_PATH_BSHELL, _PATH_BSHELL};
 use crate::env::{EnvMode, EnvStackSetResult, EnvVar, Statuses};
 use crate::env_dispatch::{env_dispatch_init, env_dispatch_var_change};
+use crate::env_universal_common::{CallbackDataList, EnvUniversal};
 use crate::event::Event;
-use crate::ffi::{self, env_universal_t, universal_notifier_t};
+use crate::ffi;
 use crate::flog::FLOG;
 use crate::global_safety::RelaxedAtomicBool;
 use crate::null_terminated_array::OwningNullTerminatedArray;
@@ -16,21 +18,22 @@ use crate::path::{
     path_emit_config_directory_messages, path_get_config, path_get_data, path_make_canonical,
     paths_are_same_file,
 };
+use crate::proc::is_interactive_session;
 use crate::termsize;
 use crate::wchar::prelude::*;
-use crate::wchar_ffi::{AsWstr, WCharFromFFI};
 use crate::wcstringutil::join_strings;
 use crate::wutil::{fish_wcstol, wgetcwd, wgettext};
+use std::sync::atomic::Ordering;
 
-use autocxx::WithinUniquePtr;
-use cxx::UniquePtr;
 use lazy_static::lazy_static;
-use libc::c_int;
+use libc::{c_int, STDOUT_FILENO, _IONBF};
 use once_cell::sync::OnceCell;
 use std::collections::HashMap;
 use std::ffi::CStr;
+use std::io::Write;
 use std::mem::MaybeUninit;
 use std::os::unix::prelude::*;
+use std::pin::Pin;
 use std::sync::{Arc, Mutex};
 
 /// TODO: migrate to history once ported.
@@ -38,21 +41,12 @@ const DFLT_FISH_HISTORY_SESSION_ID: &wstr = L!("fish");
 
 // Universal variables instance.
 lazy_static! {
-    static ref UVARS: Mutex<UniquePtr<env_universal_t>> = Mutex::new(env_universal_t::new_unique());
+    static ref UVARS: Mutex<EnvUniversal> = Mutex::new(EnvUniversal::new());
 }
 
 /// Set when a universal variable has been modified but not yet been written to disk via sync().
 static UVARS_LOCALLY_MODIFIED: RelaxedAtomicBool = RelaxedAtomicBool::new(false);
 
-/// Convert an EnvVar to an FFI env_var_t.
-pub fn env_var_to_ffi(var: Option<EnvVar>) -> cxx::UniquePtr<ffi::env_var_t> {
-    if let Some(var) = var {
-        ffi::env_var_t::new_ffi(Box::into_raw(Box::from(var)).cast()).within_unique_ptr()
-    } else {
-        cxx::UniquePtr::null()
-    }
-}
-
 /// An environment is read-only access to variable values.
 pub trait Environment {
     /// Get a variable by name using default flags.
@@ -114,6 +108,32 @@ impl Environment for EnvNull {
     }
 }
 
+/// A helper type for wrapping a type-erased Environment.
+pub struct EnvDyn {
+    inner: Box<dyn Environment + Send + Sync>,
+}
+
+impl EnvDyn {
+    // Exposed for testing.
+    pub fn new(inner: Box<dyn Environment + Send + Sync>) -> Self {
+        Self { inner }
+    }
+}
+
+impl Environment for EnvDyn {
+    fn getf(&self, key: &wstr, mode: EnvMode) -> Option<EnvVar> {
+        self.inner.getf(key, mode)
+    }
+
+    fn get_names(&self, flags: EnvMode) -> Vec<WString> {
+        self.inner.get_names(flags)
+    }
+
+    fn get_pwd_slash(&self) -> WString {
+        self.inner.get_pwd_slash()
+    }
+}
+
 /// An immutable environment, used in snapshots.
 pub struct EnvScoped {
     inner: EnvMutex<EnvScopedImpl>,
@@ -136,7 +156,7 @@ pub struct EnvStack {
 }
 
 impl EnvStack {
-    fn new() -> EnvStack {
+    pub fn new() -> EnvStack {
         EnvStack {
             inner: EnvStackImpl::new(),
         }
@@ -148,7 +168,7 @@ impl EnvStack {
 
     /// \return whether we are the principal stack.
     pub fn is_principal(&self) -> bool {
-        self as *const Self == Arc::as_ptr(&*PRINCIPAL_STACK)
+        std::ptr::eq(self, Self::principal().as_ref().get_ref())
     }
 
     /// Helpers to get and set the proc statuses.
@@ -279,9 +299,9 @@ impl EnvStack {
 
     /// Snapshot this environment. This means returning a read-only copy. Local variables are copied
     /// but globals are shared (i.e. changes to global will be visible to this snapshot).
-    pub fn snapshot(&self) -> Box<dyn Environment> {
+    pub fn snapshot(&self) -> EnvDyn {
         let scoped = EnvScoped::from_impl(self.lock().base.snapshot());
-        Box::new(scoped)
+        EnvDyn::new(Box::new(scoped) as Box<dyn Environment + Send + Sync>)
     }
 
     /// Synchronizes universal variable changes.
@@ -289,7 +309,7 @@ impl EnvStack {
     /// instance (that is, look for changes from other fish instances).
     /// \return a list of events for changed variables.
     #[allow(clippy::vec_box)]
-    pub fn universal_sync(&self, always: bool) -> Vec<Box<Event>> {
+    pub fn universal_sync(&self, always: bool) -> Vec<Event> {
         if UVAR_SCOPE_IS_GLOBAL.load() {
             return Vec::new();
         }
@@ -298,25 +318,23 @@ impl EnvStack {
         }
         UVARS_LOCALLY_MODIFIED.store(false);
 
-        let mut unused = autocxx::c_int(0);
-        let sync_res_ptr = uvars().as_mut().unwrap().sync_ffi().within_unique_ptr();
-        let sync_res = sync_res_ptr.as_ref().unwrap();
-        if sync_res.get_changed() {
-            universal_notifier_t::default_notifier_ffi(std::pin::Pin::new(&mut unused))
-                .post_notification();
+        let mut callbacks = CallbackDataList::new();
+        let changed = uvars().sync(&mut callbacks);
+        if changed {
+            ffi::env_universal_notifier_t_default_notifier_post_notification_ffi();
         }
         // React internally to changes to special variables like LANG, and populate on-variable events.
         let mut result = Vec::new();
         #[allow(unreachable_code)]
-        for idx in 0..sync_res.count() {
-            let name = sync_res.get_key(idx).from_ffi();
+        for callback in callbacks {
+            let name = callback.key;
             env_dispatch_var_change(&name, self);
-            let evt = if sync_res.get_is_erase(idx) {
+            let evt = if callback.val.is_none() {
                 Event::variable_erase(name)
             } else {
                 Event::variable_set(name)
             };
-            result.push(Box::new(evt));
+            result.push(evt);
         }
         result
     }
@@ -331,6 +349,10 @@ impl EnvStack {
     pub fn principal() -> &'static EnvStackRef {
         &PRINCIPAL_STACK
     }
+
+    pub fn set_argv(&self, argv: Vec<WString>) {
+        self.set(L!("argv"), EnvMode::LOCAL, argv);
+    }
 }
 
 impl Environment for EnvScoped {
@@ -365,17 +387,17 @@ impl Environment for EnvStack {
     }
 }
 
-pub type EnvStackRef = Arc<EnvStack>;
+pub type EnvStackRef = Pin<Arc<EnvStack>>;
 
 // A variable stack that only represents globals.
 // Do not push or pop from this.
 lazy_static! {
-    static ref GLOBALS: EnvStackRef = Arc::new(EnvStack::new());
+    static ref GLOBALS: EnvStackRef = Arc::pin(EnvStack::new());
 }
 
 // Our singleton "principal" stack.
 lazy_static! {
-    static ref PRINCIPAL_STACK: EnvStackRef = Arc::new(EnvStack::new());
+    static ref PRINCIPAL_STACK: EnvStackRef = Arc::pin(EnvStack::new());
 }
 
 /// Some configuration path environment variables.
@@ -647,7 +669,7 @@ pub fn env_init(paths: Option<&ConfigPaths>, do_uvars: bool, default_paths: bool
 
     // Set up SHLVL variable. Not we can't use vars.get() because SHLVL is read-only, and therefore
     // was not inherited from the environment.
-    if ffi::is_interactive_session() {
+    if is_interactive_session() {
         let nshlvl_str = if let Some(shlvl_var) = std::env::var_os("SHLVL") {
             // TODO: Figure out how to handle invalid numbers better. Shouldn't we issue a
             // diagnostic?
@@ -718,33 +740,23 @@ pub fn env_init(paths: Option<&ConfigPaths>, do_uvars: bool, default_paths: bool
     if !do_uvars {
         UVAR_SCOPE_IS_GLOBAL.store(true);
     } else {
+        // let vars = EnvStack::principal();
+
         // Set up universal variables using the default path.
-        let callbacks = uvars()
-            .as_mut()
-            .unwrap()
-            .initialize_ffi()
-            .within_unique_ptr();
-        let vars = EnvStack::principal();
-        let callbacks = callbacks.as_ref().unwrap();
-        for idx in 0..callbacks.count() {
-            let name = callbacks.get_key(idx).from_ffi();
-            env_dispatch_var_change(&name, vars);
+        let mut callbacks = CallbackDataList::new();
+        uvars().initialize(&mut callbacks);
+        for callback in callbacks {
+            env_dispatch_var_change(&callback.key, vars);
         }
 
         // Do not import variables that have the same name and value as
         // an exported universal variable. See issues #5258 and #5348.
-        let mut table = uvars()
-            .as_ref()
-            .unwrap()
-            .get_table_ffi()
-            .within_unique_ptr();
-        for idx in 0..table.count() {
-            // autocxx gets confused when a value goes Rust -> Cxx -> Rust.
-            let uvar = table.as_mut().unwrap().get_var(idx).from_ffi();
+        let uvars_locked = uvars();
+        let table = uvars_locked.get_table();
+        for (name, uvar) in table {
             if !uvar.exports() {
                 continue;
             }
-            let name: &wstr = table.get_name(idx).as_wstr();
 
             // Look for a global exported variable with the same name.
             let global = EnvStack::globals().getf(name, EnvMode::GLOBAL | EnvMode::EXPORT);
@@ -759,15 +771,13 @@ pub fn env_init(paths: Option<&ConfigPaths>, do_uvars: bool, default_paths: bool
         let prefix_len = prefix.char_count();
         let from_universal = true;
         let mut abbrs = abbrs_get_set();
-        for idx in 0..table.count() {
-            let name: &wstr = table.get_name(idx).as_wstr();
+        for (name, uvar) in table {
             if !name.starts_with(prefix) {
                 continue;
             }
             let escaped_name = name.slice_from(prefix_len);
             if let Some(name) = unescape_string(escaped_name, UnescapeStringStyle::Var) {
                 let key = name.clone();
-                let uvar = table.get_var(idx).from_ffi();
                 let replacement: WString = join_strings(uvar.as_list(), ' ');
                 abbrs.add(Abbreviation::new(
                     name,
@@ -780,3 +790,16 @@ pub fn env_init(paths: Option<&ConfigPaths>, do_uvars: bool, default_paths: bool
         }
     }
 }
+
+/// Various things we need to initialize at run-time that don't really fit any of the other init
+/// routines.
+pub fn misc_init() {
+    // If stdout is open on a tty ensure stdio is unbuffered. That's because those functions might
+    // be intermixed with `write()` calls and we need to ensure the writes are not reordered. See
+    // issue #3748.
+    if unsafe { libc::isatty(STDOUT_FILENO) } != 0 {
+        let _ = std::io::stdout().flush();
+        unsafe { libc::setvbuf(stdout_stream(), std::ptr::null_mut(), _IONBF, 0) };
+    }
+    _PATH_BSHELL.store(unsafe { C_PATH_BSHELL().cast_mut() }, Ordering::SeqCst);
+}

fish-rust/src/env/environment_impl.rs

@@ -3,17 +3,17 @@ use crate::env::{
     is_read_only, ElectricVar, EnvMode, EnvStackSetResult, EnvVar, EnvVarFlags, Statuses, VarTable,
     ELECTRIC_VARIABLES, PATH_ARRAY_SEP,
 };
-use crate::ffi::{self, env_universal_t};
+use crate::env_universal_common::EnvUniversal;
+use crate::ffi;
 use crate::flog::FLOG;
 use crate::global_safety::RelaxedAtomicBool;
+use crate::kill::kill_entries;
 use crate::null_terminated_array::OwningNullTerminatedArray;
 use crate::threads::{is_forked_child, is_main_thread};
 use crate::wchar::prelude::*;
 use crate::wchar_ffi::{WCharFromFFI, WCharToFFI};
 use crate::wutil::fish_wcstol_radix;
 
-use autocxx::WithinUniquePtr;
-use cxx::UniquePtr;
 use lazy_static::lazy_static;
 use std::cell::{RefCell, UnsafeCell};
 use std::collections::HashSet;
@@ -29,38 +29,23 @@ const DFLT_FISH_HISTORY_SESSION_ID: &wstr = L!("fish");
 
 // Universal variables instance.
 lazy_static! {
-    static ref UVARS: Mutex<UniquePtr<env_universal_t>> = Mutex::new(env_universal_t::new_unique());
+    static ref UVARS: Mutex<EnvUniversal> = Mutex::new(EnvUniversal::new());
 }
 
 /// Getter for universal variables.
 /// This is typically initialized in env_init(), and is considered empty before then.
-pub fn uvars() -> MutexGuard<'static, UniquePtr<env_universal_t>> {
+pub fn uvars() -> MutexGuard<'static, EnvUniversal> {
     UVARS.lock().unwrap()
 }
 
 /// Whether we were launched with no_config; in this case setting a uvar instead sets a global.
 pub static UVAR_SCOPE_IS_GLOBAL: RelaxedAtomicBool = RelaxedAtomicBool::new(false);
 
-/// Helper to get the kill ring.
-fn get_kill_ring_entries() -> Vec<WString> {
-    crate::kill::kill_entries()
-}
-
 /// Helper to get the history for a session ID.
 fn get_history_var_text(history_session_id: &wstr) -> Vec<WString> {
     ffi::get_history_variable_text_ffi(&history_session_id.to_ffi()).from_ffi()
 }
 
-/// Convert an FFI env_var_t to our EnvVar.
-impl ffi::env_var_t {
-    #[allow(clippy::wrong_self_convention)]
-    pub fn from_ffi(&self) -> EnvVar {
-        let var_ptr: *const EnvVar = self.ffi_ptr().cast();
-        let var: &EnvVar = unsafe { &*var_ptr };
-        var.clone()
-    }
-}
-
 /// Apply the pathvar behavior, splitting about colons.
 pub fn colon_split<T: AsRef<wstr>>(val: &[T]) -> Vec<WString> {
     let mut split_val = Vec::new();
@@ -70,11 +55,6 @@ pub fn colon_split<T: AsRef<wstr>>(val: &[T]) -> Vec<WString> {
     split_val
 }
 
-/// Convert an EnvVar to an FFI env_var_t.
-fn env_var_to_ffi(var: EnvVar) -> cxx::UniquePtr<ffi::env_var_t> {
-    ffi::env_var_t::new_ffi(Box::into_raw(Box::from(var)).cast()).within_unique_ptr()
-}
-
 /// Return true if a variable should become a path variable by default. See #436.
 fn variable_should_auto_pathvar(name: &wstr) -> bool {
     name.ends_with("PATH")
@@ -392,10 +372,7 @@ impl EnvScopedImpl {
             let vals = get_history_var_text(history_session_id);
             return Some(EnvVar::new_from_name_vec("history"L, vals));
         } else if key == "fish_killring"L {
-            Some(EnvVar::new_from_name_vec(
-                "fish_killring"L,
-                get_kill_ring_entries(),
-            ))
+            Some(EnvVar::new_from_name_vec("fish_killring"L, kill_entries()))
         } else if key == "pipestatus"L {
             let js = &self.perproc_data.statuses;
             let mut result = Vec::new();
@@ -473,12 +450,7 @@ impl EnvScopedImpl {
     }
 
     fn try_get_universal(&self, key: &wstr) -> Option<EnvVar> {
-        return uvars()
-            .as_ref()
-            .expect("Should have non-null uvars in this function")
-            .get_ffi(&key.to_ffi())
-            .as_ref()
-            .map(|v| v.from_ffi());
+        return uvars().get(key);
     }
 
     pub fn getf(&self, key: &wstr, mode: EnvMode) -> Option<EnvVar> {
@@ -547,11 +519,7 @@ impl EnvScopedImpl {
         }
 
         if query.universal {
-            let uni_list = uvars()
-                .as_ref()
-                .expect("Should have non-null uvars in this function")
-                .get_names_ffi(query.exports, query.unexports)
-                .from_ffi();
+            let uni_list = uvars().get_names(query.exports, query.unexports);
             names.extend(uni_list);
         }
         names.into_iter().collect()
@@ -559,13 +527,29 @@ impl EnvScopedImpl {
 
     /// Slightly optimized implementation.
     pub fn get_pwd_slash(&self) -> WString {
-        let mut pwd = self.perproc_data.pwd.clone();
+        // Return "/" if PWD is missing.
+        // See https://github.com/fish-shell/fish-shell/issues/5080
+        let mut pwd;
+        pwd = self.perproc_data.pwd.clone();
         if !pwd.ends_with('/') {
             pwd.push('/');
         }
         pwd
     }
 
+    // todo!("these two are clones from the trait")
+    fn get_unless_empty(&self, name: &wstr) -> Option<EnvVar> {
+        self.getf_unless_empty(name, EnvMode::default())
+    }
+
+    fn getf_unless_empty(&self, name: &wstr, mode: EnvMode) -> Option<EnvVar> {
+        let var = self.getf(name, mode)?;
+        if !var.is_empty() {
+            return Some(var);
+        }
+        None
+    }
+
     /// Return a copy of self, with copied locals but shared globals.
     pub fn snapshot(&self) -> EnvMutex<Self> {
         EnvMutex::new(EnvScopedImpl {
@@ -587,7 +571,7 @@ impl EnvScopedImpl {
     {
         // Our uvars generation count doesn't come from next_export_generation(), so always supply
         // it even if it's 0.
-        func(uvars().as_ref().unwrap().get_export_generation());
+        func(uvars().get_export_generation());
         if self.globals.borrow().exports() {
             func(self.globals.borrow().export_gen);
         }
@@ -648,19 +632,9 @@ impl EnvScopedImpl {
         Self::get_exported(&self.globals, &mut vals);
         Self::get_exported(&self.locals, &mut vals);
 
-        let uni = uvars()
-            .as_ref()
-            .unwrap()
-            .get_names_ffi(true, false)
-            .from_ffi();
+        let uni = uvars().get_names(true, false);
         for key in uni {
-            let var = uvars()
-                .as_ref()
-                .unwrap()
-                .get_ffi(&key.to_ffi())
-                .as_ref()
-                .map(|v| v.from_ffi())
-                .expect("Variable should be present in uvars");
+            let var = uvars().get(&key).unwrap();
             // Only insert if not already present, as uvars have lowest precedence.
             // TODO: a longstanding bug is that an unexported local variable will not mask an exported uvar.
             vals.entry(key).or_insert(var);
@@ -692,7 +666,6 @@ impl EnvScopedImpl {
 
             // Have to pull this into a local to satisfy the borrow checker.
             let mut generations = std::mem::take(&mut self.export_array_generations);
-            generations.clear();
             self.enumerate_generations(|gen| generations.push(gen));
             self.export_array_generations = generations;
         }
@@ -789,7 +762,6 @@ impl EnvStackImpl {
                 result.uvar_modified = true;
             } else if query.global || (query.universal && UVAR_SCOPE_IS_GLOBAL.load()) {
                 Self::set_in_node(&mut self.base.globals, key, val, flags);
-                result.global_modified = true;
             } else if query.local {
                 assert!(
                     !self.base.locals.ptr_eq(&self.base.globals),
@@ -824,13 +796,7 @@ impl EnvStackImpl {
             // Existing global variable.
             Self::set_in_node(&mut node, key, val, flags);
             result.global_modified = true;
-        } else if uvars()
-            .as_ref()
-            .unwrap()
-            .get_ffi(&key.to_ffi())
-            .as_ref()
-            .is_some()
-        {
+        } else if !UVAR_SCOPE_IS_GLOBAL.load() && uvars().get(key).is_some() {
             // Existing universal variable.
             self.set_universal(key, val, query);
             result.uvar_modified = true;
@@ -864,7 +830,7 @@ impl EnvStackImpl {
         if query.has_scope {
             // The user requested erasing from a particular scope.
             if query.universal {
-                if uvars().as_mut().unwrap().remove(&key.to_ffi()) {
+                if uvars().remove(key) {
                     result.status = EnvStackSetResult::ENV_OK;
                 } else {
                     result.status = EnvStackSetResult::ENV_NOT_FOUND;
@@ -892,11 +858,7 @@ impl EnvStackImpl {
             // pass
         } else if Self::remove_from_chain(&mut self.base.globals, key) {
             result.global_modified = true;
-        } else if uvars()
-            .as_mut()
-            .expect("Should have non-null uvars in this function")
-            .remove(&key.to_ffi())
-        {
+        } else if uvars().remove(key) {
             result.uvar_modified = true;
         } else {
             result.status = EnvStackSetResult::ENV_NOT_FOUND;
@@ -1038,10 +1000,7 @@ impl EnvStackImpl {
     /// Set a universal variable, inheriting as applicable from the given old variable.
     fn set_universal(&mut self, key: &wstr, mut val: Vec<WString>, query: Query) {
         let mut locked_uvars = uvars();
-        let uv = locked_uvars
-            .as_mut()
-            .expect("Should have non-null uvars in this function");
-        let oldvar = uv.get_ffi(&key.to_ffi()).as_ref().map(|v| v.from_ffi());
+        let oldvar = locked_uvars.get(key);
         let oldvar = oldvar.as_ref();
 
         // Resolve whether or not to export.
@@ -1074,7 +1033,7 @@ impl EnvStackImpl {
         varflags.set(EnvVarFlags::PATHVAR, pathvar);
         let new_var = EnvVar::new_vec(val, varflags);
 
-        uv.set(&key.to_ffi(), &env_var_to_ffi(new_var));
+        locked_uvars.set(key, new_var);
     }
 
     /// Set a variable in a given node \p node.
@@ -1202,6 +1161,7 @@ impl<T> EnvMutex<T> {
 
 // Safety: we use a global lock.
 unsafe impl<T> Sync for EnvMutex<T> {}
+unsafe impl<T> Send for EnvMutex<T> {}
 
 #[test]
 fn test_colon_split() {

fish-rust/src/env/mod.rs

@@ -4,7 +4,7 @@ mod environment_impl;
 pub mod var;
 
 use crate::common::ToCString;
-pub use env_ffi::{EnvStackRefFFI, EnvStackSetResult};
+pub use env_ffi::{EnvDynFFI, EnvStackRefFFI, EnvStackSetResult};
 pub use environment::*;
 use std::sync::atomic::{AtomicBool, AtomicUsize};
 pub use var::*;

fish-rust/src/env/var.rs

@@ -52,30 +52,6 @@ impl From<EnvMode> for u16 {
     }
 }
 
-/// Return values for `env_stack_t::set()`.
-pub mod status {
-    pub const ENV_OK: i32 = 0;
-    pub const ENV_PERM: i32 = 1;
-    pub const ENV_SCOPE: i32 = 2;
-    pub const ENV_INVALID: i32 = 3;
-    pub const ENV_NOT_FOUND: i32 = 4;
-}
-
-/// Return values for `EnvStack::set()`.
-pub enum EnvStackSetResult {
-    ENV_OK,
-    ENV_PERM,
-    ENV_SCOPE,
-    ENV_INVALID,
-    ENV_NOT_FOUND,
-}
-
-impl Default for EnvStackSetResult {
-    fn default() -> Self {
-        EnvStackSetResult::ENV_OK
-    }
-}
-
 /// A struct of configuration directories, determined in main() that fish will optionally pass to
 /// env_init.
 #[derive(Default)]

fish-rust/src/env_dispatch.rs

@@ -1,12 +1,15 @@
 use crate::common::ToCString;
+use crate::complete::complete_invalidate_path;
 use crate::curses::{self, Term};
 use crate::env::{setenv_lock, unsetenv_lock, EnvMode, EnvStack, Environment};
 use crate::env::{CURSES_INITIALIZED, READ_BYTE_LIMIT, TERM_HAS_XN};
-use crate::ffi::is_interactive_session;
 use crate::flog::FLOG;
 use crate::function;
+use crate::input_common::{update_wait_on_escape_ms, update_wait_on_sequence_key_ms};
 use crate::output::ColorSupport;
+use crate::proc::is_interactive_session;
 use crate::wchar::prelude::*;
+use crate::wchar_ffi::WCharToFFI;
 use crate::wutil::fish_wcstoi;
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -103,18 +106,6 @@ struct VarDispatchTable {
     table: HashMap<&'static wstr, EnvCallback>,
 }
 
-// TODO: Delete this after input_common is ported (and pass the input_function function directly).
-fn update_wait_on_escape_ms(vars: &EnvStack) {
-    let fish_escape_delay_ms = vars.get_unless_empty(L!("fish_escape_delay_ms"));
-    let var = crate::env::environment::env_var_to_ffi(fish_escape_delay_ms);
-    crate::ffi::update_wait_on_escape_ms_ffi(var);
-}
-fn update_wait_on_sequence_key_ms(vars: &EnvStack) {
-    let fish_sequence_key_delay_ms = vars.get_unless_empty(L!("fish_sequence_key_delay_ms"));
-    let var = crate::env::environment::env_var_to_ffi(fish_sequence_key_delay_ms);
-    crate::ffi::update_wait_on_sequence_key_ms_ffi(var);
-}
-
 impl VarDispatchTable {
     /// Add a callback for the variable `name`. We must not already be observing this variable.
     pub fn add(&mut self, name: &'static wstr, callback: NamedEnvCallback) {
@@ -249,9 +240,8 @@ fn handle_term_size_change(vars: &EnvStack) {
 }
 
 fn handle_fish_history_change(vars: &EnvStack) {
-    let fish_history = vars.get(L!("fish_history"));
-    let var = crate::env::env_var_to_ffi(fish_history);
-    crate::ffi::reader_change_history(&crate::ffi::history_session_id(var));
+    let session_id = crate::history::history_session_id(vars);
+    crate::ffi::reader_change_history(&session_id.to_ffi());
 }
 
 fn handle_fish_cursor_selection_mode_change(vars: &EnvStack) {
@@ -286,7 +276,7 @@ fn handle_function_path_change(_: &EnvStack) {
 }
 
 fn handle_complete_path_change(_: &EnvStack) {
-    crate::ffi::complete_invalidate_path();
+    complete_invalidate_path()
 }
 
 fn handle_tz_change(var_name: &wstr, vars: &EnvStack) {

fish-rust/src/event.rs

@@ -4,19 +4,19 @@
 //! defined when these functions produce output or perform memory allocations, since such functions
 //! may not be safely called by signal handlers.
 
-use autocxx::WithinUniquePtr;
-use cxx::{CxxVector, CxxWString, UniquePtr};
+use crate::ffi::wcstring_list_ffi_t;
+use cxx::{CxxWString, UniquePtr};
 use libc::pid_t;
 use std::num::NonZeroU32;
 use std::pin::Pin;
 use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
 use std::sync::{Arc, Mutex};
 
-use crate::builtins::shared::IoStreams;
-use crate::common::{escape_string, scoped_push, EscapeFlags, EscapeStringStyle, ScopeGuard};
-use crate::ffi::{self, block_t, Parser, Repin};
+use crate::common::{escape, scoped_push_replacer, ScopeGuard};
 use crate::flog::FLOG;
+use crate::io::{IoChain, IoStreams};
 use crate::job_group::{JobId, MaybeJobId};
+use crate::parser::{Block, Parser};
 use crate::signal::{signal_check_cancel, signal_handle, Signal};
 use crate::termsize;
 use crate::wchar::prelude::*;
@@ -29,8 +29,9 @@ mod event_ffi {
         include!("parser.h");
         include!("io.h");
         type wcharz_t = crate::ffi::wcharz_t;
-        type Parser = crate::ffi::Parser;
-        type IoStreams = crate::ffi::IoStreams;
+        type Parser = crate::parser::Parser;
+        type IoStreams<'a> = crate::io::IoStreams<'a>;
+        type wcstring_list_ffi_t = crate::ffi::wcstring_list_ffi_t;
     }
 
     enum event_type_t {
@@ -77,16 +78,12 @@ mod event_ffi {
         fn set_removed(self: &mut EventHandler);
 
         fn event_fire_generic_ffi(
-            parser: Pin<&mut Parser>,
+            parser: &Parser,
             name: &CxxWString,
-            arguments: &CxxVector<wcharz_t>,
+            arguments: &wcstring_list_ffi_t,
         );
-        #[cxx_name = "event_get_desc"]
-        fn event_get_desc_ffi(parser: &Parser, evt: &Event) -> UniquePtr<CxxWString>;
         #[cxx_name = "event_fire_delayed"]
-        fn event_fire_delayed_ffi(parser: Pin<&mut Parser>);
-        #[cxx_name = "event_fire"]
-        fn event_fire_ffi(parser: Pin<&mut Parser>, event: &Event);
+        fn fire_delayed(parser: &Parser);
         #[cxx_name = "event_print"]
         fn event_print_ffi(streams: Pin<&mut IoStreams>, type_filter: &CxxWString);
 
@@ -408,16 +405,14 @@ impl Event {
     }
 
     /// Test if specified event is blocked.
-    fn is_blocked(&self, parser: &mut Parser) -> bool {
-        let mut i = 0;
-        while let Some(block) = parser.get_block_at_index(i) {
-            i += 1;
-            if block.ffi_event_blocks() != 0 {
+    fn is_blocked(&self, parser: &Parser) -> bool {
+        for block in parser.blocks().iter().rev() {
+            if block.event_blocks != 0 {
                 return true;
             }
         }
 
-        parser.ffi_global_event_blocks() != 0
+        parser.global_event_blocks.load(Ordering::Relaxed) != 0
     }
 }
 
@@ -573,11 +568,7 @@ pub fn get_desc(parser: &Parser, evt: &Event) -> WString {
         EventDescription::ProcessExit { pid } => format!("exit handler for process {pid}"),
         EventDescription::JobExit { pid, .. } => {
             if let Some(job) = parser.job_get_from_pid(*pid) {
-                format!(
-                    "exit handler for job {}, '{}'",
-                    job.job_id().0,
-                    job.command()
-                )
+                format!("exit handler for job {}, '{}'", job.job_id(), job.command())
             } else {
                 format!("exit handler for job with pid {pid}")
             }
@@ -592,10 +583,6 @@ pub fn get_desc(parser: &Parser, evt: &Event) -> WString {
     WString::from_str(&s)
 }
 
-fn event_get_desc_ffi(parser: &Parser, evt: &Event) -> UniquePtr<CxxWString> {
-    get_desc(parser, evt).to_ffi()
-}
-
 /// Add an event handler.
 pub fn add_handler(eh: EventHandler) {
     if let EventDescription::Signal { signal } = eh.desc {
@@ -661,22 +648,25 @@ fn event_get_function_handler_descs_ffi(name: &CxxWString) -> Vec<event_descript
 /// Perform the specified event. Since almost all event firings will not be matched by even a single
 /// event handler, we make sure to optimize the 'no matches' path. This means that nothing is
 /// allocated/initialized unless needed.
-fn fire_internal(parser: &mut Parser, event: &Event) {
+fn fire_internal(parser: &Parser, event: &Event) {
     assert!(
-        parser.libdata_pod().is_event >= 0,
+        parser.libdata().pods.is_event >= 0,
         "is_event should not be negative"
     );
 
     // Suppress fish_trace during events.
-    let is_event = parser.libdata_pod().is_event;
-    let mut parser = scoped_push(
-        parser,
-        |parser| &mut parser.libdata_pod().is_event,
+    let is_event = parser.libdata().pods.is_event;
+    let _inc_event = scoped_push_replacer(
+        |new_value| std::mem::replace(&mut parser.libdata_mut().pods.is_event, new_value),
         is_event + 1,
     );
-    let mut parser = scoped_push(
-        &mut *parser,
-        |parser| &mut parser.libdata_pod().suppress_fish_trace,
+    let _suppress_trace = scoped_push_replacer(
+        |new_value| {
+            std::mem::replace(
+                &mut parser.libdata_mut().pods.suppress_fish_trace,
+                new_value,
+            )
+        },
         true,
     );
 
@@ -702,20 +692,17 @@ fn fire_internal(parser: &mut Parser, event: &Event) {
         let mut buffer = handler.function_name.clone();
         for arg in &event.arguments {
             buffer.push(' ');
-            buffer.push_utfstr(&escape_string(
-                arg,
-                EscapeStringStyle::Script(EscapeFlags::default()),
-            ));
+            buffer.push_utfstr(&escape(arg));
         }
 
         // Event handlers are not part of the main flow of code, so they are marked as
         // non-interactive.
         let saved_is_interactive =
-            std::mem::replace(&mut parser.libdata_pod().is_interactive, false);
-        let saved_statuses = parser.get_last_statuses().within_unique_ptr();
-        let mut parser = ScopeGuard::new(&mut *parser, |parser| {
-            parser.pin().set_last_statuses(saved_statuses);
-            parser.libdata_pod().is_interactive = saved_is_interactive;
+            std::mem::replace(&mut parser.libdata_mut().pods.is_interactive, false);
+        let saved_statuses = parser.get_last_statuses();
+        let _cleanup = ScopeGuard::new((), |()| {
+            parser.set_last_statuses(saved_statuses);
+            parser.libdata_mut().pods.is_interactive = saved_is_interactive;
         });
 
         FLOG!(
@@ -727,14 +714,9 @@ fn fire_internal(parser: &mut Parser, event: &Event) {
             "'"
         );
 
-        let b = (*parser)
-            .pin()
-            .push_block(block_t::event_block((event as *const Event).cast()).within_unique_ptr());
-        (*parser)
-            .pin()
-            .eval_string_ffi1(&buffer.to_ffi())
-            .within_unique_ptr();
-        (*parser).pin().pop_block(b);
+        let b = parser.push_block(Block::event_block(event.clone()));
+        parser.eval(&buffer, &IoChain::new());
+        parser.pop_block(b);
 
         handler.fired.store(true, Ordering::Relaxed);
         fired_one_shot |= handler.is_one_shot();
@@ -746,13 +728,16 @@ fn fire_internal(parser: &mut Parser, event: &Event) {
 }
 
 /// Fire all delayed events attached to the given parser.
-pub fn fire_delayed(parser: &mut Parser) {
-    let ld = parser.libdata_pod();
+pub fn fire_delayed(parser: &Parser) {
+    {
+        let ld = &parser.libdata().pods;
+
+        // Do not invoke new event handlers from within event handlers.
+        if ld.is_event != 0 {
+            return;
+        };
+    }
 
-    // Do not invoke new event handlers from within event handlers.
-    if ld.is_event != 0 {
-        return;
-    };
     // Do not invoke new event handlers if we are unwinding (#6649).
     if signal_check_cancel() != 0 {
         return;
@@ -760,7 +745,7 @@ pub fn fire_delayed(parser: &mut Parser) {
 
     // We unfortunately can't keep this locked until we're done with it because the SIGWINCH handler
     // code might call back into here and we would delay processing of the events, leading to a test
-    // failure under CI. (Yes, the `&mut Parser` is a lie.)
+    // failure under CI.
     let mut to_send = std::mem::take(&mut *BLOCKED_EVENTS.lock().expect("Mutex poisoned!"));
 
     // Append all signal events to to_send.
@@ -799,10 +784,6 @@ pub fn fire_delayed(parser: &mut Parser) {
     }
 }
 
-fn event_fire_delayed_ffi(parser: Pin<&mut Parser>) {
-    fire_delayed(parser.unpin())
-}
-
 /// Enqueue a signal event. Invoked from a signal handler.
 pub fn enqueue_signal(signal: libc::c_int) {
     // Beware, we are in a signal handler
@@ -810,7 +791,7 @@ pub fn enqueue_signal(signal: libc::c_int) {
 }
 
 /// Fire the specified event event, executing it on `parser`.
-pub fn fire(parser: &mut Parser, event: Event) {
+pub fn fire(parser: &Parser, event: Event) {
     // Fire events triggered by signals.
     fire_delayed(parser);
 
@@ -821,10 +802,6 @@ pub fn fire(parser: &mut Parser, event: Event) {
     }
 }
 
-fn event_fire_ffi(parser: Pin<&mut Parser>, event: &Event) {
-    fire(parser.unpin(), event.clone())
-}
-
 #[widestrs]
 pub const EVENT_FILTER_NAMES: [&wstr; 7] = [
     "signal"L,
@@ -892,13 +869,12 @@ pub fn print(streams: &mut IoStreams, type_filter: &wstr) {
     }
 }
 
-fn event_print_ffi(streams: Pin<&mut ffi::IoStreams>, type_filter: &CxxWString) {
-    let mut streams = IoStreams::new(streams);
-    print(&mut streams, type_filter.as_wstr());
+fn event_print_ffi(streams: Pin<&mut IoStreams>, type_filter: &CxxWString) {
+    print(streams.get_mut(), type_filter.as_wstr());
 }
 
 /// Fire a generic event with the specified name.
-pub fn fire_generic(parser: &mut Parser, name: WString, arguments: Vec<WString>) {
+pub fn fire_generic(parser: &Parser, name: WString, arguments: Vec<WString>) {
     fire(
         parser,
         Event {
@@ -908,14 +884,6 @@ pub fn fire_generic(parser: &mut Parser, name: WString, arguments: Vec<WString>)
     )
 }
 
-fn event_fire_generic_ffi(
-    parser: Pin<&mut Parser>,
-    name: &CxxWString,
-    arguments: &CxxVector<wcharz_t>,
-) {
-    fire_generic(
-        parser.unpin(),
-        name.from_ffi(),
-        arguments.iter().map(WString::from).collect(),
-    );
+fn event_fire_generic_ffi(parser: &Parser, name: &CxxWString, arguments: &wcstring_list_ffi_t) {
+    fire_generic(parser, name.from_ffi(), arguments.from_ffi());
 }

fish-rust/src/fds.rs

@@ -2,14 +2,13 @@ use crate::common::wcs2zstring;
 use crate::ffi;
 use crate::wchar::prelude::*;
 use crate::wutil::perror;
-use libc::EINTR;
-use libc::{fcntl, F_GETFL, F_SETFL, O_CLOEXEC, O_NONBLOCK};
+use libc::{c_int, EINTR, FD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, O_CLOEXEC, O_NONBLOCK};
 use nix::unistd;
 use std::ffi::CStr;
 use std::io::{self, Read, Write};
 use std::os::unix::prelude::*;
 
-pub const PIPE_ERROR: &wstr = L!("An error occurred while setting up pipe");
+pub const PIPE_ERROR: &str = "An error occurred while setting up pipe";
 
 /// The first "high fd", which is considered outside the range of valid user-specified redirections
 /// (like >&5).
@@ -164,6 +163,27 @@ pub fn make_autoclose_pipes() -> Option<AutoClosePipes> {
     }
 }
 
+/// Sets CLO_EXEC on a given fd according to the value of \p should_set.
+pub fn set_cloexec(fd: RawFd, should_set: bool) -> c_int {
+    // Note we don't want to overwrite existing flags like O_NONBLOCK which may be set. So fetch the
+    // existing flags and modify them.
+    let flags = unsafe { libc::fcntl(fd, F_GETFD, 0) };
+    if flags < 0 {
+        return -1;
+    }
+    let mut new_flags = flags;
+    if should_set {
+        new_flags |= FD_CLOEXEC;
+    } else {
+        new_flags &= !FD_CLOEXEC;
+    }
+    if flags == new_flags {
+        0
+    } else {
+        unsafe { libc::fcntl(fd, F_SETFD, new_flags) }
+    }
+}
+
 /// Wide character version of open() that also sets the close-on-exec flag (atomically when
 /// possible).
 pub fn wopen_cloexec(pathname: &wstr, flags: i32, mode: libc::c_int) -> RawFd {
@@ -190,12 +210,12 @@ pub fn exec_close(fd: RawFd) {
 
 /// Mark an fd as nonblocking
 pub fn make_fd_nonblocking(fd: RawFd) -> Result<(), io::Error> {
-    let flags = unsafe { fcntl(fd, F_GETFL, 0) };
+    let flags = unsafe { libc::fcntl(fd, F_GETFL, 0) };
     let nonblocking = (flags & O_NONBLOCK) == O_NONBLOCK;
     if !nonblocking {
-        match unsafe { fcntl(fd, F_SETFL, flags | O_NONBLOCK) } {
-            0 => return Ok(()),
-            _ => return Err(io::Error::last_os_error()),
+        match unsafe { libc::fcntl(fd, F_SETFL, flags | O_NONBLOCK) } {
+            -1 => return Err(io::Error::last_os_error()),
+            _ => return Ok(()),
         };
     }
     Ok(())
@@ -203,12 +223,12 @@ pub fn make_fd_nonblocking(fd: RawFd) -> Result<(), io::Error> {
 
 /// Mark an fd as blocking
 pub fn make_fd_blocking(fd: RawFd) -> Result<(), io::Error> {
-    let flags = unsafe { fcntl(fd, F_GETFL, 0) };
+    let flags = unsafe { libc::fcntl(fd, F_GETFL, 0) };
     let nonblocking = (flags & O_NONBLOCK) == O_NONBLOCK;
     if nonblocking {
-        match unsafe { fcntl(fd, F_SETFL, flags & !O_NONBLOCK) } {
-            0 => return Ok(()),
-            _ => return Err(io::Error::last_os_error()),
+        match unsafe { libc::fcntl(fd, F_SETFL, flags & !O_NONBLOCK) } {
+            -1 => return Err(io::Error::last_os_error()),
+            _ => return Ok(()),
         };
     }
     Ok(())

fish-rust/src/ffi.rs

@@ -1,29 +1,24 @@
-use crate::wchar_ffi::WCharToFFI;
+use crate::io::{IoStreams, OutputStreamFfi};
+use crate::wchar;
 #[rustfmt::skip]
 use ::std::pin::Pin;
 #[rustfmt::skip]
 use ::std::slice;
-use crate::env::{EnvMode, EnvStackRef, EnvStackRefFFI};
-use crate::job_group::JobGroup;
-pub use crate::wait_handle::{
-    WaitHandleRef, WaitHandleRefFFI, WaitHandleStore, WaitHandleStoreFFI,
-};
+pub use crate::wait_handle::{WaitHandleRef, WaitHandleStore};
 use crate::wchar::prelude::*;
 use crate::wchar_ffi::WCharFromFFI;
 use autocxx::prelude::*;
-use cxx::SharedPtr;
-use libc::pid_t;
 
 // autocxx has been hacked up to know about this.
 pub type wchar_t = u32;
 
 include_cpp! {
     #include "autoload.h"
-    #include "builtin.h"
     #include "color.h"
     #include "common.h"
     #include "complete.h"
     #include "env.h"
+    #include "env_dispatch.h"
     #include "env_universal_common.h"
     #include "event.h"
     #include "exec.h"
@@ -47,9 +42,10 @@ include_cpp! {
     #include "tokenizer.h"
     #include "wutil.h"
 
-    // We need to block these types so when exposing C++ to Rust.
-    block!("WaitHandleStoreFFI")
-    block!("WaitHandleRefFFI")
+    #include "builtins/bind.h"
+    #include "builtins/commandline.h"
+    #include "builtins/read.h"
+    #include "builtins/ulimit.h"
 
     safety!(unsafe_ffi)
 
@@ -58,34 +54,31 @@ include_cpp! {
     generate!("wperror")
     generate!("set_inheriteds_ffi")
 
-    generate!("proc_init")
-    generate!("misc_init")
     generate!("reader_init")
+    generate!("reader_run_count")
     generate!("term_copy_modes")
     generate!("set_profiling_active")
     generate!("reader_read_ffi")
+    generate!("fish_is_unwinding_for_exit")
     generate!("restore_term_mode")
-    generate!("parse_util_detect_errors_ffi")
+    generate!("update_wait_on_escape_ms_ffi")
+    generate!("read_generation_count")
     generate!("set_flog_output_file_ffi")
     generate!("flog_setlinebuf_ffi")
     generate!("activate_flog_categories_by_pattern")
     generate!("save_term_foreground_process_group")
     generate!("restore_term_foreground_process_group_for_exit")
     generate!("set_cloexec")
+    generate!("env_universal_notifier_t_default_notifier_post_notification_ffi")
+
+    generate!("builtin_bind")
+    generate!("builtin_commandline")
+    generate!("builtin_read")
+    generate!("builtin_ulimit")
 
     generate!("init_input")
 
     generate_pod!("pipes_ffi_t")
-    generate!("environment_t")
-    generate!("env_stack_t")
-    generate!("env_var_t")
-    generate!("env_universal_t")
-    generate!("env_universal_sync_result_t")
-    generate!("callback_data_t")
-    generate!("universal_notifier_t")
-    generate!("var_table_ffi_t")
-
-    generate!("event_list_ffi_t")
 
     generate!("make_pipes_ffi")
 
@@ -93,232 +86,39 @@ include_cpp! {
 
     generate!("wgettext_ptr")
 
-    generate!("block_t")
-    generate!("Parser")
-
-    generate!("job_t")
-    generate!("job_control_t")
-    generate!("get_job_control_mode")
-    generate!("set_job_control_mode")
-    generate!("get_login")
-    generate!("mark_login")
-    generate!("mark_no_exec")
-    generate!("process_t")
-    generate!("library_data_t")
-    generate_pod!("library_data_pod_t")
-
-    generate!("highlighter_t")
-
-    generate!("proc_wait_any")
-
-    generate!("output_stream_t")
-    generate!("IoStreams")
-    generate!("make_null_io_streams_ffi")
-    generate!("make_test_io_streams_ffi")
-    generate!("get_test_output_ffi")
-
-    generate_pod!("RustFFIJobList")
-    generate_pod!("RustFFIProcList")
-    generate_pod!("RustBuiltin")
-
-    generate!("builtin_exists")
-    generate!("builtin_missing_argument")
-    generate!("builtin_unknown_option")
-    generate!("builtin_print_help")
-    generate!("builtin_print_error_trailer")
-    generate!("builtin_get_names_ffi")
-
     generate!("pretty_printer_t")
 
-    generate!("escape_string")
-
     generate!("fd_event_signaller_t")
 
-    generate!("block_t")
-    generate!("block_type_t")
-    generate!("statuses_t")
-    generate!("io_chain_t")
-
-    generate!("env_var_t")
-
-    generate!("exec_subshell_ffi")
+    generate!("highlight_role_t")
+    generate!("highlight_spec_t")
 
     generate!("rgb_color_t")
     generate_pod!("color24_t")
-    generate!("colorize_shell")
     generate!("reader_status_count")
+    generate!("reader_write_title_ffi")
+    generate!("reader_push_ffi")
+    generate!("reader_readline_ffi")
+    generate!("reader_pop")
+    generate!("commandline_get_state_history_ffi")
+    generate!("commandline_set_buffer_ffi")
+    generate!("commandline_get_state_initialized_ffi")
+    generate!("commandline_get_state_text_ffi")
+    generate!("completion_apply_to_command_line")
 
     generate!("get_history_variable_text_ffi")
 
-    generate!("is_interactive_session")
-    generate!("set_interactive_session")
+    generate_pod!("escape_string_style_t")
+
+
     generate!("screen_set_midnight_commander_hack")
     generate!("screen_clear_layout_cache_ffi")
     generate!("escape_code_length_ffi")
     generate!("reader_schedule_prompt_repaint")
     generate!("reader_change_history")
-    generate!("history_session_id")
-    generate!("history_save_all")
     generate!("reader_change_cursor_selection_mode")
     generate!("reader_set_autosuggestion_enabled_ffi")
-    generate!("complete_invalidate_path")
-    generate!("complete_add_wrapper")
-    generate!("update_wait_on_escape_ms_ffi")
     generate!("update_wait_on_sequence_key_ms_ffi")
-    generate!("autoload_t")
-    generate!("make_autoload_ffi")
-    generate!("perform_autoload_ffi")
-    generate!("complete_get_wrap_targets_ffi")
-
-    generate!("is_thompson_shell_script")
-}
-
-impl Parser {
-    pub fn get_wait_handles_mut(&mut self) -> &mut WaitHandleStore {
-        let ptr = self.get_wait_handles_void() as *mut Box<WaitHandleStoreFFI>;
-        assert!(!ptr.is_null());
-        unsafe { (*ptr).from_ffi_mut() }
-    }
-
-    pub fn get_wait_handles(&self) -> &WaitHandleStore {
-        let ptr = self.get_wait_handles_void() as *const Box<WaitHandleStoreFFI>;
-        assert!(!ptr.is_null());
-        unsafe { (*ptr).from_ffi() }
-    }
-
-    pub fn get_block_at_index(&self, i: usize) -> Option<&block_t> {
-        let b = self.block_at_index(i);
-        unsafe { b.as_ref() }
-    }
-
-    pub fn get_jobs(&self) -> &[SharedPtr<job_t>] {
-        let ffi_jobs = self.ffi_jobs();
-        unsafe { slice::from_raw_parts(ffi_jobs.jobs, ffi_jobs.count) }
-    }
-
-    pub fn libdata_pod(&mut self) -> &mut library_data_pod_t {
-        let libdata = self.pin().ffi_libdata_pod();
-
-        unsafe { &mut *libdata }
-    }
-
-    pub fn remove_var(&mut self, var: &wstr, flags: c_int) -> c_int {
-        self.pin().remove_var_ffi(&var.to_ffi(), flags)
-    }
-
-    pub fn job_get_from_pid(&self, pid: pid_t) -> Option<&job_t> {
-        let job = self.ffi_job_get_from_pid(pid.into());
-        unsafe { job.as_ref() }
-    }
-
-    pub fn get_vars(&mut self) -> EnvStackRef {
-        self.pin().vars().from_ffi()
-    }
-
-    pub fn get_func_name(&mut self, level: i32) -> Option<WString> {
-        let name = self.pin().get_function_name_ffi(c_int(level));
-        name.as_ref()
-            .map(|s| s.from_ffi())
-            .filter(|s| !s.is_empty())
-    }
-}
-
-unsafe impl Send for env_universal_t {}
-
-impl env_stack_t {
-    /// Access the underlying Rust environment stack.
-    #[allow(clippy::borrowed_box)]
-    pub fn from_ffi(&self) -> EnvStackRef {
-        // Safety: get_impl_ffi returns a pointer to a Box<EnvStackRefFFI>.
-        let envref = self.get_impl_ffi();
-        assert!(!envref.is_null());
-        let env: &Box<EnvStackRefFFI> = unsafe { &*(envref.cast()) };
-        env.0.clone()
-    }
-}
-
-impl environment_t {
-    /// Helper to get a variable as a string, using the default flags.
-    pub fn get_as_string(&self, name: &wstr) -> Option<WString> {
-        self.get_as_string_flags(name, EnvMode::default())
-    }
-
-    /// Helper to get a variable as a string, using the given flags.
-    pub fn get_as_string_flags(&self, name: &wstr, flags: EnvMode) -> Option<WString> {
-        self.get_or_null(&name.to_ffi(), flags.bits())
-            .as_ref()
-            .map(|s| s.as_string().from_ffi())
-    }
-}
-
-impl env_stack_t {
-    /// Helper to get a variable as a string, using the default flags.
-    pub fn get_as_string(&self, name: &wstr) -> Option<WString> {
-        self.get_as_string_flags(name, EnvMode::default())
-    }
-
-    /// Helper to get a variable as a string, using the given flags.
-    pub fn get_as_string_flags(&self, name: &wstr, flags: EnvMode) -> Option<WString> {
-        self.get_or_null(&name.to_ffi(), flags.bits())
-            .as_ref()
-            .map(|s| s.as_string().from_ffi())
-    }
-
-    /// Helper to set a value.
-    pub fn set_var<T: AsRef<wstr>, U: AsRef<wstr>>(
-        &mut self,
-        name: T,
-        value: &[U],
-        flags: EnvMode,
-    ) -> libc::c_int {
-        use crate::wchar_ffi::{wstr_to_u32string, W0String};
-        let strings: Vec<W0String> = value.iter().map(wstr_to_u32string).collect();
-        let ptrs: Vec<*const u32> = strings.iter().map(|s| s.as_ptr()).collect();
-        self.pin()
-            .set_ffi(
-                &name.as_ref().to_ffi(),
-                flags.bits(),
-                ptrs.as_ptr() as *const c_void,
-                ptrs.len(),
-            )
-            .into()
-    }
-}
-
-impl job_t {
-    #[allow(clippy::mut_from_ref)]
-    pub fn get_procs(&self) -> &mut [UniquePtr<process_t>] {
-        let ffi_procs = self.ffi_processes();
-        unsafe { slice::from_raw_parts_mut(ffi_procs.procs, ffi_procs.count) }
-    }
-
-    pub fn get_job_group(&self) -> &JobGroup {
-        unsafe { ::std::mem::transmute::<&job_group_t, &JobGroup>(self.ffi_group()) }
-    }
-}
-
-impl process_t {
-    /// \return the wait handle for the process, if it exists.
-    pub fn get_wait_handle(&self) -> Option<WaitHandleRef> {
-        let handle_ptr = self.get_wait_handle_void() as *const Box<WaitHandleRefFFI>;
-        if handle_ptr.is_null() {
-            None
-        } else {
-            let handle: &WaitHandleRefFFI = unsafe { &*handle_ptr };
-            Some(handle.from_ffi().clone())
-        }
-    }
-
-    /// \return the wait handle for the process, creating it if necessary.
-    pub fn make_wait_handle(&mut self, jid: u64) -> Option<WaitHandleRef> {
-        let handle_ref = self.pin().make_wait_handle_void(jid) as *const Box<WaitHandleRefFFI>;
-        if handle_ref.is_null() {
-            None
-        } else {
-            let handle: &WaitHandleRefFFI = unsafe { &*handle_ref };
-            Some(handle.from_ffi().clone())
-        }
-    }
 }
 
 /// Allow wcharz_t to be "into" wstr.
@@ -339,6 +139,17 @@ impl From<wcharz_t> for WString {
     }
 }
 
+/// Allow wcstring_list_ffi_t to be "into" Vec<WString>.
+impl From<&wcstring_list_ffi_t> for Vec<wchar::WString> {
+    fn from(w: &wcstring_list_ffi_t) -> Self {
+        let mut result = Vec::with_capacity(w.size());
+        for i in 0..w.size() {
+            result.push(w.at(i).from_ffi());
+        }
+        result
+    }
+}
+
 /// A bogus trait for turning &mut Foo into Pin<&mut Foo>.
 /// autocxx enforces that non-const methods must be called through Pin,
 /// but this means we can't pass around mutable references to types like Parser.
@@ -357,16 +168,10 @@ pub trait Repin {
 }
 
 // Implement Repin for our types.
-impl Repin for autoload_t {}
-impl Repin for block_t {}
-impl Repin for env_stack_t {}
-impl Repin for env_universal_t {}
-impl Repin for IoStreams {}
-impl Repin for job_t {}
-impl Repin for output_stream_t {}
-impl Repin for Parser {}
-impl Repin for process_t {}
+impl Repin for IoStreams<'_> {}
 impl Repin for wcstring_list_ffi_t {}
+impl Repin for rgb_color_t {}
+impl Repin for OutputStreamFfi<'_> {}
 
 pub use autocxx::c_int;
 pub use ffi::*;
@@ -423,19 +228,3 @@ impl core::convert::From<void_ptr> for *const autocxx::c_void {
         value.0 as *const _
     }
 }
-
-impl TryFrom<&wstr> for job_control_t {
-    type Error = ();
-
-    fn try_from(value: &wstr) -> Result<Self, Self::Error> {
-        if value == "full" {
-            Ok(job_control_t::all)
-        } else if value == "interactive" {
-            Ok(job_control_t::interactive)
-        } else if value == "none" {
-            Ok(job_control_t::none)
-        } else {
-            Err(())
-        }
-    }
-}

fish-rust/src/fish.rs

@@ -25,28 +25,36 @@ use crate::{
         BUILTIN_ERR_MISSING, BUILTIN_ERR_UNKNOWN, STATUS_CMD_OK, STATUS_CMD_UNKNOWN,
     },
     common::{
-        escape, exit_without_destructors, get_executable_path, str2wcstring, wcs2string,
-        PROFILING_ACTIVE, PROGRAM_NAME,
+        escape, exit_without_destructors, get_executable_path, save_term_foreground_process_group,
+        scoped_push_replacer, str2wcstring, wcs2string, PROFILING_ACTIVE, PROGRAM_NAME,
     },
+    env::Statuses,
     env::{
         environment::{env_init, EnvStack, Environment},
         ConfigPaths, EnvMode,
     },
     event::{self, Event},
-    ffi::{self, Repin},
+    ffi::{self},
     flog::{self, activate_flog_categories_by_pattern, set_flog_file_fd, FLOG, FLOGF},
-    function, future_feature_flags as features,
+    function, future_feature_flags as features, history,
     history::start_private_mode,
-    parse_constants::{ParseErrorList, ParseErrorListFfi, ParseTreeFlags},
-    parse_tree::{ParsedSource, ParsedSourceRefFFI},
+    io::IoChain,
+    parse_constants::{ParseErrorList, ParseTreeFlags},
+    parse_tree::ParsedSource,
+    parse_util::parse_util_detect_errors_in_ast,
+    parser::{BlockType, Parser},
     path::path_get_config,
+    proc::{
+        get_login, is_interactive_session, mark_login, mark_no_exec, proc_init,
+        set_interactive_session,
+    },
     signal::{signal_clear_cancel, signal_unblock_all},
     threads::{self, asan_maybe_exit},
     topic_monitor,
     wchar::prelude::*,
-    wchar_ffi::{WCharFromFFI, WCharToFFI},
     wutil::waccess,
 };
+use libc::STDERR_FILENO;
 use std::env;
 use std::ffi::{CString, OsStr, OsString};
 use std::fs::File;
@@ -264,7 +272,7 @@ fn determine_config_directory_paths(argv0: impl AsRef<Path>) -> ConfigPaths {
 }
 
 // Source the file config.fish in the given directory.
-fn source_config_in_directory(parser: &mut ffi::Parser, dir: &wstr) {
+fn source_config_in_directory(parser: &Parser, dir: &wstr) {
     // If the config.fish file doesn't exist or isn't readable silently return. Fish versions up
     // thru 2.2.0 would instead try to source the file with stderr redirected to /dev/null to deal
     // with that possibility.
@@ -286,17 +294,13 @@ fn source_config_in_directory(parser: &mut ffi::Parser, dir: &wstr) {
 
     let cmd: WString = L!("builtin source ").to_owned() + escaped_pathname.as_utfstr();
 
-    parser.libdata_pod().within_fish_init = true;
-    // PORTING: you need to call `within_unique_ptr`, otherwise it is a no-op
-    let _ = parser
-        .pin()
-        .eval_string_ffi1(&cmd.to_ffi())
-        .within_unique_ptr();
-    parser.libdata_pod().within_fish_init = false;
+    parser.libdata_mut().pods.within_fish_init = true;
+    let _ = parser.eval(&cmd, &IoChain::new());
+    parser.libdata_mut().pods.within_fish_init = false;
 }
 
 /// Parse init files. exec_path is the path of fish executable as determined by argv[0].
-fn read_init(parser: &mut ffi::Parser, paths: &ConfigPaths) {
+fn read_init(parser: &Parser, paths: &ConfigPaths) {
     source_config_in_directory(parser, &str2wcstring(paths.data.as_os_str().as_bytes()));
     source_config_in_directory(parser, &str2wcstring(paths.sysconf.as_os_str().as_bytes()));
 
@@ -308,7 +312,7 @@ fn read_init(parser: &mut ffi::Parser, paths: &ConfigPaths) {
     }
 }
 
-fn run_command_list(parser: &mut ffi::Parser, cmds: &[OsString]) -> i32 {
+fn run_command_list(parser: &Parser, cmds: &[OsString]) -> i32 {
     let mut retval = STATUS_CMD_OK;
     for cmd in cmds {
         let cmd_wcs = str2wcstring(cmd.as_bytes());
@@ -316,44 +320,17 @@ fn run_command_list(parser: &mut ffi::Parser, cmds: &[OsString]) -> i32 {
         let mut errors = ParseErrorList::new();
         let ast = Ast::parse(&cmd_wcs, ParseTreeFlags::empty(), Some(&mut errors));
         let errored = ast.errored() || {
-            // parse_util_detect_errors_in_ast is just partially ported
-            // parse_util_detect_errors_in_ast(&ast, &cmd_wcs, Some(&mut errors)).is_err();
-
-            let mut errors_ffi = ParseErrorListFfi(errors.clone());
-            let res = ffi::parse_util_detect_errors_ffi(
-                &ast as *const Ast as *const _,
-                &cmd_wcs.to_ffi(),
-                &mut errors_ffi as *mut ParseErrorListFfi as *mut _,
-            );
-            errors = errors_ffi.0;
-            res != 0
+            parse_util_detect_errors_in_ast(&ast, &cmd_wcs, Some(&mut errors)).is_err()
         };
 
         if !errored {
             // Construct a parsed source ref.
-            // Be careful to transfer ownership, this could be a very large string.
-
-            let ps = ParsedSourceRefFFI(Some(Arc::new(ParsedSource::new(cmd_wcs, ast))));
-            // this casting is needed since rust defines the type, so the type is incomplete when we
-            // read the headers
-            let _ = parser
-                .pin()
-                .eval_parsed_source_ffi1(
-                    &ps as *const ParsedSourceRefFFI as *const _,
-                    ffi::block_type_t::top,
-                )
-                .within_unique_ptr();
+            let ps = Arc::new(ParsedSource::new(cmd_wcs, ast));
+            let _ = parser.eval_parsed_source(&ps, &IoChain::new(), None, BlockType::top);
             retval = STATUS_CMD_OK;
         } else {
-            let mut sb = WString::new().to_ffi();
-            let errors_ffi = ParseErrorListFfi(errors);
-            parser.pin().get_backtrace_ffi(
-                &cmd_wcs.to_ffi(),
-                &errors_ffi as *const ParseErrorListFfi as *const _,
-                sb.pin_mut(),
-            );
-            // fwprint! does not seem to work?
-            eprint!("{}", sb.from_ffi());
+            let backtrace = parser.get_backtrace(&cmd_wcs, &errors);
+            fwprintf!(STDERR_FILENO, "%s", backtrace);
             // XXX: Why is this the return for "unknown command"?
             retval = STATUS_CMD_UNKNOWN;
         }
@@ -362,7 +339,7 @@ fn run_command_list(parser: &mut ffi::Parser, cmds: &[OsString]) -> i32 {
     retval.unwrap()
 }
 
-fn fish_parse_opt(args: &mut [&wstr], opts: &mut FishCmdOpts) -> usize {
+fn fish_parse_opt(args: &mut [WString], opts: &mut FishCmdOpts) -> usize {
     use crate::wgetopt::{wgetopter_t, wopt, woption, woption_argument_t::*};
 
     const RUSAGE_ARG: char = 1 as char;
@@ -398,7 +375,8 @@ fn fish_parse_opt(args: &mut [&wstr], opts: &mut FishCmdOpts) -> usize {
         wopt(L!("version"), no_argument, 'v'),
     ];
 
-    let mut w = wgetopter_t::new(SHORT_OPTS, LONG_OPTS, args);
+    let mut shim_args: Vec<&wstr> = args.iter().map(|s| s.as_ref()).collect();
+    let mut w = wgetopter_t::new(SHORT_OPTS, LONG_OPTS, &mut shim_args);
     while let Some(c) = w.wgetopt_long() {
         match c {
             'c' => opts
@@ -498,7 +476,7 @@ fn fish_parse_opt(args: &mut [&wstr], opts: &mut FishCmdOpts) -> usize {
         && optind == args.len()
         && unsafe { libc::isatty(libc::STDIN_FILENO) != 0 }
     {
-        ffi::set_interactive_session(true);
+        set_interactive_session(true);
     }
 
     optind
@@ -554,13 +532,8 @@ fn main() -> i32 {
         ffi::activate_flog_categories_by_pattern(s);
     }
 
-    let owning_args = args;
-    let mut args_for_opts: Vec<&wstr> = owning_args.iter().map(WString::as_utfstr).collect();
-
     let mut opts = FishCmdOpts::default();
-    my_optind = fish_parse_opt(&mut args_for_opts, &mut opts);
-
-    let args = args_for_opts;
+    my_optind = fish_parse_opt(&mut args, &mut opts);
 
     // Direct any debug output right away.
     // --debug-output takes precedence, otherwise $FISH_DEBUG_OUTPUT is used.
@@ -625,13 +598,13 @@ fn main() -> i32 {
 
     // Apply our options
     if opts.is_login {
-        ffi::mark_login();
+        mark_login();
     }
     if opts.no_exec {
-        ffi::mark_no_exec();
+        mark_no_exec();
     }
     if opts.is_interactive_session {
-        ffi::set_interactive_session(true);
+        set_interactive_session(true);
     }
     if opts.enable_private_mode {
         start_private_mode(EnvStack::globals());
@@ -639,9 +612,8 @@ fn main() -> i32 {
 
     // Only save (and therefore restore) the fg process group if we are interactive. See issues
     // #197 and #1002.
-    if ffi::is_interactive_session() {
-        // save_term_foreground_process_group();
-        ffi::save_term_foreground_process_group();
+    if is_interactive_session() {
+        save_term_foreground_process_group();
     }
 
     let mut paths: Option<ConfigPaths> = None;
@@ -649,7 +621,7 @@ fn main() -> i32 {
     if !opts.no_exec {
         // PORTING: C++ had not converted, we must revert
         paths = Some(determine_config_directory_paths(OsString::from_vec(
-            wcs2string(args[0]),
+            wcs2string(&args[0]),
         )));
         env_init(
             paths.as_ref(),
@@ -667,20 +639,20 @@ fn main() -> i32 {
         }
     }
     features::set_from_string(opts.features.as_utfstr());
-    ffi::proc_init();
-    ffi::misc_init();
+    proc_init();
+    crate::env::misc_init();
     ffi::reader_init();
 
-    let parser = unsafe { &mut *ffi::Parser::principal_parser_ffi() };
-    parser.pin().set_syncs_uvars(!opts.no_config);
+    let parser = Parser::principal_parser();
+    parser.set_syncs_uvars(!opts.no_config);
 
     if !opts.no_exec && !opts.no_config {
         read_init(parser, paths.as_ref().unwrap());
     }
 
-    if ffi::is_interactive_session() && opts.no_config && !opts.no_exec {
+    if is_interactive_session() && opts.no_config && !opts.no_exec {
         // If we have no config, we default to the default key bindings.
-        parser.get_vars().set_one(
+        parser.vars().set_one(
             L!("fish_key_bindings"),
             EnvMode::UNEXPORT,
             L!("fish_default_key_bindings").to_owned(),
@@ -694,19 +666,16 @@ fn main() -> i32 {
     ffi::term_copy_modes();
 
     // Stomp the exit status of any initialization commands (issue #635).
-    // PORTING: it is actually really nice that this just compiles, assuming it works
-    parser
-        .pin()
-        .set_last_statuses(ffi::statuses_t::just(c_int(STATUS_CMD_OK.unwrap())).within_box());
+    parser.set_last_statuses(Statuses::just(STATUS_CMD_OK.unwrap()));
 
     // TODO: if-let-chains
     if opts.profile_startup_output.is_some() && opts.profile_startup_output != opts.profile_output {
         let s = cstr_from_osstr(&opts.profile_startup_output.unwrap());
-        parser.pin().emit_profiling(s.as_ptr());
+        parser.emit_profiling(s.as_bytes());
 
         // If we are profiling both, ensure the startup data only
         // ends up in the startup file.
-        parser.pin().clear_profiling();
+        parser.clear_profiling();
     }
 
     PROFILING_ACTIVE.store(opts.profile_output.is_some());
@@ -722,22 +691,21 @@ fn main() -> i32 {
 
     if !opts.batch_cmds.is_empty() {
         // Run the commands specified as arguments, if any.
-        if ffi::get_login() {
+        if get_login() {
             // Do something nasty to support OpenSUSE assuming we're bash. This may modify cmds.
             fish_xdm_login_hack_hack_hack_hack(&mut opts.batch_cmds, &args[my_optind..]);
         }
 
         // Pass additional args as $argv.
         // Note that we *don't* support setting argv[0]/$0, unlike e.g. bash.
-        // PORTING: the args were converted to WString here in C++
         let list = &args[my_optind..];
-        parser.get_vars().set(
+        parser.vars().set(
             L!("argv"),
             EnvMode::default(),
-            list.iter().map(|&s| s.to_owned()).collect(),
+            list.iter().map(|s| s.to_owned()).collect(),
         );
         res = run_command_list(parser, &opts.batch_cmds);
-        parser.libdata_pod().exit_current_script = false;
+        parser.libdata_mut().pods.exit_current_script = false;
     } else if my_optind == args.len() {
         // Implicitly interactive mode.
         if opts.no_exec && unsafe { libc::isatty(libc::STDIN_FILENO) != 0 } {
@@ -748,10 +716,15 @@ fn main() -> i32 {
             // above line should always exit
             return libc::EXIT_FAILURE;
         }
-        res = ffi::reader_read_ffi(parser.pin(), c_int(libc::STDIN_FILENO)).into();
+        res = ffi::reader_read_ffi(
+            parser as *const Parser as *const autocxx::c_void,
+            c_int(libc::STDIN_FILENO),
+            &IoChain::new() as *const _ as *const autocxx::c_void,
+        )
+        .into();
     } else {
         // C++ had not converted at this point, we must undo
-        let n = wcs2string(args[my_optind]);
+        let n = wcs2string(&args[my_optind]);
         let path = OsStr::from_bytes(&n);
         my_optind += 1;
         // Rust sets cloexec by default, see above
@@ -768,16 +741,24 @@ fn main() -> i32 {
             Ok(f) => {
                 // PORTING: the args were converted to WString here in C++
                 let list = &args[my_optind..];
-                parser.get_vars().set(
+                parser.vars().set(
                     L!("argv"),
                     EnvMode::default(),
-                    list.iter().map(|&s| s.to_owned()).collect(),
+                    list.iter().map(|s| s.to_owned()).collect(),
                 );
                 let rel_filename = &args[my_optind - 1];
-                // PORTING: this used to be `scoped_push`
-                let old_filename = parser.pin().current_filename_ffi().from_ffi();
-                parser.pin().set_filename_ffi(rel_filename.to_ffi());
-                res = ffi::reader_read_ffi(parser.pin(), c_int(f.as_raw_fd())).into();
+                let _filename_push = scoped_push_replacer(
+                    |new_value| {
+                        std::mem::replace(&mut parser.libdata_mut().current_filename, new_value)
+                    },
+                    Some(Arc::new(rel_filename.to_owned())),
+                );
+                res = ffi::reader_read_ffi(
+                    parser as *const Parser as *const autocxx::c_void,
+                    c_int(f.as_raw_fd()),
+                    &IoChain::new() as *const _ as *const autocxx::c_void,
+                )
+                .into();
                 if res != 0 {
                     FLOGF!(
                         warning,
@@ -785,7 +766,6 @@ fn main() -> i32 {
                         path.to_string_lossy()
                     );
                 }
-                parser.pin().set_filename_ffi(old_filename.to_ffi());
             }
         }
     }
@@ -793,7 +773,7 @@ fn main() -> i32 {
     let exit_status = if res != 0 {
         STATUS_CMD_UNKNOWN.unwrap()
     } else {
-        parser.pin().get_last_status().into()
+        parser.get_last_status()
     };
 
     event::fire(
@@ -814,10 +794,10 @@ fn main() -> i32 {
 
     if let Some(profile_output) = opts.profile_output {
         let s = cstr_from_osstr(&profile_output);
-        parser.pin().emit_profiling(s.as_ptr());
+        parser.emit_profiling(s.as_bytes());
     }
 
-    ffi::history_save_all();
+    history::save_all();
     if opts.print_rusage_self {
         print_rusage_self();
     }
@@ -845,7 +825,7 @@ fn escape_single_quoted_hack_hack_hack_hack(s: &wstr) -> OsString {
     return result;
 }
 
-fn fish_xdm_login_hack_hack_hack_hack(cmds: &mut Vec<OsString>, args: &[&wstr]) -> bool {
+fn fish_xdm_login_hack_hack_hack_hack(cmds: &mut Vec<OsString>, args: &[WString]) -> bool {
     if cmds.len() != 1 {
         return false;
     }

fish-rust/src/flog.rs

@@ -191,7 +191,7 @@ macro_rules! FLOG {
 
 macro_rules! FLOGF {
     ($category:ident, $fmt: expr, $($elem:expr),+ $(,)*) => {
-        crate::flog::FLOG!($category, sprintf!($fmt, $($elem),*));
+        crate::flog::FLOG!($category, crate::wutil::sprintf!($fmt, $($elem),*))
     }
 }
 

fish-rust/src/fork_exec/mod.rs

@@ -5,16 +5,17 @@
 mod flog_safe;
 pub mod postfork;
 pub mod spawn;
-use crate::ffi::job_t;
+use crate::proc::Job;
+use libc::{SIGINT, SIGQUIT};
 
 /// Get the list of signals which should be blocked for a given job.
 /// Return true if at least one signal was set.
-fn blocked_signals_for_job(job: &job_t, sigmask: &mut libc::sigset_t) -> bool {
+pub fn blocked_signals_for_job(job: &Job, sigmask: &mut libc::sigset_t) -> bool {
     // Block some signals in background jobs for which job control is turned off (#6828).
     if !job.is_foreground() && !job.wants_job_control() {
         unsafe {
-            libc::sigaddset(sigmask, libc::SIGINT);
-            libc::sigaddset(sigmask, libc::SIGQUIT);
+            libc::sigaddset(sigmask, SIGINT);
+            libc::sigaddset(sigmask, SIGQUIT);
         }
         return true;
     }

fish-rust/src/fork_exec/postfork.rs

@@ -50,7 +50,7 @@ pub fn report_setpgid_error(
     is_parent: bool,
     pid: pid_t,
     desired_pgid: pid_t,
-    job_id: c_int,
+    job_id: i64,
     command_str: *const c_char,
     argv0_str: *const c_char,
 ) {
@@ -100,7 +100,7 @@ pub fn report_setpgid_error(
 
 /// Execute setpgid, moving pid into the given pgroup.
 /// Return the result of setpgid.
-fn execute_setpgid(pid: pid_t, pgroup: pid_t, is_parent: bool) -> i32 {
+pub fn execute_setpgid(pid: pid_t, pgroup: pid_t, is_parent: bool) -> i32 {
     // There is a comment "Historically we have looped here to support WSL."
     // TODO: stop looping.
     let mut eperm_count = 0;
@@ -145,7 +145,7 @@ fn execute_setpgid(pid: pid_t, pgroup: pid_t, is_parent: bool) -> i32 {
 }
 
 /// Set up redirections and signal handling in the child process.
-fn child_setup_process(
+pub fn child_setup_process(
     claim_tty_from: pid_t,
     sigmask: Option<&libc::sigset_t>,
     is_forked: bool,
@@ -204,7 +204,7 @@ fn child_setup_process(
 /// This function is a wrapper around fork. If the fork calls fails with EAGAIN, it is retried
 /// FORK_LAPS times, with a very slight delay between each lap. If fork fails even then, the process
 /// will exit with an error message.
-fn execute_fork() -> pid_t {
+pub fn execute_fork() -> pid_t {
     let mut err = 0;
     for i in 0..FORK_LAPS {
         let pid = unsafe { libc::fork() };
@@ -242,7 +242,7 @@ fn execute_fork() -> pid_t {
     exit_without_destructors(1)
 }
 
-fn safe_report_exec_error(
+pub fn safe_report_exec_error(
     err: i32,
     actual_cmd: *const c_char,
     argvv: *const *const c_char,
@@ -596,7 +596,7 @@ mod ffi {
             is_parent,
             pid,
             desired_pgid,
-            job_id,
+            job_id.try_into().unwrap(),
             command_str,
             argv0_str,
         )

fish-rust/src/fork_exec/spawn.rs

@@ -1,7 +1,8 @@
 //! Wrappers around posix_spawn.
 
 use super::blocked_signals_for_job;
-use crate::ffi::{self, job_t};
+use crate::exec::is_thompson_shell_script;
+use crate::proc::Job;
 use crate::redirection::Dup2List;
 use crate::signal::get_signals_with_handlers;
 use errno::{self, set_errno, Errno};
@@ -96,15 +97,15 @@ pub struct PosixSpawner {
 }
 
 impl PosixSpawner {
-    pub fn new(j: &job_t, dup2s: &Dup2List) -> Result<PosixSpawner, Errno> {
+    pub fn new(j: &Job, dup2s: &Dup2List) -> Result<PosixSpawner, Errno> {
         let mut attr = Attr::new()?;
         let mut actions = FileActions::new()?;
 
         // desired_pgid tracks the pgroup for the process. If it is none, the pgroup is left unchanged.
         // If it is zero, create a new pgroup from the pid. If it is >0, join that pgroup.
-        let desired_pgid = if let Some(pgid) = j.get_job_group().get_pgid() {
+        let desired_pgid = if let Some(pgid) = j.get_pgid() {
             Some(pgid)
-        } else if j.get_procs()[0].as_ref().unwrap().get_leads_pgrp() {
+        } else if j.processes()[0].leads_pgrp {
             Some(0)
         } else {
             None
@@ -179,14 +180,15 @@ impl PosixSpawner {
         // the kernel won't run the binary we hand it off to the interpreter
         // after performing a binary safety check, recommended by POSIX: a
         // line needs to exist before the first \0 with a lowercase letter.
-        if spawn_err.0 == libc::ENOEXEC && ffi::is_thompson_shell_script(cmd) {
+        let cmdcstr = unsafe { CStr::from_ptr(cmd) };
+        if spawn_err.0 == libc::ENOEXEC && is_thompson_shell_script(cmdcstr) {
             // Create a new argv with /bin/sh prepended.
             let interp = get_path_bshell();
             let mut argv2 = vec![interp.as_ptr() as *mut c_char];
 
             // The command to call should use the full path,
             // not what we would pass as argv0.
-            let cmd2: CString = CString::new(unsafe { CStr::from_ptr(cmd).to_bytes() }).unwrap();
+            let cmd2: CString = CString::new(cmdcstr.to_bytes()).unwrap();
             argv2.push(cmd2.as_ptr() as *mut c_char);
             for i in 1.. {
                 let ptr = unsafe { argv.offset(i).read() };
@@ -218,20 +220,6 @@ fn get_path_bshell() -> CString {
     CString::new("/bin/sh").unwrap()
 }
 
-/// Returns a Box<PosixSpawner>::into_raw(), or nullptr on error, in which case errno is set.
-/// j is an unowned pointer to a job_t, dup2s is an unowned pointer to a Dup2List.
-fn new_spawner_ffi(j: *const u8, dup2s: *const u8) -> *mut PosixSpawner {
-    let j: &job_t = unsafe { &*j.cast() };
-    let dup2s: &Dup2List = unsafe { &*dup2s.cast() };
-    match PosixSpawner::new(j, dup2s) {
-        Ok(spawner) => Box::into_raw(Box::new(spawner)),
-        Err(err) => {
-            set_errno(err);
-            std::ptr::null_mut()
-        }
-    }
-}
-
 impl Drop for PosixSpawner {
     fn drop(&mut self) {
         // Necessary to define this for FFI purposes, to avoid link errors.
@@ -255,28 +243,3 @@ impl PosixSpawner {
         }
     }
 }
-
-fn force_cxx_to_generate_box_do_not_call() -> Box<PosixSpawner> {
-    unimplemented!("Do not call, for linking help only");
-}
-
-#[cxx::bridge]
-mod posix_spawn_ffi {
-    extern "Rust" {
-        type PosixSpawner;
-
-        // Required to force cxx to generate a Box destructor, to avoid link errors.
-        fn force_cxx_to_generate_box_do_not_call() -> Box<PosixSpawner>;
-
-        #[cxx_name = "new_spawner"]
-        fn new_spawner_ffi(j: *const u8, dup2s: *const u8) -> *mut PosixSpawner;
-
-        #[cxx_name = "spawn"]
-        fn spawn_ffi(
-            &mut self,
-            cmd: *const c_char,
-            argv: *const *mut c_char,
-            envp: *const *mut c_char,
-        ) -> i32;
-    }
-}

fish-rust/src/function.rs

@@ -3,12 +3,14 @@
 // the parser and to some degree the builtin handling library.
 
 use crate::ast::{self, Node};
+use crate::autoload::Autoload;
 use crate::common::{assert_sync, escape, valid_func_name, FilenameRef};
+use crate::complete::complete_get_wrap_targets;
 use crate::env::{EnvStack, Environment};
 use crate::event::{self, EventDescription};
-use crate::ffi::{self, Parser, Repin};
 use crate::global_safety::RelaxedAtomicBool;
 use crate::parse_tree::{NodeRef, ParsedSourceRefFFI};
+use crate::parser::Parser;
 use crate::parser_keywords::parser_keywords_is_reserved;
 use crate::wchar::prelude::*;
 use crate::wchar_ffi::wcstring_list_ffi_t;
@@ -68,7 +70,7 @@ struct FunctionSet {
     autoload_tombstones: HashSet<WString>,
 
     /// The autoloader for our functions.
-    autoloader: cxx::UniquePtr<ffi::autoload_t>,
+    autoloader: Autoload,
 }
 
 impl FunctionSet {
@@ -105,16 +107,16 @@ static FUNCTION_SET: Lazy<Mutex<FunctionSet>> = Lazy::new(|| {
     Mutex::new(FunctionSet {
         funcs: HashMap::new(),
         autoload_tombstones: HashSet::new(),
-        autoloader: ffi::make_autoload_ffi(L!("fish_function_path").to_ffi()),
+        autoloader: Autoload::new(L!("fish_function_path")),
     })
 });
 
-/// Necessary until autoloader has been ported to Rust.
+// Safety: global lock.
 unsafe impl Send for FunctionSet {}
 
 /// Make sure that if the specified function is a dynamically loaded function, it has been fully
 /// loaded. Note this executes fish script code.
-fn load(name: &wstr, parser: &mut Parser) -> bool {
+pub fn load(name: &wstr, parser: &Parser) -> bool {
     parser.assert_can_execute();
     let mut path_to_autoload: Option<WString> = None;
     // Note we can't autoload while holding the funcset lock.
@@ -122,13 +124,10 @@ fn load(name: &wstr, parser: &mut Parser) -> bool {
     {
         let mut funcset: std::sync::MutexGuard<FunctionSet> = FUNCTION_SET.lock().unwrap();
         if funcset.allow_autoload(name) {
-            let path = funcset
+            if let Some(path) = funcset
                 .autoloader
-                .as_mut()
-                .unwrap()
-                .resolve_command_ffi(&name.to_ffi() /* Environment::globals() */)
-                .from_ffi();
-            if !path.is_empty() {
+                .resolve_command(name, EnvStack::globals().as_ref().get_ref())
+            {
                 path_to_autoload = Some(path);
             }
         }
@@ -137,14 +136,12 @@ fn load(name: &wstr, parser: &mut Parser) -> bool {
     // Release the lock and perform any autoload, then reacquire the lock and clean up.
     if let Some(path_to_autoload) = path_to_autoload.as_ref() {
         // Crucially, the lock is acquired after perform_autoload().
-        ffi::perform_autoload_ffi(&path_to_autoload.to_ffi(), parser.pin());
+        Autoload::perform_autoload(path_to_autoload, parser);
         FUNCTION_SET
             .lock()
             .unwrap()
             .autoloader
-            .as_mut()
-            .unwrap()
-            .mark_autoload_finished(&name.to_ffi());
+            .mark_autoload_finished(name);
     }
     path_to_autoload.is_some()
 }
@@ -199,7 +196,7 @@ pub fn add(name: WString, props: Arc<FunctionProperties>) {
     // Check if this is a function that we are autoloading.
     props
         .is_autoload
-        .store(funcset.autoloader.autoload_in_progress(&name.to_ffi()));
+        .store(funcset.autoloader.autoload_in_progress(&name));
 
     // Create and store a new function.
     let existing = funcset.funcs.insert(name, props);
@@ -219,7 +216,7 @@ pub fn get_props(name: &wstr) -> Option<Arc<FunctionProperties>> {
 }
 
 /// \return the properties for a function, or None, perhaps triggering autoloading.
-pub fn get_props_autoload(name: &wstr, parser: &mut Parser) -> Option<Arc<FunctionProperties>> {
+pub fn get_props_autoload(name: &wstr, parser: &Parser) -> Option<Arc<FunctionProperties>> {
     parser.assert_can_execute();
     if parser_keywords_is_reserved(name) {
         return None;
@@ -230,7 +227,7 @@ pub fn get_props_autoload(name: &wstr, parser: &mut Parser) -> Option<Arc<Functi
 
 /// Returns true if the function named \p cmd exists.
 /// This may autoload.
-pub fn exists(cmd: &wstr, parser: &mut Parser) -> bool {
+pub fn exists(cmd: &wstr, parser: &Parser) -> bool {
     parser.assert_can_execute();
     if !valid_func_name(cmd) {
         return false;
@@ -249,12 +246,7 @@ pub fn exists_no_autoload(cmd: &wstr) -> bool {
     }
     let mut funcset = FUNCTION_SET.lock().unwrap();
     // Check if we either have the function, or it could be autoloaded.
-    funcset.get_props(cmd).is_some()
-        || funcset
-            .autoloader
-            .as_mut()
-            .unwrap()
-            .can_autoload(&cmd.to_ffi())
+    funcset.get_props(cmd).is_some() || funcset.autoloader.can_autoload(cmd)
 }
 
 /// Remove the function with the specified name.
@@ -291,7 +283,7 @@ fn get_function_body_source(props: &FunctionProperties) -> &wstr {
 
 /// Sets the description of the function with the name \c name.
 /// This triggers autoloading.
-pub fn set_desc(name: &wstr, desc: WString, parser: &mut Parser) {
+pub(crate) fn set_desc(name: &wstr, desc: WString, parser: &Parser) {
     parser.assert_can_execute();
     load(name, parser);
     let mut funcset = FUNCTION_SET.lock().unwrap();
@@ -307,7 +299,7 @@ pub fn set_desc(name: &wstr, desc: WString, parser: &mut Parser) {
 /// Creates a new function using the same definition as the specified function. Returns true if copy
 /// is successful.
 pub fn copy(name: &wstr, new_name: WString, parser: &Parser) -> bool {
-    let filename = parser.current_filename_ffi().from_ffi();
+    let filename = parser.current_filename();
     let lineno = parser.get_lineno();
 
     let mut funcset = FUNCTION_SET.lock().unwrap();
@@ -319,8 +311,8 @@ pub fn copy(name: &wstr, new_name: WString, parser: &Parser) -> bool {
     let mut new_props = props.as_ref().clone();
     new_props.is_autoload.store(false);
     new_props.is_copy = true;
-    new_props.copy_definition_file = Some(Arc::new(filename));
-    new_props.copy_definition_lineno = lineno.into();
+    new_props.copy_definition_file = filename.clone();
+    new_props.copy_definition_lineno = lineno.unwrap_or(0) as i32;
 
     // Note this will NOT overwrite an existing function with the new name.
     // TODO: rationalize if this behavior is desired.
@@ -350,7 +342,7 @@ pub fn invalidate_path() {
     // Remove all autoloaded functions and update the autoload path.
     let mut funcset = FUNCTION_SET.lock().unwrap();
     funcset.funcs.retain(|_, props| !props.is_autoload.load());
-    funcset.autoloader.as_mut().unwrap().clear();
+    funcset.autoloader.clear();
 }
 
 impl FunctionProperties {
@@ -427,7 +419,7 @@ impl FunctionProperties {
         }
 
         // Output wrap targets.
-        for wrap in ffi::complete_get_wrap_targets_ffi(&name.to_ffi()).from_ffi() {
+        for wrap in complete_get_wrap_targets(name) {
             out.push_str(" --wraps=");
             out.push_utfstr(&escape(&wrap));
         }
@@ -588,9 +580,9 @@ fn function_get_props_ffi(name: &CxxWString) -> *mut FunctionPropertiesRefFFI {
 
 fn function_get_props_autoload_ffi(
     name: &CxxWString,
-    parser: Pin<&mut Parser>,
+    parser: &Parser,
 ) -> *mut FunctionPropertiesRefFFI {
-    let props = get_props_autoload(name.as_wstr(), parser.unpin());
+    let props = get_props_autoload(name.as_wstr(), parser);
     if let Some(props) = props {
         Box::into_raw(Box::new(FunctionPropertiesRefFFI(props)))
     } else {
@@ -598,16 +590,16 @@ fn function_get_props_autoload_ffi(
     }
 }
 
-fn function_load_ffi(name: &CxxWString, parser: Pin<&mut Parser>) -> bool {
-    load(name.as_wstr(), parser.unpin())
+fn function_load_ffi(name: &CxxWString, parser: &Parser) -> bool {
+    load(name.as_wstr(), parser)
 }
 
-fn function_set_desc_ffi(name: &CxxWString, desc: &CxxWString, parser: Pin<&mut Parser>) {
-    set_desc(name.as_wstr(), desc.from_ffi(), parser.unpin());
+fn function_set_desc_ffi(name: &CxxWString, desc: &CxxWString, parser: &Parser) {
+    set_desc(name.as_wstr(), desc.from_ffi(), parser);
 }
 
-fn function_exists_ffi(cmd: &CxxWString, parser: Pin<&mut Parser>) -> bool {
-    exists(cmd.as_wstr(), parser.unpin())
+fn function_exists_ffi(cmd: &CxxWString, parser: &Parser) -> bool {
+    exists(cmd.as_wstr(), parser)
 }
 
 fn function_exists_no_autoload_ffi(cmd: &CxxWString) -> bool {
@@ -621,8 +613,8 @@ fn function_get_names_ffi(get_hidden: bool, mut out: Pin<&mut wcstring_list_ffi_
     }
 }
 
-fn function_copy_ffi(name: &CxxWString, new_name: &CxxWString, parser: Pin<&mut Parser>) -> bool {
-    copy(name.as_wstr(), new_name.from_ffi(), parser.unpin())
+fn function_copy_ffi(name: &CxxWString, new_name: &CxxWString, parser: &Parser) -> bool {
+    copy(name.as_wstr(), new_name.from_ffi(), parser)
 }
 
 #[cxx::bridge]
@@ -632,7 +624,7 @@ mod function_ffi {
         include!("parse_tree.h");
         include!("parser.h");
         include!("wutil.h");
-        type Parser = crate::ffi::Parser;
+        type Parser = crate::parser::Parser;
         type wcstring_list_ffi_t = crate::ffi::wcstring_list_ffi_t;
     }
 
@@ -675,18 +667,17 @@ mod function_ffi {
         #[cxx_name = "function_get_props_autoload_raw"]
         fn function_get_props_autoload_ffi(
             name: &CxxWString,
-            parser: Pin<&mut Parser>,
+            parser: &Parser,
         ) -> *mut FunctionPropertiesRefFFI;
 
         #[cxx_name = "function_load"]
-        fn function_load_ffi(name: &CxxWString, parser: Pin<&mut Parser>) -> bool;
+        fn function_load_ffi(name: &CxxWString, parser: &Parser) -> bool;
 
         #[cxx_name = "function_set_desc"]
-        fn function_set_desc_ffi(name: &CxxWString, desc: &CxxWString, parser: Pin<&mut Parser>);
+        fn function_set_desc_ffi(name: &CxxWString, desc: &CxxWString, parser: &Parser);
 
         #[cxx_name = "function_exists"]
-        fn function_exists_ffi(cmd: &CxxWString, parser: Pin<&mut Parser>) -> bool;
-
+        fn function_exists_ffi(cmd: &CxxWString, parser: &Parser) -> bool;
         #[cxx_name = "function_exists_no_autoload"]
         fn function_exists_no_autoload_ffi(cmd: &CxxWString) -> bool;
 
@@ -694,11 +685,7 @@ mod function_ffi {
         fn function_get_names_ffi(get_hidden: bool, out: Pin<&mut wcstring_list_ffi_t>);
 
         #[cxx_name = "function_copy"]
-        fn function_copy_ffi(
-            name: &CxxWString,
-            new_name: &CxxWString,
-            parser: Pin<&mut Parser>,
-        ) -> bool;
+        fn function_copy_ffi(name: &CxxWString, new_name: &CxxWString, parser: &Parser) -> bool;
 
         #[cxx_name = "function_invalidate_path"]
         fn invalidate_path();

fish-rust/src/future_feature_flags.rs

@@ -129,7 +129,7 @@ pub use test as feature_test;
 
 /// Set a flag.
 #[cfg(any(test, feature = "fish-ffi-tests"))]
-fn set(flag: FeatureFlag, value: bool) {
+pub fn set(flag: FeatureFlag, value: bool) {
     LOCAL_FEATURES.with(|fc| fc.borrow().as_ref().unwrap_or(&FEATURES).set(flag, value));
 }
 

fish-rust/src/global_safety.rs

@@ -1,9 +1,10 @@
 use crate::flog::FLOG;
-use std::cell::{Ref, RefMut};
+use std::cell::{Ref, RefCell, RefMut};
+use std::rc::{Rc, Weak};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::MutexGuard;
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct RelaxedAtomicBool(AtomicBool);
 
 impl RelaxedAtomicBool {
@@ -27,6 +28,30 @@ impl Clone for RelaxedAtomicBool {
     }
 }
 
+pub struct SharedFromThisBase<T> {
+    weak: RefCell<Weak<T>>,
+}
+
+impl<T> SharedFromThisBase<T> {
+    pub fn new() -> SharedFromThisBase<T> {
+        SharedFromThisBase {
+            weak: RefCell::new(Weak::new()),
+        }
+    }
+
+    pub fn initialize(&self, r: &Rc<T>) {
+        *self.weak.borrow_mut() = Rc::downgrade(r);
+    }
+}
+
+pub trait SharedFromThis<T> {
+    fn get_base(&self) -> &SharedFromThisBase<T>;
+
+    fn shared_from_this(&self) -> Rc<T> {
+        self.get_base().weak.borrow().upgrade().unwrap()
+    }
+}
+
 pub struct DebugRef<'a, T>(Ref<'a, T>);
 
 impl<'a, T> DebugRef<'a, T> {

fish-rust/src/history/file.rs

@@ -0,0 +1,558 @@
+//! Implemention of history files.
+
+use std::{
+    io::Write,
+    ops::{Deref, DerefMut},
+    os::fd::RawFd,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use errno::errno;
+use libc::{
+    c_void, lseek, mmap, munmap, EINTR, MAP_ANONYMOUS, MAP_FAILED, MAP_PRIVATE, PROT_READ,
+    PROT_WRITE, SEEK_END, SEEK_SET,
+};
+
+use crate::{
+    common::{str2wcstring, subslice_position, wcs2string},
+    flog::FLOG,
+    path::{path_get_config_remoteness, DirRemoteness},
+};
+
+use super::{history_ffi::PersistenceMode, HistoryItem};
+
+/// History file types.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum HistoryFileType {
+    Fish2_0,
+    Fish1_x,
+}
+
+/// A type wrapping up the logic around mmap and munmap.
+struct MmapRegion {
+    ptr: *mut u8,
+    len: usize,
+}
+
+impl MmapRegion {
+    /// Creates a new mmap'ed region.
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must be the result of a successful `mmap()` call with length `len`.
+    unsafe fn new(ptr: *mut u8, len: usize) -> Self {
+        assert!(ptr.cast() != MAP_FAILED);
+        assert!(len > 0);
+        Self { ptr, len }
+    }
+
+    /// Map a region `[0, len)` from an `fd`.
+    /// Returns [`None`] on failure.
+    pub fn map_file(fd: RawFd, len: usize) -> Option<Self> {
+        if len == 0 {
+            return None;
+        }
+
+        let ptr = unsafe { mmap(std::ptr::null_mut(), len, PROT_READ, MAP_PRIVATE, fd, 0) };
+        if ptr == MAP_FAILED {
+            return None;
+        }
+
+        // SAFETY: mmap of `len` was successful and returned `ptr`
+        Some(unsafe { Self::new(ptr.cast(), len) })
+    }
+
+    /// Map anonymous memory of a given length.
+    /// Returns [`None`] on failure.
+    pub fn map_anon(len: usize) -> Option<Self> {
+        if len == 0 {
+            return None;
+        }
+
+        let ptr = unsafe {
+            mmap(
+                std::ptr::null_mut(),
+                len,
+                PROT_READ | PROT_WRITE,
+                MAP_PRIVATE | MAP_ANONYMOUS,
+                -1,
+                0,
+            )
+        };
+        if ptr == MAP_FAILED {
+            return None;
+        }
+
+        // SAFETY: mmap of `len` was successful and returned `ptr`
+        Some(unsafe { Self::new(ptr.cast(), len) })
+    }
+}
+
+// SAFETY: MmapRegion has exclusive mutable access to the region
+unsafe impl Send for MmapRegion {}
+// SAFETY: MmapRegion does not offer interior mutability
+unsafe impl Sync for MmapRegion {}
+
+impl Deref for MmapRegion {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        unsafe { std::slice::from_raw_parts(self.ptr, self.len) }
+    }
+}
+
+impl DerefMut for MmapRegion {
+    fn deref_mut(&mut self) -> &mut [u8] {
+        unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) }
+    }
+}
+
+impl Drop for MmapRegion {
+    fn drop(&mut self) {
+        unsafe { munmap(self.ptr.cast(), self.len) };
+    }
+}
+
+/// HistoryFileContents holds the read-only contents of a file.
+pub struct HistoryFileContents {
+    region: MmapRegion,
+}
+
+impl HistoryFileContents {
+    /// Construct a history file contents from a file descriptor. The file descriptor is not closed.
+    pub fn create(fd: RawFd) -> Option<Self> {
+        // Check that the file is seekable, and its size.
+        let len = unsafe { lseek(fd, 0, SEEK_END) };
+        let Ok(len) = usize::try_from(len) else {
+            return None;
+        };
+        let mmap_file_directly = should_mmap();
+        let mut region = if mmap_file_directly {
+            MmapRegion::map_file(fd, len)?
+        } else {
+            MmapRegion::map_anon(len)?
+        };
+
+        // If we mapped anonymous memory, we have to read from the file.
+        if !mmap_file_directly {
+            if unsafe { lseek(fd, 0, SEEK_SET) } != 0 {
+                return None;
+            }
+            if !read_from_fd(fd, region.as_mut()) {
+                return None;
+            }
+        }
+
+        region.try_into().ok()
+    }
+
+    /// Decode an item at a given offset.
+    pub fn decode_item(&self, offset: usize) -> Option<HistoryItem> {
+        let contents = &self.region[offset..];
+        decode_item_fish_2_0(contents)
+    }
+
+    /// Support for iterating item offsets.
+    /// The cursor should initially be 0.
+    /// If cutoff is given, skip items whose timestamp is newer than cutoff.
+    /// Returns the offset of the next item, or [`None`] on end.
+    pub fn offset_of_next_item(
+        &self,
+        cursor: &mut usize,
+        cutoff: Option<SystemTime>,
+    ) -> Option<usize> {
+        offset_of_next_item_fish_2_0(self.contents(), cursor, cutoff)
+    }
+
+    /// Returns a view of the file contents.
+    pub fn contents(&self) -> &[u8] {
+        &self.region
+    }
+}
+
+/// Try to infer the history file type based on inspecting the data.
+fn infer_file_type(contents: &[u8]) -> HistoryFileType {
+    assert!(!contents.is_empty(), "File should never be empty");
+    if contents[0] == b'#' {
+        HistoryFileType::Fish1_x
+    } else {
+        // assume new fish
+        HistoryFileType::Fish2_0
+    }
+}
+
+impl TryFrom<MmapRegion> for HistoryFileContents {
+    type Error = ();
+
+    fn try_from(region: MmapRegion) -> Result<Self, Self::Error> {
+        let type_ = infer_file_type(&region);
+        if type_ == HistoryFileType::Fish1_x {
+            FLOG!(error, "unsupported history file format 1.x");
+            return Err(());
+        }
+        Ok(Self { region })
+    }
+}
+
+/// Append a history item to a buffer, in preparation for outputting it to the history file.
+pub fn append_history_item_to_buffer(item: &HistoryItem, buffer: &mut Vec<u8>) {
+    assert!(item.should_write_to_disk(), "Item should not be persisted");
+
+    let mut cmd = wcs2string(item.str());
+    escape_yaml_fish_2_0(&mut cmd);
+    buffer.extend(b"- cmd: ");
+    buffer.extend(&cmd);
+    buffer.push(b'\n');
+    writeln!(buffer, "  when: {}", time_to_seconds(item.timestamp())).unwrap();
+
+    let paths = item.get_required_paths();
+    if !paths.is_empty() {
+        writeln!(buffer, "  paths:").unwrap();
+        for path in paths {
+            let mut path = wcs2string(path);
+            escape_yaml_fish_2_0(&mut path);
+            buffer.extend(b"    - ");
+            buffer.extend(&path);
+            buffer.push(b'\n');
+        }
+    }
+}
+
+/// Check if we should mmap the fd.
+/// Don't try mmap() on non-local filesystems.
+fn should_mmap() -> bool {
+    if super::NEVER_MMAP.load() {
+        return false;
+    }
+
+    // mmap only if we are known not-remote.
+    return path_get_config_remoteness() == DirRemoteness::local;
+}
+
+/// Read from `fd` to fill `dest`, zeroing any unused space.
+// Return true on success, false on failure.
+fn read_from_fd(fd: RawFd, dest: &mut [u8]) -> bool {
+    let mut remaining = dest.len();
+    let mut nread = 0;
+    while remaining > 0 {
+        let amt =
+            unsafe { libc::read(fd, (&mut dest[nread]) as *mut u8 as *mut c_void, remaining) };
+        if amt < 0 {
+            if errno().0 != EINTR {
+                return false;
+            }
+        } else if amt == 0 {
+            break;
+        } else {
+            remaining -= amt as usize;
+            nread += amt as usize;
+        }
+    }
+    dest[nread..].fill(0u8);
+    true
+}
+
+fn replace_all(s: &mut Vec<u8>, needle: &[u8], replacement: &[u8]) {
+    let mut offset = 0;
+    while let Some(relpos) = subslice_position(&s[offset..], needle) {
+        offset += relpos;
+        s.splice(offset..(offset + needle.len()), replacement.iter().copied());
+        offset += replacement.len();
+    }
+}
+
+/// Support for escaping and unescaping the nonstandard "yaml" format introduced in fish 2.0.
+fn escape_yaml_fish_2_0(s: &mut Vec<u8>) {
+    replace_all(s, b"\\", b"\\\\"); // replace one backslash with two
+    replace_all(s, b"\n", b"\\n"); // replace newline with backslash + literal n
+}
+
+/// This function is called frequently, so it ought to be fast.
+fn unescape_yaml_fish_2_0(s: &mut Vec<u8>) {
+    let mut cursor = 0;
+    while cursor < s.len() {
+        // Look for a backslash.
+        let Some(backslash) = s[cursor..].iter().position(|&c| c == b'\\') else {
+            // No more backslashes
+            break;
+        };
+
+        // Add back the start offset
+        let backslash = backslash + cursor;
+
+        // Backslash found. Maybe we'll do something about it.
+        let Some(escaped_char) = s.get(backslash + 1) else {
+            // Backslash was final character
+            break;
+        };
+
+        match escaped_char {
+            b'\\' => {
+                // Two backslashes in a row. Delete the second one.
+                s.remove(backslash + 1);
+            }
+            b'n' => {
+                // Backslash + n. Replace with a newline.
+                s.splice(backslash..(backslash + 2), [b'\n']);
+            }
+            _ => {
+                // Unknown backslash escape, keep as-is
+            }
+        };
+
+        // The character at index backslash has now been made whole; start at the next
+        // character.
+        cursor = backslash + 1;
+    }
+
+    debug_assert!(std::str::from_utf8(s).is_ok());
+}
+
+/// Read one line, stripping off any newline, returning the number of bytes consumed.
+fn read_line(data: &[u8]) -> (usize, &[u8]) {
+    // Locate the newline.
+    if let Some(newline) = data.iter().position(|&c| c == b'\n') {
+        // we found a newline
+        let line = &data[..newline];
+        // Return the amount to advance the cursor; skip over the newline.
+        (newline + 1, line)
+    } else {
+        // We ran off the end.
+        (data.len(), b"")
+    }
+}
+
+fn trim_start(s: &[u8]) -> &[u8] {
+    &s[s.iter().take_while(|c| c.is_ascii_whitespace()).count()..]
+}
+
+/// Trims leading spaces in the given string, returning how many there were.
+fn trim_leading_spaces(s: &[u8]) -> (usize, &[u8]) {
+    let count = s.iter().take_while(|c| **c == b' ').count();
+    (count, &s[count..])
+}
+
+fn extract_prefix_and_unescape_yaml(line: &[u8]) -> Option<(Vec<u8>, Vec<u8>)> {
+    let mut split = line.splitn(2, |c| *c == b':');
+    let key = split.next().unwrap();
+    let value = split.next()?;
+    assert!(split.next().is_none());
+
+    let mut key = key.to_owned();
+    // Skip a space after the : if necessary.
+    let mut value = trim_start(value).to_owned();
+
+    unescape_yaml_fish_2_0(&mut key);
+    unescape_yaml_fish_2_0(&mut value);
+    Some((key, value))
+}
+
+/// Decode an item via the fish 2.0 format.
+fn decode_item_fish_2_0(mut data: &[u8]) -> Option<HistoryItem> {
+    let (advance, line) = read_line(data);
+    let line = trim_start(line);
+    let Some((key, value)) = extract_prefix_and_unescape_yaml(line) else {
+        return None;
+    };
+
+    if key != b"- cmd" {
+        return None;
+    }
+
+    data = &data[advance..];
+    let cmd = str2wcstring(&value);
+
+    // Read the remaining lines.
+    let mut indent = None;
+    let mut when = UNIX_EPOCH;
+    let mut paths = Vec::new();
+    loop {
+        let (advance, line) = read_line(data);
+
+        let (this_indent, line) = trim_leading_spaces(line);
+        let indent = *indent.get_or_insert(this_indent);
+        if this_indent == 0 || indent != this_indent {
+            break;
+        }
+
+        let Some((key, value)) = extract_prefix_and_unescape_yaml(line) else {
+            break;
+        };
+
+        // We are definitely going to consume this line.
+        data = &data[advance..];
+
+        if key == b"when" {
+            // Parse an int from the timestamp. Should this fail, 0 is acceptable.
+            when = time_from_seconds(
+                std::str::from_utf8(&value)
+                    .ok()
+                    .and_then(|s| s.parse().ok())
+                    .unwrap_or(0),
+            );
+        } else if key == b"paths" {
+            // Read lines starting with " - " until we can't read any more.
+            loop {
+                let (advance, line) = read_line(data);
+                let (leading_spaces, line) = trim_leading_spaces(line);
+                if leading_spaces <= indent {
+                    break;
+                }
+
+                let Some(line) = line.strip_prefix(b"- ") else {
+                    break;
+                };
+
+                // We're going to consume this line.
+                data = &data[advance..];
+
+                let mut line = line.to_owned();
+                unescape_yaml_fish_2_0(&mut line);
+                paths.push(str2wcstring(&line));
+            }
+        }
+    }
+
+    let mut result = HistoryItem::new(cmd, when, 0, PersistenceMode::Disk);
+    result.set_required_paths(paths);
+    Some(result)
+}
+
+fn time_from_seconds(offset: i64) -> SystemTime {
+    if let Ok(n) = u64::try_from(offset) {
+        UNIX_EPOCH + Duration::from_secs(n)
+    } else {
+        UNIX_EPOCH - Duration::from_secs(offset.unsigned_abs())
+    }
+}
+
+pub fn time_to_seconds(ts: SystemTime) -> i64 {
+    match ts.duration_since(UNIX_EPOCH) {
+        Ok(d) => {
+            // after epoch
+            i64::try_from(d.as_secs()).unwrap()
+        }
+        Err(e) => {
+            // before epoch
+            -i64::try_from(e.duration().as_secs()).unwrap()
+        }
+    }
+}
+
+/// Parse a timestamp line that looks like this: spaces, "when:", spaces, timestamp, newline
+/// We know the string contains a newline, so stop when we reach it.
+fn parse_timestamp(s: &[u8]) -> Option<SystemTime> {
+    let s = trim_start(s);
+
+    let Some(s) = s.strip_prefix(b"when:") else {
+        return None;
+    };
+
+    let s = trim_start(s);
+
+    std::str::from_utf8(s)
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .map(time_from_seconds)
+}
+
+fn complete_lines(s: &[u8]) -> impl Iterator<Item = &[u8]> {
+    let mut lines = s.split(|&c| c == b'\n');
+    // Remove either the last empty element (in case last line is newline-terminated) or the
+    // trailing non-newline-terminated line
+    lines.next_back();
+    lines
+}
+
+/// Support for iteratively locating the offsets of history items.
+/// Pass the file contents and a mutable reference to a `cursor`, initially 0.
+/// If `cutoff_timestamp` is given, skip items created at or after that timestamp.
+/// Returns [`None`] when done.
+fn offset_of_next_item_fish_2_0(
+    contents: &[u8],
+    cursor: &mut usize,
+    cutoff_timestamp: Option<SystemTime>,
+) -> Option<usize> {
+    let mut lines = complete_lines(&contents[*cursor..]).peekable();
+    while let Some(mut line) = lines.next() {
+        // Skip lines with a leading space, since these are in the interior of one of our items.
+        if line.starts_with(b" ") {
+            continue;
+        }
+
+        // Try to be a little YAML compatible. Skip lines with leading %, ---, or ...
+        if line.starts_with(b"%") || line.starts_with(b"---") || line.starts_with(b"...") {
+            continue;
+        }
+
+        // Hackish: fish 1.x rewriting a fish 2.0 history file can produce lines with lots of
+        // leading "- cmd: - cmd: - cmd:". Trim all but one leading "- cmd:".
+        while line.starts_with(b"- cmd: - cmd: ") {
+            // Skip over just one of the - cmd. In the end there will be just one left.
+            line = line.strip_prefix(b"- cmd: ").unwrap();
+        }
+
+        // Hackish: fish 1.x rewriting a fish 2.0 history file can produce commands like "when:
+        // 123456". Ignore those.
+        if line.starts_with(b"- cmd:    when:") {
+            continue;
+        }
+
+        // At this point, we know `line` is at the beginning of an item. But maybe we want to
+        // skip this item because of timestamps. A `None` cutoff means we don't care; if we do care,
+        // then try parsing out a timestamp.
+        if let Some(cutoff_timestamp) = cutoff_timestamp {
+            // Hackish fast way to skip items created after our timestamp. This is the mechanism by
+            // which we avoid "seeing" commands from other sessions that started after we started.
+            // We try hard to ensure that our items are sorted by their timestamps, so in theory we
+            // could just break, but I don't think that works well if (for example) the clock
+            // changes. So we'll read all subsequent items.
+            // Walk over lines that we think are interior. These lines are not null terminated, but
+            // are guaranteed to contain a newline.
+            let mut timestamp = None;
+            loop {
+                let Some(interior_line) = lines.next_if(|l| l.starts_with(b" ")) else {
+                    // If the first character is not a space, it's not an interior line, so we're done.
+                    break;
+                };
+
+                // Try parsing a timestamp from this line. If we succeed, the loop will break.
+                timestamp = parse_timestamp(interior_line);
+                if timestamp.is_some() {
+                    break;
+                }
+            }
+
+            // Skip this item if the timestamp is past our cutoff.
+            if let Some(timestamp) = timestamp {
+                if timestamp > cutoff_timestamp {
+                    continue;
+                }
+            }
+        }
+
+        // We made it through the gauntlet.
+
+        /// # Safety
+        ///
+        /// Both `from` and `to` must be derived from the same slice.
+        unsafe fn offset(from: &[u8], to: &[u8]) -> usize {
+            let from = from.as_ptr();
+            let to = to.as_ptr();
+            // SAFETY: from and to are derived from the same slice, slices can't be longer than
+            // isize::MAX
+            let offset = unsafe { to.offset_from(from) };
+            offset.try_into().unwrap()
+        }
+
+        // Advance the cursor past the last line of this entry
+        *cursor = match lines.next() {
+            Some(next_line) => unsafe { offset(contents, next_line) },
+            None => contents.len(),
+        };
+
+        return Some(unsafe { offset(contents, line) });
+    }
+
+    None
+}

fish-rust/src/input.rs

@@ -0,0 +1,4 @@
+use crate::wchar::{wstr, L};
+
+pub const FISH_BIND_MODE_VAR: &wstr = L!("fish_bind_mode");
+pub const DEFAULT_BIND_MODE: &wstr = L!("default");

fish-rust/src/input_common.rs

@@ -0,0 +1,21 @@
+use crate::env::{EnvStack, Environment};
+use crate::wchar::prelude::*;
+use crate::wchar_ffi::WCharToFFI;
+
+pub fn update_wait_on_escape_ms(vars: &EnvStack) {
+    let fish_escape_delay_ms = vars.get_unless_empty(L!("fish_escape_delay_ms"));
+    let is_empty = fish_escape_delay_ms.is_none();
+    let value = fish_escape_delay_ms
+        .map(|s| s.as_string().to_ffi())
+        .unwrap_or(L!("").to_ffi());
+    crate::ffi::update_wait_on_escape_ms_ffi(is_empty, &value);
+}
+
+pub fn update_wait_on_sequence_key_ms(vars: &EnvStack) {
+    let fish_sequence_key_delay_ms = vars.get_unless_empty(L!("fish_sequence_key_delay_ms"));
+    let is_empty = fish_sequence_key_delay_ms.is_none();
+    let value = fish_sequence_key_delay_ms
+        .map(|s| s.as_string().to_ffi())
+        .unwrap_or(L!("").to_ffi());
+    crate::ffi::update_wait_on_sequence_key_ms_ffi(is_empty, &value);
+}

fish-rust/src/io.rs

@@ -8,18 +8,24 @@ use crate::fds::{
 };
 use crate::flog::{should_flog, FLOG, FLOGF};
 use crate::global_safety::RelaxedAtomicBool;
-use crate::job_group::JobGroup;
 use crate::path::path_apply_working_directory;
+use crate::proc::JobGroupRef;
 use crate::redirection::{RedirectionMode, RedirectionSpecList};
 use crate::signal::SigChecker;
 use crate::topic_monitor::topic_t;
 use crate::wchar::prelude::*;
+use crate::wchar_ffi::WCharFromFFI;
 use crate::wutil::{perror, perror_io, wdirname, wstat, wwrite_to_fd};
+use cxx::CxxWString;
 use errno::Errno;
-use libc::{EAGAIN, EEXIST, EINTR, ENOENT, ENOTDIR, EPIPE, EWOULDBLOCK, O_EXCL, STDERR_FILENO};
-use std::cell::UnsafeCell;
-use std::sync::{Arc, Condvar, Mutex, MutexGuard, RwLock, RwLockReadGuard};
-use std::{os::fd::RawFd, rc::Rc};
+use libc::{
+    EAGAIN, EEXIST, EINTR, ENOENT, ENOTDIR, EPIPE, EWOULDBLOCK, O_EXCL, STDERR_FILENO,
+    STDOUT_FILENO,
+};
+use std::cell::{RefCell, UnsafeCell};
+use std::os::fd::RawFd;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Condvar, Mutex, MutexGuard};
 
 /// separated_buffer_t represents a buffer of output from commands, prepared to be turned into a
 /// variable. For example, command substitutions output into one of these. Most commands just
@@ -167,7 +173,7 @@ impl SeparatedBuffer {
 }
 
 /// Describes what type of IO operation an io_data_t represents.
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum IoMode {
     file,
     pipe,
@@ -188,8 +194,29 @@ pub trait IoData {
     fn print(&self);
     // The address of the object, for comparison.
     fn as_ptr(&self) -> *const ();
+    fn as_bufferfill(&self) -> Option<&IoBufferfill> {
+        None
+    }
 }
 
+// todo!("this should be safe because of how it's used. Rationalize this better.")
+pub trait IoDataSync: IoData + Send + Sync {}
+unsafe impl Send for IoClose {}
+unsafe impl Send for IoFd {}
+unsafe impl Send for IoFile {}
+unsafe impl Send for IoPipe {}
+unsafe impl Send for IoBufferfill {}
+unsafe impl Sync for IoClose {}
+unsafe impl Sync for IoFd {}
+unsafe impl Sync for IoFile {}
+unsafe impl Sync for IoPipe {}
+unsafe impl Sync for IoBufferfill {}
+impl IoDataSync for IoClose {}
+impl IoDataSync for IoFd {}
+impl IoDataSync for IoFile {}
+impl IoDataSync for IoPipe {}
+impl IoDataSync for IoBufferfill {}
+
 pub struct IoClose {
     fd: RawFd,
 }
@@ -327,14 +354,19 @@ pub struct IoBufferfill {
     write_fd: AutoCloseFd,
 
     /// The receiving buffer.
-    buffer: Arc<RwLock<IoBuffer>>,
+    buffer: Arc<IoBuffer>,
 }
 impl IoBufferfill {
+    /// Create an io_bufferfill_t which, when written from, fills a buffer with the contents.
+    /// \returns nullptr on failure, e.g. too many open fds.
+    pub fn create() -> Option<Arc<IoBufferfill>> {
+        Self::create_opts(0, STDOUT_FILENO)
+    }
     /// Create an io_bufferfill_t which, when written from, fills a buffer with the contents.
     /// \returns nullptr on failure, e.g. too many open fds.
     ///
     /// \param target the fd which this will be dup2'd to - typically stdout.
-    pub fn create(buffer_limit: usize, target: RawFd) -> Option<Rc<IoBufferfill>> {
+    pub fn create_opts(buffer_limit: usize, target: RawFd) -> Option<Arc<IoBufferfill>> {
         assert!(target >= 0, "Invalid target fd");
 
         // Construct our pipes.
@@ -351,31 +383,33 @@ impl IoBufferfill {
             }
         }
         // Our fillthread gets the read end of the pipe; out_pipe gets the write end.
-        let buffer = Arc::new(RwLock::new(IoBuffer::new(buffer_limit)));
+        let buffer = Arc::new(IoBuffer::new(buffer_limit));
         begin_filling(&buffer, pipes.read);
         assert!(pipes.write.is_valid(), "fd is not valid");
-        Some(Rc::new(IoBufferfill {
+        Some(Arc::new(IoBufferfill {
             target,
             write_fd: pipes.write,
             buffer,
         }))
     }
 
-    pub fn buffer(&self) -> RwLockReadGuard<'_, IoBuffer> {
-        self.buffer.read().unwrap()
+    pub fn buffer_ref(&self) -> &Arc<IoBuffer> {
+        &self.buffer
+    }
+
+    pub fn buffer(&self) -> &IoBuffer {
+        &self.buffer
     }
 
     /// Reset the receiver (possibly closing the write end of the pipe), and complete the fillthread
     /// of the buffer. \return the buffer.
-    pub fn finish(filler: IoBufferfill) -> SeparatedBuffer {
+    pub fn finish(filler: Arc<IoBufferfill>) -> SeparatedBuffer {
         // The io filler is passed in. This typically holds the only instance of the write side of the
         // pipe used by the buffer's fillthread (except for that side held by other processes). Get the
         // buffer out of the bufferfill and clear the shared_ptr; this will typically widow the pipe.
         // Then allow the buffer to finish.
         filler
             .buffer
-            .write()
-            .unwrap()
             .complete_background_fillthread_and_take_buffer()
     }
 }
@@ -400,6 +434,9 @@ impl IoData for IoBufferfill {
     fn as_ptr(&self) -> *const () {
         (self as *const Self).cast()
     }
+    fn as_bufferfill(&self) -> Option<&IoBufferfill> {
+        Some(self)
+    }
 }
 
 /// An io_buffer_t is a buffer which can populate itself by reading from an fd.
@@ -413,19 +450,24 @@ pub struct IoBuffer {
 
     /// A promise, allowing synchronization with the background fill operation.
     /// The operation has a reference to this as well, and fulfills this promise when it exits.
-    fill_waiter: Option<Arc<(Mutex<bool>, Condvar)>>,
+    #[allow(clippy::type_complexity)]
+    fill_waiter: RefCell<Option<Arc<(Mutex<bool>, Condvar)>>>,
 
     /// The item id of our background fillthread fd monitor item.
-    item_id: FdMonitorItemId,
+    item_id: AtomicU64,
 }
 
+// safety: todo!("rationalize why fill_waiter is safe")
+unsafe impl Send for IoBuffer {}
+unsafe impl Sync for IoBuffer {}
+
 impl IoBuffer {
     pub fn new(limit: usize) -> Self {
         IoBuffer {
             buffer: Mutex::new(SeparatedBuffer::new(limit)),
             shutdown_fillthread: RelaxedAtomicBool::new(false),
-            fill_waiter: None,
-            item_id: FdMonitorItemId::from(0),
+            fill_waiter: RefCell::new(None),
+            item_id: AtomicU64::new(0),
         }
     }
 
@@ -473,27 +515,28 @@ impl IoBuffer {
     }
 
     /// End the background fillthread operation, and return the buffer, transferring ownership.
-    pub fn complete_background_fillthread_and_take_buffer(&mut self) -> SeparatedBuffer {
+    pub fn complete_background_fillthread_and_take_buffer(&self) -> SeparatedBuffer {
         // Mark that our fillthread is done, then wake it up.
         assert!(self.fillthread_running(), "Should have a fillthread");
         assert!(
-            self.item_id != FdMonitorItemId::from(0),
+            self.item_id.load(Ordering::SeqCst) != 0,
             "Should have a valid item ID"
         );
         self.shutdown_fillthread.store(true);
-        fd_monitor().poke_item(self.item_id);
+        fd_monitor().poke_item(FdMonitorItemId::from(self.item_id.load(Ordering::SeqCst)));
 
         // Wait for the fillthread to fulfill its promise, and then clear the future so we know we no
         // longer have one.
 
-        let (mutex, condvar) = &**(self.fill_waiter.as_ref().unwrap());
+        let mut promise = self.fill_waiter.borrow_mut();
+        let (mutex, condvar) = &**promise.as_ref().unwrap();
         {
             let mut done = mutex.lock().unwrap();
             while !*done {
                 done = condvar.wait(done).unwrap();
             }
         }
-        self.fill_waiter = None;
+        *promise = None;
 
         // Return our buffer, transferring ownership.
         let mut locked_buff = self.buffer.lock().unwrap();
@@ -505,16 +548,13 @@ impl IoBuffer {
 
     /// Helper to return whether the fillthread is running.
     pub fn fillthread_running(&self) -> bool {
-        return self.fill_waiter.is_some();
+        return self.fill_waiter.borrow().is_some();
     }
 }
 
 /// Begin the fill operation, reading from the given fd in the background.
-fn begin_filling(iobuffer: &Arc<RwLock<IoBuffer>>, fd: AutoCloseFd) {
-    assert!(
-        !iobuffer.read().unwrap().fillthread_running(),
-        "Already have a fillthread"
-    );
+fn begin_filling(iobuffer: &Arc<IoBuffer>, fd: AutoCloseFd) {
+    assert!(!iobuffer.fillthread_running(), "Already have a fillthread");
 
     // We want to fill buffer_ by reading from fd. fd is the read end of a pipe; the write end is
     // owned by another process, or something else writing in fish.
@@ -536,8 +576,7 @@ fn begin_filling(iobuffer: &Arc<RwLock<IoBuffer>>, fd: AutoCloseFd) {
     // complete_background_fillthread(). Note that TSan complains if the promise's dtor races with
     // the future's call to wait(), so we store the promise, not just its future (#7681).
     let promise = Arc::new((Mutex::new(false), Condvar::new()));
-    iobuffer.write().unwrap().fill_waiter = Some(promise.clone());
-
+    iobuffer.fill_waiter.replace(Some(promise.clone()));
     // Run our function to read until the receiver is closed.
     // It's OK to capture 'buffer' because 'this' waits for the promise in its dtor.
     let item_callback: Option<NativeCallback> = {
@@ -551,16 +590,14 @@ fn begin_filling(iobuffer: &Arc<RwLock<IoBuffer>>, fd: AutoCloseFd) {
                 let mut done = false;
                 if reason == ItemWakeReason::Readable {
                     // select() reported us as readable; read a bit.
-                    let iobuf = iobuffer.write().unwrap();
-                    let mut buf = iobuf.buffer.lock().unwrap();
+                    let mut buf = iobuffer.buffer.lock().unwrap();
                     let ret = IoBuffer::read_once(fd.fd(), &mut buf);
                     done =
                         ret == 0 || (ret < 0 && ![EAGAIN, EWOULDBLOCK].contains(&errno::errno().0));
-                } else if iobuffer.read().unwrap().shutdown_fillthread.load() {
+                } else if iobuffer.shutdown_fillthread.load() {
                     // Here our caller asked us to shut down; read while we keep getting data.
                     // This will stop when the fd is closed or if we get EAGAIN.
-                    let iobuf = iobuffer.write().unwrap();
-                    let mut buf = iobuf.buffer.lock().unwrap();
+                    let mut buf = iobuffer.buffer.lock().unwrap();
                     loop {
                         let ret = IoBuffer::read_once(fd.fd(), &mut buf);
                         if ret <= 0 {
@@ -572,34 +609,45 @@ fn begin_filling(iobuffer: &Arc<RwLock<IoBuffer>>, fd: AutoCloseFd) {
                 if done {
                     fd.close();
                     let (mutex, condvar) = &*promise;
-                    let mut done = mutex.lock().unwrap();
-                    *done = true;
+                    {
+                        let mut done = mutex.lock().unwrap();
+                        *done = true;
+                    }
                     condvar.notify_one();
                 }
             },
         ))
     };
 
-    iobuffer.write().unwrap().item_id =
-        fd_monitor().add(FdMonitorItem::new(fd, None, item_callback));
+    let item_id = fd_monitor().add(FdMonitorItem::new(fd, None, item_callback));
+    iobuffer.item_id.store(u64::from(item_id), Ordering::SeqCst);
 }
 
-pub type IoDataRef = Rc<dyn IoData>;
+pub type IoDataRef = Arc<dyn IoDataSync>;
 
-#[derive(Default)]
+#[derive(Clone, Default)]
 pub struct IoChain(pub Vec<IoDataRef>);
 
+unsafe impl cxx::ExternType for IoChain {
+    type Id = cxx::type_id!("IoChain");
+    type Kind = cxx::kind::Opaque;
+}
+
 impl IoChain {
     pub fn new() -> Self {
         Default::default()
     }
-    pub fn remove(&mut self, element: &IoDataRef) {
-        let element = Rc::as_ptr(element) as *const ();
+    pub fn remove(&mut self, element: &dyn IoDataSync) {
+        let element = element as *const _;
+        let element = element as *const ();
         self.0.retain(|e| {
-            let e = Rc::as_ptr(e) as *const ();
+            let e = Arc::as_ptr(e) as *const ();
             !std::ptr::eq(e, element)
         });
     }
+    pub fn clear(&mut self) {
+        self.0.clear()
+    }
     pub fn push(&mut self, element: IoDataRef) {
         self.0.push(element);
     }
@@ -623,12 +671,12 @@ impl IoChain {
             match spec.mode {
                 RedirectionMode::fd => {
                     if spec.is_close() {
-                        self.push(Rc::new(IoClose::new(spec.fd)));
+                        self.push(Arc::new(IoClose::new(spec.fd)));
                     } else {
                         let target_fd = spec
                             .get_target_as_fd()
                             .expect("fd redirection should have been validated already");
-                        self.push(Rc::new(IoFd::new(spec.fd, target_fd)));
+                        self.push(Arc::new(IoFd::new(spec.fd, target_fd)));
                     }
                 }
                 _ => {
@@ -677,11 +725,11 @@ impl IoChain {
                         // If opening a file fails, insert a closed FD instead of the file redirection
                         // and return false. This lets execution potentially recover and at least gives
                         // the shell a chance to gracefully regain control of the shell (see #7038).
-                        self.push(Rc::new(IoClose::new(spec.fd)));
+                        self.push(Arc::new(IoClose::new(spec.fd)));
                         have_error = true;
                         continue;
                     }
-                    self.push(Rc::new(IoFile::new(spec.fd, file)));
+                    self.push(Arc::new(IoFile::new(spec.fd, file)));
                 }
             }
         }
@@ -916,7 +964,7 @@ impl BufferedOutputStream {
     }
 }
 
-pub struct NativeIoStreams<'a> {
+pub struct IoStreams<'a> {
     // Streams for out and err.
     pub out: &'a mut OutputStream,
     pub err: &'a mut OutputStream,
@@ -941,17 +989,22 @@ pub struct NativeIoStreams<'a> {
     pub err_is_redirected: bool,
 
     // Actual IO redirections. This is only used by the source builtin. Unowned.
-    io_chain: *const IoChain,
+    pub io_chain: *mut IoChain,
 
     // The job group of the job, if any. This enables builtins which run more code like eval() to
     // share pgid.
     // FIXME: this is awkwardly placed.
-    job_group: Option<Rc<JobGroup>>,
+    pub job_group: Option<JobGroupRef>,
 }
 
-impl<'a> NativeIoStreams<'a> {
+unsafe impl cxx::ExternType for IoStreams<'_> {
+    type Id = cxx::type_id!("IoStreams");
+    type Kind = cxx::kind::Opaque;
+}
+
+impl<'a> IoStreams<'a> {
     pub fn new(out: &'a mut OutputStream, err: &'a mut OutputStream) -> Self {
-        NativeIoStreams {
+        IoStreams {
             out,
             err,
             stdin_fd: -1,
@@ -960,10 +1013,13 @@ impl<'a> NativeIoStreams<'a> {
             err_is_piped: false,
             out_is_redirected: false,
             err_is_redirected: false,
-            io_chain: std::ptr::null(),
+            io_chain: std::ptr::null_mut(),
             job_group: None,
         }
     }
+    pub fn out_is_terminal(&self) -> bool {
+        !self.out_is_redirected && unsafe { libc::isatty(STDOUT_FILENO) == 1 }
+    }
 }
 
 /// File redirection error message.
@@ -985,3 +1041,69 @@ fn fd_monitor() -> &'static mut FdMonitor {
     let ptr: *mut FdMonitor = unsafe { (*FDM).get() };
     unsafe { &mut *ptr }
 }
+
+#[cxx::bridge]
+#[allow(clippy::needless_lifetimes)]
+mod io_ffi {
+    extern "Rust" {
+        type IoChain;
+        type IoStreams<'a>;
+        type OutputStreamFfi<'a>;
+
+        fn new_io_chain() -> Box<IoChain>;
+
+        #[cxx_name = "out"]
+        unsafe fn out_ffi<'a>(self: &'a mut IoStreams<'a>) -> Box<OutputStreamFfi<'a>>;
+        #[cxx_name = "err"]
+        unsafe fn err_ffi<'a>(self: &'a mut IoStreams<'a>) -> Box<OutputStreamFfi<'a>>;
+        #[cxx_name = "out_is_redirected"]
+        unsafe fn out_is_redirected_ffi<'a>(self: &IoStreams<'a>) -> bool;
+        #[cxx_name = "stdin_is_directly_redirected"]
+        unsafe fn stdin_is_directly_redirected_ffi<'a>(self: &IoStreams<'a>) -> bool;
+        #[cxx_name = "stdin_fd"]
+        unsafe fn stdin_fd_ffi<'a>(self: &IoStreams<'a>) -> i32;
+
+        #[cxx_name = "append"]
+        unsafe fn append_ffi<'a>(self: &mut OutputStreamFfi<'a>, s: &CxxWString) -> bool;
+        #[cxx_name = "push"]
+        unsafe fn push_ffi<'a>(self: &mut OutputStreamFfi<'a>, s: u32) -> bool;
+    }
+}
+
+impl<'a> IoStreams<'a> {
+    fn out_ffi(&'a mut self) -> Box<OutputStreamFfi<'a>> {
+        Box::new(OutputStreamFfi(self.out))
+    }
+    fn err_ffi(&'a mut self) -> Box<OutputStreamFfi<'a>> {
+        Box::new(OutputStreamFfi(self.err))
+    }
+    unsafe fn out_is_redirected_ffi(&self) -> bool {
+        self.out_is_redirected
+    }
+    unsafe fn stdin_is_directly_redirected_ffi(&self) -> bool {
+        self.stdin_is_directly_redirected
+    }
+    unsafe fn stdin_fd_ffi(&self) -> i32 {
+        self.stdin_fd
+    }
+}
+
+pub struct OutputStreamFfi<'a>(pub &'a mut OutputStream);
+
+unsafe impl cxx::ExternType for OutputStreamFfi<'_> {
+    type Id = cxx::type_id!("OutputStreamFfi");
+    type Kind = cxx::kind::Opaque;
+}
+
+impl<'a> OutputStreamFfi<'a> {
+    fn append_ffi(&mut self, s: &CxxWString) -> bool {
+        self.0.append(s.from_ffi())
+    }
+    fn push_ffi(&mut self, s: u32) -> bool {
+        self.0.append_char(char::from_u32(s).unwrap())
+    }
+}
+
+fn new_io_chain() -> Box<IoChain> {
+    Box::new(IoChain::new())
+}

fish-rust/src/job_group.rs

@@ -1,13 +1,14 @@
 use self::ffi::pgid_t;
-use crate::common::{assert_send, assert_sync};
 use crate::global_safety::RelaxedAtomicBool;
+use crate::proc::JobGroupRef;
 use crate::signal::Signal;
 use crate::wchar::prelude::*;
-use crate::wchar_ffi::{WCharFromFFI, WCharToFFI};
+use crate::wchar_ffi::WCharToFFI;
 use cxx::{CxxWString, UniquePtr};
+use std::cell::RefCell;
 use std::num::NonZeroU32;
 use std::sync::atomic::{AtomicI32, Ordering};
-use std::sync::Mutex;
+use std::sync::{Arc, Mutex};
 
 #[cxx::bridge]
 mod ffi {
@@ -42,34 +43,15 @@ mod ffi {
         // a SharedPtr/UniquePtr/Box and won't let us pass/return them by value/reference, either.
         unsafe fn get_modes_ffi(&self, size: usize) -> *const u8; /* actually `* const libc::termios` */
         unsafe fn set_modes_ffi(&mut self, modes: *const u8, size: usize); /* actually `* const libc::termios` */
-
-        // The C++ code uses `shared_ptr<JobGroup>` but cxx bridge doesn't support returning a
-        // `SharedPtr<OpaqueRustType>` nor does it implement `Arc<T>` so we return a box and then
-        // convert `rust::box<T>` to `std::shared_ptr<T>` with `box_to_shared_ptr()` (from ffi.h).
-        fn create_job_group_ffi(command: &CxxWString, wants_job_id: bool) -> Box<JobGroup>;
-        fn create_job_group_with_job_control_ffi(
-            command: &CxxWString,
-            wants_term: bool,
-        ) -> Box<JobGroup>;
     }
 }
 
-fn create_job_group_ffi(command: &CxxWString, wants_job_id: bool) -> Box<JobGroup> {
-    let job_group = JobGroup::create(command.from_ffi(), wants_job_id);
-    Box::new(job_group)
-}
-
-fn create_job_group_with_job_control_ffi(command: &CxxWString, wants_term: bool) -> Box<JobGroup> {
-    let job_group = JobGroup::create_with_job_control(command.from_ffi(), wants_term);
-    Box::new(job_group)
-}
-
 /// A job id, corresponding to what is printed by `jobs`. 1 is the first valid job id.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
 #[repr(transparent)]
 pub struct JobId(NonZeroU32);
 
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub struct MaybeJobId(pub Option<JobId>);
 
 impl std::ops::Deref for MaybeJobId {
@@ -80,30 +62,27 @@ impl std::ops::Deref for MaybeJobId {
     }
 }
 
+impl MaybeJobId {
+    pub fn as_num(&self) -> i64 {
+        self.0.map(|j| i64::from(u32::from(j.0))).unwrap_or(-1)
+    }
+}
+
 impl std::fmt::Display for MaybeJobId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.0
-            .map(|j| i64::from(u32::from(j.0)))
-            .unwrap_or(-1)
-            .fmt(f)
+        self.as_num().fmt(f)
     }
 }
 
 impl ToWString for MaybeJobId {
     fn to_wstring(&self) -> WString {
-        self.0
-            .map(|j| i64::from(u32::from(j.0)))
-            .unwrap_or(-1)
-            .to_wstring()
+        self.as_num().to_wstring()
     }
 }
 
 impl<'a> printf_compat::args::ToArg<'a> for MaybeJobId {
     fn to_arg(self) -> printf_compat::args::Arg<'a> {
-        self.0
-            .map(|j| i64::from(u32::from(j.0)))
-            .unwrap_or(-1)
-            .to_arg()
+        self.as_num().to_arg()
     }
 }
 
@@ -120,7 +99,7 @@ impl<'a> printf_compat::args::ToArg<'a> for MaybeJobId {
 pub struct JobGroup {
     /// If set, the saved terminal modes of this job. This needs to be saved so that we can restore
     /// the terminal to the same state when resuming a stopped job.
-    pub tmodes: Option<libc::termios>,
+    pub tmodes: RefCell<Option<libc::termios>>,
     /// Whether job control is enabled in this `JobGroup` or not.
     ///
     /// If this is set, then the first process in the root job must be external, as it will become
@@ -133,7 +112,7 @@ pub struct JobGroup {
     pub is_foreground: RelaxedAtomicBool,
     /// The pgid leading our group. This is only ever set if [`job_control`](Self::JobControl) is
     /// true. We ensure the value (when set) is always non-negative.
-    pgid: Option<libc::pid_t>,
+    pgid: RefCell<Option<libc::pid_t>>,
     /// The original command which produced this job tree.
     pub command: WString,
     /// Our job id, if any. `None` here should evaluate to `-1` for ffi purposes.
@@ -144,8 +123,9 @@ pub struct JobGroup {
     signal: AtomicI32,
 }
 
-const _: () = assert_send::<JobGroup>();
-const _: () = assert_sync::<JobGroup>();
+// safety: all fields without interior mutabillity are only written to once
+unsafe impl Send for JobGroup {}
+unsafe impl Sync for JobGroup {}
 
 impl JobGroup {
     /// Whether this job wants job control.
@@ -223,26 +203,26 @@ impl JobGroup {
     ///
     /// As such, this method takes `&mut self` rather than `&self` to enforce that this operation is
     /// only available during initial construction/initialization.
-    pub fn set_pgid(&mut self, pgid: libc::pid_t) {
+    pub fn set_pgid(&self, pgid: libc::pid_t) {
         assert!(
             self.wants_job_control(),
             "Should not set a pgid for a group that doesn't want job control!"
         );
         assert!(pgid >= 0, "Invalid pgid!");
-        assert!(self.pgid.is_none(), "JobGroup::pgid already set!");
+        assert!(self.pgid.borrow().is_none(), "JobGroup::pgid already set!");
 
-        self.pgid = Some(pgid);
+        self.pgid.replace(Some(pgid));
     }
 
     /// Returns the value of [`JobGroup::pgid`]. This is never fish's own pgid!
     pub fn get_pgid(&self) -> Option<libc::pid_t> {
-        self.pgid
+        *self.pgid.borrow()
     }
 
     /// Returns the value of [`JobGroup::pgid`] in a `UniquePtr<T>` to take the place of an
     /// `Option<T>` for ffi purposes. A null `UniquePtr` is equivalent to `None`.
     pub fn get_pgid_ffi(&self) -> cxx::UniquePtr<pgid_t> {
-        match self.pgid {
+        match *self.pgid.borrow() {
             Some(value) => UniquePtr::new(pgid_t { value }),
             None => UniquePtr::null(),
         }
@@ -257,6 +237,7 @@ impl JobGroup {
         );
 
         self.tmodes
+            .borrow()
             .as_ref()
             // Really cool that type inference works twice in a row here. The first `_` is deduced
             // from the left and the second `_` is deduced from the right (the return type).
@@ -279,9 +260,9 @@ impl JobGroup {
 
         let modes = modes as *const libc::termios;
         if modes.is_null() {
-            self.tmodes = None;
+            self.tmodes.replace(None);
         } else {
-            self.tmodes = Some(*modes);
+            self.tmodes.replace(Some(*modes));
         }
     }
 }
@@ -294,7 +275,7 @@ impl JobGroup {
 static CONSUMED_JOB_IDS: Mutex<Vec<JobId>> = Mutex::new(Vec::new());
 
 impl JobId {
-    const NONE: MaybeJobId = MaybeJobId(None);
+    pub const NONE: MaybeJobId = MaybeJobId(None);
 
     pub fn new(value: NonZeroU32) -> Self {
         JobId(value)
@@ -346,18 +327,18 @@ impl JobGroup {
             job_control,
             wants_term,
             command,
-            tmodes: None,
+            tmodes: RefCell::default(),
             signal: 0.into(),
             is_foreground: RelaxedAtomicBool::new(false),
-            pgid: None,
+            pgid: RefCell::default(),
         }
     }
 
     /// Return a new `JobGroup` with the provided `command`. The `JobGroup` is only assigned a
     /// `JobId` if `wants_job_id` is true and is created with job control disabled and
     /// [`JobGroup::wants_term`] set to false.
-    pub fn create(command: WString, wants_job_id: bool) -> JobGroup {
-        JobGroup::new(
+    pub fn create(command: WString, wants_job_id: bool) -> JobGroupRef {
+        Arc::new(JobGroup::new(
             command,
             if wants_job_id {
                 MaybeJobId(Some(JobId::acquire()))
@@ -366,19 +347,19 @@ impl JobGroup {
             },
             false, /* job_control */
             false, /* wants_term */
-        )
+        ))
     }
 
     /// Return a new `JobGroup` with the provided `command` with job control enabled. A [`JobId`] is
     /// automatically acquired and assigned. If `wants_term` is true then [`JobGroup::wants_term`]
     /// is also set to `true` accordingly.
-    pub fn create_with_job_control(command: WString, wants_term: bool) -> JobGroup {
-        JobGroup::new(
+    pub fn create_with_job_control(command: WString, wants_term: bool) -> JobGroupRef {
+        Arc::new(JobGroup::new(
             command,
             MaybeJobId(Some(JobId::acquire())),
             true, /* job_control */
             wants_term,
-        )
+        ))
     }
 }
 

fish-rust/src/lib.rs

@@ -2,13 +2,22 @@
 #![allow(dead_code)]
 #![allow(non_upper_case_globals)]
 #![allow(clippy::bool_assert_comparison)]
+#![allow(clippy::box_default)]
+#![allow(clippy::collapsible_if)]
+#![allow(clippy::comparison_chain)]
 #![allow(clippy::derivable_impls)]
 #![allow(clippy::field_reassign_with_default)]
+#![allow(clippy::if_same_then_else)]
 #![allow(clippy::manual_is_ascii_check)]
+#![allow(clippy::mut_from_ref)]
 #![allow(clippy::needless_return)]
 #![allow(clippy::option_map_unit_fn)]
 #![allow(clippy::ptr_arg)]
+#![allow(clippy::redundant_slicing)]
+#![allow(clippy::too_many_arguments)]
 #![allow(clippy::uninlined_format_args)]
+#![allow(clippy::unnecessary_to_owned)]
+#![allow(clippy::unnecessary_unwrap)]
 
 pub const BUILD_VERSION: &str = match option_env!("FISH_BUILD_VERSION") {
     Some(v) => v,
@@ -20,6 +29,7 @@ mod common;
 
 mod abbrs;
 mod ast;
+mod autoload;
 mod builtins;
 mod color;
 mod compat;
@@ -27,7 +37,9 @@ mod complete;
 mod curses;
 mod env;
 mod env_dispatch;
+mod env_universal_common;
 mod event;
+mod exec;
 mod expand;
 mod fallback;
 mod fd_monitor;
@@ -50,6 +62,8 @@ mod future_feature_flags;
 mod global_safety;
 mod highlight;
 mod history;
+mod input;
+mod input_common;
 mod io;
 mod job_group;
 mod kill;
@@ -59,11 +73,15 @@ mod null_terminated_array;
 mod operation_context;
 mod output;
 mod parse_constants;
+mod parse_execution;
 mod parse_tree;
 mod parse_util;
+mod parser;
 mod parser_keywords;
 mod path;
+mod pointer;
 mod print_help;
+mod proc;
 mod re;
 mod reader;
 mod redirection;
@@ -87,5 +105,6 @@ mod widecharwidth;
 mod wildcard;
 mod wutil;
 
-// Don't use `#[cfg(test)]` here to make sure ffi tests are built and tested
+#[cfg(any(test, feature = "fish-ffi-tests"))]
+#[allow(unused_imports)] // Easy way to suppress warnings while we have two testing modes.
 mod tests;

fish-rust/src/null_terminated_array.rs

@@ -20,6 +20,31 @@ impl NulTerminatedString for CStr {
     }
 }
 
+pub trait AsNullTerminatedArray {
+    type CharType;
+    fn get(&self) -> *mut *const Self::CharType;
+    fn iter(&self) -> NullTerminatedArrayIterator<Self::CharType> {
+        NullTerminatedArrayIterator { ptr: self.get() }
+    }
+}
+
+// TODO This should expose strings as CStr.
+pub struct NullTerminatedArrayIterator<CharType> {
+    ptr: *mut *const CharType,
+}
+impl<CharType> Iterator for NullTerminatedArrayIterator<CharType> {
+    type Item = *const CharType;
+    fn next(&mut self) -> Option<*const CharType> {
+        let result = unsafe { *self.ptr };
+        if result.is_null() {
+            None
+        } else {
+            self.ptr = unsafe { self.ptr.add(1) };
+            Some(result)
+        }
+    }
+}
+
 /// This supports the null-terminated array of NUL-terminated strings consumed by exec.
 /// Given a list of strings, construct a vector of pointers to those strings contents.
 /// This is used for building null-terminated arrays of null-terminated strings.
@@ -28,7 +53,8 @@ pub struct NullTerminatedArray<'p, T: NulTerminatedString + ?Sized> {
     _phantom: PhantomData<&'p T>,
 }
 
-impl<'p, Str: NulTerminatedString + ?Sized> NullTerminatedArray<'p, Str> {
+impl<'p, Str: NulTerminatedString + ?Sized> AsNullTerminatedArray for NullTerminatedArray<'p, Str> {
+    type CharType = Str::CharType;
     /// Return the list of pointers, appropriate for envp or argv.
     /// Note this returns a mutable array of const strings. The caller may rearrange the strings but
     /// not modify their contents.
@@ -42,7 +68,8 @@ impl<'p, Str: NulTerminatedString + ?Sized> NullTerminatedArray<'p, Str> {
         );
         self.pointers.as_ptr() as *mut *const Str::CharType
     }
-
+}
+impl<'p, Str: NulTerminatedString + ?Sized> NullTerminatedArray<'p, Str> {
     /// Construct from a list of "strings".
     /// This holds pointers into the strings.
     pub fn new<S: AsRef<Str>>(strs: &'p [S]) -> Self {
@@ -76,12 +103,18 @@ pub struct OwningNullTerminatedArray {
 const _: () = assert_send::<OwningNullTerminatedArray>();
 const _: () = assert_sync::<OwningNullTerminatedArray>();
 
-impl OwningNullTerminatedArray {
+impl AsNullTerminatedArray for OwningNullTerminatedArray {
+    type CharType = c_char;
     /// Cover over null_terminated_array.get().
     fn get(&self) -> *mut *const c_char {
         self.null_terminated_array.get()
     }
+}
 
+impl OwningNullTerminatedArray {
+    pub fn get_mut(&self) -> *mut *mut c_char {
+        self.get().cast()
+    }
     /// Construct, taking ownership of a list of strings.
     pub fn new(strs: Vec<CString>) -> Self {
         let strings = strs.into_boxed_slice();
@@ -174,6 +207,15 @@ fn test_null_terminated_array() {
         assert_eq!(*ptr.offset(2), ptr::null());
     }
 }
+#[test]
+fn test_null_terminated_array_iter() {
+    let owned_strs = &[CString::new("foo").unwrap(), CString::new("bar").unwrap()];
+    let strs: Vec<_> = owned_strs.iter().map(|s| s.as_c_str()).collect();
+    let arr = NullTerminatedArray::new(&strs);
+    let v1: Vec<_> = arr.iter().collect();
+    let v2: Vec<_> = owned_strs.iter().map(|s| s.as_ptr()).collect();
+    assert_eq!(v1, v2);
+}
 
 #[test]
 fn test_owning_null_terminated_array() {

fish-rust/src/operation_context.rs

@@ -1,7 +1,204 @@
-pub struct OperationContext {}
+use crate::common::CancelChecker;
+use crate::env::{EnvDyn, EnvDynFFI};
+use crate::env::{EnvStack, EnvStackRef, Environment};
+use crate::parser::{Parser, ParserRef};
+use crate::proc::JobGroupRef;
+use once_cell::sync::Lazy;
+use std::sync::Arc;
 
-impl OperationContext {
-    pub fn empty() -> OperationContext {
-        todo!()
+/// A common helper which always returns false.
+pub fn no_cancel() -> bool {
+    false
+}
+
+// Default limits for expansion.
+/// The default maximum number of items from expansion.
+pub const EXPANSION_LIMIT_DEFAULT: usize = 512 * 1024;
+/// A smaller limit for background operations like syntax highlighting.
+pub const EXPANSION_LIMIT_BACKGROUND: usize = 512;
+
+enum Vars<'a> {
+    // The parser, if this is a foreground operation. If this is a background operation, this may be
+    // nullptr.
+    Parser(ParserRef),
+    // A set of variables.
+    Vars(&'a dyn Environment),
+
+    TestOnly(ParserRef, &'a dyn Environment),
+}
+
+/// A operation_context_t is a simple property bag which wraps up data needed for highlighting,
+/// expansion, completion, and more.
+pub struct OperationContext<'a> {
+    vars: Vars<'a>,
+
+    // The limit in the number of expansions which should be produced.
+    pub expansion_limit: usize,
+
+    /// The job group of the parental job.
+    /// This is used only when expanding command substitutions. If this is set, any jobs created
+    /// by the command substitutions should use this tree.
+    pub job_group: Option<JobGroupRef>,
+
+    // A function which may be used to poll for cancellation.
+    pub cancel_checker: CancelChecker,
+}
+
+static nullenv: Lazy<EnvStackRef> = Lazy::new(|| Arc::pin(EnvStack::new()));
+
+impl<'a> OperationContext<'a> {
+    pub fn vars(&self) -> &dyn Environment {
+        match &self.vars {
+            Vars::Parser(parser) => &*parser.variables,
+            Vars::Vars(vars) => *vars,
+            Vars::TestOnly(_, vars) => *vars,
+        }
+    }
+
+    // \return an "empty" context which contains no variables, no parser, and never cancels.
+    pub fn empty() -> OperationContext<'static> {
+        OperationContext::background(&**nullenv, EXPANSION_LIMIT_DEFAULT)
+    }
+
+    // \return an operation context that contains only global variables, no parser, and never
+    // cancels.
+    pub fn globals() -> OperationContext<'static> {
+        OperationContext::background(&**EnvStack::globals(), EXPANSION_LIMIT_DEFAULT)
+    }
+
+    /// Construct from a full set of properties.
+    pub fn foreground(
+        parser: ParserRef,
+        cancel_checker: CancelChecker,
+        expansion_limit: usize,
+    ) -> OperationContext<'a> {
+        OperationContext {
+            vars: Vars::Parser(parser),
+            expansion_limit,
+            job_group: None,
+            cancel_checker,
+        }
+    }
+
+    pub fn test_only_foreground(
+        parser: ParserRef,
+        vars: &'a dyn Environment,
+        cancel_checker: CancelChecker,
+    ) -> OperationContext<'a> {
+        OperationContext {
+            vars: Vars::TestOnly(parser, vars),
+            expansion_limit: EXPANSION_LIMIT_DEFAULT,
+            job_group: None,
+            cancel_checker,
+        }
+    }
+
+    /// Construct from vars alone.
+    pub fn background(vars: &'a dyn Environment, expansion_limit: usize) -> OperationContext<'a> {
+        OperationContext {
+            vars: Vars::Vars(vars),
+            expansion_limit,
+            job_group: None,
+            cancel_checker: Box::new(no_cancel),
+        }
+    }
+
+    pub fn background_with_cancel_checker(
+        vars: &'a dyn Environment,
+        cancel_checker: CancelChecker,
+        expansion_limit: usize,
+    ) -> OperationContext<'a> {
+        OperationContext {
+            vars: Vars::Vars(vars),
+            expansion_limit,
+            job_group: None,
+            cancel_checker,
+        }
+    }
+
+    pub fn has_parser(&self) -> bool {
+        matches!(self.vars, Vars::Parser(_) | Vars::TestOnly(_, _))
+    }
+    pub fn maybe_parser(&self) -> Option<&Parser> {
+        match &self.vars {
+            Vars::Parser(parser) => Some(parser),
+            Vars::Vars(_) => None,
+            Vars::TestOnly(parser, _) => Some(parser),
+        }
+    }
+    pub fn parser(&self) -> &Parser {
+        match &self.vars {
+            Vars::Parser(parser) => parser,
+            Vars::Vars(_) => panic!(),
+            Vars::TestOnly(parser, _) => parser,
+        }
+    }
+    // Invoke the cancel checker. \return if we should cancel.
+    pub fn check_cancel(&self) -> bool {
+        (self.cancel_checker)()
     }
 }
+
+/// \return an operation context for a background operation..
+/// Crucially the operation context itself does not contain a parser.
+/// It is the caller's responsibility to ensure the environment lives as long as the result.
+pub fn get_bg_context(env: &EnvDyn, generation_count: u32) -> OperationContext {
+    let cancel_checker = move || {
+        // Cancel if the generation count changed.
+        generation_count != crate::ffi::read_generation_count()
+    };
+    OperationContext::background_with_cancel_checker(
+        env,
+        Box::new(cancel_checker),
+        EXPANSION_LIMIT_BACKGROUND,
+    )
+}
+
+#[cxx::bridge]
+#[allow(clippy::needless_lifetimes)]
+mod operation_context_ffi {
+    extern "C++" {
+        include!("env_fwd.h");
+        include!("parser.h");
+        #[cxx_name = "EnvStackRef"]
+        type EnvStackRefFFI = crate::env::EnvStackRefFFI;
+        #[cxx_name = "EnvDyn"]
+        type EnvDynFFI = crate::env::EnvDynFFI;
+        type Parser = crate::parser::Parser;
+    }
+    extern "Rust" {
+        type OperationContext<'a>;
+
+        fn empty_operation_context() -> Box<OperationContext<'static>>;
+        fn operation_context_globals() -> Box<OperationContext<'static>>;
+        fn check_cancel(&self) -> bool;
+
+        #[cxx_name = "get_bg_context"]
+        unsafe fn get_bg_context_ffi(
+            env: &EnvDynFFI,
+            generation_count: u32,
+        ) -> Box<OperationContext<'_>>;
+        #[cxx_name = "parser_context"]
+        fn parser_context_ffi(parser: &Parser) -> Box<OperationContext<'static>>;
+    }
+}
+
+fn get_bg_context_ffi(env: &EnvDynFFI, generation_count: u32) -> Box<OperationContext<'_>> {
+    Box::new(get_bg_context(&env.0, generation_count))
+}
+
+fn parser_context_ffi(parser: &Parser) -> Box<OperationContext<'static>> {
+    Box::new(parser.context())
+}
+
+unsafe impl cxx::ExternType for OperationContext<'_> {
+    type Id = cxx::type_id!("OperationContext");
+    type Kind = cxx::kind::Opaque;
+}
+
+fn empty_operation_context() -> Box<OperationContext<'static>> {
+    Box::new(OperationContext::empty())
+}
+fn operation_context_globals() -> Box<OperationContext<'static>> {
+    Box::new(OperationContext::globals())
+}

fish-rust/src/output.rs

@@ -523,7 +523,7 @@ pub fn best_color(candidates: &[RgbColor], support: ColorSupport) -> RgbColor {
 /// TODO: This code should be refactored to enable sharing with builtin_set_color.
 ///       In particular, the argument parsing still isn't fully capable.
 #[allow(clippy::collapsible_else_if)]
-fn parse_color(var: &EnvVar, is_background: bool) -> RgbColor {
+pub fn parse_color(var: &EnvVar, is_background: bool) -> RgbColor {
     let mut is_bold = false;
     let mut is_underline = false;
     let mut is_italics = false;
@@ -604,7 +604,7 @@ fn make_buffering_outputter_ffi() -> Box<Outputter> {
     Box::new(Outputter::new_buffering())
 }
 
-type RgbColorFFI = crate::ffi::rgb_color_t;
+pub type RgbColorFFI = crate::ffi::rgb_color_t;
 use crate::wchar_ffi::AsWstr;
 impl Outputter {
     fn set_color_ffi(&mut self, fg: &RgbColorFFI, bg: &RgbColorFFI) {

fish-rust/src/parse_constants.rs

@@ -1,9 +1,9 @@
 //! Constants used in the programmatic representation of fish code.
 
 use crate::fallback::{fish_wcswidth, fish_wcwidth};
+use crate::ffi::wcharz_t;
 use crate::tokenizer::variable_assignment_equals_pos;
 use crate::wchar::prelude::*;
-use crate::wchar_ffi::wcharz_t;
 use crate::wchar_ffi::{AsWstr, WCharFromFFI, WCharToFFI};
 use bitflags::bitflags;
 use cxx::{type_id, ExternType};
@@ -110,6 +110,7 @@ mod parse_constants_ffi {
     }
 
     // Statement decorations like 'command' or 'exec'.
+    #[derive(Clone, Copy, Eq, PartialEq)]
     pub enum StatementDecoration {
         none,
         command,
@@ -118,6 +119,7 @@ mod parse_constants_ffi {
     }
 
     // Parse error code list.
+    #[derive(Debug)]
     pub enum ParseErrorCode {
         none,
 
@@ -248,6 +250,12 @@ impl SourceRange {
     }
 }
 
+impl From<SourceRange> for std::ops::Range<usize> {
+    fn from(value: SourceRange) -> Self {
+        value.start()..value.end()
+    }
+}
+
 impl Default for ParseTokenType {
     fn default() -> Self {
         ParseTokenType::invalid
@@ -349,7 +357,7 @@ impl Default for ParseErrorCode {
     }
 }
 
-#[derive(Clone, Default)]
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct ParseError {
     /// Text of the error.
     pub text: WString,