Commands to move by entire tokens

ja: I'll try to add default bindings in a follow-up PR.

Closes #10738
Closes #2014

CHANGELOG.rst

@@ -177,7 +177,9 @@ New or improved bindings
   - :kbd:`shift-enter` now inserts a newline instead of executing the command line.
   - :kbd:`ctrl-backspace` now deletes the last word instead of only one character.
   - :kbd:`ctrl-delete` deletes the next word (same as :kbd:`alt-d`).
-- New special input functions ``forward-char-passive`` and ``backward-char-passive`` are like their non-passive variants but do not accept autosuggestions or move focus in the completion pager (:issue:`10398`).
+- New special input functions:
+  - ``forward-char-passive`` and ``backward-char-passive`` are like their non-passive variants but do not accept autosuggestions or move focus in the completion pager (:issue:`10398`).
+  - ``forward-token``, ``backward-token``, ``kill-token``, and ``backward-kill-token`` are similar to the ``*-bigword`` variants but for the whole argument token which includes escaped spaces (:issue:`2014`).
 - Vi mode has seen some improvements but continues to suffer from the lack of people working on it.
   - Insert-mode :kbd:`ctrl-n` accepts autosuggestions (:issue:`10339`).
   - Outside insert mode, the cursor will no longer be placed beyond the last character on the commandline.

doc_src/cmds/bind.rst

@@ -125,12 +125,18 @@ The following special input functions are available:
 ``backward-bigword``
     move one whitespace-delimited word to the left
 
+``backward-token``
+    move one argument to the left
+
 ``backward-delete-char``
     deletes one character of input to the left of the cursor
 
 ``backward-kill-bigword``
     move the whitespace-delimited word to the left of the cursor to the killring
 
+``backward-kill-token``
+    move the argument to the left of the cursor to the killring
+
 ``backward-kill-line``
     move everything from the beginning of the line to the cursor to the killring
 
@@ -209,6 +215,9 @@ The following special input functions are available:
 ``forward-bigword``
     move one whitespace-delimited word to the right
 
+``forward-token``
+    move one argument to the right
+
 ``forward-char``
     move one character to the right; or if at the end of the commandline, accept the current autosuggestion.
     If the completion pager is active, select the next completion instead.
@@ -253,7 +262,7 @@ The following special input functions are available:
     read another character and jump to its next occurence after/before the cursor
 
 ``forward-jump-till`` and ``backward-jump-till``
-    jump to right *before* the next occurence
+    jump to right *before* the next occurrence
 
 ``repeat-jump`` and ``repeat-jump-reverse``
     redo the last jump in the same/opposite direction
@@ -273,6 +282,9 @@ The following special input functions are available:
 ``kill-bigword``
     move the next whitespace-delimited word to the killring
 
+``kill-token``
+    move the next argument to the killring
+
 ``kill-line``
     move everything from the cursor to the end of the line to the killring
 

src/input.rs

@@ -135,7 +135,9 @@ const INPUT_FUNCTION_METADATA: &[InputFunctionMetadata] = &[
     make_md(L!("backward-kill-bigword"), ReadlineCmd::BackwardKillBigword),
     make_md(L!("backward-kill-line"), ReadlineCmd::BackwardKillLine),
     make_md(L!("backward-kill-path-component"), ReadlineCmd::BackwardKillPathComponent),
+    make_md(L!("backward-kill-token"), ReadlineCmd::BackwardKillToken),
     make_md(L!("backward-kill-word"), ReadlineCmd::BackwardKillWord),
+    make_md(L!("backward-token"), ReadlineCmd::BackwardToken),
     make_md(L!("backward-word"), ReadlineCmd::BackwardWord),
     make_md(L!("begin-selection"), ReadlineCmd::BeginSelection),
     make_md(L!("begin-undo-group"), ReadlineCmd::BeginUndoGroup),
@@ -167,6 +169,7 @@ const INPUT_FUNCTION_METADATA: &[InputFunctionMetadata] = &[
     make_md(L!("forward-jump"), ReadlineCmd::ForwardJump),
     make_md(L!("forward-jump-till"), ReadlineCmd::ForwardJumpTill),
     make_md(L!("forward-single-char"), ReadlineCmd::ForwardSingleChar),
+    make_md(L!("forward-token"), ReadlineCmd::ForwardToken),
     make_md(L!("forward-word"), ReadlineCmd::ForwardWord),
     make_md(L!("history-pager"), ReadlineCmd::HistoryPager),
     make_md(L!("history-pager-delete"), ReadlineCmd::HistoryPagerDelete),
@@ -184,6 +187,7 @@ const INPUT_FUNCTION_METADATA: &[InputFunctionMetadata] = &[
     make_md(L!("kill-inner-line"), ReadlineCmd::KillInnerLine),
     make_md(L!("kill-line"), ReadlineCmd::KillLine),
     make_md(L!("kill-selection"), ReadlineCmd::KillSelection),
+    make_md(L!("kill-token"), ReadlineCmd::KillToken),
     make_md(L!("kill-whole-line"), ReadlineCmd::KillWholeLine),
     make_md(L!("kill-word"), ReadlineCmd::KillWord),
     make_md(L!("nextd-or-forward-word"), ReadlineCmd::NextdOrForwardWord),

src/input_common.rs

@@ -52,6 +52,8 @@ pub enum ReadlineCmd {
     BackwardWord,
     ForwardBigword,
     BackwardBigword,
+    ForwardToken,
+    BackwardToken,
     NextdOrForwardWord,
     PrevdOrBackwardWord,
     HistorySearchBackward,
@@ -75,9 +77,11 @@ pub enum ReadlineCmd {
     KillInnerLine,
     KillWord,
     KillBigword,
+    KillToken,
     BackwardKillWord,
     BackwardKillPathComponent,
     BackwardKillBigword,
+    BackwardKillToken,
     HistoryTokenSearchBackward,
     HistoryTokenSearchForward,
     SelfInsert,

src/reader.rs

@@ -2795,6 +2795,56 @@ impl<'a> Reader<'a> {
                     self.rls().last_cmd != Some(c),
                 );
             }
+            rl::BackwardKillToken => {
+                let Some(new_position) = self.backward_token() else {
+                    return;
+                };
+
+                let (elt, _el) = self.active_edit_line();
+                if elt == EditableLineTag::Commandline {
+                    self.suppress_autosuggestion = true;
+                }
+
+                let (elt, el) = self.active_edit_line();
+                self.data.kill(
+                    elt,
+                    new_position..el.position(),
+                    Kill::Prepend,
+                    self.rls().last_cmd != Some(rl::BackwardKillToken),
+                );
+            }
+            rl::BackwardToken => {
+                let Some(new_position) = self.backward_token() else {
+                    return;
+                };
+                let (elt, _el) = self.active_edit_line();
+                self.update_buff_pos(elt, Some(new_position));
+            }
+            rl::KillToken => {
+                let Some(new_position) = self.forward_token() else {
+                    return;
+                };
+
+                let (elt, _el) = self.active_edit_line();
+                if elt == EditableLineTag::Commandline {
+                    self.suppress_autosuggestion = true;
+                }
+
+                let (elt, el) = self.active_edit_line();
+                self.data.kill(
+                    elt,
+                    el.position()..new_position,
+                    Kill::Append,
+                    self.rls().last_cmd != Some(rl::KillToken),
+                );
+            }
+            rl::ForwardToken => {
+                let Some(new_position) = self.forward_token() else {
+                    return;
+                };
+                let (elt, _el) = self.active_edit_line();
+                self.update_buff_pos(elt, Some(new_position));
+            }
             rl::BackwardWord | rl::BackwardBigword | rl::PrevdOrBackwardWord => {
                 if c == rl::PrevdOrBackwardWord && self.command_line.is_empty() {
                     self.eval_bind_cmd(L!("prevd"));
@@ -3332,6 +3382,54 @@ impl<'a> Reader<'a> {
             }
         }
     }
+
+    fn backward_token(&mut self) -> Option<usize> {
+        let (_elt, el) = self.active_edit_line();
+        let pos = el.position();
+        if pos == 0 {
+            return None;
+        }
+
+        let mut tok = 0..0;
+        let mut prev_tok = 0..0;
+        parse_util_token_extent(el.text(), el.position(), &mut tok, Some(&mut prev_tok));
+
+        // if we are at the start of a token, go back one
+        let new_position = if tok.start == pos {
+            if prev_tok.start == pos {
+                let cmdsub = parse_util_cmdsubst_extent(el.text(), prev_tok.start);
+                cmdsub.start.saturating_sub(1)
+            } else {
+                prev_tok.start
+            }
+        } else {
+            tok.start
+        };
+
+        Some(new_position)
+    }
+
+    fn forward_token(&self) -> Option<usize> {
+        let (_elt, el) = self.active_edit_line();
+        let pos = el.position();
+        if pos == el.len() {
+            return None;
+        }
+
+        // If we are not in a token, look for one ahead
+        let buff_pos = pos
+            + el.text()[pos..]
+                .chars()
+                .take_while(|c| c.is_ascii_whitespace())
+                .count();
+
+        let mut tok = 0..0;
+        parse_util_token_extent(el.text(), buff_pos, &mut tok, None);
+
+        let new_position = if tok.end == pos { pos + 1 } else { tok.end };
+
+        Some(new_position)
+    }
 }
 
 /// Returns true if the last token is a comment.
@@ -4909,6 +5007,8 @@ fn command_ends_paging(c: ReadlineCmd, focused_on_search_field: bool) -> bool {
         | rl::BackwardWord
         | rl::ForwardBigword
         | rl::BackwardBigword
+        | rl::ForwardToken
+        | rl::BackwardToken
         | rl::NextdOrForwardWord
         | rl::PrevdOrBackwardWord
         | rl::DeleteChar
@@ -4921,9 +5021,11 @@ fn command_ends_paging(c: ReadlineCmd, focused_on_search_field: bool) -> bool {
         | rl::KillInnerLine
         | rl::KillWord
         | rl::KillBigword
+        | rl::KillToken
         | rl::BackwardKillWord
         | rl::BackwardKillPathComponent
         | rl::BackwardKillBigword
+        | rl::BackwardKillToken
         | rl::SelfInsert
         | rl::SelfInsertNotFirst
         | rl::TransposeChars