scrollback-push to query for indn/cuu via XTGETTCAP

Some terminals like the Linux console don't support indn (scroll
forward). Let's query for the presence of these features, and fall
back to the traditional behavior if absent.

For now, break with the tradition of using the terminfo database that
we read ourselves. Instead ask the terminal directly via XTGETTCAP.
This is a fairly young feature implemented by terminals like xterm,
foot and kitty, however xterm doesn't expose these capabilities at
this point.

This is a good opportunity to try XTGETTCAP, since these are
capabilities we haven't used before. Advantages of XTGETTCAP are that
it works across SSH and is independent of $TERM (of course ignoring
$TERM may also be breaking to some users). Let's see if it sees
adoption in practice.

Tested to work on foot and kitty, allowing the default ctrl-l binding
to work without erasing any screen content.

See #11003

CHANGELOG.rst

@@ -19,7 +19,8 @@ New or improved bindings
 - :kbd:`ctrl-z` (undo) after executing a command will restore the previous cursor position instead of placing the cursor at the end of the command line.
 - The OSC 133 prompt marking feature has learned about kitty's ``click_events=1`` flag, which allows moving fish's cursor by clicking.
 - :kbd:`ctrl-l` no longer clears the screen but only pushes to the terminal's scrollback all text above the prompt (via a new special input function ``scrollback-push``).
-  You can restore previous behavior with `bind ctrl-l clear-screen`.
+  This feature depends on the terminal advertising via XTGETTCAP support for the ``indn`` and ``cuu`` terminfo capabilities.
+  If not presesnt, the binding falls back to ``clear-screen``.
 
 Completions
 ^^^^^^^^^^^

src/curses.rs

@@ -66,10 +66,8 @@ pub struct Term {
     pub cursor_down: Option<CString>,
     pub cursor_left: Option<CString>,
     pub cursor_right: Option<CString>,
-    pub parm_cursor_up: Option<CString>,
     pub parm_left_cursor: Option<CString>,
     pub parm_right_cursor: Option<CString>,
-    pub parm_index: Option<CString>,
     pub clr_eol: Option<CString>,
     pub clr_eos: Option<CString>,
 
@@ -217,10 +215,8 @@ impl Term {
             cursor_down: get_str_cap(&db, "do"),
             cursor_left: get_str_cap(&db, "le"),
             cursor_right: get_str_cap(&db, "nd"),
-            parm_cursor_up: get_str_cap(&db, "UP"),
             parm_left_cursor: get_str_cap(&db, "LE"),
             parm_right_cursor: get_str_cap(&db, "RI"),
-            parm_index: get_str_cap(&db, "SF"),
             clr_eol: get_str_cap(&db, "ce"),
             clr_eos: get_str_cap(&db, "cd"),
 
@@ -429,10 +425,8 @@ pub fn setup_fallback_term() -> Arc<Term> {
         cursor_down: Some(CString::new("\n").unwrap()),
         cursor_left: Some(CString::new("\x08").unwrap()),
         cursor_right: Some(CString::new("\x1b[C").unwrap()),
-        parm_cursor_up: Some(CString::new("\x1b[%p1%dA").unwrap()),
         parm_left_cursor: Some(CString::new("\x1b[%p1%dD").unwrap()),
         parm_right_cursor: Some(CString::new("\x1b[%p1%dC").unwrap()),
-        parm_index: Some(CString::new("\x1b[%p1%dS").unwrap()),
         clr_eol: Some(CString::new("\x1b[K").unwrap()),
         clr_eos: Some(CString::new("\x1b[J").unwrap()),
         max_colors: Some(256),

src/input_common.rs

@@ -437,6 +437,9 @@ pub fn update_wait_on_sequence_key_ms(vars: &EnvStack) {
 
 static TERMINAL_PROTOCOLS: AtomicBool = AtomicBool::new(false);
 
+pub(crate) static SCROLL_FORWARD_SUPPORTED: RelaxedAtomicBool = RelaxedAtomicBool::new(false);
+pub(crate) static CURSOR_UP_SUPPORTED: RelaxedAtomicBool = RelaxedAtomicBool::new(false);
+
 static KITTY_KEYBOARD_SUPPORTED: RelaxedAtomicBool = RelaxedAtomicBool::new(false);
 
 macro_rules! kitty_progressive_enhancements {
@@ -785,6 +788,8 @@ pub trait InputEventQueuer {
         buffer: &mut Vec<u8>,
         have_escape_prefix: &mut bool,
     ) -> Option<Key> {
+        assert!(buffer.len() <= 2);
+        let recursive_invocation = buffer.len() == 2;
         let Some(next) = self.try_readb(buffer) else {
             if !self.paste_is_buffering() {
                 return Some(Key::from_raw(key::Escape));
@@ -792,7 +797,7 @@ pub trait InputEventQueuer {
             return None;
         };
         let invalid = Key::from_raw(key::Invalid);
-        if buffer.len() == 2 && next == b'\x1b' {
+        if recursive_invocation && next == b'\x1b' {
             return Some(
                 match self.parse_escape_sequence(buffer, have_escape_prefix) {
                     Some(mut nested_sequence) => {
@@ -814,6 +819,10 @@ pub trait InputEventQueuer {
             // potential SS3
             return Some(self.parse_ss3(buffer).unwrap_or(invalid));
         }
+        if !recursive_invocation && next == b'P' {
+            // potential DCS
+            return Some(self.parse_dcs(buffer).unwrap_or(invalid));
+        }
         match canonicalize_control_char(next) {
             Some(mut key) => {
                 key.modifiers.alt = true;
@@ -1238,6 +1247,56 @@ pub trait InputEventQueuer {
         Some(key)
     }
 
+    fn parse_dcs(&mut self, buffer: &mut Vec<u8>) -> Option<Key> {
+        assert!(buffer.len() == 2);
+        let Some(success) = self.try_readb(buffer) else {
+            return Some(alt('P'));
+        };
+        let success = match success {
+            b'0' => false,
+            b'1' => true,
+            _ => return None,
+        };
+        if self.try_readb(buffer)? != b'+' {
+            return None;
+        }
+        if self.try_readb(buffer)? != b'r' {
+            return None;
+        }
+        while self.try_readb(buffer)? != b'\x1b' {}
+        if self.try_readb(buffer)? != b'\\' {
+            return None;
+        }
+        buffer.pop();
+        buffer.pop();
+        if !success {
+            return None;
+        }
+        // \e P 1 r + Pn ST
+        let mut buffer = buffer[5..].splitn(2, |&c| c == b'=');
+        let key = buffer.next().unwrap();
+        let value = buffer.next()?;
+        let key = parse_hex(key)?;
+        let value = parse_hex(value)?;
+        FLOG!(
+            reader,
+            format!(
+                "Received XTGETTCAP response: {}={:?}",
+                str2wcstring(&key),
+                str2wcstring(&value)
+            )
+        );
+        if key == b"indn" && matches!(&value[..], b"\x1b[%p1%dS" | b"\\E[%p1%dS") {
+            SCROLL_FORWARD_SUPPORTED.store(true);
+            FLOG!(reader, "Scroll forward is supported");
+        }
+        if key == b"cuu" && matches!(&value[..], b"\x1b[%p1%dA" | b"\\E[%p1%dA") {
+            CURSOR_UP_SUPPORTED.store(true);
+            FLOG!(reader, "Cursor up is supported");
+        }
+        return None;
+    }
+
     fn readch_timed_esc(&mut self) -> Option<CharEvent> {
         self.readch_timed(WAIT_ON_ESCAPE_MS.load(Ordering::Relaxed))
     }
@@ -1492,3 +1551,24 @@ impl InputEventQueuer for InputEventQueue {
         }
     }
 }
+
+fn parse_hex(hex: &[u8]) -> Option<Vec<u8>> {
+    if hex.len() % 2 != 0 {
+        return None;
+    }
+    let mut result = vec![0; hex.len() / 2];
+    let mut i = 0;
+    while i < hex.len() {
+        let d1 = char::from(hex[i]).to_digit(16)?;
+        let d2 = char::from(hex[i + 1]).to_digit(16)?;
+        let decoded = u8::try_from(16 * d1 + d2).unwrap();
+        result[i / 2] = decoded;
+        i += 2;
+    }
+    Some(result)
+}
+
+#[test]
+fn test_parse_hex() {
+    assert_eq!(parse_hex(&[b'3', b'd']), Some(vec![61]));
+}

src/reader.rs

@@ -88,6 +88,7 @@ use crate::input_common::{
     terminal_protocol_hacks, terminal_protocols_enable_ifn, CharEvent, CharInputStyle, InputData,
     ReadlineCmd,
 };
+use crate::input_common::{CURSOR_UP_SUPPORTED, SCROLL_FORWARD_SUPPORTED};
 use crate::io::IoChain;
 use crate::key::ViewportPosition;
 use crate::kill::{kill_add, kill_replace, kill_yank, kill_yank_rotate};
@@ -2095,6 +2096,11 @@ impl<'a> Reader<'a> {
             let _ = out.write(KITTY_PROGRESSIVE_ENHANCEMENTS_QUERY);
             // Query for cursor position reporting support.
             zelf.request_cursor_position(&mut out, CursorPositionWait::InitialFeatureProbe);
+            let mut xtgettcap = |cap| {
+                let _ = write!(&mut out, "\x1bP+q{}\x1b\\", DisplayAsHex(cap));
+            };
+            xtgettcap("indn");
+            xtgettcap("cuu");
             out.end_buffering();
         }
 
@@ -3627,20 +3633,26 @@ impl<'a> Reader<'a> {
             rl::ClearScreenAndRepaint => {
                 self.clear_screen_and_repaint();
             }
-            rl::ScrollbackPush => match self.cursor_position_wait() {
-                CursorPositionWait::None => self.request_cursor_position(
-                    &mut Outputter::stdoutput().borrow_mut(),
-                    CursorPositionWait::Blocking(CursorPositionBlockingWait::ScrollbackPush),
-                ),
-                CursorPositionWait::InitialFeatureProbe => self.clear_screen_and_repaint(),
-                CursorPositionWait::Blocking(_) => {
-                    // TODO: re-queue it I guess.
-                    FLOG!(
+            rl::ScrollbackPush => {
+                if !SCROLL_FORWARD_SUPPORTED.load() || !CURSOR_UP_SUPPORTED.load() {
+                    self.clear_screen_and_repaint();
+                    return;
+                }
+                match self.cursor_position_wait() {
+                    CursorPositionWait::None => self.request_cursor_position(
+                        &mut Outputter::stdoutput().borrow_mut(),
+                        CursorPositionWait::Blocking(CursorPositionBlockingWait::ScrollbackPush),
+                    ),
+                    CursorPositionWait::InitialFeatureProbe => self.clear_screen_and_repaint(),
+                    CursorPositionWait::Blocking(_) => {
+                        // TODO: re-queue it I guess.
+                        FLOG!(
                         reader,
                         "Ignoring scrollback-push received while still waiting for Cursor Position Report"
                     );
+                    }
                 }
-            },
+            }
             rl::SelfInsert | rl::SelfInsertNotFirst | rl::FuncAnd | rl::FuncOr => {
                 // This can be reached via `commandline -f and` etc
                 // panic!("should have been handled by inputter_t::readch");
@@ -4261,6 +4273,17 @@ fn reader_interactive_init(parser: &Parser) {
     );
 }
 
+struct DisplayAsHex<'a>(&'a str);
+
+impl<'a> std::fmt::Display for DisplayAsHex<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for byte in self.0.bytes() {
+            write!(f, "{:x}", byte)?;
+        }
+        Ok(())
+    }
+}
+
 /// Destroy data for interactive use.
 fn reader_interactive_destroy() {
     Outputter::stdoutput()

src/screen.rs

@@ -8,6 +8,7 @@
 //! of text around to handle text insertion.
 
 use crate::editable_line::line_at_cursor;
+use crate::input_common::{CURSOR_UP_SUPPORTED, SCROLL_FORWARD_SUPPORTED};
 use crate::key::ViewportPosition;
 use crate::pager::{PageRendering, Pager, PAGER_MIN_HEIGHT};
 use crate::FLOG;
@@ -522,17 +523,14 @@ impl Screen {
             return;
         }
         let zelf = self.scoped_buffer();
-        let Some(term) = term() else {
-            return;
-        };
         let mut out = zelf.outp.borrow_mut();
         let lines_to_scroll = i32::try_from(lines_to_scroll).unwrap();
         // Scroll down.
+        assert!(SCROLL_FORWARD_SUPPORTED.load());
         out.tputs_bytes(format!("\x1b[{}S", lines_to_scroll).as_bytes());
+        assert!(CURSOR_UP_SUPPORTED.load());
         // Reposition cursor.
-        if let Some(up) = term.parm_cursor_up.as_ref() {
-            out.tputs_if_some(&tparm1(up, lines_to_scroll));
-        }
+        out.tputs_bytes(format!("\x1b[{}A", lines_to_scroll).as_bytes());
     }
 
     fn command_line_y_given_cursor_y(&mut self, viewport_cursor_y: usize) -> usize {