prevent string_decoder from mangling non-utf output - fixes #4025, fixes #4178, fixes #4373, fixes #4282

app/lib/pty.ts

@@ -1,8 +1,9 @@
 import * as nodePTY from '@tabby-gang/node-pty'
-import { StringDecoder } from './stringDecoder'
 import { v4 as uuidv4 } from 'uuid'
 import { ipcMain } from 'electron'
 import { Application } from './app'
+import { UTF8Splitter } from './utfSplitter'
+import { Subject, debounceTime } from 'rxjs'
 
 class PTYDataQueue {
     private buffers: Buffer[] = []
@@ -10,9 +11,17 @@ class PTYDataQueue {
     private maxChunk = 1024 * 100
     private maxDelta = this.maxChunk * 5
     private flowPaused = false
-    private decoder = new StringDecoder()
+    private decoder = new UTF8Splitter()
+    private output$ = new Subject<Buffer>()
 
-    constructor (private pty: nodePTY.IPty, private onData: (data: Buffer) => void) { }
+    constructor (private pty: nodePTY.IPty, private onData: (data: Buffer) => void) {
+        this.output$.pipe(debounceTime(500)).subscribe(() => {
+            const remainder = this.decoder.flush()
+            if (remainder.length) {
+                this.onData(remainder)
+            }
+        })
+    }
 
     push (data: Buffer) {
         this.buffers.push(data)
@@ -61,7 +70,9 @@ class PTYDataQueue {
     }
 
     private emitData (data: Buffer) {
-        this.onData(this.decoder.write(data))
+        const validChunk = this.decoder.write(data)
+        this.onData(validChunk)
+        this.output$.next(validChunk)
     }
 
     private pause () {

app/lib/utfSplitter.ts

@@ -0,0 +1,32 @@
+const partials = [
+    [0b110, 5, 0],
+    [0b1110, 4, 1],
+    [0b11110, 3, 2],
+]
+
+export class UTF8Splitter {
+    private internal = Buffer.alloc(0)
+
+    write (data: Buffer): Buffer {
+        this.internal = Buffer.concat([this.internal, data])
+
+        let keep = 0
+        for (const [pattern, shift, maxOffset] of partials) {
+            for (let offset = 0; offset < maxOffset + 1; offset++) {
+                if (this.internal[this.internal.length - offset - 1] >> shift === pattern) {
+                    keep = Math.max(keep, offset + 1)
+                }
+            }
+        }
+
+        const result = this.internal.slice(0, this.internal.length - keep)
+        this.internal = this.internal.slice(this.internal.length - keep)
+        return result
+    }
+
+    flush (): Buffer {
+        const result = this.internal
+        this.internal = Buffer.alloc(0)
+        return result
+    }
+}