hv_wdt: Add hypervisor watchdog on secondary core

Signed-off-by: Hector Martin <marcan@marcan.st>

Makefile

@@ -44,7 +44,7 @@ OBJECTS := \
 	fb.o font.o font_retina.o \
 	gxf.o gxf_asm.o \
 	heapblock.o \
-	hv.o hv_vm.o hv_exc.o hv_vuart.o hv_asm.o \
+	hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o \
 	iodev.o \
 	kboot.o \
 	main.o \

proxyclient/hv.py

@@ -33,6 +33,7 @@ class HV_EVENT(IntEnum):
     HOOK_VM = 1
     VTIMER = 2
     USER_INTERRUPT = 3
+    WDT_BARK = 4
 
 VMProxyHookData = Struct(
     "flags" / RegAdapter(MMIOTraceFlags),
@@ -403,6 +404,29 @@ class HV:
         if self._sigint_pending:
             self._handle_sigint()
 
+    def handle_bark(self, reason, code, info):
+        self._in_handler = True
+        self._sigint_pending = False
+        self._stepping = False
+
+        locals = {
+            "hv": self,
+            "iface": self.iface,
+            "p": self.p,
+            "u": self.u,
+        }
+
+        for attr in dir(self):
+            a = getattr(self, attr)
+            if callable(a):
+                locals[attr] = getattr(self, attr)
+
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+        ret = shell.run_shell(locals, "Entering panic shell", "Returning from exception")
+        signal.signal(signal.SIGINT, self._handle_sigint)
+
+        self.p.exit(0)
+
     def skip(self):
         self.ctx.elr += 4
         raise shell.ExitConsole(EXC_RET.HANDLED)
@@ -521,6 +545,7 @@ class HV:
         self.iface.set_handler(START.HV, HV_EVENT.USER_INTERRUPT, self.handle_exception)
         self.iface.set_handler(START.HV, HV_EVENT.HOOK_VM, self.handle_exception)
         self.iface.set_handler(START.HV, HV_EVENT.VTIMER, self.handle_exception)
+        self.iface.set_handler(START.HV, HV_EVENT.WDT_BARK, self.handle_bark)
         self.iface.set_event_handler(EVENT.MMIOTRACE, self.handle_mmiotrace)
 
         self.map_sw(0x2_00000000,

src/hv.c

@@ -4,6 +4,7 @@
 #include "assert.h"
 #include "cpu_regs.h"
 #include "gxf.h"
+#include "smp.h"
 #include "utils.h"
 
 #define HV_TICK_RATE 1000
@@ -16,6 +17,9 @@ u64 hv_tick_interval;
 
 void hv_init(void)
 {
+    smp_start_secondaries();
+    hv_wdt_init();
+
     // Enable physical timer for EL1
     msr(CNTHCTL_EL2, CNTHCTL_EL1PTEN | CNTHCTL_EL1PCTEN);
 
@@ -54,8 +58,10 @@ void hv_start(void *entry, u64 regs[4])
     if (gxf_enabled())
         gl2_call(hv_set_gxf_vbar, 0, 0, 0, 0);
 
+    hv_wdt_start();
     hv_arm_tick();
     hv_enter_guest(regs[0], regs[1], regs[2], regs[3], entry);
+    hv_wdt_stop();
 
     printf("Exiting hypervisor.\n");
 }
@@ -124,5 +130,5 @@ void hv_arm_tick(void)
 
 void hv_tick(void)
 {
-    //     printf("HV tick!\n");
+    hv_wdt_pet();
 }

src/hv.h

@@ -31,6 +31,7 @@ typedef enum _hv_entry_type {
     HV_HOOK_VM = 1,
     HV_VTIMER,
     HV_USER_INTERRUPT,
+    HV_WDT_BARK,
 } hv_entry_type;
 
 /* VM */
@@ -52,6 +53,15 @@ void hv_map_vuart(u64 base, iodev_id_t iodev);
 void hv_exc_proxy(u64 *regs, uartproxy_boot_reason_t reason, uartproxy_exc_code_t type,
                   void *extra);
 
+/* WDT */
+void hv_wdt_pet(void);
+void hv_wdt_suspend(void);
+void hv_wdt_resume(void);
+void hv_wdt_init(void);
+void hv_wdt_start(void);
+void hv_wdt_stop(void);
+void hv_wdt_breadcrumb(char c);
+
 /* Utilities */
 void hv_write_hcr(u64 val);
 u64 hv_get_spsr(void);

src/hv_exc.c

@@ -5,6 +5,7 @@
 #include "cpu_regs.h"
 #include "exception.h"
 #include "string.h"
+#include "uart.h"
 #include "uartproxy.h"
 
 #define _SYSREG_ISS(_1, _2, op0, op1, CRn, CRm, op2)                                               \
@@ -21,6 +22,8 @@ void hv_exc_proxy(u64 *regs, uartproxy_boot_reason_t reason, uartproxy_exc_code_
 {
     int from_el = FIELD_GET(SPSR_M, hv_get_spsr()) >> 2;
 
+    hv_wdt_breadcrumb('P');
+
     struct uartproxy_exc_info exc_info = {
         .spsr = hv_get_spsr(),
         .elr = hv_get_elr(),
@@ -41,7 +44,9 @@ void hv_exc_proxy(u64 *regs, uartproxy_boot_reason_t reason, uartproxy_exc_code_
         .info = &exc_info,
     };
 
+    hv_wdt_suspend();
     int ret = uartproxy_run(&start);
+    hv_wdt_resume();
 
     switch (ret) {
         case EXC_RET_STEP:
@@ -55,6 +60,7 @@ void hv_exc_proxy(u64 *regs, uartproxy_boot_reason_t reason, uartproxy_exc_code_
                 msr(CNTV_TVAL_EL0, 100);
                 msr(CNTV_CTL_EL0, CNTx_CTL_ENABLE);
             }
+            hv_wdt_breadcrumb('p');
             return;
         case EXC_EXIT_GUEST:
             hv_exit_guest();
@@ -130,6 +136,7 @@ static bool hv_handle_msr(u64 *regs, u64 iss)
 
 static void hv_exc_exit(u64 *regs)
 {
+    hv_wdt_breadcrumb('x');
     if (iodev_can_read(uartproxy_iodev))
         hv_exc_proxy(regs, START_HV, HV_USER_INTERRUPT, NULL);
     hv_update_fiq();
@@ -137,18 +144,22 @@ static void hv_exc_exit(u64 *regs)
 
 void hv_exc_sync(u64 *regs)
 {
+    hv_wdt_breadcrumb('S');
     bool handled = false;
     u64 esr = hv_get_esr();
     u32 ec = FIELD_GET(ESR_EC, esr);
 
     switch (ec) {
         case ESR_EC_DABORT_LOWER:
+            hv_wdt_breadcrumb('D');
             handled = hv_handle_dabort(regs);
             break;
         case ESR_EC_MSR:
+            hv_wdt_breadcrumb('M');
             handled = hv_handle_msr(regs, FIELD_GET(ESR_ISS, esr));
             break;
         case ESR_EC_IMPDEF:
+            hv_wdt_breadcrumb('A');
             switch (FIELD_GET(ESR_ISS, esr)) {
                 case ESR_ISS_IMPDEF_MSR:
                     handled = hv_handle_msr(regs, mrs(AFSR1_EL1));
@@ -157,22 +168,29 @@ void hv_exc_sync(u64 *regs)
             break;
     }
 
-    if (handled)
+    if (handled) {
+        hv_wdt_breadcrumb('+');
         hv_set_elr(hv_get_elr() + 4);
-    else
+    } else {
+        hv_wdt_breadcrumb('-');
         hv_exc_proxy(regs, START_EXCEPTION_LOWER, EXC_SYNC, NULL);
+    }
 
     hv_exc_exit(regs);
+    hv_wdt_breadcrumb('s');
 }
 
 void hv_exc_irq(u64 *regs)
 {
+    hv_wdt_breadcrumb('I');
     hv_exc_proxy(regs, START_EXCEPTION_LOWER, EXC_IRQ, NULL);
     hv_exc_exit(regs);
+    hv_wdt_breadcrumb('i');
 }
 
 void hv_exc_fiq(u64 *regs)
 {
+    hv_wdt_breadcrumb('F');
     if (mrs(CNTP_CTL_EL0) == (CNTx_CTL_ISTATUS | CNTx_CTL_ENABLE)) {
         msr(CNTP_CTL_EL0, CNTx_CTL_ISTATUS | CNTx_CTL_IMASK | CNTx_CTL_ENABLE);
         hv_tick();
@@ -206,10 +224,13 @@ void hv_exc_fiq(u64 *regs)
 
     // Handles guest timers
     hv_exc_exit(regs);
+    hv_wdt_breadcrumb('f');
 }
 
 void hv_exc_serr(u64 *regs)
 {
+    hv_wdt_breadcrumb('E');
     hv_exc_proxy(regs, START_EXCEPTION_LOWER, EXC_SERROR, NULL);
     hv_exc_exit(regs);
+    hv_wdt_breadcrumb('e');
 }

src/hv_wdt.c

@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: MIT */
+
+#include "hv.h"
+#include "adt.h"
+#include "cpu_dbg_regs.h"
+#include "smp.h"
+#include "uart.h"
+#include "utils.h"
+
+#define WDT_CPU     1
+#define WDT_TIMEOUT 1
+
+static bool hv_wdt_active = false;
+static bool hv_wdt_enabled = false;
+static volatile u64 hv_wdt_timestamp = 0;
+static u64 hv_wdt_timeout = 0;
+static volatile u64 hv_wdt_breadcrumbs;
+
+static u64 cpu_dbg_base = 0;
+
+void hv_wdt_bark(void)
+{
+    u64 tmp = hv_wdt_breadcrumbs;
+    uart_puts("HV watchdog: bark!");
+
+    uart_printf("Breadcrumbs: ");
+    for (int i = 56; i >= 0; i -= 8) {
+        char c = (tmp >> i) & 0xff;
+        if (c)
+            uart_putchar(c);
+    }
+    uart_putchar('\n');
+
+    uart_puts("Attempting to enter proxy");
+
+    struct uartproxy_msg_start start = {
+        .reason = START_HV,
+        .code = HV_WDT_BARK,
+    };
+
+    uartproxy_run(&start);
+    reboot();
+}
+
+void hv_wdt_main(void)
+{
+    while (hv_wdt_active) {
+        if (hv_wdt_enabled) {
+            sysop("dmb ish");
+            u64 timestamp = hv_wdt_timestamp;
+            sysop("isb");
+            u64 now = mrs(CNTPCT_EL0);
+            sysop("isb");
+            if ((now - timestamp) > hv_wdt_timeout)
+                hv_wdt_bark();
+        }
+
+        udelay(1000);
+
+        sysop("dmb ish");
+    }
+}
+
+void hv_wdt_pet(void)
+{
+    hv_wdt_timestamp = mrs(CNTPCT_EL0);
+    sysop("dmb ish");
+}
+
+void hv_wdt_suspend(void)
+{
+    hv_wdt_enabled = false;
+    sysop("dsb ish");
+}
+
+void hv_wdt_resume(void)
+{
+    hv_wdt_pet();
+    hv_wdt_enabled = true;
+    sysop("dsb ish");
+}
+
+void hv_wdt_breadcrumb(char c)
+{
+    u64 tmp = hv_wdt_breadcrumbs;
+    tmp <<= 8;
+    tmp |= c;
+    hv_wdt_breadcrumbs = tmp;
+    sysop("dmb ish");
+}
+
+void hv_wdt_init(void)
+{
+    int node = adt_path_offset(adt, "/cpus/cpu0");
+    if (node < 0) {
+        printf("Error getting /cpus/cpu0 node\n");
+        return;
+    }
+
+    u64 reg[2];
+    if (ADT_GETPROP_ARRAY(adt, node, "cpu-uttdbg-reg", reg) < 0) {
+        printf("Error getting cpu-uttdbg-reg property\n");
+        return;
+    }
+
+    cpu_dbg_base = reg[0];
+}
+
+void hv_wdt_start(void)
+{
+    hv_wdt_breadcrumbs = 0;
+    hv_wdt_timeout = mrs(CNTFRQ_EL0) * WDT_TIMEOUT;
+    hv_wdt_pet();
+    hv_wdt_active = true;
+    hv_wdt_enabled = true;
+    smp_call4(WDT_CPU, hv_wdt_main, 0, 0, 0, 0);
+}
+
+void hv_wdt_stop(void)
+{
+    hv_wdt_active = false;
+    smp_wait(WDT_CPU);
+}