hv: Make breadcrumbs per-CPU and introduce panic handler

Signed-off-by: Hector Martin <marcan@marcan.st>

proxyclient/m1n1/hv/__init__.py

@@ -1383,6 +1383,7 @@ class HV(Reloadable):
         self.iface.set_handler(START.HV, HV_EVENT.WDT_BARK, self.handle_bark)
         self.iface.set_handler(START.HV, HV_EVENT.CPU_SWITCH, self.handle_exception)
         self.iface.set_handler(START.HV, HV_EVENT.VIRTIO, self.handle_virtio)
+        self.iface.set_handler(START.HV, HV_EVENT.PANIC, self.handle_bark)
         self.iface.set_event_handler(EVENT.MMIOTRACE, self.handle_mmiotrace)
         self.iface.set_event_handler(EVENT.IRQTRACE, self.handle_irqtrace)
 

proxyclient/m1n1/hv/types.py

@@ -38,6 +38,7 @@ class HV_EVENT(IntEnum):
     WDT_BARK = 4
     CPU_SWITCH = 5
     VIRTIO = 6
+    PANIC = 7
 
 VMProxyHookData = Struct(
     "flags" / RegAdapter(MMIOTraceFlags),

src/hv.c

@@ -263,8 +263,8 @@ void hv_rendezvous(void)
             return;
     }
 
-    panic("HV: Failed to rendezvous, missing CPUs: 0x%lx\n",
-          __atomic_load_n(&hv_cpus_in_guest, __ATOMIC_ACQUIRE));
+    hv_panic("HV: Failed to rendezvous, missing CPUs: 0x%lx (current: %d)\n",
+             __atomic_load_n(&hv_cpus_in_guest, __ATOMIC_ACQUIRE), smp_id());
 }
 
 bool hv_switch_cpu(int cpu)
@@ -369,6 +369,8 @@ void hv_tick(struct exc_info *ctx)
     hv_wdt_pet();
     iodev_handle_events(uartproxy_iodev);
     if (iodev_can_read(uartproxy_iodev)) {
+        printf("HV: User interrupt\n");
+        iodev_console_flush();
         if (hv_pinned_cpu == -1 || hv_pinned_cpu == smp_id())
             hv_exc_proxy(ctx, START_HV, HV_USER_INTERRUPT, NULL);
     }

src/hv.h

@@ -48,6 +48,7 @@ typedef enum _hv_entry_type {
     HV_WDT_BARK,
     HV_CPU_SWITCH,
     HV_VIRTIO,
+    HV_PANIC,
 } hv_entry_type;
 
 /* VM */
@@ -87,6 +88,14 @@ void hv_wdt_init(void);
 void hv_wdt_start(int cpu);
 void hv_wdt_stop(void);
 void hv_wdt_breadcrumb(char c);
+void hv_do_panic(void);
+
+#define hv_panic(fmt, ...)                                                                         \
+    do {                                                                                           \
+        debug_printf("HV panic:" fmt, ##__VA_ARGS__);                                              \
+        hv_do_panic();                                                                             \
+        flush_and_reboot();                                                                        \
+    } while (0)
 
 /* Utilities */
 void hv_write_hcr(u64 val);

src/hv_wdt.c

@@ -3,6 +3,7 @@
 #include "hv.h"
 #include "adt.h"
 #include "smp.h"
+#include "string.h"
 #include "uart.h"
 #include "utils.h"
 
@@ -12,24 +13,60 @@ static bool hv_wdt_active = false;
 static bool hv_wdt_enabled = false;
 static volatile u64 hv_wdt_timestamp = 0;
 static u64 hv_wdt_timeout = 0;
-static volatile u64 hv_wdt_breadcrumbs;
+static volatile u64 hv_wdt_breadcrumbs[MAX_CPUS] = {0};
 
 static int hv_wdt_cpu;
 
 static u64 cpu_dbg_base = 0;
 
+void hv_do_panic(void)
+{
+    printf("Breadcrumbs:\n");
+    for (int cpu = 0; cpu < MAX_CPUS; cpu++) {
+        if (cpu > 0 && !smp_is_alive(cpu))
+            continue;
+
+        u64 tmp = hv_wdt_breadcrumbs[cpu];
+
+        printf("CPU %2d: ", cpu);
+        for (int i = 56; i >= 0; i -= 8) {
+            char c = (tmp >> i) & 0xff;
+            if (c)
+                printf("%c", c);
+        }
+        printf("\n");
+    }
+
+    printf("Attempting to enter proxy\n");
+    iodev_console_flush();
+
+    struct uartproxy_msg_start start = {
+        .reason = START_HV,
+        .code = HV_WDT_BARK,
+    };
+
+    uartproxy_run(&start);
+}
+
 void hv_wdt_bark(void)
 {
-    u64 tmp = hv_wdt_breadcrumbs;
     uart_puts("HV watchdog: bark!");
 
     uart_printf("Breadcrumbs: ");
-    for (int i = 56; i >= 0; i -= 8) {
-        char c = (tmp >> i) & 0xff;
-        if (c)
-            uart_putchar(c);
+    for (int cpu = 0; cpu < MAX_CPUS; cpu++) {
+        if (cpu > 0 && !smp_is_alive(cpu))
+            continue;
+
+        u64 tmp = hv_wdt_breadcrumbs[cpu];
+
+        uart_printf("CPU %2d: ", cpu);
+        for (int i = 56; i >= 0; i -= 8) {
+            char c = (tmp >> i) & 0xff;
+            if (c)
+                uart_putchar(c);
+        }
+        uart_putchar('\n');
     }
-    uart_putchar('\n');
 
     uart_puts("Attempting to enter proxy");
 
@@ -82,10 +119,11 @@ void hv_wdt_resume(void)
 
 void hv_wdt_breadcrumb(char c)
 {
-    u64 tmp = hv_wdt_breadcrumbs;
+    u64 cpu = mrs(TPIDR_EL2);
+    u64 tmp = hv_wdt_breadcrumbs[cpu];
     tmp <<= 8;
     tmp |= c;
-    hv_wdt_breadcrumbs = tmp;
+    hv_wdt_breadcrumbs[cpu] = tmp;
     sysop("dmb ish");
 }
 
@@ -112,7 +150,7 @@ void hv_wdt_start(int cpu)
         return;
 
     hv_wdt_cpu = cpu;
-    hv_wdt_breadcrumbs = 0;
+    memset((void *)hv_wdt_breadcrumbs, 0, sizeof(hv_wdt_breadcrumbs));
     hv_wdt_timeout = mrs(CNTFRQ_EL0) * WDT_TIMEOUT;
     hv_wdt_pet();
     hv_wdt_active = true;