gxf: add support for guarded exception levels

Signed-off-by: Sven Peter <sven@svenpeter.dev>

Makefile

@@ -42,6 +42,7 @@ OBJECTS := \
 	dart.o \
 	exception.o exception_asm.o \
 	fb.o font.o font_retina.o \
+	gxf.o gxf_asm.o \
 	heapblock.o \
 	hv.o hv_vm.o hv_exc.o hv_vuart.o hv_asm.o \
 	iodev.o \

proxyclient/proxy.py

@@ -398,6 +398,8 @@ class M1N1Proxy:
     P_EL0_CALL = 0x009
     P_EL1_CALL = 0x00a
     P_VECTOR = 0x00b
+    P_GL1_CALL = 0x00c
+    P_GL2_CALL = 0x00d
 
     P_WRITE64 = 0x100
     P_WRITE32 = 0x101
@@ -602,6 +604,14 @@ class M1N1Proxy:
         if len(args) > 4:
             raise ValueError("Too many arguments")
         return self.request(self.P_EL1_CALL, addr, *args)
+    def gl1_call(self, addr, *args):
+        if len(args) > 4:
+            raise ValueError("Too many arguments")
+        return self.request(self.P_GL1_CALL, addr, *args)
+    def gl2_call(self, addr, *args):
+        if len(args) > 4:
+            raise ValueError("Too many arguments")
+        return self.request(self.P_GL2_CALL, addr, *args)
 
     def write64(self, addr, data):
         if addr & 7:

src/cpu_regs.h

@@ -123,3 +123,36 @@
 
 #define SYS_IMP_APL_UPMSR sys_reg(3, 7, 15, 6, 4)
 #define UPMSR_IACT        (BIT(0))
+
+/* SPRR and GXF registers */
+#define SYS_IMP_APL_SPRR_CONFIG_EL1 sys_reg(3, 6, 15, 1, 0)
+#define SPRR_CONFIG_EN              BIT(0)
+#define SPRR_CONFIG_LOCK_CONFIG     BIT(1)
+#define SPRR_CONFIG_LOCK_PERM_EL0   BIT(4)
+#define SPRR_CONFIG_LOCK_PERM_EL1   BIT(5)
+
+#define SYS_IMP_APL_GXF_CONFIG_EL1 sys_reg(3, 6, 15, 1, 2)
+#define GXF_CONFIG_EN              BIT(0)
+
+#define SYS_IMP_APL_GXF_STATUS sys_reg(3, 6, 15, 8, 0)
+#define GXF_STATUS_GUARDED     BIT(0)
+
+#define SYS_IMP_APL_GXF_ABORT_EL1 sys_reg(3, 6, 15, 8, 2)
+#define SYS_IMP_APL_GXF_ENTER_EL1 sys_reg(3, 6, 15, 8, 1)
+
+#define SYS_IMP_APL_GXF_ABORT_EL12 sys_reg(3, 6, 15, 15, 3)
+#define SYS_IMP_APL_GXF_ENTER_EL12 sys_reg(3, 6, 15, 15, 2)
+
+#define SYS_IMP_APL_SPRR_PERM_EL0 sys_reg(3, 6, 15, 1, 5)
+#define SYS_IMP_APL_SPRR_PERM_EL1 sys_reg(3, 6, 15, 1, 6)
+
+#define SYS_IMP_APL_TPIDR_GL1 sys_reg(3, 6, 15, 10, 1)
+#define SYS_IMP_APL_VBAR_GL1  sys_reg(3, 6, 15, 10, 2)
+#define SYS_IMP_APL_SPSR_GL1  sys_reg(3, 6, 15, 10, 3)
+#define SYS_IMP_APL_ASPSR_GL1 sys_reg(3, 6, 15, 10, 4)
+#define SYS_IMP_APL_ESR_GL1   sys_reg(3, 6, 15, 10, 5)
+#define SYS_IMP_APL_ELR_GL1   sys_reg(3, 6, 15, 10, 6)
+#define SYS_IMP_APL_FAR_GL1   sys_reg(3, 6, 15, 10, 7)
+
+#define SYS_IMP_APL_VBAR_GL12 sys_reg(3, 6, 15, 9, 2)
+#define SYS_IMP_APL_SP_GL12   sys_reg(3, 6, 15, 10, 0)

src/exception.c

@@ -2,6 +2,7 @@
 
 #include "exception.h"
 #include "cpu_regs.h"
+#include "gxf.h"
 #include "iodev.h"
 #include "uart.h"
 #include "utils.h"
@@ -28,6 +29,14 @@ static char *m_table[0x10] = {
     [0x09] = "EL2h", //
 };
 
+static char *gl_m_table[0x10] = {
+    [0x00] = "GL0t", //
+    [0x04] = "GL1t", //
+    [0x05] = "GL1h", //
+    [0x08] = "GL2t", //
+    [0x09] = "GL2h", //
+};
+
 static char *ec_table[0x40] = {
     [0x00] = "unknown",
     [0x01] = "wf*",
@@ -72,6 +81,42 @@ static char *ec_table[0x40] = {
     [0x3c] = "brk (a64)",
 };
 
+static const char *get_exception_source(int el12)
+{
+    u64 spsr = el12 ? mrs(SPSR_EL12) : mrs(SPSR_EL1);
+    u64 aspsr = in_gl12() ? mrs(SYS_IMP_APL_ASPSR_GL1) : 0;
+    const char *m_desc = NULL;
+
+    if (aspsr & 1)
+        m_desc = gl_m_table[spsr & 0xf];
+    else
+        m_desc = m_table[spsr & 0xf];
+
+    if (!m_desc)
+        m_desc = "?";
+
+    return m_desc;
+}
+
+static const char *get_exception_level(void)
+{
+    u64 lvl = mrs(CurrentEL);
+
+    if (in_gl12()) {
+        if (lvl == 0x04)
+            return "GL1";
+        else if (lvl == 0x08)
+            return "GL2";
+    } else {
+        if (lvl == 0x04)
+            return "EL1";
+        else if (lvl == 0x08)
+            return "EL2";
+    }
+
+    return "?";
+}
+
 void exception_initialize(void)
 {
     msr(VBAR_EL1, _vectors_start);
@@ -106,10 +151,8 @@ void print_regs(u64 *regs, int el12)
 
     u64 spsr = el12 ? mrs(SPSR_EL12) : mrs(SPSR_EL1);
 
-    const char *m_desc = m_table[spsr & 0xf];
+    printf("Exception taken from %s\n", get_exception_source(el12));
-    printf("Exception taken from %s\n", m_desc ? m_desc : "?");
+    printf("Running in %s\n", get_exception_level());
-
-    printf("Running in EL%lu\n", mrs(CurrentEL) >> 2);
     printf("MPIDR: 0x%lx\n", mrs(MPIDR_EL1));
     printf("Registers: (@%p)\n", regs);
     printf("  x0-x3: %016lx %016lx %016lx %016lx\n", regs[0], regs[1], regs[2], regs[3]);
@@ -127,6 +170,9 @@ void print_regs(u64 *regs, int el12)
     printf("PC:       0x%lx (rel: 0x%lx)\n", elr, elr - (u64)_base);
     printf("SP:       0x%lx\n", sp);
     printf("SPSR_EL1: 0x%lx\n", spsr);
+    if (in_gl12()) {
+        printf("ASPSR:    0x%lx\n", mrs(SYS_IMP_APL_ASPSR_GL1));
+    }
     printf("FAR_EL1:  0x%lx\n", el12 ? mrs(FAR_EL12) : mrs(FAR_EL1));
 
     const char *ec_desc = ec_table[(esr >> 26) & 0x3f];
@@ -168,7 +214,7 @@ void exc_sync(u64 *regs)
         return;
     }
 
-    if (in_el2() && (spsr & 0xf) == 5 && ((esr >> 26) & 0x3f) == 0x16) {
+    if (in_el2() && !in_gl12() && (spsr & 0xf) == 5 && ((esr >> 26) & 0x3f) == 0x16) {
         // Hypercall
         u32 imm = mrs(ESR_EL2) & 0xffff;
         switch (imm) {
@@ -241,7 +287,7 @@ void exc_irq(u64 *regs)
     ufstat = read32(0x235200018);
 #endif
 
-    uart_puts("Exception: IRQ");
+    printf("Exception: IRQ (from %s)\n", get_exception_source(0));
 
     u32 reason = read32(0x23b102004);
 
@@ -259,8 +305,7 @@ void exc_irq(u64 *regs)
 
 void exc_fiq(u64 *regs)
 {
-    const char *m_desc = m_table[mrs(SPSR_EL1) & 0xf];
+    printf("Exception: FIQ (from %s)\n", get_exception_source(0));
-    printf("Exception: FIQ (from %s)\n", m_desc ? m_desc : "?");
 
     u64 reg = mrs(CNTP_CTL_EL0);
     if (reg == 0x5) {

src/gxf.c

@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: MIT */
+
+#include "cpu_regs.h"
+#include "exception.h"
+#include "gxf.h"
+#include "memory.h"
+#include "uart.h"
+#include "utils.h"
+
+#define GL_STACK_SIZE 0x4000
+
+uint64_t gxf_enter(void *func, uint64_t a, uint64_t b, uint64_t c, uint64_t d);
+
+u8 gl1_stack[GL_STACK_SIZE] ALIGNED(64);
+void *gl1_stack_base = &gl1_stack[GL_STACK_SIZE];
+
+u8 gl2_stack[GL_STACK_SIZE] ALIGNED(64);
+void *gl2_stack_base = &gl2_stack[GL_STACK_SIZE];
+
+bool in_gl12(void)
+{
+    if (!(mrs(SYS_IMP_APL_SPRR_CONFIG_EL1) & SPRR_CONFIG_EN))
+        return false;
+    if (!(mrs(SYS_IMP_APL_GXF_CONFIG_EL1) & GXF_CONFIG_EN))
+        return false;
+    if (!(mrs(SYS_IMP_APL_GXF_STATUS) & GXF_STATUS_GUARDED))
+        return false;
+    return true;
+}
+
+static uint64_t gl_call(void *func, uint64_t a, uint64_t b, uint64_t c, uint64_t d)
+{
+    // disable the MMU first since enabling SPRR will change the meaning of all
+    // pagetable permission bits and also prevent us from having rwx pages
+    u64 mmu_state = mmu_disable();
+    u64 sprr_state = mrs(SYS_IMP_APL_SPRR_CONFIG_EL1) & SPRR_CONFIG_EN;
+    reg_set_sync(SYS_IMP_APL_SPRR_CONFIG_EL1, SPRR_CONFIG_EN);
+
+    u64 gxf_state = mrs(SYS_IMP_APL_GXF_CONFIG_EL1) & GXF_CONFIG_EN;
+    reg_set_sync(SYS_IMP_APL_GXF_CONFIG_EL1, GXF_CONFIG_EN);
+
+    uint64_t ret = gxf_enter(func, a, b, c, d);
+
+    msr_sync(SYS_IMP_APL_GXF_CONFIG_EL1, gxf_state);
+    msr_sync(SYS_IMP_APL_SPRR_CONFIG_EL1, sprr_state);
+    mmu_restore(mmu_state);
+
+    return ret;
+}
+
+uint64_t gl2_call(void *func, uint64_t a, uint64_t b, uint64_t c, uint64_t d)
+{
+    if (mrs(CurrentEL) != 0x8)
+        return -1;
+    return gl_call(func, a, b, c, d);
+}
+
+struct gl_call_argv {
+    void *func;
+    uint64_t a, b, c, d;
+};
+
+static uint64_t gl_call_wrapper(struct gl_call_argv *args)
+{
+    return gl_call(args->func, args->a, args->b, args->c, args->d);
+}
+
+uint64_t gl1_call(void *func, uint64_t a, uint64_t b, uint64_t c, uint64_t d)
+{
+    if (mrs(CurrentEL) == 0x4)
+        return gl_call(func, a, b, c, d);
+
+    struct gl_call_argv args;
+    args.func = func;
+    args.a = a;
+    args.b = b;
+    args.c = c;
+    args.d = d;
+
+    // enable EL1 here since once GXF has been enabled HCR_EL2 writes are only possible from GL2
+    reg_clr(HCR_EL2, BIT(27));
+
+    u64 mmu_state = mmu_disable();
+    u64 sprr_state = mrs(SYS_IMP_APL_SPRR_CONFIG_EL1) & SPRR_CONFIG_EN;
+    reg_set_sync(SYS_IMP_APL_SPRR_CONFIG_EL1, SPRR_CONFIG_EN);
+
+    u64 gxf_state = mrs(SYS_IMP_APL_GXF_CONFIG_EL1) & GXF_CONFIG_EN;
+    reg_set_sync(SYS_IMP_APL_GXF_CONFIG_EL1, GXF_CONFIG_EN);
+
+    uint64_t ret = el1_call(gl_call_wrapper, (uint64_t)&args, 0, 0, 0);
+
+    msr_sync(SYS_IMP_APL_GXF_CONFIG_EL1, gxf_state);
+    msr_sync(SYS_IMP_APL_SPRR_CONFIG_EL1, sprr_state);
+    mmu_restore(mmu_state);
+
+    return ret;
+}

src/gxf.h

@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: MIT */
+
+#ifndef __GXF_H__
+#define __GXF_H__
+
+#include "types.h"
+
+bool in_gl12(void);
+void gxf_init(void);
+
+uint64_t gl1_call(void *func, uint64_t a, uint64_t b, uint64_t c, uint64_t d);
+uint64_t gl2_call(void *func, uint64_t a, uint64_t b, uint64_t c, uint64_t d);
+
+#endif

src/gxf_asm.S

@@ -0,0 +1,238 @@
+/* SPDX-License-Identifier: MIT */
+
+#include "cpu_regs.h"
+
+#define genter .long 0x00201420
+#define gexit .long 0x00201400
+
+.global gxf_init
+.type gxf_init, @function
+gxf_init:
+    str x30, [sp, #-16]!
+    mov x0, 1
+    msr SYS_IMP_APL_SPRR_CONFIG_EL1, x0
+    isb
+    msr SYS_IMP_APL_GXF_CONFIG_EL1, x0
+    isb
+    ldr x0, =_gxf_setup
+    msr SYS_IMP_APL_GXF_ENTER_EL1, x0
+    isb
+    genter
+    msr SYS_IMP_APL_GXF_CONFIG_EL1, xzr
+    isb
+    msr SYS_IMP_APL_SPRR_CONFIG_EL1, xzr
+    isb
+    ldr x30, [sp], #16
+    ret
+
+.globl gxf_enter
+.type gxf_enter, @function
+gxf_enter:
+    genter
+    ret  
+
+_gxf_setup:
+    ldr x0, =gl2_stack_base
+    ldr x0, [x0]
+    mov sp, x0
+    ldr x1, =_gxf_vectors
+    ldr x2, =_gxf_exc_sync
+    ldr x3, =_gxf_entry
+    msr SYS_IMP_APL_VBAR_GL1, x1
+    msr SYS_IMP_APL_GXF_ABORT_EL1, x2
+    msr SYS_IMP_APL_GXF_ENTER_EL1, x3
+
+    mrs x4, CurrentEL
+    cmp x4, #8
+    bne 1f
+
+    ldr x0, =gl1_stack_base
+    ldr x0, [x0]
+    msr SYS_IMP_APL_SP_GL12, x0
+    msr SYS_IMP_APL_VBAR_GL12, x1
+    msr SYS_IMP_APL_GXF_ABORT_EL12, x2
+    msr SYS_IMP_APL_GXF_ENTER_EL12, x3
+
+    1:
+    isb
+    gexit
+
+_gxf_entry:
+    stp x29, x30, [sp, #-16]!
+    stp x23, x24, [sp, #-16]!
+    stp x21, x22, [sp, #-16]!
+    stp x19, x20, [sp, #-16]!
+
+    // these registers would be overwritten by each exception happening in GL1/2
+    // but we need them to gexit correctly again
+    mrs x20, SYS_IMP_APL_SPSR_GL1
+    mrs x21, SYS_IMP_APL_ASPSR_GL1
+    mrs x22, SYS_IMP_APL_ESR_GL1
+    mrs x23, SYS_IMP_APL_ELR_GL1
+    mrs x24, SYS_IMP_APL_FAR_GL1
+
+    mov x5, x0
+    mov x0, x1
+    mov x1, x2
+    mov x2, x3
+    mov x3, x4
+
+    blr x5
+
+    msr SYS_IMP_APL_SPSR_GL1, x20
+    msr SYS_IMP_APL_ASPSR_GL1, x21
+    msr SYS_IMP_APL_ESR_GL1, x22
+    msr SYS_IMP_APL_ELR_GL1, x23
+    msr SYS_IMP_APL_FAR_GL1, x24
+
+    ldp x19, x20, [sp], #16
+    ldp x21, x22, [sp], #16
+    ldp x23, x24, [sp], #16
+    ldp x29, x30, [sp], #16
+
+    isb
+    gexit
+
+.align 11
+_gxf_vectors:
+    mov x9, '0'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, '1'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, '2'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, '3'
+    b _gxf_exc_unk
+    .align 7
+    b _gxf_exc_sync
+    .align 7
+    mov x9, '5'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, '6'
+    b _gxf_exc_unk
+    .align 7
+    b _gxf_serr
+    .align 7
+    b _gxf_exc_sync
+    .align 7
+    mov x9, '9'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, 'a'
+    b _gxf_exc_unk
+    .align 7
+    b _gxf_serr
+    .align 7
+    mov x9, 'c'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, 'd'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, 'e'
+    b _gxf_exc_unk
+    .align 7
+    mov x9, 'f'
+    b _gxf_exc_unk
+    .align 7
+
+_gxf_exc_sync:
+    str x30, [sp, #-16]!
+    bl _gxf_exc_entry
+    bl exc_sync
+    b _gxf_exc_return
+
+_gxf_serr:
+    str x30, [sp, #-16]!
+    bl _gxf_exc_entry
+    bl exc_serr
+    b _gxf_exc_return
+
+_gxf_exc_entry:
+    stp x28, x29, [sp, #-16]!
+    stp x26, x27, [sp, #-16]!
+    stp x24, x25, [sp, #-16]!
+    stp x22, x23, [sp, #-16]!
+    stp x20, x21, [sp, #-16]!
+    stp x18, x19, [sp, #-16]!
+    stp x16, x17, [sp, #-16]!
+    stp x14, x15, [sp, #-16]!
+    stp x12, x13, [sp, #-16]!
+    stp x10, x11, [sp, #-16]!
+    stp x8, x9, [sp, #-16]!
+    stp x6, x7, [sp, #-16]!
+    stp x4, x5, [sp, #-16]!
+    stp x2, x3, [sp, #-16]!
+    stp x0, x1, [sp, #-16]!
+
+    mov x0, sp
+
+    mrs x1, SYS_IMP_APL_SPSR_GL1
+    msr SPSR_EL1, x1
+    mrs x1, SYS_IMP_APL_ELR_GL1
+    msr ELR_EL1, x1
+    mrs x1, SYS_IMP_APL_ESR_GL1
+    msr ESR_EL1, x1
+    mrs x1, SYS_IMP_APL_FAR_GL1
+    msr FAR_EL1, x1
+
+    ret
+
+_gxf_exc_return:
+    mrs x0, SPSR_EL1
+    msr SYS_IMP_APL_SPSR_GL1, x0
+    mrs x0, ELR_EL1
+    msr SYS_IMP_APL_ELR_GL1, x0
+
+    ldp x0, x1, [sp], #16
+    ldp x2, x3, [sp], #16
+    ldp x4, x5, [sp], #16
+    ldp x6, x7, [sp], #16
+    ldp x8, x9, [sp], #16
+    ldp x10, x11, [sp], #16
+    ldp x12, x13, [sp], #16
+    ldp x14, x15, [sp], #16
+    ldp x16, x17, [sp], #16
+    ldp x18, x19, [sp], #16
+    ldp x20, x21, [sp], #16
+    ldp x22, x23, [sp], #16
+    ldp x24, x25, [sp], #16
+    ldp x26, x27, [sp], #16
+    ldp x28, x29, [sp], #16
+    ldr x30, [sp], #16
+    isb
+
+    gexit
+
+_gxf_exc_unk:
+    mov w0, 0xd /* '\r', clang compat */
+    bl debug_putc
+    mov w0, '\n'
+    bl debug_putc
+    mov w0, '!'
+    bl debug_putc
+    mov w0, 'G'
+    bl debug_putc
+    mov w0, 'L'
+    bl debug_putc
+    mov w0, 'E'
+    bl debug_putc
+    mov w0, 'X'
+    bl debug_putc
+    mov w0, 'C'
+    bl debug_putc
+    mov w0, ':'
+    bl debug_putc
+    mov w0, w9
+    bl debug_putc
+    mov w0, '!'
+    bl debug_putc
+    mov w0, 0xd /* '\r', clang compat */
+    bl debug_putc
+    mov w0, '\n'
+    bl debug_putc
+    b reboot

src/proxy.c

@@ -4,6 +4,7 @@
 #include "dart.h"
 #include "exception.h"
 #include "fb.h"
+#include "gxf.h"
 #include "heapblock.h"
 #include "hv.h"
 #include "iodev.h"
@@ -80,6 +81,14 @@ int proxy_process(ProxyRequest *request, ProxyReply *reply)
             reply->retval = el1_call((void *)request->args[0], request->args[1], request->args[2],
                                      request->args[3], request->args[4]);
             break;
+        case P_GL1_CALL:
+            reply->retval = gl1_call((void *)request->args[0], request->args[1], request->args[2],
+                                     request->args[3], request->args[4]);
+            break;
+        case P_GL2_CALL:
+            reply->retval = gl2_call((void *)request->args[0], request->args[1], request->args[2],
+                                     request->args[3], request->args[4]);
+            break;
         case P_VECTOR:
             next_stage.entry = (generic_func *)request->args[0];
             memcpy(next_stage.args, &request->args[1], 4 * sizeof(u64));

src/proxy.h

@@ -18,6 +18,8 @@ typedef enum {
     P_EL0_CALL,
     P_EL1_CALL,
     P_VECTOR,
+    P_GL1_CALL,
+    P_GL2_CALL,
 
     P_WRITE64 = 0x100, // Generic register functions
     P_WRITE32,

src/startup.c

@@ -2,6 +2,7 @@
 
 #include "chickens.h"
 #include "exception.h"
+#include "gxf.h"
 #include "smp.h"
 #include "string.h"
 #include "types.h"
@@ -91,6 +92,7 @@ void _start_c(void *boot_args, void *base)
         (void *)(((u64)cur_boot_args.devtree) - cur_boot_args.virt_base + cur_boot_args.phys_base);
 
     exception_initialize();
+    gxf_init();
     m1n1_main();
 }
 

tools/gen_reg_include.py

@@ -1,10 +1,21 @@
 import json, sys
+import argparse
 
-data = json.load(open(sys.argv[1]))
+parser = argparse.ArgumentParser()
+parser.add_argument("--imp-apl-prefix", action="store_true")
+parser.add_argument("regfile")
+args = parser.parse_args()
+
+if args.imp_apl_prefix:
+    prefix = "IMP_APL_"
+else:
+    prefix = ""
+
+data = json.load(open(args.regfile))
 for reg in data:
     name = reg['name']
 
-    print(f"#define SYS_{name} sys_reg({', '.join(str(i) for i in reg['enc'])})")
+    print(f"#define SYS_{prefix}{name} sys_reg({', '.join(str(i) for i in reg['enc'])})")
 
     if name[-4:-1] == "_EL":
         name = name[:-4]