| // Copyright Microsoft and CHERIoT Contributors. |
| // SPDX-License-Identifier: MIT |
| |
| #include "export-table-assembly.h" |
| #include "trusted-stack-assembly.h" |
| #include "misc-assembly.h" |
| #include <errno.h> |
| |
| .include "assembly-helpers.s" |
| |
| # Symbolic names for the stack high water mark registers until |
| # the assembler knows about them. |
| |
| /** |
| * Machine-mode stack high water mark CSR |
| */ |
| #define CSR_MSHWM 0xbc1 |
| /** |
| * Machine mode stack high water mark stack base CSR |
| */ |
| #define CSR_MSHWMB 0xbc2 |
| |
| #define MAX_FAULTS_PER_COMPARTMENT_CALL 1024 |
| |
| #define SPILL_SLOT_cs0 0 |
| #define SPILL_SLOT_cs1 8 |
| #define SPILL_SLOT_cgp 16 |
| #define SPILL_SLOT_pcc 24 |
| #define SPILL_SLOT_SIZE 32 |
| |
| /* |
| * The switcher uniformly speaks of registers using their RISC-V ELF psABI names |
| * and not their raw index, as, broadly speaking, we use registers in a similar |
| * way to C functions. However, it's probably convenient to have a mapping |
| * readily accessible, so here 'tis: |
| * |
| * # x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 x15 |
| * psABI zero ra sp gp tp t0 t1 t2 s0 s1 a0 a1 a2 a3 a4 a5 |
| * |
| * When we use the psABI name without a 'c' prefix, we are sometimes meaning to |
| * refer to the address component of the capability. |
| * |
| * Despite the use of psABI names and conformance at the interface (argument |
| * registers used for arguments, return address register used for its canonical |
| * purpose, &c), one should not read too much of the psABI calling convention |
| * into the code here. Within the switcher, the machine is a raw register |
| * machine and C is a distant, high-level language. |
| */ |
| |
| switcher_code_start: |
| |
| # Global for the sealing key. Stored in the switcher's code section. |
| .section .text, "ax", @progbits |
| .globl compartment_switcher_sealing_key |
| .p2align 3 |
| compartment_switcher_sealing_key: |
| .long 0 |
| .long 0 |
| # Global for the scheduler's PCC. Stored in the switcher's code section. |
| .section .text, "ax", @progbits |
| .globl switcher_scheduler_entry_pcc |
| .p2align 3 |
| switcher_scheduler_entry_pcc: |
| .long 0 |
| .long 0 |
| # Global for the scheduler's CGP. Stored in the switcher's code section. |
| .section .text, "ax", @progbits |
| .globl switcher_scheduler_entry_cgp |
| .p2align 3 |
| switcher_scheduler_entry_cgp: |
| .long 0 |
| .long 0 |
| # Global for the scheduler's CSP. Stored in the switcher's code section. |
| .section .text, "ax", @progbits |
| .globl switcher_scheduler_entry_csp |
| .p2align 2 |
| switcher_scheduler_entry_csp: |
| .long 0 |
| .long 0 |
| |
| /** |
| * Copy a register context from `src` to `dst` using `scratch` as the register |
| * to hold loaded capabilities and `counter` as the register to hold the loop |
| * counter. All four registers are clobbered by this macro. |
| */ |
| .macro copyContext dst, src, scratch, counter |
| addi \counter, zero, 15 |
| 1: |
| clc \scratch, 0(\src) |
| csc \scratch, 0(\dst) |
| addi \counter, \counter, -1 |
| cincoffset \dst, \dst, 8 |
| cincoffset \src, \src, 8 |
| bnez \counter, 1b |
| .endm |
| |
| /// Spill a single register to a trusted stack pointed to by csp. |
| .macro spillOne, reg |
| csc \reg, TrustedStack_offset_\reg(csp) |
| .endm |
| |
| /** |
| * Spill all of the registers in the list (in order) to a trusted stack pointed |
| * to by csp. |
| */ |
| .macro spillRegisters reg1, regs:vararg |
| forall spillOne, \reg1, \regs |
| .endm |
| |
| /// Reload a single register from a trusted stack pointed to by csp. |
| .macro reloadOne, reg |
| clc \reg, TrustedStack_offset_\reg(csp) |
| .endm |
| |
| /** |
| * Reload all of the registers in the list (in order) to a trusted stack pointed |
| * to by csp. |
| */ |
| .macro reloadRegisters reg1, regs:vararg |
| forall reloadOne, \reg1, \regs |
| .endm |
| |
| /** |
| * Verify the compartment stack is valid, with the expected permissions and |
| * unsealed. |
| * This macro assumes t2 and tp are available to use. |
| */ |
| .macro check_compartment_stack_integrity reg |
| // Check that the caller's CSP is a tagged, unsealed capability (with at |
| // least load permission - we'll check the other permissions properly |
| // later) by loading a byte. If this doesn't work, we'll fall off this |
| // path into the exception handler and force unwind. |
| clb t2, 0(\reg) |
| // make sure the caller's CSP has the expected permissions |
| cgetperm t2, \reg |
| li tp, COMPARTMENT_STACK_PERMISSIONS |
| bne tp, t2, .Lforce_unwind |
| // Check that the top and base are 16-byte aligned |
| cgetbase t2, csp |
| or t2, t2, sp |
| andi t2, t2, 0xf |
| bnez t2, .Lforce_unwind |
| .endm |
| |
| /** |
| * Zero the stack. The three operands are the base address, the top address, |
| * and a scratch register to use. The base must be a capability but it must |
| * be provided without the c prefix because it is used as both a capability |
| * and integer register. All three registers are clobbered. |
| */ |
| .macro zero_stack base top scratch |
| addi \scratch, \top, -32 |
| addi \top, \top, -16 |
| bgt \base, \scratch, 1f |
| // Zero the stack in 32-byte chunks |
| 0: |
| csc cnull, 0(c\base) |
| csc cnull, 8(c\base) |
| csc cnull, 16(c\base) |
| csc cnull, 24(c\base) |
| cincoffset c\base, c\base, 32 |
| ble \base, \scratch, 0b |
| 1: |
| bgt \base, \top, 2f |
| // Zero any 16-byte tail |
| csc cnull, 0(c\base) |
| csc cnull, 8(c\base) |
| 2: |
| .endm |
| |
| /** |
| * Clear the hazard pointers associated with this thread. We don't care about |
| * leaks here (they're store-only from anywhere except the allocator), so just |
| * write a 32-bit zero over half of each one to clobber the tags. |
| */ |
| .macro clear_hazard_slots trustedStack, scratch |
| clc \scratch, TrustedStack_offset_hazardPointers(\trustedStack) |
| csw zero, 0(\scratch) |
| csw zero, 8(\scratch) |
| .endm |
| |
| .section .text, "ax", @progbits |
| .globl __Z26compartment_switcher_entryz |
| .p2align 2 |
| .type __Z26compartment_switcher_entryz,@function |
| __Z26compartment_switcher_entryz: |
| /* |
| * Spill caller-save registers carefully. If we find ourselves unable to do |
| * so, we'll return an error to the caller (via the exception path; see |
| * .Lhandle_error_in_switcher). The error handling path assumes that the |
| * first spill is to the lowest address and guaranteed to trap if any would. |
| */ |
| cincoffset ct2, csp, -SPILL_SLOT_SIZE |
| .Lswitcher_entry_first_spill: |
| csc cs0, SPILL_SLOT_cs0(ct2) |
| csc cs1, SPILL_SLOT_cs1(ct2) |
| csc cgp, SPILL_SLOT_cgp(ct2) |
| csc cra, SPILL_SLOT_pcc(ct2) |
| cmove csp, ct2 |
| // before we access any privileged state, we can verify the |
| // compartment's csp is valid. If not, force unwind. |
| // Note that this check is purely to protect the callee, not the switcher |
| // itself. |
| check_compartment_stack_integrity csp |
| // The caller should back up all callee saved registers. |
| // mtdc should always have an offset of 0. |
| cspecialr ct2, mtdc |
| #ifndef NDEBUG |
| // XXX: This line is useless, only for mtdc to show up in debugging. |
| cmove ct2, ct2 |
| #endif |
| clear_hazard_slots ct2, ctp |
| |
| // make sure the trusted stack is still in bounds |
| clhu tp, TrustedStack_offset_frameoffset(ct2) |
| cgetlen t2, ct2 |
| bgeu tp, t2, .Lout_of_trusted_stack |
| // we are past the stacks checks. Reload ct2; tp is still as it was |
| cspecialr ct2, mtdc |
| // ctp points to the current available trusted stack frame. |
| cincoffset ctp, ct2, tp |
| csc csp, TrustedStackFrame_offset_csp(ctp) |
| // We have just entered this call, so no faults triggered during this call |
| // yet. |
| csh zero, TrustedStackFrame_offset_errorHandlerCount(ctp) |
| // For now, store a null export entry so that we don't ever try to pass |
| // switcher state to an error handler. |
| csc cnull, TrustedStackFrame_offset_calleeExportTable(ctp) |
| clhu s1, TrustedStack_offset_frameoffset(ct2) |
| addi s1, s1, TrustedStackFrame_size |
| // Update the frame offset. |
| // Any fault before this point (wrong target cap, unaligned stack, etc.) is |
| // seen as a fault in the caller. From this point after writing the new |
| // tstack offset, any fault is seen as a callee fault. With a null export |
| // table entry on the trusted stack, a fault here will cause a forced |
| // unwind until we set the correct one. |
| csh s1, TrustedStack_offset_frameoffset(ct2) |
| // Chop off the stack. |
| cgetaddr s0, csp |
| cgetbase s1, csp |
| csetaddr csp, csp, s1 |
| sub s1, s0, s1 |
| csetboundsexact ct2, csp, s1 |
| csetaddr csp, ct2, s0 |
| #ifdef CONFIG_MSHWM |
| // Read the stack high water mark (which is 16-byte aligned) |
| csrr gp, CSR_MSHWM |
| // Skip zeroing if high water mark >= stack pointer |
| bge gp, sp, .Lafter_zero |
| // Use stack high water mark as base address for zeroing. If this faults |
| // then it will trigger a force unwind. This can happen only if the caller |
| // is doing something bad. |
| csetaddr ct2, csp, gp |
| #endif |
| zero_stack t2, s0, gp |
| .Lafter_zero: |
| |
| // Fetch the sealing key |
| LoadCapPCC cs0, compartment_switcher_sealing_key |
| li gp, SEAL_TYPE_SealedImportTableEntries |
| csetaddr cs0, cs0, gp |
| // The target capability is in ct1. Unseal, check tag and load the entry point offset. |
| cunseal ct1, ct1, cs0 |
| // Load the entry point offset. If cunseal failed then this will fault and |
| // we will force unwind. |
| clhu s0, ExportEntry_offset_functionStart(ct1) |
| // At this point, we know that the cunseal has succeeded (we didn't trap on |
| // the load) and so it's safe to store the unsealed value of the export |
| // table pointer. Nothing between this point and transition to the callee |
| // should fault. |
| csc ct1, TrustedStackFrame_offset_calleeExportTable(ctp) |
| |
| // Load the minimum stack size required by the callee. |
| clbu tp, ExportEntry_offset_minimumStackSize(ct1) |
| // The stack size is in 8-byte units, so multiply by 8. |
| slli tp, tp, 3 |
| // Check that the stack is large enough for the callee. |
| // At this point, we have already truncated the stack and so the length of |
| // the stack is the length that the callee can use. |
| cgetlen t2, csp |
| /* |
| * Include the space we reserved for the unwind state. |
| * |
| * tp holds the number of required stack bytes, a value between 0 and 0x7F8 |
| * (the result of an unsigned byte load left shifted by 3). Given this |
| * extremely limited range, adding STACK_ENTRY_RESERVED_SPACE will not cause |
| * overflow (while instead subtracting it from the available length, in t2, |
| * might underflow). |
| */ |
| addi tp, tp, STACK_ENTRY_RESERVED_SPACE |
| bgtu tp, t2, .Lstack_too_small |
| |
| // Reserve space for unwind state and so on. |
| cincoffset csp, csp, -STACK_ENTRY_RESERVED_SPACE |
| #ifdef CONFIG_MSHWM |
| // store new stack top as stack high water mark |
| csrw CSR_MSHWM, sp |
| #endif |
| |
| // Get the flags field into tp |
| clbu tp, ExportEntry_offset_flags(ct1) |
| cgetbase s1, ct1 |
| csetaddr ct1, ct1, s1 |
| // Load the target CGP |
| clc cgp, ExportTable_offset_cgp(ct1) |
| // Load the target PCC and point to the function. |
| clc cra, ExportTable_offset_pcc(ct1) |
| cincoffset cra, cra, s0 |
| // Get the number of registers to zero in t2 |
| andi t2, tp, 0x7 |
| // Get the interrupt-disable bit in t1 |
| andi t1, tp, 0x10 |
| // Zero any unused argument registers |
| // The low 3 bits of the flags field contain the number of arguments to |
| // pass. We create a small sled that zeroes them and jump into the middle |
| // of it at an offset defined by the number of registers that the export |
| // entry told us to pass. |
| .Lload_zero_arguments_start: |
| auipcc cs0, %cheriot_compartment_hi(.Lzero_arguments_start) |
| cincoffset cs0, cs0, %cheriot_compartment_lo_i(.Lload_zero_arguments_start) |
| // Change from the number of registers to pass into the number of 2-byte |
| // instructions to skip. |
| sll t2, t2, 1 |
| // Offset the jump target by the number of registers that we should be |
| // passing. |
| cincoffset cs0, cs0, t2 |
| // Jump into the sled. |
| cjr cs0 |
| .Lzero_arguments_start: |
| zeroRegisters a0, a1, a2, a3, a4, a5, t0 |
| // Enable interrupts of the interrupt-disable bit is not set in flags |
| bnez t1, .Lskip_interrupt_disable |
| csrsi mstatus, 0x8 |
| .Lskip_interrupt_disable: |
| // Registers passed to the callee are: |
| // cra (c1), csp (c2), and cgp (c3) are passed unconditionally. |
| // ca0-ca5 (c10-c15) and ct0 (c5) are either passed as arguments or cleared |
| // above. This should add up to 10 registers, with the remaining 5 being |
| // cleared now: |
| zeroRegisters tp, t1, t2, s0, s1 |
| cjalr cra |
| |
| .globl switcher_skip_compartment_call |
| switcher_skip_compartment_call: |
| // If we are doing a forced unwind of the trusted stack then we do almost |
| // exactly the same as a normal unwind. We will jump here from the |
| // exception path (.Lforce_unwind) |
| |
| /* |
| * Pop a frame from the trusted stack, leaving all registers in the state |
| * expected by the caller of a cross-compartment call. The callee is |
| * responsible for zeroing argument and temporary registers. |
| * |
| * The below should not fault before returning back to the caller. If a |
| * fault occurs there must be a serious bug elsewhere. |
| */ |
| |
| cspecialr ctp, mtdc |
| clear_hazard_slots ctp, ct2 |
| // make sure there is a frame left in the trusted stack |
| clhu t2, TrustedStack_offset_frameoffset(ctp) |
| li tp, TrustedStack_offset_frames |
| // Move to the previous trusted stack frame. |
| addi t2, t2, -TrustedStackFrame_size |
| // If this is the first trusted stack frame, then the csp that we would be |
| // loading is the csp on entry, which does not have a spilled area. In |
| // this case, we would fault when loading, so would exit the thread, but we |
| // should instead gracefully exit the thread. |
| bgeu tp, t2, .Lcommon_defer_irqs_and_thread_exit |
| cspecialr ctp, mtdc |
| cincoffset ct1, ctp, t2 |
| // Restore the stack pointer. All other spilled values are spilled there. |
| clc csp, TrustedStackFrame_offset_csp(ct1) |
| // Update the current frame offset. |
| csh t2, TrustedStack_offset_frameoffset(ctp) |
| // Do the loads *after* moving the trusted stack pointer. In theory, the |
| // checks in `check_compartment_stack_integrity` make it impossible for |
| // this to fault, but if we do fault here then we'd end up in an infinite |
| // loop trying repeatedly to pop the same trusted stack frame. This would |
| // be bad. Instead, we move the trusted stack pointer *first* and so, if |
| // the accesses to the untrusted stack fault, we will detect a fault in the |
| // switcher, enter the force-unwind path, and pop the frame for the |
| // compartment that gave us a malicious csp. |
| clc cs0, SPILL_SLOT_cs0(csp) |
| clc cs1, SPILL_SLOT_cs1(csp) |
| clc cra, SPILL_SLOT_pcc(csp) |
| clc cgp, SPILL_SLOT_cgp(csp) |
| cincoffset csp, csp, SPILL_SLOT_SIZE |
| #ifdef CONFIG_MSHWM |
| // read the stack high water mark, which is 16-byte aligned |
| // we will use this as base address for stack clearing |
| // note that it cannot be greater than stack top as we |
| // we set it to stack top when we pushed to trusted stack frame |
| csrr tp, CSR_MSHWM |
| #else |
| cgetbase tp, csp |
| #endif |
| cgetaddr t1, csp |
| csetaddr ct2, csp, tp |
| zero_stack t2, t1, tp |
| #ifdef CONFIG_MSHWM |
| csrw CSR_MSHWM, sp |
| #endif |
| |
| // Zero all registers apart from RA, GP, SP and return args. |
| // cra, csp and cgp needed for the compartment |
| // cs0 saved and restored on trusted stack |
| // cs1 saved and restored on trusted stack |
| // ca0, used for first return value |
| // ca1, used for second return value |
| zeroAllRegistersExcept ra, sp, gp, s0, s1, a0, a1 |
| .Ljust_return: |
| cret |
| |
| // If the stack is too small, we don't do the call, but to avoid leaking |
| // any other state we still go through the same return path as normal. We |
| // set the return registers to -ENOTENOUGHSTACK and 0, so users can see |
| // that this is the failure reason. |
| .Lstack_too_small: |
| li a0, -ENOTENOUGHSTACK |
| li a1, 0 |
| j switcher_skip_compartment_call |
| .size compartment_switcher_entry, . - compartment_switcher_entry |
| |
| // the entry point of all exceptions and interrupts |
| // For now, the entire routine is run with interrupts disabled. |
| .global exception_entry_asm |
| .p2align 2 |
| exception_entry_asm: |
| // We do not trust the interruptee's context. We cannot use its stack in any way. |
| // The save reg frame we can use is fetched from the tStack. |
| // In general, mtdc holds the trusted stack register. We are here with |
| // interrupts off and precious few registers available to us, so swap it |
| // with the csp (we'll put it back, later). |
| cspecialrw csp, mtdc, csp |
| #ifndef NDEBUG |
| // XXX: This move is useless, but just for debugging in the simulator. |
| cmove csp, csp |
| #endif |
| |
| // If we read out zero, we've reentered the exception and are about to |
| // trap. Make sure that we end up in an architectural trap loop: clobber |
| // mtcc, so that trapping attempts to vector to an untagged PCC, thereby |
| // causing another (i.e., a third) trap in spillRegisters, below. |
| // |
| // While that's a good start, it does not guarantee that we end up in a |
| // trap loop: the reentry will probably have put something non-zero into |
| // mtdc, so we wouldn't hit this, and wouldn't loop, when we take that |
| // third trap. (Exactly what we'd do instead is hard to say; we'd try |
| // spilling registers to an attacker-controlled pointer, at the very |
| // least.) Therefore, clobber mtcc (!) to ensure that the certainly |
| // upcoming third trap puts us in an architectural trap loop. This is |
| // slightly preferable to clearing mtdc, which would also ensure that we |
| // looped, because the architectural loop is tighter and involves no |
| // program text, making it easier for microarchitecture to detect. |
| bnez sp, .Lexception_entry_still_alive |
| cspecialw mtcc, csp |
| .Lexception_entry_still_alive: |
| |
| // csp now points to the save reg frame that we can use. |
| // The guest csp (c2) is now in mtdc. Will be spilled later, but we |
| // spill all the other 14 registers now. |
| spillRegisters cra, cgp, ctp, ct0, ct1, ct2, cs0, cs1, ca0, ca1, ca2, ca3, ca4, ca5 |
| |
| // If a thread has exited then it will set a fake value in the mcause so |
| // that the scheduler knows not to try to resume it. |
| .Lthread_exit: |
| // mtdc got swapped with the thread's csp, store it and clobber mtdc with |
| // zero. The trusted stack pointer is solely in csp, now; if we take |
| // another trap before a new one is installed, or if the scheduler enables |
| // interrupts and we take one, we'll pull this zero out of mtdc, above. |
| zeroOne t1 |
| cspecialrw ct1, mtdc, ct1 |
| csc ct1, TrustedStack_offset_csp(csp) |
| |
| // Store the rest of the special registers |
| cspecialr ct0, mepcc |
| csc ct0, TrustedStack_offset_mepcc(csp) |
| csrr t1, mstatus |
| csw t1, TrustedStack_offset_mstatus(csp) |
| #ifdef CONFIG_MSHWM |
| csrr t1, CSR_MSHWM |
| csw t1, TrustedStack_offset_mshwm(csp) |
| csrr t1, CSR_MSHWMB |
| csw t1, TrustedStack_offset_mshwmb(csp) |
| #endif |
| csrr t1, mcause |
| csw t1, TrustedStack_offset_mcause(csp) |
| |
| // If we hit one of the exception conditions that we should let |
| // compartments handle then deliver it to the compartment. |
| // CHERI exception code. |
| li a0, MCAUSE_CHERI |
| beq a0, t1, .Lhandle_error |
| // Misaligned instruction, instruction access, illegal instruction, |
| // breakpoint, misaligned load, load fault, misaligned store, and store |
| // access faults are in the range 0-7 |
| li a0, 0x8 |
| bltu t1, a0, .Lhandle_error |
| |
| // TODO: On an ecall, we don't need to save any caller-save registers |
| |
| // At this point, thread state is completely saved. Now prepare the |
| // scheduler context. |
| // Function signature of the scheduler entry point: |
| // TrustedStack *exception_entry(TrustedStack *sealedTStack, |
| // size_t mcause, size_t mepc, size_t mtval) |
| |
| LoadCapPCC ca5, compartment_switcher_sealing_key |
| li gp, 10 |
| csetaddr ca5, ca5, gp |
| cseal ca0, csp, ca5 // sealed trusted stack |
| mv a1, t1 // mcause |
| cgetaddr a2, ct0 // mepcc address |
| csrr a3, mtval |
| // Fetch the stack, cgp and the trusted stack for the scheduler. |
| LoadCapPCC csp, switcher_scheduler_entry_csp |
| LoadCapPCC cgp, switcher_scheduler_entry_cgp |
| LoadCapPCC cra, switcher_scheduler_entry_pcc |
| |
| // Zero everything apart from things explicitly passed to scheduler. |
| // cra, csp and cgp needed for the scheduler compartment |
| // ca0, used for the sealed trusted stack argument |
| // ca1, used for mcause |
| // ca2, used for mepc |
| // ca3, used for mtval |
| zeroAllRegistersExcept ra, sp, gp, a0, a1, a2, a3 |
| |
| // Call the scheduler. This returns the new thread in ca0. |
| cjalr cra |
| // The scheduler may change interrupt posture or may trap, but if it |
| // returns to us (that is, we reach here), the use of the sentry created by |
| // cjalr will have restored us to deferring interrupts, and we will remain |
| // in that posture until the mret in install_context. |
| |
| // Switch onto the new thread's trusted stack |
| LoadCapPCC ct0, compartment_switcher_sealing_key |
| li gp, SEAL_TYPE_SealedTrustedStacks |
| csetaddr ct0, ct0, gp |
| cunseal csp, ca0, ct0 |
| clw t0, TrustedStack_offset_mcause(csp) |
| |
| // Only now that we have done something that actually requires the tag of |
| // csp be set, put it into mtdc. If the scheduler has returned something |
| // untagged or something with the wrong otype, the cunseal will have left |
| // csp untagged and clw will trap with mtdc still 0. If we made it here, |
| // though, csp is tagged and so was tagged and correctly typed, and so it |
| // is safe to install it to mtdc. We won't cause traps between here and |
| // mret, so reentrancy is no longer a concern. |
| cspecialw mtdc, csp |
| |
| // If mcause is MCAUSE_THREAD_INTERRUPT, then we will jump into the error |
| // handler: another thread has signalled that this thread should be |
| // interrupted. MCAUSE_THREAD_INTERRUPT is a reserved exception number that |
| // we repurpose to indicate explicit interruption. |
| li t1, MCAUSE_THREAD_INTERRUPT |
| beq t0, t1, .Lhandle_injected_error |
| |
| // Environment call from M-mode is exception code 11. |
| // We need to skip the ecall instruction to avoid an infinite loop. |
| li t1, 11 |
| clc ct2, TrustedStack_offset_mepcc(csp) |
| bne t0, t1, .Linstall_context |
| cincoffset ct2, ct2, 4 |
| // Fall through to install context |
| |
| // Install context expects csp and mtdc to point to the trusted stack and for |
| // ct2 to be the pcc to jump to. All other registers are in unspecified states |
| // and will be overwritten when we install the context. |
| .Linstall_context: |
| clw ra, TrustedStack_offset_mstatus(csp) |
| csrw mstatus, ra |
| #ifdef CONFIG_MSHWM |
| clw ra, TrustedStack_offset_mshwm(csp) |
| csrw CSR_MSHWM, ra |
| clw ra, TrustedStack_offset_mshwmb(csp) |
| csrw CSR_MSHWMB, ra |
| #endif |
| cspecialw mepcc, ct2 |
| // csp (c2) will be loaded last and will overwrite the trusted stack pointer |
| // with the thread's stack pointer. |
| reloadRegisters cra, cgp, ctp, ct0, ct1, ct2, cs0, cs1, ca0, ca1, ca2, ca3, ca4, ca5, csp |
| mret |
| |
| // We are starting a forced unwind. This is reached either when we are unable |
| // to run an error handler, or when we do run an error handler and it instructs |
| // us to return. This treats all register values as undefined on entry. |
| .Lforce_unwind: |
| li a0, -ECOMPARTMENTFAIL |
| li a1, 0 |
| j switcher_skip_compartment_call |
| |
| |
| // If we have run out of trusted stack, then just restore the caller's state |
| // and return an error value. |
| .Lout_of_trusted_stack: |
| // Restore the spilled values |
| clc cs0, SPILL_SLOT_cs0(csp) |
| clc cs1, SPILL_SLOT_cs1(csp) |
| clc cra, SPILL_SLOT_pcc(csp) |
| clc cgp, SPILL_SLOT_cgp(csp) |
| cincoffset csp, csp, SPILL_SLOT_SIZE |
| // Set the return registers |
| li a0, -ENOTENOUGHTRUSTEDSTACK |
| li a1, 0 |
| // Zero everything else |
| zeroAllRegistersExcept ra, sp, gp, s0, s1, a0, a1 |
| cret |
| |
| // If we have a possibly recoverable error, see if we have a useful error |
| // handler. At this point, the register state will have been saved in the |
| // register-save area and so we just need to set up the environment. |
| // |
| // On entry to this block, csp contains the trusted stack pointer, all other |
| // registers are undefined. |
| // |
| // The handler will have this type signature: |
| // enum ErrorRecoveryBehaviour compartment_error_handler(struct ErrorState *frame, |
| // size_t mcause, |
| // size_t mtval); |
| .Lhandle_error: |
| // We're now out of the exception path, so make sure that mtdc contains |
| // the trusted stack pointer. |
| cspecialw mtdc, csp |
| // Store an error value in return registers, which will be passed to the |
| // caller on unwind. They are currently undefined, if we leave this path |
| // for a forced unwind then we will return whatever is in ca0 and ca1 to |
| // the caller so must ensure that we don't leak anything. |
| li a0, -1 |
| li a1, 0 |
| |
| // We want to make sure we can't leak any switcher state into error |
| // handlers, so if we're faulting in the switcher then we should force |
| // unwind. We never change the base of PCC in the switcher, so we can |
| // check for this case by ensuring that the spilled mepcc and our current |
| // pcc have the same base. |
| auipcc ct0, 0 |
| clc ct1, TrustedStack_offset_mepcc(csp) |
| cgetbase t0, ct0 |
| cgetbase tp, ct1 |
| beq t0, tp, .Lhandle_error_in_switcher |
| |
| // Load the interrupted thread's stack pointer into ct0 |
| clc ct0, TrustedStack_offset_csp(csp) |
| // See if we can find a handler: |
| clhu tp, TrustedStack_offset_frameoffset(csp) |
| li t1, TrustedStack_offset_frames |
| beq tp, t1, .Lset_mcause_and_exit_thread |
| addi tp, tp, -TrustedStackFrame_size |
| |
| // ctp points to the current available trusted stack frame. |
| cincoffset ctp, csp, tp |
| // a0 indicates whether we're calling a stackless error handler (0: stack, |
| // 1: stackless) |
| li a0, 0 |
| |
| // Allocate space for the register save frame on the stack. |
| cincoffset ct0, ct0, -(16*8) |
| |
| // WARNING: ENCODING SPECIFIC. |
| // The following depends on the fact that before-the-start values are not |
| // representable in the CHERIoT encoding and so will clear the tag. If |
| // this property changes then this will need to be replaced by a check that |
| // against the base of the stack. Note that this check can't be a simple |
| // cgetbase on ct0, because moving the address below the base sufficiently |
| // far that it's out of *representable* bounds will move the reported base |
| // value (base is a displacement from the address). |
| cgettag t1, ct0 |
| // If there isn't enough space on the stack, see if there's a stackless |
| // handler. |
| beqz t1, .Ltry_stackless_handler |
| |
| clc ct1, TrustedStackFrame_offset_calleeExportTable(ctp) |
| // Set the export table pointer to point to the *start* of the export |
| // table. It will currently point to the entry point that was raised. |
| // TODO: We might want to pass this to the error handler, it might be |
| // useful for providing per-entry-point error results. |
| cgetbase s0, ct1 |
| csetaddr ct1, ct1, s0 |
| clhu s0, ExportTable_offset_errorHandler(ct1) |
| // A value of 0xffff indicates no error handler |
| // If we found one, use it, otherwise fall through and try to find a |
| // stackless handler. |
| li s1, 0xffff |
| bne s0, s1, .Lhandler_found |
| |
| .Ltry_stackless_handler: |
| clc ct1, TrustedStackFrame_offset_calleeExportTable(ctp) |
| // Set the export table pointer to point to the *start* of the export |
| // table. It will currently point to the entry point that was raised. |
| cgetbase s0, ct1 |
| csetaddr ct1, ct1, s0 |
| clhu s0, ExportTable_offset_errorHandlerStackless(ct1) |
| // A value of 0xffff indicates no error handler |
| // Give up if there is no error handler for this compartment. |
| li s1, 0xffff |
| beq s0, s1, .Lforce_unwind |
| |
| // The stack may have had its tag cleared at this point, so for stackless |
| // handlers we need to restore the on-entry stack. |
| // Get the previous trusted stack frame |
| |
| // Load the caller's csp |
| clc ct0, TrustedStackFrame_offset_csp(ctp) |
| |
| // If this is the top stack frame, then the csp field is the value on |
| // entry. If it's any other frame then we need to go to the previous one |
| cincoffset cs1, csp, TrustedStack_offset_frames |
| beq s1, tp, .Lrecovered_stack |
| |
| // The address of the stack pointer will point to the bottom of the |
| // caller's save area, so we set the bounds to be the base up to the |
| // current address. |
| cgetaddr a1, ct0 |
| cgetbase a2, ct0 |
| sub a1, a1, a2 |
| csetaddr ct0, ct0, a2 |
| // The code that installs the context expects csp to be in ct0 |
| csetboundsexact ct0, ct0, a1 |
| .Lrecovered_stack: |
| li a0, 1 |
| |
| .Lhandler_found: |
| |
| // Increment the handler invocation count. |
| clhu s1, TrustedStackFrame_offset_errorHandlerCount(ctp) |
| addi s1, s1, 1 |
| csh s1, TrustedStackFrame_offset_errorHandlerCount(ctp) |
| |
| // If we are in a double fault, unwind now. The low bit should be 1 while |
| // we are handling a fault. |
| andi ra, s1, 1 |
| beqz ra, .Lforce_unwind |
| // If we have reached some arbitrary limit on the number of faults in a |
| // singe compartment calls, give up now. |
| // TODO: Make this a number based on something sensible, possibly something |
| // set per entry point. Some compartments (especially top-level ones) |
| // should be allowed to fault an unbounded number of times. |
| li ra, MAX_FAULTS_PER_COMPARTMENT_CALL |
| bgtu s1, ra, .Lforce_unwind |
| |
| // Load the pristine pcc and cgp for the invoked compartment. |
| clc cra, ExportTable_offset_pcc(ct1) |
| clc cgp, ExportTable_offset_cgp(ct1) |
| // Set the jump target to the error handler entry point |
| // This may result in something out-of-bounds if the compartment has a |
| // malicious value for their error handler (hopefully caught at link or |
| // load time), but if it does then we will double-fault and force unwind. |
| cgetbase s1, cra |
| csetaddr cra, cra, s1 |
| cincoffset cra, cra, s0 |
| |
| // If we're in an error handler with a stack, set up the stack, otherwise |
| // we just need to set up argument registers. |
| beqz a0, .Lset_up_stack_handler |
| clw a0, TrustedStack_offset_mcause(csp) |
| csrr a1, mtval |
| li a2, 0 |
| cmove csp, ct0 |
| j .Linvoke_error_handler |
| |
| .Lset_up_stack_handler: |
| // Set up the on-stack context for the callee |
| clc cs1, 0(csp) |
| ccleartag cs1, cs1 |
| csc cs1, 0(ct0) |
| // Source for context copy. |
| cincoffset ca2, csp, TrustedStack_offset_cra |
| // Destination for context copy |
| cincoffset ca3, ct0, TrustedStack_offset_cra |
| copyContext ca3, ca2, cs1, a4 |
| |
| // Set up the arguments for the call |
| cmove ca0, ct0 |
| clw a1, TrustedStack_offset_mcause(csp) |
| csrr a2, mtval |
| cmove csp, ca0 |
| |
| .Linvoke_error_handler: |
| // Enable interrupts before invoking the handler |
| csrsi mstatus, 0x8 |
| |
| // Clear all registers except: |
| // cra is set by cjalr. csp and cgp are needed for the called compartment. |
| // ca0, used for the register state |
| // ca1, used for mcause |
| // ca2, used for mtval |
| zeroAllRegistersExcept ra, sp, gp, a0, a1, a2 |
| // Call the handler. |
| cjalr cra |
| |
| /* |
| * Now that we're back, defer interrupts again before we do anything that |
| * manipulates the TrustedStack. |
| * |
| * TODO: Eventually we'd like to move this down onto the paths where it |
| * actually matters and let most of this code run with IRQs enabled. |
| */ |
| csrci mstatus, 0x8 |
| |
| // Move the return value to a register that will be cleared in a forced |
| // unwind and zero the return registers. |
| move s0, a0 |
| // Store an error value in return registers, which will be passed to the |
| // caller on unwind. |
| li a0, -1 |
| li a1, 0 |
| // Return values are 0 for install context, 1 for forced unwind. Anything |
| // that is not either of these is invalid and so we should do a forced |
| // unwind anyway. |
| bne s0, zero, .Lforce_unwind |
| |
| // We have been asked to install the new register context and resume. |
| // We do this by copying the register frame over the save area and entering |
| // the exception resume path. This may fault, but if it does then we will |
| // detect it as a double fault and forcibly unwind. |
| |
| // Load the trusted stack pointer to ct1 |
| cspecialr ct1, mtdc |
| #ifdef CONFIG_MSHWM |
| // Update the spilled copy of the stack high water mark to ensure that we |
| // will clear all of the stack used by the error handler and the spilled |
| // context. |
| csrr t0, CSR_MSHWM |
| csw t0, TrustedStack_offset_mshwm(ct1) |
| #endif |
| clhu tp, TrustedStack_offset_frameoffset(ct1) |
| addi tp, tp, -TrustedStackFrame_size |
| // ctp points to the current available trusted stack frame. |
| cincoffset ctp, ct1, tp |
| |
| // ct0 now contains the export table for the callee |
| clc ct0, TrustedStackFrame_offset_calleeExportTable(ctp) |
| cgetbase s0, ct0 |
| csetaddr ct0, ct0, s0 |
| // ct0 now contains the PCC for the returning compartment. |
| clc ct0, ExportTable_offset_pcc(ct0) |
| // This is the *untagged* destination pcc. Install its address into the |
| // real one |
| clc cra, 0(csp) |
| cgetaddr ra, cra |
| csetaddr ct2, ct0, ra |
| // Now copy everything else from the stack into the saved context |
| // Source |
| cincoffset ca2, csp, TrustedStack_offset_cra |
| // Destination |
| cincoffset ca3, ct1, TrustedStack_offset_cra |
| copyContext ca3, ca2, cs1, a4 |
| // Increment the handler invocation count. We have now returned and |
| // finished touching any data from the error handler that might cause a |
| // fault. Any subsequent fault is not treated as a double fault. It might |
| // be a fault loop, but that will be caught by the fault limit check. |
| clh s1, TrustedStackFrame_offset_errorHandlerCount(ctp) |
| addi s1, s1, 1 |
| csh s1, TrustedStackFrame_offset_errorHandlerCount(ctp) |
| |
| // Now that the context is set up, let the exception handler code deal with |
| // it. It expects the context to be in csp, so move the context pointer there. |
| cmove csp, ct1 |
| j .Linstall_context |
| |
| .Lhandle_injected_error: |
| #ifdef CONFIG_MSHWM |
| clw ra, TrustedStack_offset_mshwm(csp) |
| csrw CSR_MSHWM, ra |
| clw ra, TrustedStack_offset_mshwmb(csp) |
| csrw CSR_MSHWMB, ra |
| #endif |
| j .Lhandle_error |
| |
| .Lcommon_defer_irqs_and_thread_exit: |
| csrci mstatus, 0x8 |
| // Fall-through, now that IRQs are off |
| |
| // Value 24 is reserved for custom use. |
| .Lset_mcause_and_exit_thread: |
| csrw mcause, MCAUSE_THREAD_EXIT |
| // The thread exit code expects the trusted stack pointer to be in csp and |
| // the stack pointer to be in mtdc. After thread exit, we don't need the |
| // stack pointer so just put zero there. |
| zeroOne sp |
| cspecialrw csp, mtdc, csp |
| j .Lthread_exit |
| |
| /* |
| * Some switcher instructions' traps are handled specially, by looking at |
| * the offset of mepcc. Otherwise, we're off to a force unwind. |
| */ |
| .Lhandle_error_in_switcher: |
| auipcc ctp, %cheriot_compartment_hi(.Lswitcher_entry_first_spill) |
| cincoffset ctp, ctp, %cheriot_compartment_lo_i(.Lhandle_error_in_switcher) |
| bne t1, tp, .Lforce_unwind |
| li a0, -ENOTENOUGHSTACK |
| li a1, 0 |
| |
| /* |
| * Cause the interrupted thread to resume as if a return had just executed. |
| * We do this by vectoring to a `cjalr ra` (`cret`) instruction through |
| * `mepcc`; whee! Overwrites the stored context a0 and a1 with the current |
| * values of those registers, effectively passing them through |
| * .Linstall_context. |
| */ |
| .Linstall_return_context: |
| auipcc ct2, %cheriot_compartment_hi(.Ljust_return) |
| cincoffset ct2, ct2, %cheriot_compartment_lo_i(.Linstall_return_context) |
| csc ca0, TrustedStack_offset_ca0(csp) |
| csc ca1, TrustedStack_offset_ca1(csp) |
| j .Linstall_context |
| |
| |
| .size exception_entry_asm, . - exception_entry_asm |
| |
| /******************************************************************************* |
| * Switcher-exported library functions. |
| * |
| * These all provide some reflection on the switcher's state. |
| * |
| * At the moment, all of these avoid touching any registers except the argument |
| * registers, which means that we can define an alternative calling convention |
| * for them in the future to allow the compiler to preserve values in the |
| * temporary registers across calls. |
| * |
| * These are all part of the switcher's PCC and so will be covered by the same |
| * defence that the switcher has against being made to trap at unexpected |
| * times: any trap in the switcher will force unwind the caller's trusted stack |
| * frame. As such, no trap here can leak data. |
| * |
| * These functions must not use the stack and must ensure that the clobber all |
| * registers that hold sensitive state on the way out. |
| ******************************************************************************/ |
| |
| // Returns whether the trusted stack has space for N more calls. |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z23trusted_stack_has_spacei,@function |
| __Z23trusted_stack_has_spacei: |
| li a2, TrustedStackFrame_size |
| mul a2, a0, a2 |
| // Load the trusted stack into a register that we will clobber on the way |
| // out. |
| cspecialr ca0, mtdc |
| clhu a1, TrustedStack_offset_frameoffset(ca0) |
| cgetlen a0, ca0 |
| sub a0, a0, a1 |
| sltu a0, a2, a0 |
| cret |
| |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z22switcher_recover_stackv,@function |
| __Z22switcher_recover_stackv: |
| // Load the trusted stack pointer into a register that we will clobber in |
| // two instructions. |
| cspecialr ca0, mtdc |
| clhu a1, TrustedStack_offset_frameoffset(ca0) |
| addi a1, a1, -TrustedStackFrame_size |
| cincoffset ca0, ca0, a1 |
| clc ca0, TrustedStackFrame_offset_csp(ca0) |
| // If this is the first frame, then the recovered stack will be the stack |
| // on entry. If this is not the first frame then then we need to find the |
| // saved CSP from the caller and reset the bounds. The address of the |
| // saved CSP will be the value after the switcher spilled registers and so |
| // will be the top of the callee's stack. |
| li a2, TrustedStack_offset_frames |
| beq a1, a2, 0f |
| |
| // Find the previous frame's csp and reset the bounds |
| cgetaddr a1, ca0 |
| cgetbase a2, ca0 |
| sub a1, a1, a2 |
| csetaddr ca0, ca0, a2 |
| csetboundsexact ca0, ca0, a1 |
| 0: |
| cret |
| |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z30trusted_stack_interrupt_threadPv,@function |
| __Z25switcher_interrupt_threadPv: |
| // Load the unsealing key into a register that we will clobber two |
| // instructions later. |
| LoadCapPCC ca1, compartment_switcher_sealing_key |
| li a2, SEAL_TYPE_SealedTrustedStacks |
| csetaddr ca1, ca1, a2 |
| // The target capability is in ct1. Unseal, check tag and load the entry point offset. |
| cunseal ca1, ca0, ca1 |
| cgettag a0, ca1 |
| // a0 (return register) now contains the tag. We return false on failure |
| // so can just branch to the place where we zero non-return registers from |
| // here and it will contain faluse on failure. |
| beqz a0, .Lreturn |
| |
| // A thread can't interrupt itself, return failure if it tries. |
| cspecialr ca2, mtdc |
| li a0, 0 |
| beq a2, a1, .Lreturn |
| |
| // ca1 now contains the unsealed capability for the target thread. We |
| // allow the target thread to be interrupted if (and only if) the caller is |
| // in the same compartment as the interrupted thread. We will determine |
| // this by checking if the base of the two export table entries from the |
| // top of the trusted stack frames match. |
| |
| // Helper macro that loads the export table from the register containing the |
| // trusted stack. The two arguments must be different registers. |
| .macro LoadExportTable result, trustedStack |
| clhu \result, TrustedStack_offset_frameoffset(\trustedStack) |
| addi \result, \result, -TrustedStackFrame_size |
| cincoffset c\result, \trustedStack, \result |
| clc c\result, TrustedStackFrame_offset_calleeExportTable(c\result) |
| cgetbase \result, c\result |
| .endm |
| |
| LoadExportTable a3, ca1 |
| cspecialr ca0, mtdc |
| LoadExportTable a2, ca0 |
| |
| // ca1 now contains the unsealed capability for the target thread, a3 |
| // contains the base of the export table entry for that thread, a2 the base |
| // of the export table for our thread. |
| li a0, 42 |
| |
| // If the two export table entries differ, return. |
| bne a2, a3, .Lreturn |
| // After this point, we no longer care about the values in a0, a2, and a3. |
| |
| // Mark the thread as interrupted. |
| // Store a magic value in mcause |
| li a2, MCAUSE_THREAD_INTERRUPT |
| csw a2, TrustedStack_offset_mcause(ca1) |
| // Return success |
| li a0, 1 |
| .Lreturn: |
| zeroRegisters a1, a2, a3 |
| cret |
| |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z23switcher_current_threadv,@function |
| __Z23switcher_current_threadv: |
| LoadCapPCC ca0, compartment_switcher_sealing_key |
| li a1, SEAL_TYPE_SealedTrustedStacks |
| csetaddr ca0, ca0, a1 |
| cspecialr ca1, mtdc |
| cseal ca0, ca1, ca0 |
| li a1, 0 |
| cret |
| |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z28switcher_thread_hazard_slotsv,@function |
| __Z28switcher_thread_hazard_slotsv: |
| // Load the trusted stack pointer into a register that we will clobber in |
| // two instructions. |
| cspecialr ca0, mtdc |
| clc ca0, TrustedStack_offset_hazardPointers(ca0) |
| cret |
| |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z13thread_id_getv,@function |
| __Z13thread_id_getv: |
| // Load the trusted stack pointer into a register that we will clobber in |
| // the next instruction when we load the thread ID. |
| cspecialr ca0, mtdc |
| cgettag a1, ca0 |
| // If this is a null pointer, don't try to dereference it and report that |
| // we are thread 0. This permits the debug code to work even from things |
| // that are not real threads. |
| beqz a1, .Lend |
| clh a0, TrustedStack_offset_threadID(ca0) |
| .Lend: |
| cret |
| |
| |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z25stack_lowest_used_addressv,@function |
| __Z25stack_lowest_used_addressv: |
| // Read the stack high-water mark into the return register. |
| csrr a0, CSR_MSHWM |
| cret |
| |
| .section .text, "ax", @progbits |
| .p2align 2 |
| .type __Z39switcher_handler_invocation_count_resetv,@function |
| __Z39switcher_handler_invocation_count_resetv: |
| // Trusted stack pointer in ca1 |
| cspecialr ca1, mtdc |
| // Offset of the current trusted stack frame to a1 |
| clhu a0, TrustedStack_offset_frameoffset(ca1) |
| addi a0, a0, -TrustedStackFrame_size |
| // Current trusted stack frame to ca1, a0 is dead |
| cincoffset ca1, ca1, a0 |
| // Current invocation count (for return) in a0 |
| clh a0, TrustedStackFrame_offset_errorHandlerCount(ca1) |
| // Reset invocation count |
| csh zero, TrustedStackFrame_offset_errorHandlerCount(ca1) |
| // Zero trusted stack frame pointer register |
| li a1, 0 |
| cret |
| |
| // The linker expects export tables to start with space for cgp and pcc, then |
| // the compartment error handler. We should eventually remove that requirement |
| // for library export tables, but since they don't consume RAM after loading |
| // it's not urgent. |
| .section .compartment_export_table,"a",@progbits |
| export_table_start: |
| .space 20, 0 |
| |
| /** |
| * Helper that exports a switcher function as a library call. |
| */ |
| .macro export function, prefix=__library_export_libcalls |
| .type \prefix\function,@object |
| .globl \prefix\function |
| .p2align 2 |
| \prefix\function: |
| .half \function-switcher_code_start |
| // Number of registers to clear (ignored for library exports) |
| .byte 0 |
| // Interrupts disabled. |
| .byte 16 |
| .size \prefix\function, 4 |
| .endm |
| |
| // Switcher entry point must be first. |
| // We mangle the switcher export as if it were a compartment call. |
| export __Z26compartment_switcher_entryz, __export_switcher |
| export __Z23trusted_stack_has_spacei |
| export __Z22switcher_recover_stackv |
| export __Z25switcher_interrupt_threadPv |
| export __Z23switcher_current_threadv |
| export __Z28switcher_thread_hazard_slotsv |
| export __Z13thread_id_getv |
| export __Z25stack_lowest_used_addressv |
| export __Z39switcher_handler_invocation_count_resetv |
