diff --git a/core/arch/riscv/include/kernel/thread_arch.h b/core/arch/riscv/include/kernel/thread_arch.h index 3f43950619e..5e2f100f359 100644 --- a/core/arch/riscv/include/kernel/thread_arch.h +++ b/core/arch/riscv/include/kernel/thread_arch.h @@ -83,14 +83,14 @@ struct thread_abort_regs { unsigned long t4; unsigned long t5; unsigned long t6; + unsigned long epc; unsigned long status; + unsigned long ie; unsigned long cause; - unsigned long epc; unsigned long tval; - unsigned long satp; -}; +} __aligned(16); -struct thread_trap_regs { +struct thread_scall_regs { unsigned long ra; unsigned long sp; unsigned long gp; @@ -98,8 +98,6 @@ struct thread_trap_regs { unsigned long t0; unsigned long t1; unsigned long t2; - unsigned long s0; - unsigned long s1; unsigned long a0; unsigned long a1; unsigned long a2; @@ -108,16 +106,6 @@ struct thread_trap_regs { unsigned long a5; unsigned long a6; unsigned long a7; - unsigned long s2; - unsigned long s3; - unsigned long s4; - unsigned long s5; - unsigned long s6; - unsigned long s7; - unsigned long s8; - unsigned long s9; - unsigned long s10; - unsigned long s11; unsigned long t3; unsigned long t4; unsigned long t5; @@ -127,22 +115,6 @@ struct thread_trap_regs { unsigned long ie; } __aligned(16); -struct thread_scall_regs { - unsigned long a0; - unsigned long a1; - unsigned long a2; - unsigned long a3; - unsigned long a4; - unsigned long a5; - unsigned long a6; - unsigned long a7; - unsigned long t0; - unsigned long t1; - unsigned long ra; - unsigned long sp; - unsigned long status; -} __aligned(16); - struct thread_ctx_regs { unsigned long ra; unsigned long sp; diff --git a/core/arch/riscv/include/kernel/thread_private_arch.h b/core/arch/riscv/include/kernel/thread_private_arch.h index edf27a4e792..1a4fc15613e 100644 --- a/core/arch/riscv/include/kernel/thread_private_arch.h +++ b/core/arch/riscv/include/kernel/thread_private_arch.h @@ -49,9 +49,8 @@ struct thread_user_mode_rec { extern long thread_user_kcode_offset; -void thread_trap_handler(long cause, unsigned long epc, - struct thread_trap_regs *regs, - bool user); +void thread_interrupt_handler(unsigned long cause, + struct thread_ctx_regs *regs); /* * Initializes TVEC for current hart. Called by thread_init_per_cpu() */ diff --git a/core/arch/riscv/kernel/asm-defines.c b/core/arch/riscv/kernel/asm-defines.c index 6dfdfcaca1c..471e5041153 100644 --- a/core/arch/riscv/kernel/asm-defines.c +++ b/core/arch/riscv/kernel/asm-defines.c @@ -43,6 +43,8 @@ DEFINES DEFINE(THREAD_CTX_REG_IE, offsetof(struct thread_ctx_regs, ie)); DEFINE(THREAD_CTX_REG_RA, offsetof(struct thread_ctx_regs, ra)); DEFINE(THREAD_CTX_REG_SP, offsetof(struct thread_ctx_regs, sp)); + DEFINE(THREAD_CTX_REG_GP, offsetof(struct thread_ctx_regs, gp)); + DEFINE(THREAD_CTX_REG_TP, offsetof(struct thread_ctx_regs, tp)); DEFINE(THREAD_CTX_REG_T0, offsetof(struct thread_ctx_regs, t0)); DEFINE(THREAD_CTX_REG_S0, offsetof(struct thread_ctx_regs, s0)); DEFINE(THREAD_CTX_REG_A0, offsetof(struct thread_ctx_regs, a0)); @@ -63,27 +65,36 @@ DEFINES offsetof(struct thread_user_mode_rec, x[6])); DEFINE(THREAD_USER_MODE_REC_SIZE, sizeof(struct thread_user_mode_rec)); - /* struct thread_trap_regs */ - DEFINE(THREAD_TRAP_REG_SP, offsetof(struct thread_trap_regs, sp)); - DEFINE(THREAD_TRAP_REG_RA, offsetof(struct thread_trap_regs, ra)); - DEFINE(THREAD_TRAP_REG_GP, offsetof(struct thread_trap_regs, gp)); - DEFINE(THREAD_TRAP_REG_TP, offsetof(struct thread_trap_regs, tp)); - DEFINE(THREAD_TRAP_REG_T0, offsetof(struct thread_trap_regs, t0)); - DEFINE(THREAD_TRAP_REG_S0, offsetof(struct thread_trap_regs, s0)); - DEFINE(THREAD_TRAP_REG_A0, offsetof(struct thread_trap_regs, a0)); - DEFINE(THREAD_TRAP_REG_T3, offsetof(struct thread_trap_regs, t3)); - DEFINE(THREAD_TRAP_REG_EPC, offsetof(struct thread_trap_regs, epc)); - DEFINE(THREAD_TRAP_REG_STATUS, - offsetof(struct thread_trap_regs, status)); - DEFINE(THREAD_TRAP_REG_IE, offsetof(struct thread_trap_regs, ie)); - DEFINE(THREAD_TRAP_REGS_SIZE, sizeof(struct thread_trap_regs)); + /* struct thread_abort_regs */ + DEFINE(THREAD_ABT_REG_RA, offsetof(struct thread_abort_regs, ra)); + DEFINE(THREAD_ABT_REG_SP, offsetof(struct thread_abort_regs, sp)); + DEFINE(THREAD_ABT_REG_GP, offsetof(struct thread_abort_regs, gp)); + DEFINE(THREAD_ABT_REG_TP, offsetof(struct thread_abort_regs, tp)); + DEFINE(THREAD_ABT_REG_T0, offsetof(struct thread_abort_regs, t0)); + DEFINE(THREAD_ABT_REG_S0, offsetof(struct thread_abort_regs, s0)); + DEFINE(THREAD_ABT_REG_A0, offsetof(struct thread_abort_regs, a0)); + DEFINE(THREAD_ABT_REG_S2, offsetof(struct thread_abort_regs, s2)); + DEFINE(THREAD_ABT_REG_T3, offsetof(struct thread_abort_regs, t3)); + DEFINE(THREAD_ABT_REG_EPC, offsetof(struct thread_abort_regs, epc)); + DEFINE(THREAD_ABT_REG_STATUS, + offsetof(struct thread_abort_regs, status)); + DEFINE(THREAD_ABT_REG_IE, offsetof(struct thread_abort_regs, ie)); + DEFINE(THREAD_ABT_REG_CAUSE, offsetof(struct thread_abort_regs, cause)); + DEFINE(THREAD_ABT_REG_TVAL, offsetof(struct thread_abort_regs, tval)); + DEFINE(THREAD_ABT_REGS_SIZE, sizeof(struct thread_abort_regs)); /* struct thread_scall_regs */ - DEFINE(THREAD_SCALL_REG_STATUS, - offsetof(struct thread_scall_regs, status)); DEFINE(THREAD_SCALL_REG_RA, offsetof(struct thread_scall_regs, ra)); DEFINE(THREAD_SCALL_REG_SP, offsetof(struct thread_scall_regs, sp)); + DEFINE(THREAD_SCALL_REG_GP, offsetof(struct thread_scall_regs, gp)); + DEFINE(THREAD_SCALL_REG_TP, offsetof(struct thread_scall_regs, tp)); + DEFINE(THREAD_SCALL_REG_T0, offsetof(struct thread_scall_regs, t0)); DEFINE(THREAD_SCALL_REG_A0, offsetof(struct thread_scall_regs, a0)); + DEFINE(THREAD_SCALL_REG_T3, offsetof(struct thread_scall_regs, t3)); + DEFINE(THREAD_SCALL_REG_EPC, offsetof(struct thread_scall_regs, epc)); + DEFINE(THREAD_SCALL_REG_STATUS, + offsetof(struct thread_scall_regs, status)); + DEFINE(THREAD_SCALL_REG_IE, offsetof(struct thread_scall_regs, ie)); DEFINE(THREAD_SCALL_REGS_SIZE, sizeof(struct thread_scall_regs)); /* struct core_mmu_config */ diff --git a/core/arch/riscv/kernel/thread_arch.c b/core/arch/riscv/kernel/thread_arch.c index 8654aa6631e..394940ab50a 100644 --- a/core/arch/riscv/kernel/thread_arch.c +++ b/core/arch/riscv/kernel/thread_arch.c @@ -103,15 +103,15 @@ static void setup_unwind_user_mode(struct thread_scall_regs *regs) regs->sp = thread_get_saved_thread_sp(); } -static void thread_unhandled_trap(struct thread_trap_regs *regs __unused, - unsigned long cause __unused) +static void thread_unhandled_trap(unsigned long cause __unused, + struct thread_ctx_regs *regs __unused) { DMSG("Unhandled trap xepc:0x%016lx xcause:0x%016lx xtval:0x%016lx", read_csr(CSR_XEPC), read_csr(CSR_XCAUSE), read_csr(CSR_XTVAL)); panic(); } -void thread_scall_handler(struct thread_scall_regs *regs) +void thread_scall_handler(struct thread_scall_regs *regs) { struct ts_session *sess = NULL; uint32_t state = 0; @@ -137,171 +137,28 @@ void thread_scall_handler(struct thread_scall_regs *regs) } } -static void copy_scall_to_trap(struct thread_scall_regs *scall_regs, - struct thread_trap_regs *trap_regs) -{ - trap_regs->a0 = scall_regs->a0; - trap_regs->a1 = scall_regs->a1; - trap_regs->a2 = scall_regs->a2; - trap_regs->a3 = scall_regs->a3; - trap_regs->a4 = scall_regs->a4; - trap_regs->a5 = scall_regs->a5; - trap_regs->a6 = scall_regs->a6; - trap_regs->a7 = scall_regs->a7; - trap_regs->t0 = scall_regs->t0; - trap_regs->t1 = scall_regs->t1; -} - -static void copy_trap_to_scall(struct thread_trap_regs *trap_regs, - struct thread_scall_regs *scall_regs) -{ - *scall_regs = (struct thread_scall_regs) { - .status = trap_regs->status, - .ra = trap_regs->ra, - .a0 = trap_regs->a0, - .a1 = trap_regs->a1, - .a2 = trap_regs->a2, - .a3 = trap_regs->a3, - .a4 = trap_regs->a4, - .a5 = trap_regs->a5, - .a6 = trap_regs->a6, - .a7 = trap_regs->a7, - .t0 = trap_regs->t0, - .t1 = trap_regs->t1, - }; -} - -static void thread_user_ecall_handler(struct thread_trap_regs *trap_regs) -{ - struct thread_scall_regs scall_regs; - struct thread_core_local *l = thread_get_core_local(); - int ct = l->curr_thread; - - copy_trap_to_scall(trap_regs, &scall_regs); - thread_scall_handler(&scall_regs); - copy_scall_to_trap(&scall_regs, trap_regs); - /* - * Save kernel sp we'll had at the beginning of this function. - * This is when this TA has called another TA because - * __thread_enter_user_mode() also saves the stack pointer in this - * field. - */ - threads[ct].kern_sp = (unsigned long)(trap_regs + 1); - /* - * We are returning to U-Mode, on return, the program counter - * is set to xsepc (pc=xepc), we add 4 (size of an instruction) - * to continue to next instruction. - */ - trap_regs->epc += 4; -} - -static void copy_trap_to_abort(struct thread_trap_regs *trap_regs, - struct thread_abort_regs *abort_regs) -{ - *abort_regs = (struct thread_abort_regs) { - .status = trap_regs->status, - .ra = trap_regs->ra, - .sp = trap_regs->sp, - .gp = trap_regs->gp, - .tp = trap_regs->tp, - .t0 = trap_regs->t0, - .t1 = trap_regs->t1, - .t2 = trap_regs->t2, - .s0 = trap_regs->s0, - .s1 = trap_regs->s1, - .a0 = trap_regs->a0, - .a1 = trap_regs->a1, - .a2 = trap_regs->a2, - .a3 = trap_regs->a3, - .a4 = trap_regs->a4, - .a5 = trap_regs->a5, - .a6 = trap_regs->a6, - .a7 = trap_regs->a7, - .s2 = trap_regs->s2, - .s3 = trap_regs->s3, - .s4 = trap_regs->s4, - .s5 = trap_regs->s5, - .s6 = trap_regs->s6, - .s7 = trap_regs->s7, - .s8 = trap_regs->s8, - .s9 = trap_regs->s9, - .s10 = trap_regs->s10, - .s11 = trap_regs->s11, - .t3 = trap_regs->t3, - .t4 = trap_regs->t4, - .t5 = trap_regs->t5, - .t6 = trap_regs->t6, - }; -} - -static void thread_abort_handler(struct thread_trap_regs *trap_regs, - unsigned long cause) -{ - struct thread_abort_regs abort_regs = { }; - - assert(cause == read_csr(CSR_XCAUSE)); - copy_trap_to_abort(trap_regs, &abort_regs); - abort_regs.cause = read_csr(CSR_XCAUSE); - abort_regs.epc = read_csr(CSR_XEPC); - abort_regs.tval = read_csr(CSR_XTVAL); - abort_regs.satp = read_csr(CSR_SATP); - abort_handler(cause, &abort_regs); -} - -static void thread_exception_handler(unsigned long cause, - struct thread_trap_regs *regs) -{ - switch (cause) { - case CAUSE_USER_ECALL: - thread_user_ecall_handler(regs); - break; - default: - thread_abort_handler(regs, cause); - break; - } -} - static void thread_irq_handler(void) { interrupt_main_handler(); } -static void thread_interrupt_handler(unsigned long cause, - struct thread_trap_regs *regs) +void thread_interrupt_handler(unsigned long cause, struct thread_ctx_regs *regs) { switch (cause & LONG_MAX) { case IRQ_XTIMER: clear_csr(CSR_XIE, CSR_XIE_TIE); break; case IRQ_XSOFT: - thread_unhandled_trap(regs, cause); + thread_unhandled_trap(cause, regs); break; case IRQ_XEXT: thread_irq_handler(); break; default: - thread_unhandled_trap(regs, cause); + thread_unhandled_trap(cause, regs); } } -void thread_trap_handler(long cause, unsigned long epc __unused, - struct thread_trap_regs *regs, - bool user __maybe_unused) -{ - /* - * The Interrupt bit (XLEN-1) in the cause register is set - * if the trap was caused by an interrupt. - */ - if (cause < 0) - thread_interrupt_handler(cause, regs); - /* - * Otherwise, cause is never written by the implementation, - * though it may be explicitly written by software. - */ - else - thread_exception_handler(cause, regs); -} - unsigned long xstatus_for_xret(uint8_t pie, uint8_t pp) { unsigned long xstatus = read_csr(CSR_XSTATUS); @@ -611,7 +468,6 @@ void thread_init_tvec(void) { unsigned long tvec = (unsigned long)get_trap_vect(); - static_assert(sizeof(struct thread_trap_regs) % 16 == 0); write_csr(CSR_XTVEC, tvec); assert(read_csr(CSR_XTVEC) == tvec); } diff --git a/core/arch/riscv/kernel/thread_rv.S b/core/arch/riscv/kernel/thread_rv.S index eeddc3467ab..5b41fa2b307 100644 --- a/core/arch/riscv/kernel/thread_rv.S +++ b/core/arch/riscv/kernel/thread_rv.S @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-2-Clause */ /* * Copyright 2022-2023 NXP + * Copyright 2024 Andes Technology Corporation */ #include @@ -28,26 +29,227 @@ beqz \reg, \label .endm -.macro save_regs, mode -.if \mode == TRAP_MODE_USER +/* size_t __get_core_pos(void); */ +FUNC __get_core_pos , : , .identity_map + lw a0, THREAD_CORE_LOCAL_HART_ID(tp) + ret +END_FUNC __get_core_pos + +FUNC thread_trap_vect , : + csrrw tp, CSR_XSCRATCH, tp + bnez tp, 0f + /* Read tp back */ + csrrw tp, CSR_XSCRATCH, tp + j trap_from_kernel +0: + /* Now tp is thread_core_local */ + j trap_from_user +thread_trap_vect_end: +END_FUNC thread_trap_vect + +LOCAL_FUNC trap_from_kernel, : + /* Save sp, a0, a1 into temporary spaces of thread_core_local */ + store_xregs tp, THREAD_CORE_LOCAL_X0, REG_SP + store_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 + + csrr a0, CSR_XCAUSE + /* MSB of cause differentiates between interrupts and exceptions */ + bge a0, zero, exception_from_kernel + +interrupt_from_kernel: + /* Get thread context as sp */ + get_thread_ctx sp, a0 + + /* Load and save kernel sp */ + load_xregs tp, THREAD_CORE_LOCAL_X0, REG_A0 + store_xregs sp, THREAD_CTX_REG_SP, REG_A0 + + /* Restore user a0, a1 which can be saved later */ + load_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 + + /* Save all other GPRs */ + store_xregs sp, THREAD_CTX_REG_RA, REG_RA + store_xregs sp, THREAD_CTX_REG_GP, REG_GP + store_xregs sp, THREAD_CTX_REG_T0, REG_T0, REG_T2 + store_xregs sp, THREAD_CTX_REG_S0, REG_S0, REG_S1 + store_xregs sp, THREAD_CTX_REG_A0, REG_A0, REG_A7 + store_xregs sp, THREAD_CTX_REG_S2, REG_S2, REG_S11 + store_xregs sp, THREAD_CTX_REG_T3, REG_T3, REG_T6 + /* Save XIE */ + csrr t0, CSR_XIE + store_xregs sp, THREAD_CTX_REG_IE, REG_T0 + /* Mask all interrupts */ + csrw CSR_XIE, x0 + /* Save XSTATUS */ + csrr t0, CSR_XSTATUS + store_xregs sp, THREAD_CTX_REG_STATUS, REG_T0 + /* Save XEPC */ + csrr t0, CSR_XEPC + store_xregs sp, THREAD_CTX_REG_EPC, REG_T0 + + /* + * a0 = cause + * a1 = sp + * Call thread_interrupt_handler(cause, regs) + */ + csrr a0, CSR_XCAUSE + mv a1, sp + /* Load tmp_stack_va_end as current sp. */ + load_xregs tp, THREAD_CORE_LOCAL_TMP_STACK_VA_END, REG_SP + call thread_interrupt_handler + + /* Get thread context as sp */ + get_thread_ctx sp, t0 + /* Restore XEPC */ + load_xregs sp, THREAD_CTX_REG_EPC, REG_T0 + csrw CSR_XEPC, t0 + /* Restore XIE */ + load_xregs sp, THREAD_CTX_REG_IE, REG_T0 + csrw CSR_XIE, t0 + /* Restore XSTATUS */ + load_xregs sp, THREAD_CTX_REG_STATUS, REG_T0 + csrw CSR_XSTATUS, t0 + /* Set scratch as thread_core_local */ + csrw CSR_XSCRATCH, tp + /* Restore all GPRs */ + load_xregs sp, THREAD_CTX_REG_RA, REG_RA + load_xregs sp, THREAD_CTX_REG_GP, REG_GP + load_xregs sp, THREAD_CTX_REG_T0, REG_T0, REG_T2 + load_xregs sp, THREAD_CTX_REG_S0, REG_S0, REG_S1 + load_xregs sp, THREAD_CTX_REG_A0, REG_A0, REG_A7 + load_xregs sp, THREAD_CTX_REG_S2, REG_S2, REG_S11 + load_xregs sp, THREAD_CTX_REG_T3, REG_T3, REG_T6 + load_xregs sp, THREAD_CTX_REG_SP, REG_SP + XRET + +exception_from_kernel: + /* + * Update core local flags. + * flags = (flags << THREAD_CLF_SAVED_SHIFT) | THREAD_CLF_ABORT; + */ + lw a0, THREAD_CORE_LOCAL_FLAGS(tp) + slli a0, a0, THREAD_CLF_SAVED_SHIFT + ori a0, a0, THREAD_CLF_ABORT + li a1, (THREAD_CLF_ABORT << THREAD_CLF_SAVED_SHIFT) + and a1, a0, a1 + bnez a1, sel_tmp_sp + + /* Select abort stack */ + load_xregs tp, THREAD_CORE_LOCAL_ABT_STACK_VA_END, REG_A1 + j set_sp + +sel_tmp_sp: + /* We have an abort while using the abort stack, select tmp stack */ + load_xregs tp, THREAD_CORE_LOCAL_TMP_STACK_VA_END, REG_A1 + ori a0, a0, THREAD_CLF_TMP /* flags |= THREAD_CLF_TMP; */ + +set_sp: + mv sp, a1 + sw a0, THREAD_CORE_LOCAL_FLAGS(tp) + + /* + * Save state on stack + */ + addi sp, sp, -THREAD_ABT_REGS_SIZE + + /* Save kernel sp */ + load_xregs tp, THREAD_CORE_LOCAL_X0, REG_A0 + store_xregs sp, THREAD_ABT_REG_SP, REG_A0 + + /* Restore kernel a0, a1 which can be saved later */ + load_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 + + /* Save all other GPRs */ + store_xregs sp, THREAD_ABT_REG_RA, REG_RA + store_xregs sp, THREAD_ABT_REG_GP, REG_GP + store_xregs sp, THREAD_ABT_REG_TP, REG_TP + store_xregs sp, THREAD_ABT_REG_T0, REG_T0, REG_T2 + store_xregs sp, THREAD_ABT_REG_S0, REG_S0, REG_S1 + store_xregs sp, THREAD_ABT_REG_A0, REG_A0, REG_A7 + store_xregs sp, THREAD_ABT_REG_S2, REG_S2, REG_S11 + store_xregs sp, THREAD_ABT_REG_T3, REG_T3, REG_T6 + /* Save XIE */ + csrr t0, CSR_XIE + store_xregs sp, THREAD_ABT_REG_IE, REG_T0 + /* Mask all interrupts */ + csrw CSR_XIE, x0 + /* Save XSTATUS */ + csrr t0, CSR_XSTATUS + store_xregs sp, THREAD_ABT_REG_STATUS, REG_T0 + /* Save XEPC */ + csrr t0, CSR_XEPC + store_xregs sp, THREAD_ABT_REG_EPC, REG_T0 + /* Save XTVAL */ + csrr t0, CSR_XTVAL + store_xregs sp, THREAD_ABT_REG_TVAL, REG_T0 + /* Save XCAUSE */ + csrr a0, CSR_XCAUSE + store_xregs sp, THREAD_ABT_REG_CAUSE, REG_A0 + + /* + * a0 = cause + * a1 = sp (struct thread_abort_regs *regs) + * Call abort_handler(cause, regs) + */ + mv a1, sp + call abort_handler + + /* + * Restore state from stack + */ + + /* Restore XEPC */ + load_xregs sp, THREAD_ABT_REG_EPC, REG_T0 + csrw CSR_XEPC, t0 + /* Restore XIE */ + load_xregs sp, THREAD_ABT_REG_IE, REG_T0 + csrw CSR_XIE, t0 + /* Restore XSTATUS */ + load_xregs sp, THREAD_ABT_REG_STATUS, REG_T0 + csrw CSR_XSTATUS, t0 + /* Set scratch as thread_core_local */ + csrw CSR_XSCRATCH, tp + + /* Update core local flags */ + lw a0, THREAD_CORE_LOCAL_FLAGS(tp) + srli a0, a0, THREAD_CLF_SAVED_SHIFT + sw a0, THREAD_CORE_LOCAL_FLAGS(tp) + + /* Restore all GPRs */ + load_xregs sp, THREAD_ABT_REG_RA, REG_RA + load_xregs sp, THREAD_ABT_REG_GP, REG_GP + load_xregs sp, THREAD_ABT_REG_TP, REG_TP + load_xregs sp, THREAD_ABT_REG_T0, REG_T0, REG_T2 + load_xregs sp, THREAD_ABT_REG_S0, REG_S0, REG_S1 + load_xregs sp, THREAD_ABT_REG_A0, REG_A0, REG_A7 + load_xregs sp, THREAD_ABT_REG_S2, REG_S2, REG_S11 + load_xregs sp, THREAD_ABT_REG_T3, REG_T3, REG_T6 + load_xregs sp, THREAD_ABT_REG_SP, REG_SP + XRET +END_FUNC trap_from_kernel + +LOCAL_FUNC trap_from_user, : /* Save user sp, a0, a1 into temporary spaces of thread_core_local */ store_xregs tp, THREAD_CORE_LOCAL_X0, REG_SP store_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 - /* Load and set kernel sp from thread context */ - get_thread_ctx a0, a1 - load_xregs a0, THREAD_CTX_KERN_SP, REG_SP - /* Now sp is kernel sp, create stack frame to save user context */ - addi sp, sp, -THREAD_TRAP_REGS_SIZE + + csrr a0, CSR_XCAUSE + /* MSB of cause differentiates between interrupts and exceptions */ + bge a0, zero, exception_from_user + +interrupt_from_user: + /* Get thread context as sp */ + get_thread_ctx sp, a0 /* Save user sp */ load_xregs tp, THREAD_CORE_LOCAL_X0, REG_A0 - store_xregs sp, THREAD_TRAP_REG_SP, REG_A0 + store_xregs sp, THREAD_CTX_REG_SP, REG_A0 /* Restore user a0, a1 which can be saved later */ load_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 /* Save user gp */ - store_xregs sp, THREAD_TRAP_REG_GP, REG_GP + store_xregs sp, THREAD_CTX_REG_GP, REG_GP /* * Set the scratch register to 0 such in case of a recursive @@ -55,115 +257,266 @@ */ csrrw gp, CSR_XSCRATCH, zero /* Save user tp we previously swapped into CSR_XSCRATCH */ - store_xregs sp, THREAD_TRAP_REG_TP, REG_GP + store_xregs sp, THREAD_CTX_REG_TP, REG_GP + /* Set kernel gp */ .option push .option norelax la gp, __global_pointer$ .option pop -.else - /* sp is kernel sp */ - addi sp, sp, -THREAD_TRAP_REGS_SIZE - store_xregs sp, THREAD_TRAP_REG_GP, REG_GP - store_xregs sp, THREAD_TRAP_REG_SP, REG_SP -.endif - store_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 - store_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 - store_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 - store_xregs sp, THREAD_TRAP_REG_RA, REG_RA -#if defined(CFG_UNWIND) - /* To unwind stack we need s0, which is frame pointer. */ - store_xregs sp, THREAD_TRAP_REG_S0, REG_S0 -#endif - + /* Save all other GPRs */ + store_xregs sp, THREAD_CTX_REG_RA, REG_RA + store_xregs sp, THREAD_CTX_REG_T0, REG_T0, REG_T2 + store_xregs sp, THREAD_CTX_REG_S0, REG_S0, REG_S1 + store_xregs sp, THREAD_CTX_REG_A0, REG_A0, REG_A7 + store_xregs sp, THREAD_CTX_REG_S2, REG_S2, REG_S11 + store_xregs sp, THREAD_CTX_REG_T3, REG_T3, REG_T6 + /* Save XIE */ csrr t0, CSR_XIE - store_xregs sp, THREAD_TRAP_REG_IE, REG_T0 - + store_xregs sp, THREAD_CTX_REG_IE, REG_T0 /* Mask all interrupts */ csrw CSR_XIE, x0 - + /* Save XSTATUS */ csrr t0, CSR_XSTATUS - store_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 + store_xregs sp, THREAD_CTX_REG_STATUS, REG_T0 + /* Save XEPC */ + csrr t0, CSR_XEPC + store_xregs sp, THREAD_CTX_REG_EPC, REG_T0 + /* + * a0 = cause + * a1 = sp + * Call thread_interrupt_handler(cause, regs) + */ csrr a0, CSR_XCAUSE - csrr a1, CSR_XEPC + mv a1, sp + /* Load tmp_stack_va_end as current sp. */ + load_xregs tp, THREAD_CORE_LOCAL_TMP_STACK_VA_END, REG_SP + call thread_interrupt_handler + + /* Get thread context as sp */ + get_thread_ctx sp, t0 + /* Restore XEPC */ + load_xregs sp, THREAD_CTX_REG_EPC, REG_T0 + csrw CSR_XEPC, t0 + /* Restore XIE */ + load_xregs sp, THREAD_CTX_REG_IE, REG_T0 + csrw CSR_XIE, t0 + /* Restore XSTATUS */ + load_xregs sp, THREAD_CTX_REG_STATUS, REG_T0 + csrw CSR_XSTATUS, t0 + /* Set scratch as thread_core_local */ + csrw CSR_XSCRATCH, tp + /* Restore all GPRs */ + load_xregs sp, THREAD_CTX_REG_RA, REG_RA + load_xregs sp, THREAD_CTX_REG_GP, REG_GP + load_xregs sp, THREAD_CTX_REG_TP, REG_TP + load_xregs sp, THREAD_CTX_REG_T0, REG_T0, REG_T2 + load_xregs sp, THREAD_CTX_REG_S0, REG_S0, REG_S1 + load_xregs sp, THREAD_CTX_REG_A0, REG_A0, REG_A7 + load_xregs sp, THREAD_CTX_REG_S2, REG_S2, REG_S11 + load_xregs sp, THREAD_CTX_REG_T3, REG_T3, REG_T6 + load_xregs sp, THREAD_CTX_REG_SP, REG_SP + XRET - store_xregs sp, THREAD_TRAP_REG_EPC, REG_A1 +exception_from_user: + /* a0 is CSR_XCAUSE */ + li a1, CAUSE_USER_ECALL + bne a0, a1, abort_from_user +ecall_from_user: + /* Load and set kernel sp from thread context */ + get_thread_ctx a0, a1 + load_xregs a0, THREAD_CTX_KERN_SP, REG_SP + + /* Now sp is kernel sp, create stack for struct thread_scall_regs */ + addi sp, sp, -THREAD_SCALL_REGS_SIZE + /* Save user sp */ + load_xregs tp, THREAD_CORE_LOCAL_X0, REG_A0 + store_xregs sp, THREAD_SCALL_REG_SP, REG_A0 - mv a2, sp + /* Restore user a0, a1 which can be saved later */ + load_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 - /* a0 = cause - * a1 = epc - * a2 = sp - * a3 = user - * thread_trap_handler(cause, epc, sp, user) + /* Save user gp */ + store_xregs sp, THREAD_SCALL_REG_GP, REG_GP + /* + * Set the scratch register to 0 such in case of a recursive + * exception thread_trap_vect() knows that it is emitted from kernel. */ -.endm + csrrw gp, CSR_XSCRATCH, zero + /* Save user tp we previously swapped into CSR_XSCRATCH */ + store_xregs sp, THREAD_SCALL_REG_TP, REG_GP + /* Set kernel gp */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop -.macro restore_regs, mode - load_xregs sp, THREAD_TRAP_REG_EPC, REG_T0 - csrw CSR_XEPC, t0 + /* Save other caller-saved registers */ + store_xregs sp, THREAD_SCALL_REG_RA, REG_RA + store_xregs sp, THREAD_SCALL_REG_T0, REG_T0, REG_T2 + store_xregs sp, THREAD_SCALL_REG_A0, REG_A0, REG_A7 + store_xregs sp, THREAD_SCALL_REG_T3, REG_T3, REG_T6 + /* Save XIE */ + csrr a0, CSR_XIE + store_xregs sp, THREAD_SCALL_REG_IE, REG_A0 + /* Mask all interrupts */ + csrw CSR_XIE, zero + /* Save XSTATUS */ + csrr a0, CSR_XSTATUS + store_xregs sp, THREAD_SCALL_REG_STATUS, REG_A0 + /* Save XEPC */ + csrr a0, CSR_XEPC + store_xregs sp, THREAD_SCALL_REG_EPC, REG_A0 - load_xregs sp, THREAD_TRAP_REG_IE, REG_T0 - csrw CSR_XIE, t0 + /* + * a0 = struct thread_scall_regs *regs + * Call thread_scall_handler(regs) + */ + mv a0, sp + call thread_scall_handler - load_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 - csrw CSR_XSTATUS, t0 + /* + * Save kernel sp we'll had at the beginning of this function. + * This is when this TA has called another TA because + * __thread_enter_user_mode() also saves the stack pointer in this + * field. + */ + get_thread_ctx a0, a1 + addi t0, sp, THREAD_SCALL_REGS_SIZE + store_xregs a0, THREAD_CTX_KERN_SP, REG_T0 - load_xregs sp, THREAD_TRAP_REG_RA, REG_RA - load_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 - load_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 - load_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 -#if defined(CFG_UNWIND) - /* To unwind stack we need s0, which is frame pointer. */ - load_xregs sp, THREAD_TRAP_REG_S0, REG_S0 -#endif + /* + * We are returning to U-Mode, on return, the program counter + * is set to xsepc (pc=xepc), we add 4 (size of an instruction) + * to continue to next instruction. + */ + load_xregs sp, THREAD_SCALL_REG_EPC, REG_T0 + addi t0, t0, 4 + csrw CSR_XEPC, t0 -.if \mode == TRAP_MODE_USER + /* Restore XIE */ + load_xregs sp, THREAD_SCALL_REG_IE, REG_T0 + csrw CSR_XIE, t0 + /* Restore XSTATUS */ + load_xregs sp, THREAD_SCALL_REG_STATUS, REG_T0 + csrw CSR_XSTATUS, t0 /* Set scratch as thread_core_local */ csrw CSR_XSCRATCH, tp + /* Restore caller-saved registers */ + load_xregs sp, THREAD_SCALL_REG_RA, REG_RA + load_xregs sp, THREAD_SCALL_REG_GP, REG_GP + load_xregs sp, THREAD_SCALL_REG_TP, REG_TP + load_xregs sp, THREAD_SCALL_REG_T0, REG_T0, REG_T2 + load_xregs sp, THREAD_SCALL_REG_A0, REG_A0, REG_A7 + load_xregs sp, THREAD_SCALL_REG_T3, REG_T3, REG_T6 + load_xregs sp, THREAD_SCALL_REG_SP, REG_SP + XRET - load_xregs sp, THREAD_TRAP_REG_TP, REG_TP - load_xregs sp, THREAD_TRAP_REG_GP, REG_GP - load_xregs sp, THREAD_TRAP_REG_SP, REG_SP +abort_from_user: + /* + * Update core local flags + */ + lw a0, THREAD_CORE_LOCAL_FLAGS(tp) + slli a0, a0, THREAD_CLF_SAVED_SHIFT + ori a0, a0, THREAD_CLF_ABORT + sw a0, THREAD_CORE_LOCAL_FLAGS(tp) -.else - load_xregs sp, THREAD_TRAP_REG_GP, REG_GP - load_xregs sp, THREAD_TRAP_REG_SP, REG_SP - addi sp, sp, THREAD_TRAP_REGS_SIZE -.endif -.endm + /* + * Save state on stack + */ -/* size_t __get_core_pos(void); */ -FUNC __get_core_pos , : , .identity_map - lw a0, THREAD_CORE_LOCAL_HART_ID(tp) - ret -END_FUNC __get_core_pos + /* Load abt_stack_va_end and set it as sp */ + load_xregs tp, THREAD_CORE_LOCAL_ABT_STACK_VA_END, REG_SP -FUNC thread_trap_vect , : - csrrw tp, CSR_XSCRATCH, tp - bnez tp, 0f - /* Read tp back */ - csrrw tp, CSR_XSCRATCH, tp - j trap_from_kernel -0: - /* Now tp is thread_core_local */ - j trap_from_user -thread_trap_vect_end: -END_FUNC thread_trap_vect + /* Now sp is abort sp, create stack for struct thread_abort_regs */ + addi sp, sp, -THREAD_ABT_REGS_SIZE -LOCAL_FUNC trap_from_kernel, : - save_regs TRAP_MODE_KERNEL - li a3, 0 - jal thread_trap_handler - restore_regs TRAP_MODE_KERNEL - XRET -END_FUNC trap_from_kernel + /* Save user sp */ + load_xregs tp, THREAD_CORE_LOCAL_X0, REG_A0 + store_xregs sp, THREAD_ABT_REG_SP, REG_A0 -LOCAL_FUNC trap_from_user, : - save_regs TRAP_MODE_USER - li a3, 1 - jal thread_trap_handler - restore_regs TRAP_MODE_USER + /* Restore user a0, a1 which can be saved later */ + load_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 + + /* Save user gp */ + store_xregs sp, THREAD_ABT_REG_GP, REG_GP + + /* + * Set the scratch register to 0 such in case of a recursive + * exception thread_trap_vect() knows that it is emitted from kernel. + */ + csrrw gp, CSR_XSCRATCH, zero + /* Save user tp we previously swapped into CSR_XSCRATCH */ + store_xregs sp, THREAD_ABT_REG_TP, REG_GP + /* Set kernel gp */ +.option push +.option norelax + la gp, __global_pointer$ +.option pop + /* Save all other GPRs */ + store_xregs sp, THREAD_ABT_REG_RA, REG_RA + store_xregs sp, THREAD_ABT_REG_T0, REG_T0, REG_T2 + store_xregs sp, THREAD_ABT_REG_S0, REG_S0, REG_S1 + store_xregs sp, THREAD_ABT_REG_A0, REG_A0, REG_A7 + store_xregs sp, THREAD_ABT_REG_S2, REG_S2, REG_S11 + store_xregs sp, THREAD_ABT_REG_T3, REG_T3, REG_T6 + /* Save XIE */ + csrr t0, CSR_XIE + store_xregs sp, THREAD_ABT_REG_IE, REG_T0 + /* Mask all interrupts */ + csrw CSR_XIE, x0 + /* Save XSTATUS */ + csrr t0, CSR_XSTATUS + store_xregs sp, THREAD_ABT_REG_STATUS, REG_T0 + /* Save XEPC */ + csrr t0, CSR_XEPC + store_xregs sp, THREAD_ABT_REG_EPC, REG_T0 + /* Save XTVAL */ + csrr t0, CSR_XTVAL + store_xregs sp, THREAD_ABT_REG_TVAL, REG_T0 + /* Save XCAUSE */ + csrr a0, CSR_XCAUSE + store_xregs sp, THREAD_ABT_REG_CAUSE, REG_A0 + + /* + * a0 = cause + * a1 = sp (struct thread_abort_regs *regs) + * Call abort_handler(cause, regs) + */ + mv a1, sp + call abort_handler + + /* + * Restore state from stack + */ + + /* Restore XEPC */ + load_xregs sp, THREAD_ABT_REG_EPC, REG_T0 + csrw CSR_XEPC, t0 + /* Restore XIE */ + load_xregs sp, THREAD_ABT_REG_IE, REG_T0 + csrw CSR_XIE, t0 + /* Restore XSTATUS */ + load_xregs sp, THREAD_ABT_REG_STATUS, REG_T0 + csrw CSR_XSTATUS, t0 + /* Set scratch as thread_core_local */ + csrw CSR_XSCRATCH, tp + + /* Update core local flags */ + lw a0, THREAD_CORE_LOCAL_FLAGS(tp) + srli a0, a0, THREAD_CLF_SAVED_SHIFT + sw a0, THREAD_CORE_LOCAL_FLAGS(tp) + + /* Restore all GPRs */ + load_xregs sp, THREAD_ABT_REG_RA, REG_RA + load_xregs sp, THREAD_ABT_REG_GP, REG_GP + load_xregs sp, THREAD_ABT_REG_TP, REG_TP + load_xregs sp, THREAD_ABT_REG_T0, REG_T0, REG_T2 + load_xregs sp, THREAD_ABT_REG_S0, REG_S0, REG_S1 + load_xregs sp, THREAD_ABT_REG_A0, REG_A0, REG_A7 + load_xregs sp, THREAD_ABT_REG_S2, REG_S2, REG_S11 + load_xregs sp, THREAD_ABT_REG_T3, REG_T3, REG_T6 + load_xregs sp, THREAD_ABT_REG_SP, REG_SP XRET END_FUNC trap_from_user