1/* SPDX-License-Identifier: BSD-2-Clause */ 2/* 3 * Copyright 2022-2023 NXP 4 */ 5 6#include <asm.S> 7#include <generated/asm-defines.h> 8#include <keep.h> 9#include <kernel/thread.h> 10#include <kernel/thread_private.h> 11#include <mm/core_mmu.h> 12#include <riscv.h> 13#include <riscv_macros.S> 14 15.macro get_thread_ctx res, tmp0 16 lw \tmp0, THREAD_CORE_LOCAL_CURR_THREAD(tp) 17 la \res, threads 181: 19 beqz \tmp0, 2f 20 addi \res, \res, THREAD_CTX_SIZE 21 addi \tmp0, \tmp0, -1 22 bnez \tmp0, 1b 232: 24.endm 25 26.macro b_if_prev_priv_is_u reg, label 27 andi \reg, \reg, CSR_XSTATUS_SPP 28 beqz \reg, \label 29.endm 30 31.macro save_regs, mode 32 addi sp, sp, -THREAD_TRAP_REGS_SIZE 33.if \mode == TRAP_MODE_USER 34 35 /* Save user thread pointer and load kernel thread pointer */ 36 store_xregs sp, THREAD_TRAP_REG_TP, REG_TP 37 addi tp, sp, THREAD_TRAP_REGS_SIZE 38 /* Now tp is at struct thread_user_mode_rec, which has kernel tp */ 39 load_xregs tp, THREAD_USER_MODE_REC_X4, REG_TP 40 41 store_xregs sp, THREAD_TRAP_REG_GP, REG_GP 42 43 /* 44 * Set the scratch register to 0 such in case of a recursive 45 * exception thread_trap_vect() knows that it is emitted from kernel. 46 */ 47 csrrw gp, CSR_XSCRATCH, zero 48 store_xregs sp, THREAD_TRAP_REG_SP, REG_GP 49.option push 50.option norelax 51 la gp, __global_pointer$ 52.option pop 53.endif 54 store_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 55 store_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 56 store_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 57 store_xregs sp, THREAD_TRAP_REG_RA, REG_RA 58#if defined(CFG_UNWIND) 59 /* To unwind stack we need s0, which is frame pointer. */ 60 store_xregs sp, THREAD_TRAP_REG_S0, REG_S0 61#endif 62 63 csrr t0, CSR_XSTATUS 64 store_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 65 66 csrr a0, CSR_XCAUSE 67 csrr a1, CSR_XEPC 68 69 store_xregs sp, THREAD_TRAP_REG_EPC, REG_A1 70 71 mv a2, sp 72 73 /* a0 = cause 74 * a1 = epc 75 * a2 = sp 76 * a3 = user 77 * thread_trap_handler(cause, epc, sp, user) 78 */ 79.endm 80 81.macro restore_regs, mode 82 load_xregs sp, THREAD_TRAP_REG_EPC, REG_T0 83 csrw CSR_XEPC, t0 84 85 load_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 86 csrw CSR_XSTATUS, t0 87 88 load_xregs sp, THREAD_TRAP_REG_RA, REG_RA 89 load_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 90 load_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 91 load_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 92#if defined(CFG_UNWIND) 93 /* To unwind stack we need s0, which is frame pointer. */ 94 load_xregs sp, THREAD_TRAP_REG_S0, REG_S0 95#endif 96 97.if \mode == TRAP_MODE_USER 98 addi gp, sp, THREAD_TRAP_REGS_SIZE 99 csrw CSR_XSCRATCH, gp 100 101 load_xregs sp, THREAD_TRAP_REG_TP, REG_TP 102 load_xregs sp, THREAD_TRAP_REG_GP, REG_GP 103 load_xregs sp, THREAD_TRAP_REG_SP, REG_SP 104 105.else 106 addi sp, sp, THREAD_TRAP_REGS_SIZE 107.endif 108.endm 109 110/* size_t __get_core_pos(void); */ 111FUNC __get_core_pos , : , .identity_map 112 lw a0, THREAD_CORE_LOCAL_HART_ID(tp) 113 ret 114END_FUNC __get_core_pos 115 116FUNC thread_trap_vect , : 117 csrrw sp, CSR_XSCRATCH, sp 118 bnez sp, 0f 119 csrrw sp, CSR_XSCRATCH, sp 120 j trap_from_kernel 1210: 122 j trap_from_user 123thread_trap_vect_end: 124END_FUNC thread_trap_vect 125 126LOCAL_FUNC trap_from_kernel, : 127 save_regs TRAP_MODE_KERNEL 128 li a3, 0 129 jal thread_trap_handler 130 restore_regs TRAP_MODE_KERNEL 131 XRET 132END_FUNC trap_from_kernel 133 134LOCAL_FUNC trap_from_user, : 135 save_regs TRAP_MODE_USER 136 li a3, 1 137 jal thread_trap_handler 138 restore_regs TRAP_MODE_USER 139 XRET 140END_FUNC trap_from_user 141 142/* 143 * void thread_unwind_user_mode(uint32_t ret, uint32_t exit_status0, 144 * uint32_t exit_status1); 145 * See description in thread.h 146 */ 147FUNC thread_unwind_user_mode , : 148 149 /* Store the exit status */ 150 load_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A3, REG_A5 151 sw a1, (a4) 152 sw a2, (a5) 153 154 /* Save user callee regs */ 155 store_xregs a3, THREAD_CTX_REG_S0, REG_S0, REG_S1 156 store_xregs a3, THREAD_CTX_REG_S2, REG_S2, REG_S11 157 store_xregs a3, THREAD_CTX_REG_SP, REG_SP, REG_TP 158 159 /* Restore kernel callee regs */ 160 mv a1, sp 161 162 load_xregs a1, THREAD_USER_MODE_REC_X1, REG_RA, REG_TP 163 load_xregs a1, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 164 load_xregs a1, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 165 166 add sp, sp, THREAD_USER_MODE_REC_SIZE 167 168 /* 169 * Zeroize xSCRATCH to indicate to thread_trap_vect() 170 * that we are executing in kernel. 171 */ 172 csrw CSR_XSCRATCH, zero 173 174 /* Return from the call of thread_enter_user_mode() */ 175 ret 176END_FUNC thread_unwind_user_mode 177 178/* 179 * void thread_exit_user_mode(unsigned long a0, unsigned long a1, 180 * unsigned long a2, unsigned long a3, 181 * unsigned long sp, unsigned long pc, 182 * unsigned long status); 183 */ 184FUNC thread_exit_user_mode , : 185 /* Set kernel stack pointer */ 186 mv sp, a4 187 188 /* Set xSTATUS */ 189 csrw CSR_XSTATUS, a6 190 191 /* Set return address thread_unwind_user_mode() */ 192 mv ra, a5 193 ret 194END_FUNC thread_exit_user_mode 195 196/* 197 * uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, 198 * uint32_t *exit_status0, 199 * uint32_t *exit_status1); 200 */ 201FUNC __thread_enter_user_mode , : 202 /* 203 * Create and fill in the struct thread_user_mode_rec 204 */ 205 addi sp, sp, -THREAD_USER_MODE_REC_SIZE 206 store_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A0, REG_A2 207 store_xregs sp, THREAD_USER_MODE_REC_X1, REG_RA, REG_TP 208 store_xregs sp, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 209 store_xregs sp, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 210 211 /* 212 * Save the kernel stack pointer in the thread context 213 */ 214 215 /* Get pointer to current thread context */ 216 get_thread_ctx s0, s1 217 218 /* 219 * Save kernel stack pointer to ensure that 220 * thread_exit_user_mode() uses correct stack pointer. 221 */ 222 223 store_xregs s0, THREAD_CTX_KERN_SP, REG_SP 224 /* 225 * Save kernel stack pointer in xSCRATCH to ensure that 226 * thread_trap_vect() uses correct stack pointer. 227 */ 228 csrw CSR_XSCRATCH, sp 229 230 /* Set user status */ 231 load_xregs a0, THREAD_CTX_REG_STATUS, REG_S0 232 csrw CSR_XSTATUS, s0 233 234 /* 235 * Save the values for a1 and a2 in struct thread_core_local to be 236 * restored later just before the xRET. 237 */ 238 store_xregs tp, THREAD_CORE_LOCAL_X10, REG_A1, REG_A2 239 240 /* Load the rest of the general purpose registers */ 241 load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_TP 242 load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 243 load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 244 load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 245 load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 246 load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 247 248 /* Set exception program counter */ 249 csrw CSR_XEPC, ra 250 251 /* Jump into user mode */ 252 XRET 253END_FUNC __thread_enter_user_mode 254 255/* void thread_resume(struct thread_ctx_regs *regs) */ 256FUNC thread_resume , : 257 /* Disable global interrupts first */ 258 csrc CSR_XSTATUS, CSR_XSTATUS_IE 259 260 /* Restore epc */ 261 load_xregs a0, THREAD_CTX_REG_EPC, REG_T0 262 csrw CSR_XEPC, t0 263 264 /* Restore ie */ 265 load_xregs a0, THREAD_CTX_REG_IE, REG_T0 266 csrw CSR_XIE, t0 267 268 /* Restore status */ 269 load_xregs a0, THREAD_CTX_REG_STATUS, REG_T0 270 csrw CSR_XSTATUS, t0 271 272 /* Check if previous privilege mode by status.SPP */ 273 b_if_prev_priv_is_u t0, 1f 274 /* Set scratch as zero to indicate that we are in kernel mode */ 275 csrw CSR_XSCRATCH, zero 276 j 2f 2771: 278 /* Resume to U-mode, set scratch as tp to be used in the trap handler */ 279 csrw CSR_XSCRATCH, tp 2802: 281 /* Restore all general-purpose registers */ 282 load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_TP 283 load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 284 load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 285 load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 286 load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 287 load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 288 289 XRET 290END_FUNC thread_resume 291