1/* SPDX-License-Identifier: BSD-2-Clause */ 2/* 3 * Copyright 2022-2023 NXP 4 */ 5 6#include <asm.S> 7#include <generated/asm-defines.h> 8#include <keep.h> 9#include <kernel/thread.h> 10#include <kernel/thread_private.h> 11#include <mm/core_mmu.h> 12#include <riscv.h> 13#include <riscv_macros.S> 14 15.macro get_thread_ctx res, tmp0 16 lw \tmp0, THREAD_CORE_LOCAL_CURR_THREAD(tp) 17 la \res, threads 181: 19 beqz \tmp0, 2f 20 addi \res, \res, THREAD_CTX_SIZE 21 addi \tmp0, \tmp0, -1 22 bnez \tmp0, 1b 232: 24.endm 25 26.macro b_if_prev_priv_is_u reg, label 27 andi \reg, \reg, CSR_XSTATUS_SPP 28 beqz \reg, \label 29.endm 30 31.macro save_regs, mode 32.if \mode == TRAP_MODE_USER 33 /* Save user sp, a0, a1 into temporary spaces of thread_core_local */ 34 store_xregs tp, THREAD_CORE_LOCAL_X0, REG_SP 35 store_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 36 /* Load and set kernel sp from thread context */ 37 get_thread_ctx a0, a1 38 load_xregs a0, THREAD_CTX_KERN_SP, REG_SP 39 /* Now sp is kernel sp, create stack frame to save user context */ 40 addi sp, sp, -THREAD_TRAP_REGS_SIZE 41 42 /* Save user sp */ 43 load_xregs tp, THREAD_CORE_LOCAL_X0, REG_A0 44 store_xregs sp, THREAD_TRAP_REG_SP, REG_A0 45 46 /* Restore user a0, a1 which can be saved later */ 47 load_xregs tp, THREAD_CORE_LOCAL_X1, REG_A0, REG_A1 48 49 /* Save user gp */ 50 store_xregs sp, THREAD_TRAP_REG_GP, REG_GP 51 52 /* 53 * Set the scratch register to 0 such in case of a recursive 54 * exception thread_trap_vect() knows that it is emitted from kernel. 55 */ 56 csrrw gp, CSR_XSCRATCH, zero 57 /* Save user tp we previously swapped into CSR_XSCRATCH */ 58 store_xregs sp, THREAD_TRAP_REG_TP, REG_GP 59.option push 60.option norelax 61 la gp, __global_pointer$ 62.option pop 63.else 64 /* sp is kernel sp */ 65 addi sp, sp, -THREAD_TRAP_REGS_SIZE 66 store_xregs sp, THREAD_TRAP_REG_GP, REG_GP 67 store_xregs sp, THREAD_TRAP_REG_SP, REG_SP 68.endif 69 store_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 70 store_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 71 store_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 72 store_xregs sp, THREAD_TRAP_REG_RA, REG_RA 73#if defined(CFG_UNWIND) 74 /* To unwind stack we need s0, which is frame pointer. */ 75 store_xregs sp, THREAD_TRAP_REG_S0, REG_S0 76#endif 77 78 csrr t0, CSR_XIE 79 store_xregs sp, THREAD_TRAP_REG_IE, REG_T0 80 81 /* Mask all interrupts */ 82 csrw CSR_XIE, x0 83 84 csrr t0, CSR_XSTATUS 85 store_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 86 87 csrr a0, CSR_XCAUSE 88 csrr a1, CSR_XEPC 89 90 store_xregs sp, THREAD_TRAP_REG_EPC, REG_A1 91 92 mv a2, sp 93 94 /* a0 = cause 95 * a1 = epc 96 * a2 = sp 97 * a3 = user 98 * thread_trap_handler(cause, epc, sp, user) 99 */ 100.endm 101 102.macro restore_regs, mode 103 load_xregs sp, THREAD_TRAP_REG_EPC, REG_T0 104 csrw CSR_XEPC, t0 105 106 load_xregs sp, THREAD_TRAP_REG_IE, REG_T0 107 csrw CSR_XIE, t0 108 109 load_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 110 csrw CSR_XSTATUS, t0 111 112 load_xregs sp, THREAD_TRAP_REG_RA, REG_RA 113 load_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 114 load_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 115 load_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 116#if defined(CFG_UNWIND) 117 /* To unwind stack we need s0, which is frame pointer. */ 118 load_xregs sp, THREAD_TRAP_REG_S0, REG_S0 119#endif 120 121.if \mode == TRAP_MODE_USER 122 /* Set scratch as thread_core_local */ 123 csrw CSR_XSCRATCH, tp 124 125 load_xregs sp, THREAD_TRAP_REG_TP, REG_TP 126 load_xregs sp, THREAD_TRAP_REG_GP, REG_GP 127 load_xregs sp, THREAD_TRAP_REG_SP, REG_SP 128 129.else 130 load_xregs sp, THREAD_TRAP_REG_GP, REG_GP 131 load_xregs sp, THREAD_TRAP_REG_SP, REG_SP 132 addi sp, sp, THREAD_TRAP_REGS_SIZE 133.endif 134.endm 135 136/* size_t __get_core_pos(void); */ 137FUNC __get_core_pos , : , .identity_map 138 lw a0, THREAD_CORE_LOCAL_HART_ID(tp) 139 ret 140END_FUNC __get_core_pos 141 142FUNC thread_trap_vect , : 143 csrrw tp, CSR_XSCRATCH, tp 144 bnez tp, 0f 145 /* Read tp back */ 146 csrrw tp, CSR_XSCRATCH, tp 147 j trap_from_kernel 1480: 149 /* Now tp is thread_core_local */ 150 j trap_from_user 151thread_trap_vect_end: 152END_FUNC thread_trap_vect 153 154LOCAL_FUNC trap_from_kernel, : 155 save_regs TRAP_MODE_KERNEL 156 li a3, 0 157 jal thread_trap_handler 158 restore_regs TRAP_MODE_KERNEL 159 XRET 160END_FUNC trap_from_kernel 161 162LOCAL_FUNC trap_from_user, : 163 save_regs TRAP_MODE_USER 164 li a3, 1 165 jal thread_trap_handler 166 restore_regs TRAP_MODE_USER 167 XRET 168END_FUNC trap_from_user 169 170/* 171 * void thread_unwind_user_mode(uint32_t ret, uint32_t exit_status0, 172 * uint32_t exit_status1); 173 * See description in thread.h 174 */ 175FUNC thread_unwind_user_mode , : 176 177 /* Store the exit status */ 178 load_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A3, REG_A5 179 sw a1, (a4) 180 sw a2, (a5) 181 182 /* Save user callee regs */ 183 store_xregs a3, THREAD_CTX_REG_S0, REG_S0, REG_S1 184 store_xregs a3, THREAD_CTX_REG_S2, REG_S2, REG_S11 185 store_xregs a3, THREAD_CTX_REG_SP, REG_SP, REG_TP 186 187 /* Restore kernel callee regs */ 188 mv a1, sp 189 190 load_xregs a1, THREAD_USER_MODE_REC_X1, REG_RA, REG_GP 191 load_xregs a1, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 192 load_xregs a1, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 193 194 add sp, sp, THREAD_USER_MODE_REC_SIZE 195 196 /* Return from the call of thread_enter_user_mode() */ 197 ret 198END_FUNC thread_unwind_user_mode 199 200/* 201 * void thread_exit_user_mode(unsigned long a0, unsigned long a1, 202 * unsigned long a2, unsigned long a3, 203 * unsigned long sp, unsigned long pc, 204 * unsigned long status); 205 */ 206FUNC thread_exit_user_mode , : 207 /* Set kernel stack pointer */ 208 mv sp, a4 209 210 /* Set xSTATUS */ 211 csrw CSR_XSTATUS, a6 212 213 /* 214 * Zeroize xSCRATCH to indicate to thread_trap_vect() 215 * that we are executing in kernel. 216 */ 217 csrw CSR_XSCRATCH, zero 218 219 /* 220 * Mask all interrupts first. Interrupts will be unmasked after 221 * returning from __thread_enter_user_mode(). 222 */ 223 csrw CSR_XIE, zero 224 225 /* Set epc as thread_unwind_user_mode() */ 226 csrw CSR_XEPC, a5 227 228 XRET 229END_FUNC thread_exit_user_mode 230 231/* 232 * uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, 233 * uint32_t *exit_status0, 234 * uint32_t *exit_status1); 235 */ 236FUNC __thread_enter_user_mode , : 237 /* Disable kernel mode exceptions first */ 238 csrc CSR_XSTATUS, CSR_XSTATUS_IE 239 240 /* 241 * Create and fill in the struct thread_user_mode_rec 242 */ 243 addi sp, sp, -THREAD_USER_MODE_REC_SIZE 244 store_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A0, REG_A2 245 store_xregs sp, THREAD_USER_MODE_REC_X1, REG_RA, REG_GP 246 store_xregs sp, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 247 store_xregs sp, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 248 249 /* 250 * Save the kernel stack pointer in the thread context 251 */ 252 253 /* Get pointer to current thread context */ 254 get_thread_ctx s0, s1 255 256 /* 257 * Save kernel stack pointer to ensure that 258 * thread_exit_user_mode() uses correct stack pointer. 259 */ 260 261 store_xregs s0, THREAD_CTX_KERN_SP, REG_SP 262 /* 263 * Save thread_core_local in xSCRATCH to ensure that thread_trap_vect() 264 * uses correct core local structure. 265 */ 266 csrw CSR_XSCRATCH, tp 267 268 /* Set user ie */ 269 load_xregs a0, THREAD_CTX_REG_IE, REG_S0 270 csrw CSR_XIE, s0 271 272 /* Set user status */ 273 load_xregs a0, THREAD_CTX_REG_STATUS, REG_S0 274 csrw CSR_XSTATUS, s0 275 276 /* Load the rest of the general purpose registers */ 277 load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_TP 278 load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 279 load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 280 load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 281 load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 282 load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 283 284 /* Set exception program counter */ 285 csrw CSR_XEPC, ra 286 287 /* Jump into user mode */ 288 XRET 289END_FUNC __thread_enter_user_mode 290 291/* void thread_resume(struct thread_ctx_regs *regs) */ 292FUNC thread_resume , : 293 /* Disable global interrupts first */ 294 csrc CSR_XSTATUS, CSR_XSTATUS_IE 295 296 /* Restore epc */ 297 load_xregs a0, THREAD_CTX_REG_EPC, REG_T0 298 csrw CSR_XEPC, t0 299 300 /* Restore ie */ 301 load_xregs a0, THREAD_CTX_REG_IE, REG_T0 302 csrw CSR_XIE, t0 303 304 /* Restore status */ 305 load_xregs a0, THREAD_CTX_REG_STATUS, REG_T0 306 csrw CSR_XSTATUS, t0 307 308 /* Check if previous privilege mode by status.SPP */ 309 b_if_prev_priv_is_u t0, 1f 310 /* Set scratch as zero to indicate that we are in kernel mode */ 311 csrw CSR_XSCRATCH, zero 312 j 2f 3131: 314 /* Resume to U-mode, set scratch as tp to be used in the trap handler */ 315 csrw CSR_XSCRATCH, tp 3162: 317 /* Restore all general-purpose registers */ 318 load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_TP 319 load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 320 load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 321 load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 322 load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 323 load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 324 325 XRET 326END_FUNC thread_resume 327