1/* SPDX-License-Identifier: BSD-2-Clause */ 2/* 3 * Copyright 2022-2023 NXP 4 */ 5 6#include <asm.S> 7#include <generated/asm-defines.h> 8#include <keep.h> 9#include <kernel/thread.h> 10#include <kernel/thread_private.h> 11#include <mm/core_mmu.h> 12#include <riscv.h> 13#include <riscv_macros.S> 14 15.macro get_thread_ctx res, tmp0 16 lw \tmp0, THREAD_CORE_LOCAL_CURR_THREAD(tp) 17 la \res, threads 181: 19 beqz \tmp0, 2f 20 addi \res, \res, THREAD_CTX_SIZE 21 addi \tmp0, \tmp0, -1 22 bnez \tmp0, 1b 232: 24.endm 25 26.macro b_if_prev_priv_is_u reg, label 27 andi \reg, \reg, CSR_XSTATUS_SPP 28 beqz \reg, \label 29.endm 30 31.macro save_regs, mode 32 addi sp, sp, -THREAD_TRAP_REGS_SIZE 33.if \mode == TRAP_MODE_USER 34 35 /* Save user thread pointer and load kernel thread pointer */ 36 store_xregs sp, THREAD_TRAP_REG_TP, REG_TP 37 addi tp, sp, THREAD_TRAP_REGS_SIZE 38 /* Now tp is at struct thread_user_mode_rec, which has kernel tp */ 39 load_xregs tp, THREAD_USER_MODE_REC_X4, REG_TP 40 41 store_xregs sp, THREAD_TRAP_REG_GP, REG_GP 42 43 /* 44 * Set the scratch register to 0 such in case of a recursive 45 * exception thread_trap_vect() knows that it is emitted from kernel. 46 */ 47 csrrw gp, CSR_XSCRATCH, zero 48 store_xregs sp, THREAD_TRAP_REG_SP, REG_GP 49.option push 50.option norelax 51 la gp, __global_pointer$ 52.option pop 53.endif 54 store_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 55 store_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 56 store_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 57 store_xregs sp, THREAD_TRAP_REG_RA, REG_RA 58#if defined(CFG_UNWIND) 59 /* To unwind stack we need s0, which is frame pointer. */ 60 store_xregs sp, THREAD_TRAP_REG_S0, REG_S0 61#endif 62 63 csrr t0, CSR_XSTATUS 64 store_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 65 66 csrr a0, CSR_XCAUSE 67 csrr a1, CSR_XEPC 68 69 store_xregs sp, THREAD_TRAP_REG_EPC, REG_A1 70 71 mv a2, sp 72 73 /* a0 = cause 74 * a1 = epc 75 * a2 = sp 76 * a3 = user 77 * thread_trap_handler(cause, epc, sp, user) 78 */ 79.endm 80 81.macro restore_regs, mode 82 load_xregs sp, THREAD_TRAP_REG_EPC, REG_T0 83 csrw CSR_XEPC, t0 84 85 load_xregs sp, THREAD_TRAP_REG_STATUS, REG_T0 86 csrw CSR_XSTATUS, t0 87 88 load_xregs sp, THREAD_TRAP_REG_RA, REG_RA 89 load_xregs sp, THREAD_TRAP_REG_A0, REG_A0, REG_A7 90 load_xregs sp, THREAD_TRAP_REG_T0, REG_T0, REG_T2 91 load_xregs sp, THREAD_TRAP_REG_T3, REG_T3, REG_T6 92#if defined(CFG_UNWIND) 93 /* To unwind stack we need s0, which is frame pointer. */ 94 load_xregs sp, THREAD_TRAP_REG_S0, REG_S0 95#endif 96 97.if \mode == TRAP_MODE_USER 98 addi gp, sp, THREAD_TRAP_REGS_SIZE 99 csrw CSR_XSCRATCH, gp 100 101 load_xregs sp, THREAD_TRAP_REG_TP, REG_TP 102 load_xregs sp, THREAD_TRAP_REG_GP, REG_GP 103 load_xregs sp, THREAD_TRAP_REG_SP, REG_SP 104 105.else 106 addi sp, sp, THREAD_TRAP_REGS_SIZE 107.endif 108.endm 109 110/* size_t __get_core_pos(void); */ 111FUNC __get_core_pos , : , .identity_map 112 lw a0, THREAD_CORE_LOCAL_HART_ID(tp) 113 ret 114END_FUNC __get_core_pos 115 116FUNC thread_trap_vect , : 117 csrrw sp, CSR_XSCRATCH, sp 118 bnez sp, 0f 119 csrrw sp, CSR_XSCRATCH, sp 120 j trap_from_kernel 1210: 122 j trap_from_user 123thread_trap_vect_end: 124END_FUNC thread_trap_vect 125 126LOCAL_FUNC trap_from_kernel, : 127 save_regs TRAP_MODE_KERNEL 128 li a3, 0 129 jal thread_trap_handler 130 restore_regs TRAP_MODE_KERNEL 131 XRET 132END_FUNC trap_from_kernel 133 134LOCAL_FUNC trap_from_user, : 135 save_regs TRAP_MODE_USER 136 li a3, 1 137 jal thread_trap_handler 138 restore_regs TRAP_MODE_USER 139 XRET 140END_FUNC trap_from_user 141 142/* 143 * void thread_unwind_user_mode(uint32_t ret, uint32_t exit_status0, 144 * uint32_t exit_status1); 145 * See description in thread.h 146 */ 147FUNC thread_unwind_user_mode , : 148 149 /* Store the exit status */ 150 load_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A3, REG_A5 151 sw a1, (a4) 152 sw a2, (a5) 153 154 /* Save user callee regs */ 155 store_xregs a3, THREAD_CTX_REG_S0, REG_S0, REG_S1 156 store_xregs a3, THREAD_CTX_REG_S2, REG_S2, REG_S11 157 store_xregs a3, THREAD_CTX_REG_SP, REG_SP, REG_TP 158 159 /* Restore kernel callee regs */ 160 mv a1, sp 161 162 load_xregs a1, THREAD_USER_MODE_REC_X1, REG_RA, REG_GP 163 load_xregs a1, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 164 load_xregs a1, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 165 166 add sp, sp, THREAD_USER_MODE_REC_SIZE 167 168 /* Return from the call of thread_enter_user_mode() */ 169 ret 170END_FUNC thread_unwind_user_mode 171 172/* 173 * void thread_exit_user_mode(unsigned long a0, unsigned long a1, 174 * unsigned long a2, unsigned long a3, 175 * unsigned long sp, unsigned long pc, 176 * unsigned long status); 177 */ 178FUNC thread_exit_user_mode , : 179 /* Set kernel stack pointer */ 180 mv sp, a4 181 182 /* Set xSTATUS */ 183 csrw CSR_XSTATUS, a6 184 185 /* 186 * Zeroize xSCRATCH to indicate to thread_trap_vect() 187 * that we are executing in kernel. 188 */ 189 csrw CSR_XSCRATCH, zero 190 191 /* 192 * Mask all interrupts first. Interrupts will be unmasked after 193 * returning from __thread_enter_user_mode(). 194 */ 195 csrw CSR_XIE, zero 196 197 /* Set epc as thread_unwind_user_mode() */ 198 csrw CSR_XEPC, a5 199 200 XRET 201END_FUNC thread_exit_user_mode 202 203/* 204 * uint32_t __thread_enter_user_mode(struct thread_ctx_regs *regs, 205 * uint32_t *exit_status0, 206 * uint32_t *exit_status1); 207 */ 208FUNC __thread_enter_user_mode , : 209 /* Disable kernel mode exceptions first */ 210 csrc CSR_XSTATUS, CSR_XSTATUS_IE 211 212 /* 213 * Create and fill in the struct thread_user_mode_rec 214 */ 215 addi sp, sp, -THREAD_USER_MODE_REC_SIZE 216 store_xregs sp, THREAD_USER_MODE_REC_CTX_REGS_PTR, REG_A0, REG_A2 217 store_xregs sp, THREAD_USER_MODE_REC_X1, REG_RA, REG_GP 218 store_xregs sp, THREAD_USER_MODE_REC_X8, REG_S0, REG_S1 219 store_xregs sp, THREAD_USER_MODE_REC_X18, REG_S2, REG_S11 220 221 /* 222 * Save the kernel stack pointer in the thread context 223 */ 224 225 /* Get pointer to current thread context */ 226 get_thread_ctx s0, s1 227 228 /* 229 * Save kernel stack pointer to ensure that 230 * thread_exit_user_mode() uses correct stack pointer. 231 */ 232 233 store_xregs s0, THREAD_CTX_KERN_SP, REG_SP 234 /* 235 * Save thread_core_local in xSCRATCH to ensure that thread_trap_vect() 236 * uses correct core local structure. 237 */ 238 csrw CSR_XSCRATCH, tp 239 240 /* Set user ie */ 241 load_xregs a0, THREAD_CTX_REG_IE, REG_S0 242 csrw CSR_XIE, s0 243 244 /* Set user status */ 245 load_xregs a0, THREAD_CTX_REG_STATUS, REG_S0 246 csrw CSR_XSTATUS, s0 247 248 /* Load the rest of the general purpose registers */ 249 load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_TP 250 load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 251 load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 252 load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 253 load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 254 load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 255 256 /* Set exception program counter */ 257 csrw CSR_XEPC, ra 258 259 /* Jump into user mode */ 260 XRET 261END_FUNC __thread_enter_user_mode 262 263/* void thread_resume(struct thread_ctx_regs *regs) */ 264FUNC thread_resume , : 265 /* Disable global interrupts first */ 266 csrc CSR_XSTATUS, CSR_XSTATUS_IE 267 268 /* Restore epc */ 269 load_xregs a0, THREAD_CTX_REG_EPC, REG_T0 270 csrw CSR_XEPC, t0 271 272 /* Restore ie */ 273 load_xregs a0, THREAD_CTX_REG_IE, REG_T0 274 csrw CSR_XIE, t0 275 276 /* Restore status */ 277 load_xregs a0, THREAD_CTX_REG_STATUS, REG_T0 278 csrw CSR_XSTATUS, t0 279 280 /* Check if previous privilege mode by status.SPP */ 281 b_if_prev_priv_is_u t0, 1f 282 /* Set scratch as zero to indicate that we are in kernel mode */ 283 csrw CSR_XSCRATCH, zero 284 j 2f 2851: 286 /* Resume to U-mode, set scratch as tp to be used in the trap handler */ 287 csrw CSR_XSCRATCH, tp 2882: 289 /* Restore all general-purpose registers */ 290 load_xregs a0, THREAD_CTX_REG_RA, REG_RA, REG_TP 291 load_xregs a0, THREAD_CTX_REG_T0, REG_T0, REG_T2 292 load_xregs a0, THREAD_CTX_REG_S0, REG_S0, REG_S1 293 load_xregs a0, THREAD_CTX_REG_S2, REG_S2, REG_S11 294 load_xregs a0, THREAD_CTX_REG_T3, REG_T3, REG_T6 295 load_xregs a0, THREAD_CTX_REG_A0, REG_A0, REG_A7 296 297 XRET 298END_FUNC thread_resume 299