1/* 2 * Copyright (c) 2013-2022, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <platform_def.h> 8 9#include <arch.h> 10#include <asm_macros.S> 11#include <bl32/tsp/tsp.h> 12#include <lib/xlat_tables/xlat_tables_defs.h> 13#include <smccc_helpers.h> 14 15#include "../tsp_private.h" 16 17 18 .globl tsp_entrypoint 19 .globl tsp_vector_table 20#if SPMC_AT_EL3 21 .globl tsp_cpu_on_entry 22#endif 23 24 25 26 /* --------------------------------------------- 27 * Populate the params in x0-x7 from the pointer 28 * to the smc args structure in x0. 29 * --------------------------------------------- 30 */ 31 .macro restore_args_call_smc 32 ldp x6, x7, [x0, #SMC_ARG6] 33 ldp x4, x5, [x0, #SMC_ARG4] 34 ldp x2, x3, [x0, #SMC_ARG2] 35 ldp x0, x1, [x0, #SMC_ARG0] 36 smc #0 37 .endm 38 39 .macro save_eret_context reg1 reg2 40 mrs \reg1, elr_el1 41 mrs \reg2, spsr_el1 42 stp \reg1, \reg2, [sp, #-0x10]! 43 stp x30, x18, [sp, #-0x10]! 44 .endm 45 46 .macro restore_eret_context reg1 reg2 47 ldp x30, x18, [sp], #0x10 48 ldp \reg1, \reg2, [sp], #0x10 49 msr elr_el1, \reg1 50 msr spsr_el1, \reg2 51 .endm 52 53func tsp_entrypoint _align=3 54 55#if ENABLE_PIE 56 /* 57 * ------------------------------------------------------------ 58 * If PIE is enabled fixup the Global descriptor Table only 59 * once during primary core cold boot path. 60 * 61 * Compile time base address, required for fixup, is calculated 62 * using "pie_fixup" label present within first page. 63 * ------------------------------------------------------------ 64 */ 65 pie_fixup: 66 ldr x0, =pie_fixup 67 and x0, x0, #~(PAGE_SIZE_MASK) 68 mov_imm x1, (BL32_LIMIT - BL32_BASE) 69 add x1, x1, x0 70 bl fixup_gdt_reloc 71#endif /* ENABLE_PIE */ 72 73 /* --------------------------------------------- 74 * Set the exception vector to something sane. 75 * --------------------------------------------- 76 */ 77 adr x0, tsp_exceptions 78 msr vbar_el1, x0 79 isb 80 81 /* --------------------------------------------- 82 * Enable the SError interrupt now that the 83 * exception vectors have been setup. 84 * --------------------------------------------- 85 */ 86 msr daifclr, #DAIF_ABT_BIT 87 88 /* --------------------------------------------- 89 * Enable the instruction cache, stack pointer 90 * and data access alignment checks and disable 91 * speculative loads. 92 * --------------------------------------------- 93 */ 94 mov x1, #(SCTLR_I_BIT | SCTLR_A_BIT | SCTLR_SA_BIT) 95 mrs x0, sctlr_el1 96 orr x0, x0, x1 97#if ENABLE_BTI 98 /* Enable PAC branch type compatibility */ 99 bic x0, x0, #(SCTLR_BT0_BIT | SCTLR_BT1_BIT) 100#endif 101 bic x0, x0, #SCTLR_DSSBS_BIT 102 msr sctlr_el1, x0 103 isb 104 105 /* --------------------------------------------- 106 * Invalidate the RW memory used by the BL32 107 * image. This includes the data and NOBITS 108 * sections. This is done to safeguard against 109 * possible corruption of this memory by dirty 110 * cache lines in a system cache as a result of 111 * use by an earlier boot loader stage. If PIE 112 * is enabled however, RO sections including the 113 * GOT may be modified during pie fixup. 114 * Therefore, to be on the safe side, invalidate 115 * the entire image region if PIE is enabled. 116 * --------------------------------------------- 117 */ 118#if ENABLE_PIE 119#if SEPARATE_CODE_AND_RODATA 120 adrp x0, __TEXT_START__ 121 add x0, x0, :lo12:__TEXT_START__ 122#else 123 adrp x0, __RO_START__ 124 add x0, x0, :lo12:__RO_START__ 125#endif /* SEPARATE_CODE_AND_RODATA */ 126#else 127 adrp x0, __RW_START__ 128 add x0, x0, :lo12:__RW_START__ 129#endif /* ENABLE_PIE */ 130 adrp x1, __RW_END__ 131 add x1, x1, :lo12:__RW_END__ 132 sub x1, x1, x0 133 bl inv_dcache_range 134 135 /* --------------------------------------------- 136 * Zero out NOBITS sections. There are 2 of them: 137 * - the .bss section; 138 * - the coherent memory section. 139 * --------------------------------------------- 140 */ 141 adrp x0, __BSS_START__ 142 add x0, x0, :lo12:__BSS_START__ 143 adrp x1, __BSS_END__ 144 add x1, x1, :lo12:__BSS_END__ 145 sub x1, x1, x0 146 bl zeromem 147 148#if USE_COHERENT_MEM 149 adrp x0, __COHERENT_RAM_START__ 150 add x0, x0, :lo12:__COHERENT_RAM_START__ 151 adrp x1, __COHERENT_RAM_END_UNALIGNED__ 152 add x1, x1, :lo12:__COHERENT_RAM_END_UNALIGNED__ 153 sub x1, x1, x0 154 bl zeromem 155#endif 156 157 /* -------------------------------------------- 158 * Allocate a stack whose memory will be marked 159 * as Normal-IS-WBWA when the MMU is enabled. 160 * There is no risk of reading stale stack 161 * memory after enabling the MMU as only the 162 * primary cpu is running at the moment. 163 * -------------------------------------------- 164 */ 165 bl plat_set_my_stack 166 167 /* --------------------------------------------- 168 * Initialize the stack protector canary before 169 * any C code is called. 170 * --------------------------------------------- 171 */ 172#if STACK_PROTECTOR_ENABLED 173 bl update_stack_protector_canary 174#endif 175 176 /* --------------------------------------------- 177 * Perform TSP setup 178 * --------------------------------------------- 179 */ 180 bl tsp_setup 181 182#if ENABLE_PAUTH 183 /* --------------------------------------------- 184 * Program APIAKey_EL1 185 * and enable pointer authentication 186 * --------------------------------------------- 187 */ 188 bl pauth_init_enable_el1 189#endif /* ENABLE_PAUTH */ 190 191 /* --------------------------------------------- 192 * Jump to main function. 193 * --------------------------------------------- 194 */ 195 bl tsp_main 196 197 /* --------------------------------------------- 198 * Tell TSPD that we are done initialising 199 * --------------------------------------------- 200 */ 201 mov x1, x0 202 mov x0, #TSP_ENTRY_DONE 203 smc #0 204 205tsp_entrypoint_panic: 206 b tsp_entrypoint_panic 207endfunc tsp_entrypoint 208 209 210 /* ------------------------------------------- 211 * Table of entrypoint vectors provided to the 212 * TSPD for the various entrypoints 213 * ------------------------------------------- 214 */ 215vector_base tsp_vector_table 216 b tsp_yield_smc_entry 217 b tsp_fast_smc_entry 218 b tsp_cpu_on_entry 219 b tsp_cpu_off_entry 220 b tsp_cpu_resume_entry 221 b tsp_cpu_suspend_entry 222 b tsp_sel1_intr_entry 223 b tsp_system_off_entry 224 b tsp_system_reset_entry 225 b tsp_abort_yield_smc_entry 226 227 /*--------------------------------------------- 228 * This entrypoint is used by the TSPD when this 229 * cpu is to be turned off through a CPU_OFF 230 * psci call to ask the TSP to perform any 231 * bookeeping necessary. In the current 232 * implementation, the TSPD expects the TSP to 233 * re-initialise its state so nothing is done 234 * here except for acknowledging the request. 235 * --------------------------------------------- 236 */ 237func tsp_cpu_off_entry 238 bl tsp_cpu_off_main 239 restore_args_call_smc 240endfunc tsp_cpu_off_entry 241 242 /*--------------------------------------------- 243 * This entrypoint is used by the TSPD when the 244 * system is about to be switched off (through 245 * a SYSTEM_OFF psci call) to ask the TSP to 246 * perform any necessary bookkeeping. 247 * --------------------------------------------- 248 */ 249func tsp_system_off_entry 250 bl tsp_system_off_main 251 restore_args_call_smc 252endfunc tsp_system_off_entry 253 254 /*--------------------------------------------- 255 * This entrypoint is used by the TSPD when the 256 * system is about to be reset (through a 257 * SYSTEM_RESET psci call) to ask the TSP to 258 * perform any necessary bookkeeping. 259 * --------------------------------------------- 260 */ 261func tsp_system_reset_entry 262 bl tsp_system_reset_main 263 restore_args_call_smc 264endfunc tsp_system_reset_entry 265 266 /*--------------------------------------------- 267 * This entrypoint is used by the TSPD when this 268 * cpu is turned on using a CPU_ON psci call to 269 * ask the TSP to initialise itself i.e. setup 270 * the mmu, stacks etc. Minimal architectural 271 * state will be initialised by the TSPD when 272 * this function is entered i.e. Caches and MMU 273 * will be turned off, the execution state 274 * will be aarch64 and exceptions masked. 275 * --------------------------------------------- 276 */ 277func tsp_cpu_on_entry 278 /* --------------------------------------------- 279 * Set the exception vector to something sane. 280 * --------------------------------------------- 281 */ 282 adr x0, tsp_exceptions 283 msr vbar_el1, x0 284 isb 285 286 /* Enable the SError interrupt */ 287 msr daifclr, #DAIF_ABT_BIT 288 289 /* --------------------------------------------- 290 * Enable the instruction cache, stack pointer 291 * and data access alignment checks 292 * --------------------------------------------- 293 */ 294 mov x1, #(SCTLR_I_BIT | SCTLR_A_BIT | SCTLR_SA_BIT) 295 mrs x0, sctlr_el1 296 orr x0, x0, x1 297 msr sctlr_el1, x0 298 isb 299 300 /* -------------------------------------------- 301 * Give ourselves a stack whose memory will be 302 * marked as Normal-IS-WBWA when the MMU is 303 * enabled. 304 * -------------------------------------------- 305 */ 306 bl plat_set_my_stack 307 308 /* -------------------------------------------- 309 * Enable MMU and D-caches together. 310 * -------------------------------------------- 311 */ 312 mov x0, #0 313 bl bl32_plat_enable_mmu 314 315#if ENABLE_PAUTH 316 /* --------------------------------------------- 317 * Program APIAKey_EL1 318 * and enable pointer authentication 319 * --------------------------------------------- 320 */ 321 bl pauth_init_enable_el1 322#endif /* ENABLE_PAUTH */ 323 324 /* --------------------------------------------- 325 * Enter C runtime to perform any remaining 326 * book keeping 327 * --------------------------------------------- 328 */ 329 bl tsp_cpu_on_main 330 restore_args_call_smc 331 332 /* Should never reach here */ 333tsp_cpu_on_entry_panic: 334 b tsp_cpu_on_entry_panic 335endfunc tsp_cpu_on_entry 336 337 /*--------------------------------------------- 338 * This entrypoint is used by the TSPD when this 339 * cpu is to be suspended through a CPU_SUSPEND 340 * psci call to ask the TSP to perform any 341 * bookeeping necessary. In the current 342 * implementation, the TSPD saves and restores 343 * the EL1 state. 344 * --------------------------------------------- 345 */ 346func tsp_cpu_suspend_entry 347 bl tsp_cpu_suspend_main 348 restore_args_call_smc 349endfunc tsp_cpu_suspend_entry 350 351 /*------------------------------------------------- 352 * This entrypoint is used by the TSPD to pass 353 * control for `synchronously` handling a S-EL1 354 * Interrupt which was triggered while executing 355 * in normal world. 'x0' contains a magic number 356 * which indicates this. TSPD expects control to 357 * be handed back at the end of interrupt 358 * processing. This is done through an SMC. 359 * The handover agreement is: 360 * 361 * 1. PSTATE.DAIF are set upon entry. 'x1' has 362 * the ELR_EL3 from the non-secure state. 363 * 2. TSP has to preserve the callee saved 364 * general purpose registers, SP_EL1/EL0 and 365 * LR. 366 * 3. TSP has to preserve the system and vfp 367 * registers (if applicable). 368 * 4. TSP can use 'x0-x18' to enable its C 369 * runtime. 370 * 5. TSP returns to TSPD using an SMC with 371 * 'x0' = TSP_HANDLED_S_EL1_INTR 372 * ------------------------------------------------ 373 */ 374func tsp_sel1_intr_entry 375#if DEBUG 376 mov_imm x2, TSP_HANDLE_SEL1_INTR_AND_RETURN 377 cmp x0, x2 378 b.ne tsp_sel1_int_entry_panic 379#endif 380 /*------------------------------------------------- 381 * Save any previous context needed to perform 382 * an exception return from S-EL1 e.g. context 383 * from a previous Non secure Interrupt. 384 * Update statistics and handle the S-EL1 385 * interrupt before returning to the TSPD. 386 * IRQ/FIQs are not enabled since that will 387 * complicate the implementation. Execution 388 * will be transferred back to the normal world 389 * in any case. The handler can return 0 390 * if the interrupt was handled or TSP_PREEMPTED 391 * if the expected interrupt was preempted 392 * by an interrupt that should be handled in EL3 393 * e.g. Group 0 interrupt in GICv3. In both 394 * the cases switch to EL3 using SMC with id 395 * TSP_HANDLED_S_EL1_INTR. Any other return value 396 * from the handler will result in panic. 397 * ------------------------------------------------ 398 */ 399 save_eret_context x2 x3 400 bl tsp_update_sync_sel1_intr_stats 401 bl tsp_common_int_handler 402 /* Check if the S-EL1 interrupt has been handled */ 403 cbnz x0, tsp_sel1_intr_check_preemption 404 b tsp_sel1_intr_return 405tsp_sel1_intr_check_preemption: 406 /* Check if the S-EL1 interrupt has been preempted */ 407 mov_imm x1, TSP_PREEMPTED 408 cmp x0, x1 409 b.ne tsp_sel1_int_entry_panic 410tsp_sel1_intr_return: 411 mov_imm x0, TSP_HANDLED_S_EL1_INTR 412 restore_eret_context x2 x3 413 smc #0 414 415 /* Should never reach here */ 416tsp_sel1_int_entry_panic: 417 no_ret plat_panic_handler 418endfunc tsp_sel1_intr_entry 419 420 /*--------------------------------------------- 421 * This entrypoint is used by the TSPD when this 422 * cpu resumes execution after an earlier 423 * CPU_SUSPEND psci call to ask the TSP to 424 * restore its saved context. In the current 425 * implementation, the TSPD saves and restores 426 * EL1 state so nothing is done here apart from 427 * acknowledging the request. 428 * --------------------------------------------- 429 */ 430func tsp_cpu_resume_entry 431 bl tsp_cpu_resume_main 432 restore_args_call_smc 433 434 /* Should never reach here */ 435 no_ret plat_panic_handler 436endfunc tsp_cpu_resume_entry 437 438 /*--------------------------------------------- 439 * This entrypoint is used by the TSPD to ask 440 * the TSP to service a fast smc request. 441 * --------------------------------------------- 442 */ 443func tsp_fast_smc_entry 444 bl tsp_smc_handler 445 restore_args_call_smc 446 447 /* Should never reach here */ 448 no_ret plat_panic_handler 449endfunc tsp_fast_smc_entry 450 451 /*--------------------------------------------- 452 * This entrypoint is used by the TSPD to ask 453 * the TSP to service a Yielding SMC request. 454 * We will enable preemption during execution 455 * of tsp_smc_handler. 456 * --------------------------------------------- 457 */ 458func tsp_yield_smc_entry 459 msr daifclr, #DAIF_FIQ_BIT | DAIF_IRQ_BIT 460 bl tsp_smc_handler 461 msr daifset, #DAIF_FIQ_BIT | DAIF_IRQ_BIT 462 restore_args_call_smc 463 464 /* Should never reach here */ 465 no_ret plat_panic_handler 466endfunc tsp_yield_smc_entry 467 468 /*--------------------------------------------------------------------- 469 * This entrypoint is used by the TSPD to abort a pre-empted Yielding 470 * SMC. It could be on behalf of non-secure world or because a CPU 471 * suspend/CPU off request needs to abort the preempted SMC. 472 * -------------------------------------------------------------------- 473 */ 474func tsp_abort_yield_smc_entry 475 476 /* 477 * Exceptions masking is already done by the TSPD when entering this 478 * hook so there is no need to do it here. 479 */ 480 481 /* Reset the stack used by the pre-empted SMC */ 482 bl plat_set_my_stack 483 484 /* 485 * Allow some cleanup such as releasing locks. 486 */ 487 bl tsp_abort_smc_handler 488 489 restore_args_call_smc 490 491 /* Should never reach here */ 492 bl plat_panic_handler 493endfunc tsp_abort_yield_smc_entry 494