1/* 2 * Copyright (c) 2013-2026, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6#ifndef ASM_MACROS_S 7#define ASM_MACROS_S 8 9#include <arch.h> 10#include <common/asm_macros_common.S> 11#include <lib/cpus/cpu_ops.h> 12#include <lib/spinlock.h> 13 14/* 15 * TLBI instruction with type specifier that implements the workaround for 16 * errata 813419 of Cortex-A57 or errata 1286807 of Cortex-A76. 17 */ 18#if ERRATA_A57_813419 || ERRATA_A76_1286807 19#define TLB_INVALIDATE(_type) \ 20 tlbi _type; \ 21 dsb ish; \ 22 tlbi _type 23#else 24#define TLB_INVALIDATE(_type) \ 25 tlbi _type 26#endif 27 28 29 /* 30 * Create a stack frame at the start of an assembly function. Will also 31 * add all necessary call frame information (cfi) directives for a 32 * pretty stack trace. This is necessary as there is quite a bit of 33 * flexibility within a stack frame and the stack pointer can move 34 * around throughout the function. If the debugger isn't told where to 35 * find things, it gets lost, gives up and displays nothing. So inform 36 * the debugger of what's where. Anchor the Canonical Frame Address 37 * (CFA; the thing used to track what's where) to the frame pointer as 38 * that's not expected to change in the function body and no extra 39 * bookkeeping will be necessary, allowing free movement of the sp 40 * 41 * _frame_size: requested space for caller to use. Must be a mutliple 42 * of 16 for stack pointer alignment 43 */ 44 .macro func_prologue _frame_size=0 45 .if \_frame_size & 0xf 46 .error "frame_size must have stack pointer alignment (multiple of 16)" 47 .endif 48 49 /* put frame record at top of frame */ 50 stp x29, x30, [sp, #-0x10]! 51 mov x29,sp 52 .if \_frame_size 53 sub sp, sp, #\_frame_size 54 .endif 55 56 /* point CFA to start of frame record, i.e. x29 + 0x10 */ 57 .cfi_def_cfa x29, 0x10 58 /* inform it about x29, x30 locations */ 59 .cfi_offset x30, -0x8 60 .cfi_offset x29, -0x10 61 .endm 62 63 /* 64 * Clear stack frame at the end of an assembly function. 65 * 66 * _frame_size: the value passed to func_prologue 67 */ 68 .macro func_epilogue _frame_size=0 69 /* remove requested space */ 70 .if \_frame_size 71 add sp, sp, #\_frame_size 72 .endif 73 ldp x29, x30, [sp], #0x10 74 .endm 75 76 77 .macro dcache_line_size reg, tmp 78 mrs \tmp, ctr_el0 79 ubfx \tmp, \tmp, #16, #4 80 mov \reg, #4 81 lsl \reg, \reg, \tmp 82 .endm 83 84 85 .macro icache_line_size reg, tmp 86 mrs \tmp, ctr_el0 87 and \tmp, \tmp, #0xf 88 mov \reg, #4 89 lsl \reg, \reg, \tmp 90 .endm 91 92 93 .macro smc_check label 94 mrs x0, esr_el3 95 ubfx x0, x0, #ESR_EC_SHIFT, #ESR_EC_LENGTH 96 cmp x0, #EC_AARCH64_SMC 97 b.ne $label 98 .endm 99 100 /* 101 * Declare the exception vector table, enforcing it is aligned on a 102 * 2KB boundary, as required by the ARMv8 architecture. 103 * Use zero bytes as the fill value to be stored in the padding bytes 104 * so that it inserts illegal AArch64 instructions. This increases 105 * security, robustness and potentially facilitates debugging. 106 */ 107 .macro vector_base label, section_name=.vectors 108 .section \section_name, "ax" 109 .align 11, 0 110 \label: 111 .endm 112 113 /* 114 * Create an entry in the exception vector table, enforcing it is 115 * aligned on a 128-byte boundary, as required by the ARMv8 architecture. 116 * Use zero bytes as the fill value to be stored in the padding bytes 117 * so that it inserts illegal AArch64 instructions. This increases 118 * security, robustness and potentially facilitates debugging. 119 */ 120 .macro vector_entry label, section_name=.vectors 121 .cfi_sections .debug_frame 122 .section \section_name, "ax" 123 .align 7, 0 124 .type \label, %function 125 .cfi_startproc 126 \label: 127 .endm 128 129 /* 130 * Add the bytes until fill the full exception vector, whose size is always 131 * 32 instructions. If there are more than 32 instructions in the 132 * exception vector then an error is emitted. 133 */ 134 .macro end_vector_entry label 135 .size \label, . - \label 136 .cfi_endproc 137 .iflt \label + (32 * 4) - . 138 .error "\label won't fit in exception vector" 139 .endif 140 .fill \label + (32 * 4) - . 141 .endm 142 143 /* 144 * This macro calculates the base address of the current CPU's MP stack 145 * using the plat_my_core_pos() index, the name of the stack storage 146 * and the size of each stack 147 * Out: X0 = physical address of stack base 148 * Clobber: X30, X1, X2 149 */ 150 .macro get_my_mp_stack _name, _size 151 bl plat_my_core_pos 152 adrp x2, (\_name + \_size) 153 add x2, x2, :lo12:(\_name + \_size) 154 mov x1, #\_size 155 madd x0, x0, x1, x2 156 .endm 157 158 /* 159 * This macro calculates the base address of a UP stack using the 160 * name of the stack storage and the size of the stack 161 * Out: X0 = physical address of stack base 162 */ 163 .macro get_up_stack _name, _size 164 adrp x0, (\_name + \_size) 165 add x0, x0, :lo12:(\_name + \_size) 166 .endm 167 168 /* 169 * Helper macro to generate the best mov/movk combinations according 170 * the value to be moved. The 16 bits from '_shift' are tested and 171 * if not zero, they are moved into '_reg' without affecting 172 * other bits. 173 */ 174 .macro _mov_imm16 _reg, _val, _shift 175 .if (\_val >> \_shift) & 0xffff 176 .if (\_val & (1 << \_shift - 1)) 177 movk \_reg, (\_val >> \_shift) & 0xffff, LSL \_shift 178 .else 179 mov \_reg, \_val & (0xffff << \_shift) 180 .endif 181 .endif 182 .endm 183 184 /* 185 * Helper macro to load arbitrary values into 32 or 64-bit registers 186 * which generates the best mov/movk combinations. Many base addresses 187 * are 64KB aligned the macro will eliminate updating bits 15:0 in 188 * that case 189 */ 190 .macro mov_imm _reg, _val 191 .if (\_val) == 0 192 mov \_reg, #0 193 .else 194 _mov_imm16 \_reg, (\_val), 0 195 _mov_imm16 \_reg, (\_val), 16 196 _mov_imm16 \_reg, (\_val), 32 197 _mov_imm16 \_reg, (\_val), 48 198 .endif 199 .endm 200 201 /* 202 * Macro to mark instances where we're jumping to a function and don't 203 * expect a return. To provide the function being jumped to with 204 * additional information, we use 'bl' instruction to jump rather than 205 * 'b'. 206 * 207 * Debuggers infer the location of a call from where LR points to, which 208 * is usually the instruction after 'bl'. If this macro expansion 209 * happens to be the last location in a function, that'll cause the LR 210 * to point a location beyond the function, thereby misleading debugger 211 * back trace. We therefore insert a 'nop' after the function call for 212 * debug builds, unless 'skip_nop' parameter is non-zero. 213 */ 214 .macro no_ret _func:req, skip_nop=0 215 bl \_func 216#if DEBUG 217 .ifeq \skip_nop 218 nop 219 .endif 220#endif 221 .endm 222 223 /* 224 * Reserve space for a spin lock in assembly file. 225 */ 226 .macro define_asm_spinlock _name:req 227 .align SPINLOCK_ASM_ALIGN 228 \_name: 229 .space SPINLOCK_ASM_SIZE 230 .endm 231 232 /* 233 * Helper macro to read system register value into x0 234 */ 235 .macro read reg:req 236#if ENABLE_BTI 237 BTI j 238#endif 239 mrs x0, \reg 240 ret 241 .endm 242 243 /* 244 * Helper macro to write value from x1 to system register 245 */ 246 .macro write reg:req 247#if ENABLE_BTI 248 BTI j 249#endif 250 msr \reg, x1 251 ret 252 .endm 253 254 /* 255 * The "sb" instruction was introduced later into the architecture, 256 * so not all toolchains understand it. Some deny its usage unless 257 * a supported processor is specified on the build command line. 258 * Use sb's system register encoding to work around this, we already 259 * guard the sb execution with a feature flag. 260 */ 261 262 .macro sb_barrier_insn 263 msr SYSREG_SB, xzr 264 .endm 265 266 .macro psb_csync 267 hint #17 /* use the hint synonym for compatibility */ 268 .endm 269 270 .macro tsb_csync 271 hint #18 /* use the hint synonym for compatibility */ 272 .endm 273 274 /* 275 * Macro for using speculation barrier instruction introduced by 276 * FEAT_SB, if it's enabled. 277 */ 278 .macro speculation_barrier 279 /* fallback to the old slower variant for FEAT_STATE_CHECKED */ 280#if ENABLE_FEAT_SB == 1 281 sb_barrier_insn 282#else 283 dsb sy 284 isb 285#endif 286 .endm 287 288 /* 289 * Clear Branch History instruction, translates to NOP on CPUs 290 * that don't implement the clrbhb instruction. 291 */ 292 .macro clrbhb 293 hint #22 294 .endm 295 296 /* 297 * Macro for mitigating against speculative execution beyond ERET. Uses the 298 * speculation barrier instruction introduced by FEAT_SB, if it's enabled. 299 */ 300 .macro exception_return 301 eret 302 speculation_barrier 303 .endm 304 305 /* 306 * Macro to unmask External Aborts by changing PSTATE.A bit. 307 * Put explicit synchronization event to ensure newly unmasked interrupt 308 * is taken immediately. 309 */ 310 .macro unmask_async_ea 311 msr daifclr, #DAIF_ABT_BIT 312 isb 313 .endm 314 315 /* 316 * Macro for error synchronization on exception boundaries. 317 * With FEAT_RAS enabled, it is assumed that FEAT_IESB is also present 318 * and enabled. 319 * FEAT_IESB provides an implicit error synchronization event at 320 * exception entry and exception return, so there is no need for any 321 * explicit instruction. In the FEAT_STATE_CHECKED case the explicit 322 * synchronisation will be done anyway because there isn't enough space 323 * in the exception vectors to do the check. Any platform looking for 324 * maximum performance should not use FEAT_STATE_CHECKED. 325 */ 326 .macro synchronize_errors 327#if ENABLE_FEAT_RAS == 0 || ENABLE_FEAT_RAS == 2 328 /* Complete any stores that may return an abort */ 329 dsb sy 330 /* Synchronise the CPU context with the completion of the dsb */ 331 isb 332#endif 333 .endm 334 335 /* 336 * Helper macro to instruction adr <reg>, <symbol> where <symbol> is 337 * within the range +/- 4 GB. 338 */ 339 .macro adr_l, dst, sym 340 adrp \dst, \sym 341 add \dst, \dst, :lo12:\sym 342 .endm 343 344 /* 345 * is_feat_XYZ_present_asm - Set flags and reg if FEAT_XYZ 346 * is enabled at runtime. 347 * 348 * Arguments: 349 * reg: Register for temporary use. 350 * 351 * Clobbers: reg 352 */ 353 .macro is_feat_sysreg128_present_asm reg:req 354 mrs \reg, ID_AA64ISAR2_EL1 355 ands \reg, \reg, #(ID_AA64ISAR2_SYSREG128_MASK << ID_AA64ISAR2_SYSREG128_SHIFT) 356 .endm 357 358 .macro is_feat_pauth_present_asm reg:req, clobber:req 359 mrs \reg, ID_AA64ISAR1_EL1 360 mov_imm \clobber, ((ID_AA64ISAR1_GPI_MASK << ID_AA64ISAR1_GPI_SHIFT) \ 361 | (ID_AA64ISAR1_GPA_MASK << ID_AA64ISAR1_GPA_SHIFT) \ 362 | (ID_AA64ISAR1_API_MASK << ID_AA64ISAR1_API_SHIFT) \ 363 | (ID_AA64ISAR1_APA_MASK << ID_AA64ISAR1_APA_SHIFT)) 364 tst \reg, \clobber 365 .endm 366 367 .macro is_feat_sctlr2_present_asm reg:req 368 mrs \reg, ID_AA64MMFR3_EL1 369 ands \reg, \reg, #(ID_AA64MMFR3_EL1_SCTLR2_MASK << ID_AA64MMFR3_EL1_SCTLR2_SHIFT) 370 .endm 371 372 .macro is_feat_sme_present_asm reg:req 373 mrs \reg, id_aa64pfr1_el1 374 ands \reg, \reg, #(ID_AA64PFR1_EL1_SME_MASK << ID_AA64PFR1_EL1_SME_SHIFT) 375 .endm 376 377 .macro is_feat_amu_present_asm reg:req 378 mrs \reg, ID_AA64PFR0_EL1 379 ands \reg, \reg, #(ID_AA64PFR0_AMU_MASK << ID_AA64PFR0_AMU_SHIFT) 380 .endm 381 382 .macro is_feat_morello_present_asm reg:req 383 mrs \reg, id_aa64pfr1_el1 384 ands \reg, \reg, #(ID_AA64PFR1_EL1_CE_MASK << ID_AA64PFR1_EL1_CE_SHIFT) 385 .endm 386 387 /* 388 * There are various places where Morello requires simple switchovers for 389 * system registers. For eg morello uses celr_el3 vs elr_el3, cvbar_el3 vs 390 * vbar_el3 etc., this macro simplifies that operation and makes the code 391 * more readable. 392 */ 393 .macro msr_wide_reg _sysreg:req, _gp_reg:req 394#if ENABLE_FEAT_MORELLO 395#if ENABLE_FEAT_MORELLO == 2 396 is_feat_morello_present_asm x10 397 beq 1f 398#endif /* ENABLE_FEAT_MORELLO == 2 */ 399 cvtp c\_gp_reg, x\_gp_reg 400 msr c\_sysreg, c\_gp_reg 401 b 2f 4021: 403#endif /* ENABLE_FEAT_MORELLO */ 404#if ENABLE_FEAT_MORELLO != 1 405 msr \_sysreg, x\_gp_reg 406#endif /* ENABLE_FEAT_MORELLO != 1 */ 4072: 408 .endm 409 410.macro call_reset_handler 411#if defined(IMAGE_BL1) || defined(IMAGE_BL31) || (defined(IMAGE_BL2) && RESET_TO_BL2) 412 /* --------------------------------------------------------------------- 413 * It is a cold boot. 414 * Perform any processor specific actions upon reset e.g. cache, TLB 415 * invalidations etc. 416 * --------------------------------------------------------------------- 417 */ 418 /* The plat_reset_handler can clobber x0 - x18, x30 */ 419 bl plat_reset_handler 420 421 /* Get the matching cpu_ops pointer */ 422 bl get_cpu_ops_ptr 423 424 /* Get the cpu_ops reset handler */ 425 ldr x2, [x0, #CPU_RESET_FUNC] 426 427 /* The cpu_ops reset handler can clobber x0 - x19, x30 */ 428 blr x2 429#endif 430.endm 431#endif /* ASM_MACROS_S */ 432