1/* 2 * Copyright (c) 2013-2024, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <assert_macros.S> 10#include <common/bl_common.h> 11#include <lib/xlat_tables/xlat_tables_defs.h> 12 13 .globl smc 14 15 .globl zero_normalmem 16 .globl zeromem 17 .globl memcpy16 18 19 .globl disable_mmu_el1 20 .globl disable_mmu_el3 21 .globl disable_mmu_icache_el1 22 .globl disable_mmu_icache_el3 23 .globl fixup_gdt_reloc 24func smc 25 smc #0 26endfunc smc 27 28/* ----------------------------------------------------------------------- 29 * void zero_normalmem(void *mem, unsigned int length); 30 * 31 * Initialise a region in normal memory to 0. This functions complies with the 32 * AAPCS and can be called from C code. 33 * 34 * NOTE: MMU must be enabled when using this function as it can only operate on 35 * normal memory. It is intended to be mainly used from C code when MMU 36 * is usually enabled. 37 * ----------------------------------------------------------------------- 38 */ 39.equ zero_normalmem, zeromem_dczva 40 41/* ----------------------------------------------------------------------- 42 * void zeromem(void *mem, unsigned int length); 43 * 44 * Initialise a region of device memory to 0. This functions complies with the 45 * AAPCS and can be called from C code. 46 * 47 * NOTE: When data caches and MMU are enabled, zero_normalmem can usually be 48 * used instead for faster zeroing. 49 * 50 * ----------------------------------------------------------------------- 51 */ 52func zeromem 53 /* x2 is the address past the last zeroed address */ 54 add x2, x0, x1 55 /* 56 * Uses the fallback path that does not use DC ZVA instruction and 57 * therefore does not need enabled MMU 58 */ 59 b .Lzeromem_dczva_fallback_entry 60endfunc zeromem 61 62/* ----------------------------------------------------------------------- 63 * void zeromem_dczva(void *mem, unsigned int length); 64 * 65 * Fill a region of normal memory of size "length" in bytes with null bytes. 66 * MMU must be enabled and the memory be of 67 * normal type. This is because this function internally uses the DC ZVA 68 * instruction, which generates an Alignment fault if used on any type of 69 * Device memory (see section D3.4.9 of the ARMv8 ARM, issue k). When the MMU 70 * is disabled, all memory behaves like Device-nGnRnE memory (see section 71 * D4.2.8), hence the requirement on the MMU being enabled. 72 * NOTE: The code assumes that the block size as defined in DCZID_EL0 73 * register is at least 16 bytes. 74 * 75 * ----------------------------------------------------------------------- 76 */ 77func zeromem_dczva 78 79 /* 80 * The function consists of a series of loops that zero memory one byte 81 * at a time, 16 bytes at a time or using the DC ZVA instruction to 82 * zero aligned block of bytes, which is assumed to be more than 16. 83 * In the case where the DC ZVA instruction cannot be used or if the 84 * first 16 bytes loop would overflow, there is fallback path that does 85 * not use DC ZVA. 86 * Note: The fallback path is also used by the zeromem function that 87 * branches to it directly. 88 * 89 * +---------+ zeromem_dczva 90 * | entry | 91 * +----+----+ 92 * | 93 * v 94 * +---------+ 95 * | checks |>o-------+ (If any check fails, fallback) 96 * +----+----+ | 97 * | |---------------+ 98 * v | Fallback path | 99 * +------+------+ |---------------+ 100 * | 1 byte loop | | 101 * +------+------+ .Lzeromem_dczva_initial_1byte_aligned_end 102 * | | 103 * v | 104 * +-------+-------+ | 105 * | 16 bytes loop | | 106 * +-------+-------+ | 107 * | | 108 * v | 109 * +------+------+ .Lzeromem_dczva_blocksize_aligned 110 * | DC ZVA loop | | 111 * +------+------+ | 112 * +--------+ | | 113 * | | | | 114 * | v v | 115 * | +-------+-------+ .Lzeromem_dczva_final_16bytes_aligned 116 * | | 16 bytes loop | | 117 * | +-------+-------+ | 118 * | | | 119 * | v | 120 * | +------+------+ .Lzeromem_dczva_final_1byte_aligned 121 * | | 1 byte loop | | 122 * | +-------------+ | 123 * | | | 124 * | v | 125 * | +---+--+ | 126 * | | exit | | 127 * | +------+ | 128 * | | 129 * | +--------------+ +------------------+ zeromem 130 * | | +----------------| zeromem function | 131 * | | | +------------------+ 132 * | v v 133 * | +-------------+ .Lzeromem_dczva_fallback_entry 134 * | | 1 byte loop | 135 * | +------+------+ 136 * | | 137 * +-----------+ 138 */ 139 140 /* 141 * Readable names for registers 142 * 143 * Registers x0, x1 and x2 are also set by zeromem which 144 * branches into the fallback path directly, so cursor, length and 145 * stop_address should not be retargeted to other registers. 146 */ 147 cursor .req x0 /* Start address and then current address */ 148 length .req x1 /* Length in bytes of the region to zero out */ 149 /* Reusing x1 as length is never used after block_mask is set */ 150 block_mask .req x1 /* Bitmask of the block size read in DCZID_EL0 */ 151 stop_address .req x2 /* Address past the last zeroed byte */ 152 block_size .req x3 /* Size of a block in bytes as read in DCZID_EL0 */ 153 tmp1 .req x4 154 tmp2 .req x5 155 156#if ENABLE_ASSERTIONS 157 /* 158 * Check for M bit (MMU enabled) of the current SCTLR_EL(1|3) 159 * register value and panic if the MMU is disabled. 160 */ 161#if defined(IMAGE_BL1) || defined(IMAGE_BL31) || (defined(IMAGE_BL2) && \ 162 BL2_RUNS_AT_EL3) 163 mrs tmp1, sctlr_el3 164#else 165 mrs tmp1, sctlr_el1 166#endif 167 168 tst tmp1, #SCTLR_M_BIT 169 ASM_ASSERT(ne) 170#endif /* ENABLE_ASSERTIONS */ 171 172 /* stop_address is the address past the last to zero */ 173 add stop_address, cursor, length 174 175 /* 176 * Get block_size = (log2(<block size>) >> 2) (see encoding of 177 * dczid_el0 reg) 178 */ 179 mrs block_size, dczid_el0 180 181 /* 182 * Select the 4 lowest bits and convert the extracted log2(<block size 183 * in words>) to <block size in bytes> 184 */ 185 ubfx block_size, block_size, #0, #4 186 mov tmp2, #(1 << 2) 187 lsl block_size, tmp2, block_size 188 189#if ENABLE_ASSERTIONS 190 /* 191 * Assumes block size is at least 16 bytes to avoid manual realignment 192 * of the cursor at the end of the DCZVA loop. 193 */ 194 cmp block_size, #16 195 ASM_ASSERT(hs) 196#endif 197 /* 198 * Not worth doing all the setup for a region less than a block and 199 * protects against zeroing a whole block when the area to zero is 200 * smaller than that. Also, as it is assumed that the block size is at 201 * least 16 bytes, this also protects the initial aligning loops from 202 * trying to zero 16 bytes when length is less than 16. 203 */ 204 cmp length, block_size 205 b.lo .Lzeromem_dczva_fallback_entry 206 207 /* 208 * Calculate the bitmask of the block alignment. It will never 209 * underflow as the block size is between 4 bytes and 2kB. 210 * block_mask = block_size - 1 211 */ 212 sub block_mask, block_size, #1 213 214 /* 215 * length alias should not be used after this point unless it is 216 * defined as a register other than block_mask's. 217 */ 218 .unreq length 219 220 /* 221 * If the start address is already aligned to zero block size, go 222 * straight to the cache zeroing loop. This is safe because at this 223 * point, the length cannot be smaller than a block size. 224 */ 225 tst cursor, block_mask 226 b.eq .Lzeromem_dczva_blocksize_aligned 227 228 /* 229 * Calculate the first block-size-aligned address. It is assumed that 230 * the zero block size is at least 16 bytes. This address is the last 231 * address of this initial loop. 232 */ 233 orr tmp1, cursor, block_mask 234 add tmp1, tmp1, #1 235 236 /* 237 * If the addition overflows, skip the cache zeroing loops. This is 238 * quite unlikely however. 239 */ 240 cbz tmp1, .Lzeromem_dczva_fallback_entry 241 242 /* 243 * If the first block-size-aligned address is past the last address, 244 * fallback to the simpler code. 245 */ 246 cmp tmp1, stop_address 247 b.hi .Lzeromem_dczva_fallback_entry 248 249 /* 250 * If the start address is already aligned to 16 bytes, skip this loop. 251 * It is safe to do this because tmp1 (the stop address of the initial 252 * 16 bytes loop) will never be greater than the final stop address. 253 */ 254 tst cursor, #0xf 255 b.eq .Lzeromem_dczva_initial_1byte_aligned_end 256 257 /* Calculate the next address aligned to 16 bytes */ 258 orr tmp2, cursor, #0xf 259 add tmp2, tmp2, #1 260 /* If it overflows, fallback to the simple path (unlikely) */ 261 cbz tmp2, .Lzeromem_dczva_fallback_entry 262 /* 263 * Next aligned address cannot be after the stop address because the 264 * length cannot be smaller than 16 at this point. 265 */ 266 267 /* First loop: zero byte per byte */ 2681: 269 strb wzr, [cursor], #1 270 cmp cursor, tmp2 271 b.ne 1b 272.Lzeromem_dczva_initial_1byte_aligned_end: 273 274 /* 275 * Second loop: we need to zero 16 bytes at a time from cursor to tmp1 276 * before being able to use the code that deals with block-size-aligned 277 * addresses. 278 */ 279 cmp cursor, tmp1 280 b.hs 2f 2811: 282 stp xzr, xzr, [cursor], #16 283 cmp cursor, tmp1 284 b.lo 1b 2852: 286 287 /* 288 * Third loop: zero a block at a time using DC ZVA cache block zeroing 289 * instruction. 290 */ 291.Lzeromem_dczva_blocksize_aligned: 292 /* 293 * Calculate the last block-size-aligned address. If the result equals 294 * to the start address, the loop will exit immediately. 295 */ 296 bic tmp1, stop_address, block_mask 297 298 cmp cursor, tmp1 299 b.hs 2f 3001: 301 /* Zero the block containing the cursor */ 302 dc zva, cursor 303 /* Increment the cursor by the size of a block */ 304 add cursor, cursor, block_size 305 cmp cursor, tmp1 306 b.lo 1b 3072: 308 309 /* 310 * Fourth loop: zero 16 bytes at a time and then byte per byte the 311 * remaining area 312 */ 313.Lzeromem_dczva_final_16bytes_aligned: 314 /* 315 * Calculate the last 16 bytes aligned address. It is assumed that the 316 * block size will never be smaller than 16 bytes so that the current 317 * cursor is aligned to at least 16 bytes boundary. 318 */ 319 bic tmp1, stop_address, #15 320 321 cmp cursor, tmp1 322 b.hs 2f 3231: 324 stp xzr, xzr, [cursor], #16 325 cmp cursor, tmp1 326 b.lo 1b 3272: 328 329 /* Fifth and final loop: zero byte per byte */ 330.Lzeromem_dczva_final_1byte_aligned: 331 cmp cursor, stop_address 332 b.eq 2f 3331: 334 strb wzr, [cursor], #1 335 cmp cursor, stop_address 336 b.ne 1b 3372: 338 ret 339 340 /* Fallback for unaligned start addresses */ 341.Lzeromem_dczva_fallback_entry: 342 /* 343 * If the start address is already aligned to 16 bytes, skip this loop. 344 */ 345 tst cursor, #0xf 346 b.eq .Lzeromem_dczva_final_16bytes_aligned 347 348 /* Calculate the next address aligned to 16 bytes */ 349 orr tmp1, cursor, #15 350 add tmp1, tmp1, #1 351 /* If it overflows, fallback to byte per byte zeroing */ 352 cbz tmp1, .Lzeromem_dczva_final_1byte_aligned 353 /* If the next aligned address is after the stop address, fall back */ 354 cmp tmp1, stop_address 355 b.hs .Lzeromem_dczva_final_1byte_aligned 356 357 /* Fallback entry loop: zero byte per byte */ 3581: 359 strb wzr, [cursor], #1 360 cmp cursor, tmp1 361 b.ne 1b 362 363 b .Lzeromem_dczva_final_16bytes_aligned 364 365 .unreq cursor 366 /* 367 * length is already unreq'ed to reuse the register for another 368 * variable. 369 */ 370 .unreq stop_address 371 .unreq block_size 372 .unreq block_mask 373 .unreq tmp1 374 .unreq tmp2 375endfunc zeromem_dczva 376 377/* -------------------------------------------------------------------------- 378 * void memcpy16(void *dest, const void *src, unsigned int length) 379 * 380 * Copy length bytes from memory area src to memory area dest. 381 * The memory areas should not overlap. 382 * Destination and source addresses must be 16-byte aligned. 383 * -------------------------------------------------------------------------- 384 */ 385func memcpy16 386#if ENABLE_ASSERTIONS 387 orr x3, x0, x1 388 tst x3, #0xf 389 ASM_ASSERT(eq) 390#endif 391/* copy 16 bytes at a time */ 392m_loop16: 393 cmp x2, #16 394 b.lo m_loop1 395 ldp x3, x4, [x1], #16 396 stp x3, x4, [x0], #16 397 sub x2, x2, #16 398 b m_loop16 399/* copy byte per byte */ 400m_loop1: 401 cbz x2, m_end 402 ldrb w3, [x1], #1 403 strb w3, [x0], #1 404 subs x2, x2, #1 405 b.ne m_loop1 406m_end: 407 ret 408endfunc memcpy16 409 410/* --------------------------------------------------------------------------- 411 * Disable the MMU at EL3 412 * --------------------------------------------------------------------------- 413 */ 414 415func disable_mmu_el3 416 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) 417do_disable_mmu_el3: 418 mrs x0, sctlr_el3 419 bic x0, x0, x1 420 msr sctlr_el3, x0 421 isb /* ensure MMU is off */ 422 dsb sy 423 ret 424endfunc disable_mmu_el3 425 426 427func disable_mmu_icache_el3 428 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) 429 b do_disable_mmu_el3 430endfunc disable_mmu_icache_el3 431 432/* --------------------------------------------------------------------------- 433 * Disable the MMU at EL1 434 * --------------------------------------------------------------------------- 435 */ 436 437func disable_mmu_el1 438 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) 439do_disable_mmu_el1: 440 mrs x0, sctlr_el1 441 bic x0, x0, x1 442 msr sctlr_el1, x0 443 isb /* ensure MMU is off */ 444 dsb sy 445 ret 446endfunc disable_mmu_el1 447 448 449func disable_mmu_icache_el1 450 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) 451 b do_disable_mmu_el1 452endfunc disable_mmu_icache_el1 453 454/* --------------------------------------------------------------------------- 455 * Helper to fixup Global Descriptor table (GDT) and dynamic relocations 456 * (.rela.dyn) at runtime. 457 * 458 * This function is meant to be used when the firmware is compiled with -fpie 459 * and linked with -pie options. We rely on the linker script exporting 460 * appropriate markers for start and end of the section. For GOT, we 461 * expect __GOT_START__ and __GOT_END__. Similarly for .rela.dyn, we expect 462 * __RELA_START__ and __RELA_END__. 463 * 464 * The function takes the limits of the memory to apply fixups to as 465 * arguments (which is usually the limits of the relocable BL image). 466 * x0 - the start of the fixup region 467 * x1 - the limit of the fixup region 468 * These addresses have to be 4KB page aligned. 469 * --------------------------------------------------------------------------- 470 */ 471 472/* Relocation codes */ 473#define R_AARCH64_NONE 0 474#define R_AARCH64_RELATIVE 1027 475 476func fixup_gdt_reloc 477 mov x6, x0 478 mov x7, x1 479 480#if ENABLE_ASSERTIONS 481 /* Test if the limits are 4KB aligned */ 482 orr x0, x0, x1 483 tst x0, #(PAGE_SIZE_MASK) 484 ASM_ASSERT(eq) 485#endif 486 /* 487 * Calculate the offset based on return address in x30. 488 * Assume that this function is called within a page at the start of 489 * fixup region. 490 */ 491 and x2, x30, #~(PAGE_SIZE_MASK) 492 subs x0, x2, x6 /* Diff(S) = Current Address - Compiled Address */ 493 b.eq 3f /* Diff(S) = 0. No relocation needed */ 494 495 adrp x1, __GOT_START__ 496 add x1, x1, :lo12:__GOT_START__ 497 adrp x2, __GOT_END__ 498 add x2, x2, :lo12:__GOT_END__ 499 500 /* 501 * GOT is an array of 64_bit addresses which must be fixed up as 502 * new_addr = old_addr + Diff(S). 503 * The new_addr is the address currently the binary is executing from 504 * and old_addr is the address at compile time. 505 */ 5061: ldr x3, [x1] 507 508 /* Skip adding offset if address is < lower limit */ 509 cmp x3, x6 510 b.lo 2f 511 512 /* Skip adding offset if address is > upper limit */ 513 cmp x3, x7 514 b.hi 2f 515 add x3, x3, x0 516 str x3, [x1] 517 5182: add x1, x1, #8 519 cmp x1, x2 520 b.lo 1b 521 522 /* Starting dynamic relocations. Use adrp/adr to get RELA_START and END */ 5233: adrp x1, __RELA_START__ 524 add x1, x1, :lo12:__RELA_START__ 525 adrp x2, __RELA_END__ 526 add x2, x2, :lo12:__RELA_END__ 527 528 /* 529 * According to ELF-64 specification, the RELA data structure is as 530 * follows: 531 * typedef struct { 532 * Elf64_Addr r_offset; 533 * Elf64_Xword r_info; 534 * Elf64_Sxword r_addend; 535 * } Elf64_Rela; 536 * 537 * r_offset is address of reference 538 * r_info is symbol index and type of relocation (in this case 539 * code 1027 which corresponds to R_AARCH64_RELATIVE). 540 * r_addend is constant part of expression. 541 * 542 * Size of Elf64_Rela structure is 24 bytes. 543 */ 544 545 /* Skip R_AARCH64_NONE entry with code 0 */ 5461: ldr x3, [x1, #8] 547 cbz x3, 2f 548 549#if ENABLE_ASSERTIONS 550 /* Assert that the relocation type is R_AARCH64_RELATIVE */ 551 cmp x3, #R_AARCH64_RELATIVE 552 ASM_ASSERT(eq) 553#endif 554 ldr x3, [x1] /* r_offset */ 555 add x3, x0, x3 556 ldr x4, [x1, #16] /* r_addend */ 557 558 /* Skip adding offset if r_addend is < lower limit */ 559 cmp x4, x6 560 b.lo 2f 561 562 /* Skip adding offset if r_addend entry is > upper limit */ 563 cmp x4, x7 564 b.hi 2f 565 566 add x4, x0, x4 /* Diff(S) + r_addend */ 567 str x4, [x3] 568 5692: add x1, x1, #24 570 cmp x1, x2 571 b.lo 1b 572 ret 573endfunc fixup_gdt_reloc 574