1/* 2 * Copyright (c) 2013-2021, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <assert_macros.S> 10#include <common/bl_common.h> 11#include <lib/xlat_tables/xlat_tables_defs.h> 12 13 .globl smc 14 15 .globl zero_normalmem 16 .globl zeromem 17 .globl memcpy16 18 .globl gpt_tlbi_by_pa 19 20 .globl disable_mmu_el1 21 .globl disable_mmu_el3 22 .globl disable_mmu_icache_el1 23 .globl disable_mmu_icache_el3 24 .globl fixup_gdt_reloc 25#if SUPPORT_VFP 26 .globl enable_vfp 27#endif 28 29func smc 30 smc #0 31endfunc smc 32 33/* ----------------------------------------------------------------------- 34 * void zero_normalmem(void *mem, unsigned int length); 35 * 36 * Initialise a region in normal memory to 0. This functions complies with the 37 * AAPCS and can be called from C code. 38 * 39 * NOTE: MMU must be enabled when using this function as it can only operate on 40 * normal memory. It is intended to be mainly used from C code when MMU 41 * is usually enabled. 42 * ----------------------------------------------------------------------- 43 */ 44.equ zero_normalmem, zeromem_dczva 45 46/* ----------------------------------------------------------------------- 47 * void zeromem(void *mem, unsigned int length); 48 * 49 * Initialise a region of device memory to 0. This functions complies with the 50 * AAPCS and can be called from C code. 51 * 52 * NOTE: When data caches and MMU are enabled, zero_normalmem can usually be 53 * used instead for faster zeroing. 54 * 55 * ----------------------------------------------------------------------- 56 */ 57func zeromem 58 /* x2 is the address past the last zeroed address */ 59 add x2, x0, x1 60 /* 61 * Uses the fallback path that does not use DC ZVA instruction and 62 * therefore does not need enabled MMU 63 */ 64 b .Lzeromem_dczva_fallback_entry 65endfunc zeromem 66 67/* ----------------------------------------------------------------------- 68 * void zeromem_dczva(void *mem, unsigned int length); 69 * 70 * Fill a region of normal memory of size "length" in bytes with null bytes. 71 * MMU must be enabled and the memory be of 72 * normal type. This is because this function internally uses the DC ZVA 73 * instruction, which generates an Alignment fault if used on any type of 74 * Device memory (see section D3.4.9 of the ARMv8 ARM, issue k). When the MMU 75 * is disabled, all memory behaves like Device-nGnRnE memory (see section 76 * D4.2.8), hence the requirement on the MMU being enabled. 77 * NOTE: The code assumes that the block size as defined in DCZID_EL0 78 * register is at least 16 bytes. 79 * 80 * ----------------------------------------------------------------------- 81 */ 82func zeromem_dczva 83 84 /* 85 * The function consists of a series of loops that zero memory one byte 86 * at a time, 16 bytes at a time or using the DC ZVA instruction to 87 * zero aligned block of bytes, which is assumed to be more than 16. 88 * In the case where the DC ZVA instruction cannot be used or if the 89 * first 16 bytes loop would overflow, there is fallback path that does 90 * not use DC ZVA. 91 * Note: The fallback path is also used by the zeromem function that 92 * branches to it directly. 93 * 94 * +---------+ zeromem_dczva 95 * | entry | 96 * +----+----+ 97 * | 98 * v 99 * +---------+ 100 * | checks |>o-------+ (If any check fails, fallback) 101 * +----+----+ | 102 * | |---------------+ 103 * v | Fallback path | 104 * +------+------+ |---------------+ 105 * | 1 byte loop | | 106 * +------+------+ .Lzeromem_dczva_initial_1byte_aligned_end 107 * | | 108 * v | 109 * +-------+-------+ | 110 * | 16 bytes loop | | 111 * +-------+-------+ | 112 * | | 113 * v | 114 * +------+------+ .Lzeromem_dczva_blocksize_aligned 115 * | DC ZVA loop | | 116 * +------+------+ | 117 * +--------+ | | 118 * | | | | 119 * | v v | 120 * | +-------+-------+ .Lzeromem_dczva_final_16bytes_aligned 121 * | | 16 bytes loop | | 122 * | +-------+-------+ | 123 * | | | 124 * | v | 125 * | +------+------+ .Lzeromem_dczva_final_1byte_aligned 126 * | | 1 byte loop | | 127 * | +-------------+ | 128 * | | | 129 * | v | 130 * | +---+--+ | 131 * | | exit | | 132 * | +------+ | 133 * | | 134 * | +--------------+ +------------------+ zeromem 135 * | | +----------------| zeromem function | 136 * | | | +------------------+ 137 * | v v 138 * | +-------------+ .Lzeromem_dczva_fallback_entry 139 * | | 1 byte loop | 140 * | +------+------+ 141 * | | 142 * +-----------+ 143 */ 144 145 /* 146 * Readable names for registers 147 * 148 * Registers x0, x1 and x2 are also set by zeromem which 149 * branches into the fallback path directly, so cursor, length and 150 * stop_address should not be retargeted to other registers. 151 */ 152 cursor .req x0 /* Start address and then current address */ 153 length .req x1 /* Length in bytes of the region to zero out */ 154 /* Reusing x1 as length is never used after block_mask is set */ 155 block_mask .req x1 /* Bitmask of the block size read in DCZID_EL0 */ 156 stop_address .req x2 /* Address past the last zeroed byte */ 157 block_size .req x3 /* Size of a block in bytes as read in DCZID_EL0 */ 158 tmp1 .req x4 159 tmp2 .req x5 160 161#if ENABLE_ASSERTIONS 162 /* 163 * Check for M bit (MMU enabled) of the current SCTLR_EL(1|3) 164 * register value and panic if the MMU is disabled. 165 */ 166#if defined(IMAGE_BL1) || defined(IMAGE_BL31) || (defined(IMAGE_BL2) && BL2_AT_EL3) 167 mrs tmp1, sctlr_el3 168#else 169 mrs tmp1, sctlr_el1 170#endif 171 172 tst tmp1, #SCTLR_M_BIT 173 ASM_ASSERT(ne) 174#endif /* ENABLE_ASSERTIONS */ 175 176 /* stop_address is the address past the last to zero */ 177 add stop_address, cursor, length 178 179 /* 180 * Get block_size = (log2(<block size>) >> 2) (see encoding of 181 * dczid_el0 reg) 182 */ 183 mrs block_size, dczid_el0 184 185 /* 186 * Select the 4 lowest bits and convert the extracted log2(<block size 187 * in words>) to <block size in bytes> 188 */ 189 ubfx block_size, block_size, #0, #4 190 mov tmp2, #(1 << 2) 191 lsl block_size, tmp2, block_size 192 193#if ENABLE_ASSERTIONS 194 /* 195 * Assumes block size is at least 16 bytes to avoid manual realignment 196 * of the cursor at the end of the DCZVA loop. 197 */ 198 cmp block_size, #16 199 ASM_ASSERT(hs) 200#endif 201 /* 202 * Not worth doing all the setup for a region less than a block and 203 * protects against zeroing a whole block when the area to zero is 204 * smaller than that. Also, as it is assumed that the block size is at 205 * least 16 bytes, this also protects the initial aligning loops from 206 * trying to zero 16 bytes when length is less than 16. 207 */ 208 cmp length, block_size 209 b.lo .Lzeromem_dczva_fallback_entry 210 211 /* 212 * Calculate the bitmask of the block alignment. It will never 213 * underflow as the block size is between 4 bytes and 2kB. 214 * block_mask = block_size - 1 215 */ 216 sub block_mask, block_size, #1 217 218 /* 219 * length alias should not be used after this point unless it is 220 * defined as a register other than block_mask's. 221 */ 222 .unreq length 223 224 /* 225 * If the start address is already aligned to zero block size, go 226 * straight to the cache zeroing loop. This is safe because at this 227 * point, the length cannot be smaller than a block size. 228 */ 229 tst cursor, block_mask 230 b.eq .Lzeromem_dczva_blocksize_aligned 231 232 /* 233 * Calculate the first block-size-aligned address. It is assumed that 234 * the zero block size is at least 16 bytes. This address is the last 235 * address of this initial loop. 236 */ 237 orr tmp1, cursor, block_mask 238 add tmp1, tmp1, #1 239 240 /* 241 * If the addition overflows, skip the cache zeroing loops. This is 242 * quite unlikely however. 243 */ 244 cbz tmp1, .Lzeromem_dczva_fallback_entry 245 246 /* 247 * If the first block-size-aligned address is past the last address, 248 * fallback to the simpler code. 249 */ 250 cmp tmp1, stop_address 251 b.hi .Lzeromem_dczva_fallback_entry 252 253 /* 254 * If the start address is already aligned to 16 bytes, skip this loop. 255 * It is safe to do this because tmp1 (the stop address of the initial 256 * 16 bytes loop) will never be greater than the final stop address. 257 */ 258 tst cursor, #0xf 259 b.eq .Lzeromem_dczva_initial_1byte_aligned_end 260 261 /* Calculate the next address aligned to 16 bytes */ 262 orr tmp2, cursor, #0xf 263 add tmp2, tmp2, #1 264 /* If it overflows, fallback to the simple path (unlikely) */ 265 cbz tmp2, .Lzeromem_dczva_fallback_entry 266 /* 267 * Next aligned address cannot be after the stop address because the 268 * length cannot be smaller than 16 at this point. 269 */ 270 271 /* First loop: zero byte per byte */ 2721: 273 strb wzr, [cursor], #1 274 cmp cursor, tmp2 275 b.ne 1b 276.Lzeromem_dczva_initial_1byte_aligned_end: 277 278 /* 279 * Second loop: we need to zero 16 bytes at a time from cursor to tmp1 280 * before being able to use the code that deals with block-size-aligned 281 * addresses. 282 */ 283 cmp cursor, tmp1 284 b.hs 2f 2851: 286 stp xzr, xzr, [cursor], #16 287 cmp cursor, tmp1 288 b.lo 1b 2892: 290 291 /* 292 * Third loop: zero a block at a time using DC ZVA cache block zeroing 293 * instruction. 294 */ 295.Lzeromem_dczva_blocksize_aligned: 296 /* 297 * Calculate the last block-size-aligned address. If the result equals 298 * to the start address, the loop will exit immediately. 299 */ 300 bic tmp1, stop_address, block_mask 301 302 cmp cursor, tmp1 303 b.hs 2f 3041: 305 /* Zero the block containing the cursor */ 306 dc zva, cursor 307 /* Increment the cursor by the size of a block */ 308 add cursor, cursor, block_size 309 cmp cursor, tmp1 310 b.lo 1b 3112: 312 313 /* 314 * Fourth loop: zero 16 bytes at a time and then byte per byte the 315 * remaining area 316 */ 317.Lzeromem_dczva_final_16bytes_aligned: 318 /* 319 * Calculate the last 16 bytes aligned address. It is assumed that the 320 * block size will never be smaller than 16 bytes so that the current 321 * cursor is aligned to at least 16 bytes boundary. 322 */ 323 bic tmp1, stop_address, #15 324 325 cmp cursor, tmp1 326 b.hs 2f 3271: 328 stp xzr, xzr, [cursor], #16 329 cmp cursor, tmp1 330 b.lo 1b 3312: 332 333 /* Fifth and final loop: zero byte per byte */ 334.Lzeromem_dczva_final_1byte_aligned: 335 cmp cursor, stop_address 336 b.eq 2f 3371: 338 strb wzr, [cursor], #1 339 cmp cursor, stop_address 340 b.ne 1b 3412: 342 ret 343 344 /* Fallback for unaligned start addresses */ 345.Lzeromem_dczva_fallback_entry: 346 /* 347 * If the start address is already aligned to 16 bytes, skip this loop. 348 */ 349 tst cursor, #0xf 350 b.eq .Lzeromem_dczva_final_16bytes_aligned 351 352 /* Calculate the next address aligned to 16 bytes */ 353 orr tmp1, cursor, #15 354 add tmp1, tmp1, #1 355 /* If it overflows, fallback to byte per byte zeroing */ 356 cbz tmp1, .Lzeromem_dczva_final_1byte_aligned 357 /* If the next aligned address is after the stop address, fall back */ 358 cmp tmp1, stop_address 359 b.hs .Lzeromem_dczva_final_1byte_aligned 360 361 /* Fallback entry loop: zero byte per byte */ 3621: 363 strb wzr, [cursor], #1 364 cmp cursor, tmp1 365 b.ne 1b 366 367 b .Lzeromem_dczva_final_16bytes_aligned 368 369 .unreq cursor 370 /* 371 * length is already unreq'ed to reuse the register for another 372 * variable. 373 */ 374 .unreq stop_address 375 .unreq block_size 376 .unreq block_mask 377 .unreq tmp1 378 .unreq tmp2 379endfunc zeromem_dczva 380 381/* -------------------------------------------------------------------------- 382 * void memcpy16(void *dest, const void *src, unsigned int length) 383 * 384 * Copy length bytes from memory area src to memory area dest. 385 * The memory areas should not overlap. 386 * Destination and source addresses must be 16-byte aligned. 387 * -------------------------------------------------------------------------- 388 */ 389func memcpy16 390#if ENABLE_ASSERTIONS 391 orr x3, x0, x1 392 tst x3, #0xf 393 ASM_ASSERT(eq) 394#endif 395/* copy 16 bytes at a time */ 396m_loop16: 397 cmp x2, #16 398 b.lo m_loop1 399 ldp x3, x4, [x1], #16 400 stp x3, x4, [x0], #16 401 sub x2, x2, #16 402 b m_loop16 403/* copy byte per byte */ 404m_loop1: 405 cbz x2, m_end 406 ldrb w3, [x1], #1 407 strb w3, [x0], #1 408 subs x2, x2, #1 409 b.ne m_loop1 410m_end: 411 ret 412endfunc memcpy16 413 414/* --------------------------------------------------------------------------- 415 * Disable the MMU at EL3 416 * --------------------------------------------------------------------------- 417 */ 418 419func disable_mmu_el3 420 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) 421do_disable_mmu_el3: 422 mrs x0, sctlr_el3 423 bic x0, x0, x1 424 msr sctlr_el3, x0 425 isb /* ensure MMU is off */ 426 dsb sy 427 ret 428endfunc disable_mmu_el3 429 430 431func disable_mmu_icache_el3 432 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) 433 b do_disable_mmu_el3 434endfunc disable_mmu_icache_el3 435 436/* --------------------------------------------------------------------------- 437 * Disable the MMU at EL1 438 * --------------------------------------------------------------------------- 439 */ 440 441func disable_mmu_el1 442 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) 443do_disable_mmu_el1: 444 mrs x0, sctlr_el1 445 bic x0, x0, x1 446 msr sctlr_el1, x0 447 isb /* ensure MMU is off */ 448 dsb sy 449 ret 450endfunc disable_mmu_el1 451 452 453func disable_mmu_icache_el1 454 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) 455 b do_disable_mmu_el1 456endfunc disable_mmu_icache_el1 457 458/* --------------------------------------------------------------------------- 459 * Enable the use of VFP at EL3 460 * --------------------------------------------------------------------------- 461 */ 462#if SUPPORT_VFP 463func enable_vfp 464 mrs x0, cpacr_el1 465 orr x0, x0, #CPACR_VFP_BITS 466 msr cpacr_el1, x0 467 mrs x0, cptr_el3 468 mov x1, #AARCH64_CPTR_TFP 469 bic x0, x0, x1 470 msr cptr_el3, x0 471 isb 472 ret 473endfunc enable_vfp 474#endif 475 476/* --------------------------------------------------------------------------- 477 * Helper to fixup Global Descriptor table (GDT) and dynamic relocations 478 * (.rela.dyn) at runtime. 479 * 480 * This function is meant to be used when the firmware is compiled with -fpie 481 * and linked with -pie options. We rely on the linker script exporting 482 * appropriate markers for start and end of the section. For GOT, we 483 * expect __GOT_START__ and __GOT_END__. Similarly for .rela.dyn, we expect 484 * __RELA_START__ and __RELA_END__. 485 * 486 * The function takes the limits of the memory to apply fixups to as 487 * arguments (which is usually the limits of the relocable BL image). 488 * x0 - the start of the fixup region 489 * x1 - the limit of the fixup region 490 * These addresses have to be 4KB page aligned. 491 * --------------------------------------------------------------------------- 492 */ 493 494/* Relocation codes */ 495#define R_AARCH64_NONE 0 496#define R_AARCH64_RELATIVE 1027 497 498func fixup_gdt_reloc 499 mov x6, x0 500 mov x7, x1 501 502#if ENABLE_ASSERTIONS 503 /* Test if the limits are 4KB aligned */ 504 orr x0, x0, x1 505 tst x0, #(PAGE_SIZE_MASK) 506 ASM_ASSERT(eq) 507#endif 508 /* 509 * Calculate the offset based on return address in x30. 510 * Assume that this function is called within a page at the start of 511 * fixup region. 512 */ 513 and x2, x30, #~(PAGE_SIZE_MASK) 514 subs x0, x2, x6 /* Diff(S) = Current Address - Compiled Address */ 515 b.eq 3f /* Diff(S) = 0. No relocation needed */ 516 517 adrp x1, __GOT_START__ 518 add x1, x1, :lo12:__GOT_START__ 519 adrp x2, __GOT_END__ 520 add x2, x2, :lo12:__GOT_END__ 521 522 /* 523 * GOT is an array of 64_bit addresses which must be fixed up as 524 * new_addr = old_addr + Diff(S). 525 * The new_addr is the address currently the binary is executing from 526 * and old_addr is the address at compile time. 527 */ 5281: ldr x3, [x1] 529 530 /* Skip adding offset if address is < lower limit */ 531 cmp x3, x6 532 b.lo 2f 533 534 /* Skip adding offset if address is >= upper limit */ 535 cmp x3, x7 536 b.hs 2f 537 add x3, x3, x0 538 str x3, [x1] 539 5402: add x1, x1, #8 541 cmp x1, x2 542 b.lo 1b 543 544 /* Starting dynamic relocations. Use adrp/adr to get RELA_START and END */ 5453: adrp x1, __RELA_START__ 546 add x1, x1, :lo12:__RELA_START__ 547 adrp x2, __RELA_END__ 548 add x2, x2, :lo12:__RELA_END__ 549 550 /* 551 * According to ELF-64 specification, the RELA data structure is as 552 * follows: 553 * typedef struct { 554 * Elf64_Addr r_offset; 555 * Elf64_Xword r_info; 556 * Elf64_Sxword r_addend; 557 * } Elf64_Rela; 558 * 559 * r_offset is address of reference 560 * r_info is symbol index and type of relocation (in this case 561 * code 1027 which corresponds to R_AARCH64_RELATIVE). 562 * r_addend is constant part of expression. 563 * 564 * Size of Elf64_Rela structure is 24 bytes. 565 */ 566 567 /* Skip R_AARCH64_NONE entry with code 0 */ 5681: ldr x3, [x1, #8] 569 cbz x3, 2f 570 571#if ENABLE_ASSERTIONS 572 /* Assert that the relocation type is R_AARCH64_RELATIVE */ 573 cmp x3, #R_AARCH64_RELATIVE 574 ASM_ASSERT(eq) 575#endif 576 ldr x3, [x1] /* r_offset */ 577 add x3, x0, x3 578 ldr x4, [x1, #16] /* r_addend */ 579 580 /* Skip adding offset if r_addend is < lower limit */ 581 cmp x4, x6 582 b.lo 2f 583 584 /* Skip adding offset if r_addend entry is >= upper limit */ 585 cmp x4, x7 586 b.hs 2f 587 588 add x4, x0, x4 /* Diff(S) + r_addend */ 589 str x4, [x3] 590 5912: add x1, x1, #24 592 cmp x1, x2 593 b.lo 1b 594 ret 595endfunc fixup_gdt_reloc 596 597/* 598 * TODO: Currently only supports size of 4KB, 599 * support other sizes as well. 600 */ 601func gpt_tlbi_by_pa 602#if ENABLE_ASSERTIONS 603 cmp x1, #PAGE_SIZE_4KB 604 ASM_ASSERT(eq) 605 tst x0, #(PAGE_SIZE_MASK) 606 ASM_ASSERT(eq) 607#endif 608 lsr x0, x0, #FOUR_KB_SHIFT /* 4KB size encoding is zero */ 609 sys #6, c8, c4, #3, x0 /* TLBI RPAOS, <Xt> */ 610 dsb sy 611 ret 612endfunc gpt_tlbi_by_pa 613