1/* 2 * Copyright (c) 2013-2017, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <assert_macros.S> 10 11 .globl get_afflvl_shift 12 .globl mpidr_mask_lower_afflvls 13 .globl eret 14 .globl smc 15 16 .globl zero_normalmem 17 .globl zeromem 18 .globl zeromem16 19 .globl memcpy16 20 21 .globl disable_mmu_el1 22 .globl disable_mmu_el3 23 .globl disable_mmu_icache_el1 24 .globl disable_mmu_icache_el3 25 26#if SUPPORT_VFP 27 .globl enable_vfp 28#endif 29 30func get_afflvl_shift 31 cmp x0, #3 32 cinc x0, x0, eq 33 mov x1, #MPIDR_AFFLVL_SHIFT 34 lsl x0, x0, x1 35 ret 36endfunc get_afflvl_shift 37 38func mpidr_mask_lower_afflvls 39 cmp x1, #3 40 cinc x1, x1, eq 41 mov x2, #MPIDR_AFFLVL_SHIFT 42 lsl x2, x1, x2 43 lsr x0, x0, x2 44 lsl x0, x0, x2 45 ret 46endfunc mpidr_mask_lower_afflvls 47 48 49func eret 50 eret 51endfunc eret 52 53 54func smc 55 smc #0 56endfunc smc 57 58/* ----------------------------------------------------------------------- 59 * void zero_normalmem(void *mem, unsigned int length); 60 * 61 * Initialise a region in normal memory to 0. This functions complies with the 62 * AAPCS and can be called from C code. 63 * 64 * NOTE: MMU must be enabled when using this function as it can only operate on 65 * normal memory. It is intended to be mainly used from C code when MMU 66 * is usually enabled. 67 * ----------------------------------------------------------------------- 68 */ 69.equ zero_normalmem, zeromem_dczva 70 71/* ----------------------------------------------------------------------- 72 * void zeromem(void *mem, unsigned int length); 73 * 74 * Initialise a region of device memory to 0. This functions complies with the 75 * AAPCS and can be called from C code. 76 * 77 * NOTE: When data caches and MMU are enabled, zero_normalmem can usually be 78 * used instead for faster zeroing. 79 * 80 * ----------------------------------------------------------------------- 81 */ 82func zeromem 83 /* x2 is the address past the last zeroed address */ 84 add x2, x0, x1 85 /* 86 * Uses the fallback path that does not use DC ZVA instruction and 87 * therefore does not need enabled MMU 88 */ 89 b .Lzeromem_dczva_fallback_entry 90endfunc zeromem 91 92/* ----------------------------------------------------------------------- 93 * void zeromem_dczva(void *mem, unsigned int length); 94 * 95 * Fill a region of normal memory of size "length" in bytes with null bytes. 96 * MMU must be enabled and the memory be of 97 * normal type. This is because this function internally uses the DC ZVA 98 * instruction, which generates an Alignment fault if used on any type of 99 * Device memory (see section D3.4.9 of the ARMv8 ARM, issue k). When the MMU 100 * is disabled, all memory behaves like Device-nGnRnE memory (see section 101 * D4.2.8), hence the requirement on the MMU being enabled. 102 * NOTE: The code assumes that the block size as defined in DCZID_EL0 103 * register is at least 16 bytes. 104 * 105 * ----------------------------------------------------------------------- 106 */ 107func zeromem_dczva 108 109 /* 110 * The function consists of a series of loops that zero memory one byte 111 * at a time, 16 bytes at a time or using the DC ZVA instruction to 112 * zero aligned block of bytes, which is assumed to be more than 16. 113 * In the case where the DC ZVA instruction cannot be used or if the 114 * first 16 bytes loop would overflow, there is fallback path that does 115 * not use DC ZVA. 116 * Note: The fallback path is also used by the zeromem function that 117 * branches to it directly. 118 * 119 * +---------+ zeromem_dczva 120 * | entry | 121 * +----+----+ 122 * | 123 * v 124 * +---------+ 125 * | checks |>o-------+ (If any check fails, fallback) 126 * +----+----+ | 127 * | |---------------+ 128 * v | Fallback path | 129 * +------+------+ |---------------+ 130 * | 1 byte loop | | 131 * +------+------+ .Lzeromem_dczva_initial_1byte_aligned_end 132 * | | 133 * v | 134 * +-------+-------+ | 135 * | 16 bytes loop | | 136 * +-------+-------+ | 137 * | | 138 * v | 139 * +------+------+ .Lzeromem_dczva_blocksize_aligned 140 * | DC ZVA loop | | 141 * +------+------+ | 142 * +--------+ | | 143 * | | | | 144 * | v v | 145 * | +-------+-------+ .Lzeromem_dczva_final_16bytes_aligned 146 * | | 16 bytes loop | | 147 * | +-------+-------+ | 148 * | | | 149 * | v | 150 * | +------+------+ .Lzeromem_dczva_final_1byte_aligned 151 * | | 1 byte loop | | 152 * | +-------------+ | 153 * | | | 154 * | v | 155 * | +---+--+ | 156 * | | exit | | 157 * | +------+ | 158 * | | 159 * | +--------------+ +------------------+ zeromem 160 * | | +----------------| zeromem function | 161 * | | | +------------------+ 162 * | v v 163 * | +-------------+ .Lzeromem_dczva_fallback_entry 164 * | | 1 byte loop | 165 * | +------+------+ 166 * | | 167 * +-----------+ 168 */ 169 170 /* 171 * Readable names for registers 172 * 173 * Registers x0, x1 and x2 are also set by zeromem which 174 * branches into the fallback path directly, so cursor, length and 175 * stop_address should not be retargeted to other registers. 176 */ 177 cursor .req x0 /* Start address and then current address */ 178 length .req x1 /* Length in bytes of the region to zero out */ 179 /* Reusing x1 as length is never used after block_mask is set */ 180 block_mask .req x1 /* Bitmask of the block size read in DCZID_EL0 */ 181 stop_address .req x2 /* Address past the last zeroed byte */ 182 block_size .req x3 /* Size of a block in bytes as read in DCZID_EL0 */ 183 tmp1 .req x4 184 tmp2 .req x5 185 186#if ENABLE_ASSERTIONS 187 /* 188 * Check for M bit (MMU enabled) of the current SCTLR_EL(1|3) 189 * register value and panic if the MMU is disabled. 190 */ 191#if defined(IMAGE_BL1) || defined(IMAGE_BL31) || (defined(IMAGE_BL2) && BL2_AT_EL3) 192 mrs tmp1, sctlr_el3 193#else 194 mrs tmp1, sctlr_el1 195#endif 196 197 tst tmp1, #SCTLR_M_BIT 198 ASM_ASSERT(ne) 199#endif /* ENABLE_ASSERTIONS */ 200 201 /* stop_address is the address past the last to zero */ 202 add stop_address, cursor, length 203 204 /* 205 * Get block_size = (log2(<block size>) >> 2) (see encoding of 206 * dczid_el0 reg) 207 */ 208 mrs block_size, dczid_el0 209 210 /* 211 * Select the 4 lowest bits and convert the extracted log2(<block size 212 * in words>) to <block size in bytes> 213 */ 214 ubfx block_size, block_size, #0, #4 215 mov tmp2, #(1 << 2) 216 lsl block_size, tmp2, block_size 217 218#if ENABLE_ASSERTIONS 219 /* 220 * Assumes block size is at least 16 bytes to avoid manual realignment 221 * of the cursor at the end of the DCZVA loop. 222 */ 223 cmp block_size, #16 224 ASM_ASSERT(hs) 225#endif 226 /* 227 * Not worth doing all the setup for a region less than a block and 228 * protects against zeroing a whole block when the area to zero is 229 * smaller than that. Also, as it is assumed that the block size is at 230 * least 16 bytes, this also protects the initial aligning loops from 231 * trying to zero 16 bytes when length is less than 16. 232 */ 233 cmp length, block_size 234 b.lo .Lzeromem_dczva_fallback_entry 235 236 /* 237 * Calculate the bitmask of the block alignment. It will never 238 * underflow as the block size is between 4 bytes and 2kB. 239 * block_mask = block_size - 1 240 */ 241 sub block_mask, block_size, #1 242 243 /* 244 * length alias should not be used after this point unless it is 245 * defined as a register other than block_mask's. 246 */ 247 .unreq length 248 249 /* 250 * If the start address is already aligned to zero block size, go 251 * straight to the cache zeroing loop. This is safe because at this 252 * point, the length cannot be smaller than a block size. 253 */ 254 tst cursor, block_mask 255 b.eq .Lzeromem_dczva_blocksize_aligned 256 257 /* 258 * Calculate the first block-size-aligned address. It is assumed that 259 * the zero block size is at least 16 bytes. This address is the last 260 * address of this initial loop. 261 */ 262 orr tmp1, cursor, block_mask 263 add tmp1, tmp1, #1 264 265 /* 266 * If the addition overflows, skip the cache zeroing loops. This is 267 * quite unlikely however. 268 */ 269 cbz tmp1, .Lzeromem_dczva_fallback_entry 270 271 /* 272 * If the first block-size-aligned address is past the last address, 273 * fallback to the simpler code. 274 */ 275 cmp tmp1, stop_address 276 b.hi .Lzeromem_dczva_fallback_entry 277 278 /* 279 * If the start address is already aligned to 16 bytes, skip this loop. 280 * It is safe to do this because tmp1 (the stop address of the initial 281 * 16 bytes loop) will never be greater than the final stop address. 282 */ 283 tst cursor, #0xf 284 b.eq .Lzeromem_dczva_initial_1byte_aligned_end 285 286 /* Calculate the next address aligned to 16 bytes */ 287 orr tmp2, cursor, #0xf 288 add tmp2, tmp2, #1 289 /* If it overflows, fallback to the simple path (unlikely) */ 290 cbz tmp2, .Lzeromem_dczva_fallback_entry 291 /* 292 * Next aligned address cannot be after the stop address because the 293 * length cannot be smaller than 16 at this point. 294 */ 295 296 /* First loop: zero byte per byte */ 2971: 298 strb wzr, [cursor], #1 299 cmp cursor, tmp2 300 b.ne 1b 301.Lzeromem_dczva_initial_1byte_aligned_end: 302 303 /* 304 * Second loop: we need to zero 16 bytes at a time from cursor to tmp1 305 * before being able to use the code that deals with block-size-aligned 306 * addresses. 307 */ 308 cmp cursor, tmp1 309 b.hs 2f 3101: 311 stp xzr, xzr, [cursor], #16 312 cmp cursor, tmp1 313 b.lo 1b 3142: 315 316 /* 317 * Third loop: zero a block at a time using DC ZVA cache block zeroing 318 * instruction. 319 */ 320.Lzeromem_dczva_blocksize_aligned: 321 /* 322 * Calculate the last block-size-aligned address. If the result equals 323 * to the start address, the loop will exit immediately. 324 */ 325 bic tmp1, stop_address, block_mask 326 327 cmp cursor, tmp1 328 b.hs 2f 3291: 330 /* Zero the block containing the cursor */ 331 dc zva, cursor 332 /* Increment the cursor by the size of a block */ 333 add cursor, cursor, block_size 334 cmp cursor, tmp1 335 b.lo 1b 3362: 337 338 /* 339 * Fourth loop: zero 16 bytes at a time and then byte per byte the 340 * remaining area 341 */ 342.Lzeromem_dczva_final_16bytes_aligned: 343 /* 344 * Calculate the last 16 bytes aligned address. It is assumed that the 345 * block size will never be smaller than 16 bytes so that the current 346 * cursor is aligned to at least 16 bytes boundary. 347 */ 348 bic tmp1, stop_address, #15 349 350 cmp cursor, tmp1 351 b.hs 2f 3521: 353 stp xzr, xzr, [cursor], #16 354 cmp cursor, tmp1 355 b.lo 1b 3562: 357 358 /* Fifth and final loop: zero byte per byte */ 359.Lzeromem_dczva_final_1byte_aligned: 360 cmp cursor, stop_address 361 b.eq 2f 3621: 363 strb wzr, [cursor], #1 364 cmp cursor, stop_address 365 b.ne 1b 3662: 367 ret 368 369 /* Fallback for unaligned start addresses */ 370.Lzeromem_dczva_fallback_entry: 371 /* 372 * If the start address is already aligned to 16 bytes, skip this loop. 373 */ 374 tst cursor, #0xf 375 b.eq .Lzeromem_dczva_final_16bytes_aligned 376 377 /* Calculate the next address aligned to 16 bytes */ 378 orr tmp1, cursor, #15 379 add tmp1, tmp1, #1 380 /* If it overflows, fallback to byte per byte zeroing */ 381 cbz tmp1, .Lzeromem_dczva_final_1byte_aligned 382 /* If the next aligned address is after the stop address, fall back */ 383 cmp tmp1, stop_address 384 b.hs .Lzeromem_dczva_final_1byte_aligned 385 386 /* Fallback entry loop: zero byte per byte */ 3871: 388 strb wzr, [cursor], #1 389 cmp cursor, tmp1 390 b.ne 1b 391 392 b .Lzeromem_dczva_final_16bytes_aligned 393 394 .unreq cursor 395 /* 396 * length is already unreq'ed to reuse the register for another 397 * variable. 398 */ 399 .unreq stop_address 400 .unreq block_size 401 .unreq block_mask 402 .unreq tmp1 403 .unreq tmp2 404endfunc zeromem_dczva 405 406/* -------------------------------------------------------------------------- 407 * void memcpy16(void *dest, const void *src, unsigned int length) 408 * 409 * Copy length bytes from memory area src to memory area dest. 410 * The memory areas should not overlap. 411 * Destination and source addresses must be 16-byte aligned. 412 * -------------------------------------------------------------------------- 413 */ 414func memcpy16 415#if ENABLE_ASSERTIONS 416 orr x3, x0, x1 417 tst x3, #0xf 418 ASM_ASSERT(eq) 419#endif 420/* copy 16 bytes at a time */ 421m_loop16: 422 cmp x2, #16 423 b.lo m_loop1 424 ldp x3, x4, [x1], #16 425 stp x3, x4, [x0], #16 426 sub x2, x2, #16 427 b m_loop16 428/* copy byte per byte */ 429m_loop1: 430 cbz x2, m_end 431 ldrb w3, [x1], #1 432 strb w3, [x0], #1 433 subs x2, x2, #1 434 b.ne m_loop1 435m_end: 436 ret 437endfunc memcpy16 438 439/* --------------------------------------------------------------------------- 440 * Disable the MMU at EL3 441 * --------------------------------------------------------------------------- 442 */ 443 444func disable_mmu_el3 445 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) 446do_disable_mmu_el3: 447 mrs x0, sctlr_el3 448 bic x0, x0, x1 449 msr sctlr_el3, x0 450 isb /* ensure MMU is off */ 451 dsb sy 452 ret 453endfunc disable_mmu_el3 454 455 456func disable_mmu_icache_el3 457 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) 458 b do_disable_mmu_el3 459endfunc disable_mmu_icache_el3 460 461/* --------------------------------------------------------------------------- 462 * Disable the MMU at EL1 463 * --------------------------------------------------------------------------- 464 */ 465 466func disable_mmu_el1 467 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT) 468do_disable_mmu_el1: 469 mrs x0, sctlr_el1 470 bic x0, x0, x1 471 msr sctlr_el1, x0 472 isb /* ensure MMU is off */ 473 dsb sy 474 ret 475endfunc disable_mmu_el1 476 477 478func disable_mmu_icache_el1 479 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT) 480 b do_disable_mmu_el1 481endfunc disable_mmu_icache_el1 482 483/* --------------------------------------------------------------------------- 484 * Enable the use of VFP at EL3 485 * --------------------------------------------------------------------------- 486 */ 487#if SUPPORT_VFP 488func enable_vfp 489 mrs x0, cpacr_el1 490 orr x0, x0, #CPACR_VFP_BITS 491 msr cpacr_el1, x0 492 mrs x0, cptr_el3 493 mov x1, #AARCH64_CPTR_TFP 494 bic x0, x0, x1 495 msr cptr_el3, x0 496 isb 497 ret 498endfunc enable_vfp 499#endif 500