1 /* 2 * Copyright (c) 2016-2017, Linaro Limited. All rights reserved. 3 * Copyright (c) 2014-2019, Arm Limited. All rights reserved. 4 * Copyright (c) 2014, STMicroelectronics International N.V. 5 * All rights reserved. 6 * 7 * SPDX-License-Identifier: BSD-3-Clause 8 */ 9 10 #include <assert.h> 11 #include <stdio.h> 12 #include <string.h> 13 14 #include <platform_def.h> 15 16 #include <arch.h> 17 #include <arch_helpers.h> 18 #include <common/debug.h> 19 #include <lib/cassert.h> 20 #include <lib/utils.h> 21 #include <lib/xlat_tables/xlat_tables.h> 22 23 #include "../xlat_tables_private.h" 24 25 #ifdef ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING 26 #error "ARMV7_SUPPORTS_LARGE_PAGE_ADDRESSING flag is set. \ 27 This module is to be used when LPAE is not supported" 28 #endif 29 30 CASSERT(PLAT_VIRT_ADDR_SPACE_SIZE == (1ULL << 32), invalid_vaddr_space_size); 31 CASSERT(PLAT_PHY_ADDR_SPACE_SIZE == (1ULL << 32), invalid_paddr_space_size); 32 33 #define MMU32B_UNSET_DESC ~0ul 34 #define MMU32B_INVALID_DESC 0ul 35 36 #define MT_UNKNOWN ~0U 37 38 /* 39 * MMU related values 40 */ 41 42 /* Sharable */ 43 #define MMU32B_TTB_S (1 << 1) 44 45 /* Not Outer Sharable */ 46 #define MMU32B_TTB_NOS (1 << 5) 47 48 /* Normal memory, Inner Non-cacheable */ 49 #define MMU32B_TTB_IRGN_NC 0 50 51 /* Normal memory, Inner Write-Back Write-Allocate Cacheable */ 52 #define MMU32B_TTB_IRGN_WBWA (1 << 6) 53 54 /* Normal memory, Inner Write-Through Cacheable */ 55 #define MMU32B_TTB_IRGN_WT 1 56 57 /* Normal memory, Inner Write-Back no Write-Allocate Cacheable */ 58 #define MMU32B_TTB_IRGN_WB (1 | (1 << 6)) 59 60 /* Normal memory, Outer Write-Back Write-Allocate Cacheable */ 61 #define MMU32B_TTB_RNG_WBWA (1 << 3) 62 63 #define MMU32B_DEFAULT_ATTRS \ 64 (MMU32B_TTB_S | MMU32B_TTB_NOS | \ 65 MMU32B_TTB_IRGN_WBWA | MMU32B_TTB_RNG_WBWA) 66 67 /* armv7 memory mapping attributes: section mapping */ 68 #define SECTION_SECURE (0 << 19) 69 #define SECTION_NOTSECURE (1 << 19) 70 #define SECTION_SHARED (1 << 16) 71 #define SECTION_NOTGLOBAL (1 << 17) 72 #define SECTION_ACCESS_FLAG (1 << 10) 73 #define SECTION_UNPRIV (1 << 11) 74 #define SECTION_RO (1 << 15) 75 #define SECTION_TEX(tex) ((((tex) >> 2) << 12) | \ 76 ((((tex) >> 1) & 0x1) << 3) | \ 77 (((tex) & 0x1) << 2)) 78 #define SECTION_DEVICE SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX) 79 #define SECTION_NORMAL SECTION_TEX(MMU32B_ATTR_DEVICE_INDEX) 80 #define SECTION_NORMAL_CACHED \ 81 SECTION_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX) 82 83 #define SECTION_XN (1 << 4) 84 #define SECTION_PXN (1 << 0) 85 #define SECTION_SECTION (2 << 0) 86 87 #define SECTION_PT_NOTSECURE (1 << 3) 88 #define SECTION_PT_PT (1 << 0) 89 90 #define SMALL_PAGE_SMALL_PAGE (1 << 1) 91 #define SMALL_PAGE_SHARED (1 << 10) 92 #define SMALL_PAGE_NOTGLOBAL (1 << 11) 93 #define SMALL_PAGE_TEX(tex) ((((tex) >> 2) << 6) | \ 94 ((((tex) >> 1) & 0x1) << 3) | \ 95 (((tex) & 0x1) << 2)) 96 #define SMALL_PAGE_DEVICE \ 97 SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX) 98 #define SMALL_PAGE_NORMAL \ 99 SMALL_PAGE_TEX(MMU32B_ATTR_DEVICE_INDEX) 100 #define SMALL_PAGE_NORMAL_CACHED \ 101 SMALL_PAGE_TEX(MMU32B_ATTR_IWBWA_OWBWA_INDEX) 102 #define SMALL_PAGE_ACCESS_FLAG (1 << 4) 103 #define SMALL_PAGE_UNPRIV (1 << 5) 104 #define SMALL_PAGE_RO (1 << 9) 105 #define SMALL_PAGE_XN (1 << 0) 106 107 /* The TEX, C and B bits concatenated */ 108 #define MMU32B_ATTR_DEVICE_INDEX 0x0 109 #define MMU32B_ATTR_IWBWA_OWBWA_INDEX 0x1 110 111 #define MMU32B_PRRR_IDX(idx, tr, nos) (((tr) << (2 * (idx))) | \ 112 ((uint32_t)(nos) << ((idx) + 24))) 113 #define MMU32B_NMRR_IDX(idx, ir, or) (((ir) << (2 * (idx))) | \ 114 ((uint32_t)(or) << (2 * (idx) + 16))) 115 #define MMU32B_PRRR_DS0 (1 << 16) 116 #define MMU32B_PRRR_DS1 (1 << 17) 117 #define MMU32B_PRRR_NS0 (1 << 18) 118 #define MMU32B_PRRR_NS1 (1 << 19) 119 120 #define DACR_DOMAIN(num, perm) ((perm) << ((num) * 2)) 121 #define DACR_DOMAIN_PERM_NO_ACCESS 0x0 122 #define DACR_DOMAIN_PERM_CLIENT 0x1 123 #define DACR_DOMAIN_PERM_MANAGER 0x3 124 125 #define NUM_1MB_IN_4GB (1U << 12) 126 #define NUM_4K_IN_1MB (1U << 8) 127 128 #define ONE_MB_SHIFT 20 129 130 /* mmu 32b integration */ 131 #define MMU32B_L1_TABLE_SIZE (NUM_1MB_IN_4GB * 4) 132 #define MMU32B_L2_TABLE_SIZE (NUM_4K_IN_1MB * 4) 133 #define MMU32B_L1_TABLE_ALIGN (1 << 14) 134 #define MMU32B_L2_TABLE_ALIGN (1 << 10) 135 136 static unsigned int next_xlat; 137 static unsigned long long xlat_max_pa; 138 static uintptr_t xlat_max_va; 139 140 static uint32_t mmu_l1_base[NUM_1MB_IN_4GB] 141 __aligned(MMU32B_L1_TABLE_ALIGN) __attribute__((section("xlat_table"))); 142 143 static uint32_t mmu_l2_base[MAX_XLAT_TABLES][NUM_4K_IN_1MB] 144 __aligned(MMU32B_L2_TABLE_ALIGN) __attribute__((section("xlat_table"))); 145 146 /* 147 * Array of all memory regions stored in order of ascending base address. 148 * The list is terminated by the first entry with size == 0. 149 */ 150 static mmap_region_t mmap[MAX_MMAP_REGIONS + 1]; 151 152 void print_mmap(void) 153 { 154 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 155 mmap_region_t *mm = mmap; 156 157 printf("init xlat - l1:%p l2:%p (%d)\n", 158 (void *)mmu_l1_base, (void *)mmu_l2_base, MAX_XLAT_TABLES); 159 printf("mmap:\n"); 160 while (mm->size) { 161 printf(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", 162 (void *)mm->base_va, mm->base_pa, 163 mm->size, mm->attr); 164 ++mm; 165 }; 166 printf("\n"); 167 #endif 168 } 169 170 void mmap_add(const mmap_region_t *mm) 171 { 172 const mmap_region_t *mm_cursor = mm; 173 174 while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) { 175 mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va, 176 mm_cursor->size, mm_cursor->attr); 177 mm_cursor++; 178 } 179 } 180 181 void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, 182 size_t size, unsigned int attr) 183 { 184 mmap_region_t *mm = mmap; 185 const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U; 186 unsigned long long end_pa = base_pa + size - 1U; 187 uintptr_t end_va = base_va + size - 1U; 188 189 assert(IS_PAGE_ALIGNED(base_pa)); 190 assert(IS_PAGE_ALIGNED(base_va)); 191 assert(IS_PAGE_ALIGNED(size)); 192 193 if (size == 0U) 194 return; 195 196 assert(base_pa < end_pa); /* Check for overflows */ 197 assert(base_va < end_va); 198 199 assert((base_va + (uintptr_t)size - (uintptr_t)1) <= 200 (PLAT_VIRT_ADDR_SPACE_SIZE - 1U)); 201 assert((base_pa + (unsigned long long)size - 1ULL) <= 202 (PLAT_PHY_ADDR_SPACE_SIZE - 1U)); 203 204 #if ENABLE_ASSERTIONS 205 206 /* Check for PAs and VAs overlaps with all other regions */ 207 for (mm = mmap; mm->size; ++mm) { 208 209 uintptr_t mm_end_va = mm->base_va + mm->size - 1U; 210 211 /* 212 * Check if one of the regions is completely inside the other 213 * one. 214 */ 215 bool fully_overlapped_va = 216 ((base_va >= mm->base_va) && (end_va <= mm_end_va)) || 217 ((mm->base_va >= base_va) && (mm_end_va <= end_va)); 218 219 /* 220 * Full VA overlaps are only allowed if both regions are 221 * identity mapped (zero offset) or have the same VA to PA 222 * offset. Also, make sure that it's not the exact same area. 223 */ 224 if (fully_overlapped_va) { 225 assert((mm->base_va - mm->base_pa) == 226 (base_va - base_pa)); 227 assert((base_va != mm->base_va) || (size != mm->size)); 228 } else { 229 /* 230 * If the regions do not have fully overlapping VAs, 231 * then they must have fully separated VAs and PAs. 232 * Partial overlaps are not allowed 233 */ 234 235 unsigned long long mm_end_pa = 236 mm->base_pa + mm->size - 1; 237 238 bool separated_pa = (end_pa < mm->base_pa) || 239 (base_pa > mm_end_pa); 240 bool separated_va = (end_va < mm->base_va) || 241 (base_va > mm_end_va); 242 243 assert(separated_va && separated_pa); 244 } 245 } 246 247 mm = mmap; /* Restore pointer to the start of the array */ 248 249 #endif /* ENABLE_ASSERTIONS */ 250 251 /* Find correct place in mmap to insert new region */ 252 while ((mm->base_va < base_va) && (mm->size != 0U)) 253 ++mm; 254 255 /* 256 * If a section is contained inside another one with the same base 257 * address, it must be placed after the one it is contained in: 258 * 259 * 1st |-----------------------| 260 * 2nd |------------| 261 * 3rd |------| 262 * 263 * This is required for mmap_region_attr() to get the attributes of the 264 * small region correctly. 265 */ 266 while ((mm->base_va == base_va) && (mm->size > size)) 267 ++mm; 268 269 /* Make room for new region by moving other regions up by one place */ 270 (void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm); 271 272 /* Check we haven't lost the empty sentinal from the end of the array */ 273 assert(mm_last->size == 0U); 274 275 mm->base_pa = base_pa; 276 mm->base_va = base_va; 277 mm->size = size; 278 mm->attr = attr; 279 280 if (end_pa > xlat_max_pa) 281 xlat_max_pa = end_pa; 282 if (end_va > xlat_max_va) 283 xlat_max_va = end_va; 284 } 285 286 /* map all memory as shared/global/domain0/no-usr access */ 287 static uint32_t mmap_desc(unsigned attr, unsigned int addr_pa, 288 unsigned int level) 289 { 290 uint32_t desc; 291 292 switch (level) { 293 case 1: 294 assert(!(addr_pa & (MMU32B_L1_TABLE_ALIGN - 1))); 295 296 desc = SECTION_SECTION | SECTION_SHARED; 297 298 desc |= attr & MT_NS ? SECTION_NOTSECURE : 0; 299 300 desc |= SECTION_ACCESS_FLAG; 301 desc |= attr & MT_RW ? 0 : SECTION_RO; 302 303 desc |= attr & MT_MEMORY ? 304 SECTION_NORMAL_CACHED : SECTION_DEVICE; 305 306 if ((attr & MT_RW) || !(attr & MT_MEMORY)) 307 desc |= SECTION_XN; 308 break; 309 case 2: 310 assert(!(addr_pa & (MMU32B_L2_TABLE_ALIGN - 1))); 311 312 desc = SMALL_PAGE_SMALL_PAGE | SMALL_PAGE_SHARED; 313 314 desc |= SMALL_PAGE_ACCESS_FLAG; 315 desc |= attr & MT_RW ? 0 : SMALL_PAGE_RO; 316 317 desc |= attr & MT_MEMORY ? 318 SMALL_PAGE_NORMAL_CACHED : SMALL_PAGE_DEVICE; 319 320 if ((attr & MT_RW) || !(attr & MT_MEMORY)) 321 desc |= SMALL_PAGE_XN; 322 break; 323 default: 324 panic(); 325 } 326 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 327 /* dump only the non-lpae level 2 tables */ 328 if (level == 2) { 329 printf(attr & MT_MEMORY ? "MEM" : "dev"); 330 printf(attr & MT_RW ? "-rw" : "-RO"); 331 printf(attr & MT_NS ? "-NS" : "-S"); 332 } 333 #endif 334 return desc | addr_pa; 335 } 336 337 static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va, 338 size_t size, unsigned int *attr) 339 { 340 /* Don't assume that the area is contained in the first region */ 341 unsigned int ret = MT_UNKNOWN; 342 343 /* 344 * Get attributes from last (innermost) region that contains the 345 * requested area. Don't stop as soon as one region doesn't contain it 346 * because there may be other internal regions that contain this area: 347 * 348 * |-----------------------------1-----------------------------| 349 * |----2----| |-------3-------| |----5----| 350 * |--4--| 351 * 352 * |---| <- Area we want the attributes of. 353 * 354 * In this example, the area is contained in regions 1, 3 and 4 but not 355 * in region 2. The loop shouldn't stop at region 2 as inner regions 356 * have priority over outer regions, it should stop at region 5. 357 */ 358 for ( ; ; ++mm) { 359 360 if (mm->size == 0U) 361 return ret; /* Reached end of list */ 362 363 if (mm->base_va > (base_va + size - 1U)) 364 return ret; /* Next region is after area so end */ 365 366 if ((mm->base_va + mm->size - 1U) < base_va) 367 continue; /* Next region has already been overtaken */ 368 369 if ((ret == 0U) && (mm->attr == *attr)) 370 continue; /* Region doesn't override attribs so skip */ 371 372 if ((mm->base_va > base_va) || 373 ((mm->base_va + mm->size - 1U) < (base_va + size - 1U))) 374 return MT_UNKNOWN; /* Region doesn't fully cover area */ 375 376 *attr = mm->attr; 377 ret = 0U; 378 } 379 return ret; 380 } 381 382 static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm, 383 unsigned int base_va, 384 uint32_t *table, 385 unsigned int level) 386 { 387 unsigned int level_size_shift = (level == 1) ? 388 ONE_MB_SHIFT : FOUR_KB_SHIFT; 389 unsigned int level_size = 1 << level_size_shift; 390 unsigned int level_index_mask = (level == 1) ? 391 (NUM_1MB_IN_4GB - 1) << ONE_MB_SHIFT : 392 (NUM_4K_IN_1MB - 1) << FOUR_KB_SHIFT; 393 394 assert(level == 1 || level == 2); 395 396 VERBOSE("init xlat table at %p (level%1d)\n", (void *)table, level); 397 398 do { 399 uint32_t desc = MMU32B_UNSET_DESC; 400 401 if (mm->base_va + mm->size <= base_va) { 402 /* Area now after the region so skip it */ 403 ++mm; 404 continue; 405 } 406 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 407 /* dump only non-lpae level 2 tables content */ 408 if (level == 2) 409 printf(" 0x%lx %x " + 6 - 2 * level, 410 base_va, level_size); 411 #endif 412 if (mm->base_va >= base_va + level_size) { 413 /* Next region is after area so nothing to map yet */ 414 desc = MMU32B_INVALID_DESC; 415 } else if (mm->base_va <= base_va && mm->base_va + mm->size >= 416 base_va + level_size) { 417 /* Next region covers all of area */ 418 unsigned int attr = mm->attr; 419 unsigned int r = mmap_region_attr(mm, base_va, 420 level_size, &attr); 421 422 if (r == 0U) { 423 desc = mmap_desc(attr, 424 base_va - mm->base_va + mm->base_pa, 425 level); 426 } 427 } 428 429 if (desc == MMU32B_UNSET_DESC) { 430 uintptr_t xlat_table; 431 432 /* 433 * Area not covered by a region so need finer table 434 * Reuse next level table if any (assert attrib matching). 435 * Otherwise allocate a xlat table. 436 */ 437 if (*table) { 438 assert((*table & 3) == SECTION_PT_PT); 439 assert(!(*table & SECTION_PT_NOTSECURE) == 440 !(mm->attr & MT_NS)); 441 442 xlat_table = (*table) & 443 ~(MMU32B_L1_TABLE_ALIGN - 1); 444 desc = *table; 445 } else { 446 xlat_table = (uintptr_t)mmu_l2_base + 447 next_xlat * MMU32B_L2_TABLE_SIZE; 448 next_xlat++; 449 assert(next_xlat <= MAX_XLAT_TABLES); 450 memset((char *)xlat_table, 0, 451 MMU32B_L2_TABLE_SIZE); 452 453 desc = xlat_table | SECTION_PT_PT; 454 desc |= mm->attr & MT_NS ? 455 SECTION_PT_NOTSECURE : 0; 456 } 457 /* Recurse to fill in new table */ 458 mm = init_xlation_table_inner(mm, base_va, 459 (uint32_t *)xlat_table, 460 level + 1); 461 } 462 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 463 /* dump only non-lpae level 2 tables content */ 464 if (level == 2) 465 printf("\n"); 466 #endif 467 *table++ = desc; 468 base_va += level_size; 469 } while (mm->size && (base_va & level_index_mask)); 470 471 return mm; 472 } 473 474 void init_xlat_tables(void) 475 { 476 print_mmap(); 477 478 assert(!((unsigned int)mmu_l1_base & (MMU32B_L1_TABLE_ALIGN - 1))); 479 assert(!((unsigned int)mmu_l2_base & (MMU32B_L2_TABLE_ALIGN - 1))); 480 481 memset(mmu_l1_base, 0, MMU32B_L1_TABLE_SIZE); 482 483 init_xlation_table_inner(mmap, 0, (uint32_t *)mmu_l1_base, 1); 484 485 VERBOSE("init xlat - max_va=%p, max_pa=%llx\n", 486 (void *)xlat_max_va, xlat_max_pa); 487 assert(xlat_max_va <= PLAT_VIRT_ADDR_SPACE_SIZE - 1); 488 assert(xlat_max_pa <= PLAT_VIRT_ADDR_SPACE_SIZE - 1); 489 } 490 491 /******************************************************************************* 492 * Function for enabling the MMU in Secure PL1, assuming that the 493 * page-tables have already been created. 494 ******************************************************************************/ 495 void enable_mmu_svc_mon(unsigned int flags) 496 { 497 unsigned int prrr; 498 unsigned int nmrr; 499 unsigned int sctlr; 500 501 assert(IS_IN_SECURE()); 502 assert((read_sctlr() & SCTLR_M_BIT) == 0); 503 504 /* Enable Access flag (simplified access permissions) and TEX remap */ 505 write_sctlr(read_sctlr() | SCTLR_AFE_BIT | SCTLR_TRE_BIT); 506 507 prrr = MMU32B_PRRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 1, 0) \ 508 | MMU32B_PRRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 2, 1); 509 nmrr = MMU32B_NMRR_IDX(MMU32B_ATTR_DEVICE_INDEX, 0, 0) \ 510 | MMU32B_NMRR_IDX(MMU32B_ATTR_IWBWA_OWBWA_INDEX, 1, 1); 511 512 prrr |= MMU32B_PRRR_NS1 | MMU32B_PRRR_DS1; 513 514 write_prrr(prrr); 515 write_nmrr(nmrr); 516 517 /* Program Domain access control register: domain 0 only */ 518 write_dacr(DACR_DOMAIN(0, DACR_DOMAIN_PERM_CLIENT)); 519 520 /* Invalidate TLBs at the current exception level */ 521 tlbiall(); 522 523 /* set MMU base xlat table entry (use only TTBR0) */ 524 write_ttbr0((uint32_t)mmu_l1_base | MMU32B_DEFAULT_ATTRS); 525 write_ttbr1(0); 526 527 /* 528 * Ensure all translation table writes have drained 529 * into memory, the TLB invalidation is complete, 530 * and translation register writes are committed 531 * before enabling the MMU 532 */ 533 dsb(); 534 isb(); 535 536 sctlr = read_sctlr(); 537 sctlr |= SCTLR_M_BIT; 538 #if ARMV7_SUPPORTS_VIRTUALIZATION 539 sctlr |= SCTLR_WXN_BIT; 540 #endif 541 542 if (flags & DISABLE_DCACHE) 543 sctlr &= ~SCTLR_C_BIT; 544 else 545 sctlr |= SCTLR_C_BIT; 546 547 write_sctlr(sctlr); 548 549 /* Ensure the MMU enable takes effect immediately */ 550 isb(); 551 } 552