1 /* 2 * Copyright (c) 2017-2021, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <errno.h> 9 #include <stdbool.h> 10 #include <stdint.h> 11 #include <stdio.h> 12 13 #include <platform_def.h> 14 15 #include <arch_helpers.h> 16 #include <common/debug.h> 17 #include <lib/utils_def.h> 18 #include <lib/xlat_tables/xlat_tables_defs.h> 19 #include <lib/xlat_tables/xlat_tables_v2.h> 20 21 #include "xlat_tables_private.h" 22 23 #if LOG_LEVEL < LOG_LEVEL_VERBOSE 24 25 void xlat_mmap_print(__unused const mmap_region_t *mmap) 26 { 27 /* Empty */ 28 } 29 30 void xlat_tables_print(__unused xlat_ctx_t *ctx) 31 { 32 /* Empty */ 33 } 34 35 #else /* if LOG_LEVEL >= LOG_LEVEL_VERBOSE */ 36 37 void xlat_mmap_print(const mmap_region_t *mmap) 38 { 39 printf("mmap:\n"); 40 const mmap_region_t *mm = mmap; 41 42 while (mm->size != 0U) { 43 printf(" VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x granularity:0x%zx\n", 44 mm->base_va, mm->base_pa, mm->size, mm->attr, 45 mm->granularity); 46 ++mm; 47 }; 48 printf("\n"); 49 } 50 51 /* Print the attributes of the specified block descriptor. */ 52 static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc) 53 { 54 uint64_t mem_type_index = ATTR_INDEX_GET(desc); 55 int xlat_regime = ctx->xlat_regime; 56 57 if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { 58 printf("MEM"); 59 } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) { 60 printf("NC"); 61 } else { 62 assert(mem_type_index == ATTR_DEVICE_INDEX); 63 printf("DEV"); 64 } 65 66 if ((xlat_regime == EL3_REGIME) || (xlat_regime == EL2_REGIME)) { 67 /* For EL3 and EL2 only check the AP[2] and XN bits. */ 68 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW"); 69 printf(((desc & UPPER_ATTRS(XN)) != 0ULL) ? "-XN" : "-EXEC"); 70 } else { 71 assert(xlat_regime == EL1_EL0_REGIME); 72 /* 73 * For EL0 and EL1: 74 * - In AArch64 PXN and UXN can be set independently but in 75 * AArch32 there is no UXN (XN affects both privilege levels). 76 * For consistency, we set them simultaneously in both cases. 77 * - RO and RW permissions must be the same in EL1 and EL0. If 78 * EL0 can access that memory region, so can EL1, with the 79 * same permissions. 80 */ 81 #if ENABLE_ASSERTIONS 82 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(EL1_EL0_REGIME); 83 uint64_t xn_perm = desc & xn_mask; 84 85 assert((xn_perm == xn_mask) || (xn_perm == 0ULL)); 86 #endif 87 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW"); 88 /* Only check one of PXN and UXN, the other one is the same. */ 89 printf(((desc & UPPER_ATTRS(PXN)) != 0ULL) ? "-XN" : "-EXEC"); 90 /* 91 * Privileged regions can only be accessed from EL1, user 92 * regions can be accessed from EL1 and EL0. 93 */ 94 printf(((desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) != 0ULL) 95 ? "-USER" : "-PRIV"); 96 } 97 98 #if ENABLE_RME 99 switch (desc & LOWER_ATTRS(EL3_S1_NSE | NS)) { 100 case 0ULL: 101 printf("-S"); 102 break; 103 case LOWER_ATTRS(NS): 104 printf("-NS"); 105 break; 106 case LOWER_ATTRS(EL3_S1_NSE): 107 printf("-RT"); 108 break; 109 default: /* LOWER_ATTRS(EL3_S1_NSE | NS) */ 110 printf("-RL"); 111 } 112 #else 113 printf(((LOWER_ATTRS(NS) & desc) != 0ULL) ? "-NS" : "-S"); 114 #endif 115 116 #ifdef __aarch64__ 117 /* Check Guarded Page bit */ 118 if ((desc & GP) != 0ULL) { 119 printf("-GP"); 120 } 121 #endif 122 } 123 124 static const char * const level_spacers[] = { 125 "[LV0] ", 126 " [LV1] ", 127 " [LV2] ", 128 " [LV3] " 129 }; 130 131 static const char *invalid_descriptors_ommited = 132 "%s(%d invalid descriptors omitted)\n"; 133 134 /* 135 * Recursive function that reads the translation tables passed as an argument 136 * and prints their status. 137 */ 138 static void xlat_tables_print_internal(xlat_ctx_t *ctx, uintptr_t table_base_va, 139 const uint64_t *table_base, unsigned int table_entries, 140 unsigned int level) 141 { 142 assert(level <= XLAT_TABLE_LEVEL_MAX); 143 144 uint64_t desc; 145 uintptr_t table_idx_va = table_base_va; 146 unsigned int table_idx = 0U; 147 size_t level_size = XLAT_BLOCK_SIZE(level); 148 149 /* 150 * Keep track of how many invalid descriptors are counted in a row. 151 * Whenever multiple invalid descriptors are found, only the first one 152 * is printed, and a line is added to inform about how many descriptors 153 * have been omitted. 154 */ 155 int invalid_row_count = 0; 156 157 while (table_idx < table_entries) { 158 159 desc = table_base[table_idx]; 160 161 if ((desc & DESC_MASK) == INVALID_DESC) { 162 163 if (invalid_row_count == 0) { 164 printf("%sVA:0x%lx size:0x%zx\n", 165 level_spacers[level], 166 table_idx_va, level_size); 167 } 168 invalid_row_count++; 169 170 } else { 171 172 if (invalid_row_count > 1) { 173 printf(invalid_descriptors_ommited, 174 level_spacers[level], 175 invalid_row_count - 1); 176 } 177 invalid_row_count = 0; 178 179 /* 180 * Check if this is a table or a block. Tables are only 181 * allowed in levels other than 3, but DESC_PAGE has the 182 * same value as DESC_TABLE, so we need to check. 183 */ 184 if (((desc & DESC_MASK) == TABLE_DESC) && 185 (level < XLAT_TABLE_LEVEL_MAX)) { 186 /* 187 * Do not print any PA for a table descriptor, 188 * as it doesn't directly map physical memory 189 * but instead points to the next translation 190 * table in the translation table walk. 191 */ 192 printf("%sVA:0x%lx size:0x%zx\n", 193 level_spacers[level], 194 table_idx_va, level_size); 195 196 uintptr_t addr_inner = desc & TABLE_ADDR_MASK; 197 198 xlat_tables_print_internal(ctx, table_idx_va, 199 (uint64_t *)addr_inner, 200 XLAT_TABLE_ENTRIES, level + 1U); 201 } else { 202 printf("%sVA:0x%lx PA:0x%llx size:0x%zx ", 203 level_spacers[level], table_idx_va, 204 (uint64_t)(desc & TABLE_ADDR_MASK), 205 level_size); 206 xlat_desc_print(ctx, desc); 207 printf("\n"); 208 } 209 } 210 211 table_idx++; 212 table_idx_va += level_size; 213 } 214 215 if (invalid_row_count > 1) { 216 printf(invalid_descriptors_ommited, 217 level_spacers[level], invalid_row_count - 1); 218 } 219 } 220 221 void xlat_tables_print(xlat_ctx_t *ctx) 222 { 223 const char *xlat_regime_str; 224 int used_page_tables; 225 226 if (ctx->xlat_regime == EL1_EL0_REGIME) { 227 xlat_regime_str = "1&0"; 228 } else if (ctx->xlat_regime == EL2_REGIME) { 229 xlat_regime_str = "2"; 230 } else { 231 assert(ctx->xlat_regime == EL3_REGIME); 232 xlat_regime_str = "3"; 233 } 234 VERBOSE("Translation tables state:\n"); 235 VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str); 236 VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address); 237 VERBOSE(" Max allowed VA: 0x%lx\n", ctx->va_max_address); 238 VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa); 239 VERBOSE(" Max mapped VA: 0x%lx\n", ctx->max_va); 240 241 VERBOSE(" Initial lookup level: %u\n", ctx->base_level); 242 VERBOSE(" Entries @initial lookup level: %u\n", 243 ctx->base_table_entries); 244 245 #if PLAT_XLAT_TABLES_DYNAMIC 246 used_page_tables = 0; 247 for (int i = 0; i < ctx->tables_num; ++i) { 248 if (ctx->tables_mapped_regions[i] != 0) 249 ++used_page_tables; 250 } 251 #else 252 used_page_tables = ctx->next_table; 253 #endif 254 VERBOSE(" Used %d sub-tables out of %d (spare: %d)\n", 255 used_page_tables, ctx->tables_num, 256 ctx->tables_num - used_page_tables); 257 258 xlat_tables_print_internal(ctx, 0U, ctx->base_table, 259 ctx->base_table_entries, ctx->base_level); 260 } 261 262 #endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ 263 264 /* 265 * Do a translation table walk to find the block or page descriptor that maps 266 * virtual_addr. 267 * 268 * On success, return the address of the descriptor within the translation 269 * table. Its lookup level is stored in '*out_level'. 270 * On error, return NULL. 271 * 272 * xlat_table_base 273 * Base address for the initial lookup level. 274 * xlat_table_base_entries 275 * Number of entries in the translation table for the initial lookup level. 276 * virt_addr_space_size 277 * Size in bytes of the virtual address space. 278 */ 279 static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr, 280 void *xlat_table_base, 281 unsigned int xlat_table_base_entries, 282 unsigned long long virt_addr_space_size, 283 unsigned int *out_level) 284 { 285 unsigned int start_level; 286 uint64_t *table; 287 unsigned int entries; 288 289 start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size); 290 291 table = xlat_table_base; 292 entries = xlat_table_base_entries; 293 294 for (unsigned int level = start_level; 295 level <= XLAT_TABLE_LEVEL_MAX; 296 ++level) { 297 uint64_t idx, desc, desc_type; 298 299 idx = XLAT_TABLE_IDX(virtual_addr, level); 300 if (idx >= entries) { 301 WARN("Missing xlat table entry at address 0x%lx\n", 302 virtual_addr); 303 return NULL; 304 } 305 306 desc = table[idx]; 307 desc_type = desc & DESC_MASK; 308 309 if (desc_type == INVALID_DESC) { 310 VERBOSE("Invalid entry (memory not mapped)\n"); 311 return NULL; 312 } 313 314 if (level == XLAT_TABLE_LEVEL_MAX) { 315 /* 316 * Only page descriptors allowed at the final lookup 317 * level. 318 */ 319 assert(desc_type == PAGE_DESC); 320 *out_level = level; 321 return &table[idx]; 322 } 323 324 if (desc_type == BLOCK_DESC) { 325 *out_level = level; 326 return &table[idx]; 327 } 328 329 assert(desc_type == TABLE_DESC); 330 table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); 331 entries = XLAT_TABLE_ENTRIES; 332 } 333 334 /* 335 * This shouldn't be reached, the translation table walk should end at 336 * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop. 337 */ 338 assert(false); 339 340 return NULL; 341 } 342 343 344 static int xlat_get_mem_attributes_internal(const xlat_ctx_t *ctx, 345 uintptr_t base_va, uint32_t *attributes, uint64_t **table_entry, 346 unsigned long long *addr_pa, unsigned int *table_level) 347 { 348 uint64_t *entry; 349 uint64_t desc; 350 unsigned int level; 351 unsigned long long virt_addr_space_size; 352 353 /* 354 * Sanity-check arguments. 355 */ 356 assert(ctx != NULL); 357 assert(ctx->initialized); 358 assert((ctx->xlat_regime == EL1_EL0_REGIME) || 359 (ctx->xlat_regime == EL2_REGIME) || 360 (ctx->xlat_regime == EL3_REGIME)); 361 362 virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1ULL; 363 assert(virt_addr_space_size > 0U); 364 365 entry = find_xlat_table_entry(base_va, 366 ctx->base_table, 367 ctx->base_table_entries, 368 virt_addr_space_size, 369 &level); 370 if (entry == NULL) { 371 WARN("Address 0x%lx is not mapped.\n", base_va); 372 return -EINVAL; 373 } 374 375 if (addr_pa != NULL) { 376 *addr_pa = *entry & TABLE_ADDR_MASK; 377 } 378 379 if (table_entry != NULL) { 380 *table_entry = entry; 381 } 382 383 if (table_level != NULL) { 384 *table_level = level; 385 } 386 387 desc = *entry; 388 389 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 390 VERBOSE("Attributes: "); 391 xlat_desc_print(ctx, desc); 392 printf("\n"); 393 #endif /* LOG_LEVEL >= LOG_LEVEL_VERBOSE */ 394 395 assert(attributes != NULL); 396 *attributes = 0U; 397 398 uint64_t attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; 399 400 if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { 401 *attributes |= MT_MEMORY; 402 } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) { 403 *attributes |= MT_NON_CACHEABLE; 404 } else { 405 assert(attr_index == ATTR_DEVICE_INDEX); 406 *attributes |= MT_DEVICE; 407 } 408 409 uint64_t ap2_bit = (desc >> AP2_SHIFT) & 1U; 410 411 if (ap2_bit == AP2_RW) 412 *attributes |= MT_RW; 413 414 if (ctx->xlat_regime == EL1_EL0_REGIME) { 415 uint64_t ap1_bit = (desc >> AP1_SHIFT) & 1U; 416 417 if (ap1_bit == AP1_ACCESS_UNPRIVILEGED) 418 *attributes |= MT_USER; 419 } 420 421 uint64_t ns_bit = (desc >> NS_SHIFT) & 1U; 422 423 if (ns_bit == 1U) 424 *attributes |= MT_NS; 425 426 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime); 427 428 if ((desc & xn_mask) == xn_mask) { 429 *attributes |= MT_EXECUTE_NEVER; 430 } else { 431 assert((desc & xn_mask) == 0U); 432 } 433 434 return 0; 435 } 436 437 438 int xlat_get_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va, 439 uint32_t *attr) 440 { 441 return xlat_get_mem_attributes_internal(ctx, base_va, attr, 442 NULL, NULL, NULL); 443 } 444 445 446 int xlat_change_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va, 447 size_t size, uint32_t attr) 448 { 449 /* Note: This implementation isn't optimized. */ 450 451 assert(ctx != NULL); 452 assert(ctx->initialized); 453 454 unsigned long long virt_addr_space_size = 455 (unsigned long long)ctx->va_max_address + 1U; 456 assert(virt_addr_space_size > 0U); 457 458 if (!IS_PAGE_ALIGNED(base_va)) { 459 WARN("%s: Address 0x%lx is not aligned on a page boundary.\n", 460 __func__, base_va); 461 return -EINVAL; 462 } 463 464 if (size == 0U) { 465 WARN("%s: Size is 0.\n", __func__); 466 return -EINVAL; 467 } 468 469 if ((size % PAGE_SIZE) != 0U) { 470 WARN("%s: Size 0x%zx is not a multiple of a page size.\n", 471 __func__, size); 472 return -EINVAL; 473 } 474 475 if (((attr & MT_EXECUTE_NEVER) == 0U) && ((attr & MT_RW) != 0U)) { 476 WARN("%s: Mapping memory as read-write and executable not allowed.\n", 477 __func__); 478 return -EINVAL; 479 } 480 481 size_t pages_count = size / PAGE_SIZE; 482 483 VERBOSE("Changing memory attributes of %zu pages starting from address 0x%lx...\n", 484 pages_count, base_va); 485 486 uintptr_t base_va_original = base_va; 487 488 /* 489 * Sanity checks. 490 */ 491 for (unsigned int i = 0U; i < pages_count; ++i) { 492 const uint64_t *entry; 493 uint64_t desc, attr_index; 494 unsigned int level; 495 496 entry = find_xlat_table_entry(base_va, 497 ctx->base_table, 498 ctx->base_table_entries, 499 virt_addr_space_size, 500 &level); 501 if (entry == NULL) { 502 WARN("Address 0x%lx is not mapped.\n", base_va); 503 return -EINVAL; 504 } 505 506 desc = *entry; 507 508 /* 509 * Check that all the required pages are mapped at page 510 * granularity. 511 */ 512 if (((desc & DESC_MASK) != PAGE_DESC) || 513 (level != XLAT_TABLE_LEVEL_MAX)) { 514 WARN("Address 0x%lx is not mapped at the right granularity.\n", 515 base_va); 516 WARN("Granularity is 0x%lx, should be 0x%lx.\n", 517 XLAT_BLOCK_SIZE(level), PAGE_SIZE); 518 return -EINVAL; 519 } 520 521 /* 522 * If the region type is device, it shouldn't be executable. 523 */ 524 attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; 525 if (attr_index == ATTR_DEVICE_INDEX) { 526 if ((attr & MT_EXECUTE_NEVER) == 0U) { 527 WARN("Setting device memory as executable at address 0x%lx.", 528 base_va); 529 return -EINVAL; 530 } 531 } 532 533 base_va += PAGE_SIZE; 534 } 535 536 /* Restore original value. */ 537 base_va = base_va_original; 538 539 for (unsigned int i = 0U; i < pages_count; ++i) { 540 541 uint32_t old_attr = 0U, new_attr; 542 uint64_t *entry = NULL; 543 unsigned int level = 0U; 544 unsigned long long addr_pa = 0ULL; 545 546 (void) xlat_get_mem_attributes_internal(ctx, base_va, &old_attr, 547 &entry, &addr_pa, &level); 548 549 /* 550 * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and 551 * MT_USER/MT_PRIVILEGED are taken into account. Any other 552 * information is ignored. 553 */ 554 555 /* Clean the old attributes so that they can be rebuilt. */ 556 new_attr = old_attr & ~(MT_RW | MT_EXECUTE_NEVER | MT_USER); 557 558 /* 559 * Update attributes, but filter out the ones this function 560 * isn't allowed to change. 561 */ 562 new_attr |= attr & (MT_RW | MT_EXECUTE_NEVER | MT_USER); 563 564 /* 565 * The break-before-make sequence requires writing an invalid 566 * descriptor and making sure that the system sees the change 567 * before writing the new descriptor. 568 */ 569 *entry = INVALID_DESC; 570 #if !HW_ASSISTED_COHERENCY 571 dccvac((uintptr_t)entry); 572 #endif 573 /* Invalidate any cached copy of this mapping in the TLBs. */ 574 xlat_arch_tlbi_va(base_va, ctx->xlat_regime); 575 576 /* Ensure completion of the invalidation. */ 577 xlat_arch_tlbi_va_sync(); 578 579 /* Write new descriptor */ 580 *entry = xlat_desc(ctx, new_attr, addr_pa, level); 581 #if !HW_ASSISTED_COHERENCY 582 dccvac((uintptr_t)entry); 583 #endif 584 base_va += PAGE_SIZE; 585 } 586 587 /* Ensure that the last descriptor writen is seen by the system. */ 588 dsbish(); 589 590 return 0; 591 } 592