1 /* 2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <errno.h> 9 #include <inttypes.h> 10 #include <stdbool.h> 11 #include <stdint.h> 12 #include <stdio.h> 13 14 #include <platform_def.h> 15 16 #include <arch_features.h> 17 #include <arch_helpers.h> 18 #include <common/debug.h> 19 #include <lib/utils_def.h> 20 #include <lib/xlat_tables/xlat_tables_defs.h> 21 #include <lib/xlat_tables/xlat_tables_v2.h> 22 23 #include "xlat_tables_private.h" 24 25 /* Uncomment, when xlat mmap details prints required*/ 26 //#define LOG_DEBUG 27 28 #ifndef LOG_DEBUG 29 void xlat_mmap_print(__unused const mmap_region_t *mmap) 30 { 31 /* Empty */ 32 } 33 34 void xlat_tables_print(__unused xlat_ctx_t *ctx) 35 { 36 /* Empty */ 37 } 38 39 #else /* ifndef LOG_DEBUG */ 40 41 void xlat_mmap_print(const mmap_region_t *mmap) 42 { 43 printf("mmap:\n"); 44 const mmap_region_t *mm = mmap; 45 46 while (mm->size != 0U) { 47 printf(" VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x granularity:0x%zx\n", 48 mm->base_va, mm->base_pa, mm->size, mm->attr, 49 mm->granularity); 50 ++mm; 51 }; 52 printf("\n"); 53 } 54 55 /* Print the attributes of the specified block descriptor. */ 56 static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc) 57 { 58 uint64_t mem_type_index = ATTR_INDEX_GET(desc); 59 int xlat_regime = ctx->xlat_regime; 60 61 if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { 62 printf("MEM"); 63 } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) { 64 printf("NC"); 65 } else { 66 assert(mem_type_index == ATTR_DEVICE_INDEX); 67 printf("DEV"); 68 } 69 70 if ((xlat_regime == EL3_REGIME) || (xlat_regime == EL2_REGIME)) { 71 /* For EL3 and EL2 only check the AP[2] and XN bits. */ 72 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW"); 73 printf(((desc & UPPER_ATTRS(XN)) != 0ULL) ? "-XN" : "-EXEC"); 74 } else { 75 assert(xlat_regime == EL1_EL0_REGIME); 76 /* 77 * For EL0 and EL1: 78 * - In AArch64 PXN and UXN can be set independently but in 79 * AArch32 there is no UXN (XN affects both privilege levels). 80 * For consistency, we set them simultaneously in both cases. 81 * - RO and RW permissions must be the same in EL1 and EL0. If 82 * EL0 can access that memory region, so can EL1, with the 83 * same permissions. 84 */ 85 #if ENABLE_ASSERTIONS 86 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(EL1_EL0_REGIME); 87 uint64_t xn_perm = desc & xn_mask; 88 89 assert((xn_perm == xn_mask) || (xn_perm == 0ULL)); 90 #endif 91 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW"); 92 /* Only check one of PXN and UXN, the other one is the same. */ 93 printf(((desc & UPPER_ATTRS(PXN)) != 0ULL) ? "-XN" : "-EXEC"); 94 /* 95 * Privileged regions can only be accessed from EL1, user 96 * regions can be accessed from EL1 and EL0. 97 */ 98 printf(((desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) != 0ULL) 99 ? "-USER" : "-PRIV"); 100 } 101 102 #if ENABLE_RME 103 switch (desc & LOWER_ATTRS(EL3_S1_NSE | NS)) { 104 case 0ULL: 105 printf("-S"); 106 break; 107 case LOWER_ATTRS(NS): 108 printf("-NS"); 109 break; 110 case LOWER_ATTRS(EL3_S1_NSE): 111 printf("-RT"); 112 break; 113 default: /* LOWER_ATTRS(EL3_S1_NSE | NS) */ 114 printf("-RL"); 115 } 116 #else 117 printf(((LOWER_ATTRS(NS) & desc) != 0ULL) ? "-NS" : "-S"); 118 #endif 119 120 #ifdef __aarch64__ 121 /* Check Guarded Page bit */ 122 if ((desc & GP) != 0ULL) { 123 printf("-GP"); 124 } 125 #endif 126 } 127 128 static const char * const level_spacers[] = { 129 "[LV0] ", 130 " [LV1] ", 131 " [LV2] ", 132 " [LV3] " 133 }; 134 135 static const char *invalid_descriptors_ommited = 136 "%s(%d invalid descriptors omitted)\n"; 137 138 /* 139 * Recursive function that reads the translation tables passed as an argument 140 * and prints their status. 141 */ 142 static void xlat_tables_print_internal(xlat_ctx_t *ctx, uintptr_t table_base_va, 143 const uint64_t *table_base, unsigned int table_entries, 144 unsigned int level) 145 { 146 assert(level <= XLAT_TABLE_LEVEL_MAX); 147 148 uint64_t desc; 149 uintptr_t table_idx_va = table_base_va; 150 unsigned int table_idx = 0U; 151 size_t level_size = XLAT_BLOCK_SIZE(level); 152 153 /* 154 * Keep track of how many invalid descriptors are counted in a row. 155 * Whenever multiple invalid descriptors are found, only the first one 156 * is printed, and a line is added to inform about how many descriptors 157 * have been omitted. 158 */ 159 int invalid_row_count = 0; 160 161 while (table_idx < table_entries) { 162 163 desc = table_base[table_idx]; 164 165 if ((desc & DESC_MASK) == INVALID_DESC) { 166 167 if (invalid_row_count == 0) { 168 printf("%sVA:0x%lx size:0x%zx\n", 169 level_spacers[level], 170 table_idx_va, level_size); 171 } 172 invalid_row_count++; 173 174 } else { 175 176 if (invalid_row_count > 1) { 177 printf(invalid_descriptors_ommited, 178 level_spacers[level], 179 invalid_row_count - 1); 180 } 181 invalid_row_count = 0; 182 183 /* 184 * Check if this is a table or a block. Tables are only 185 * allowed in levels other than 3, but DESC_PAGE has the 186 * same value as DESC_TABLE, so we need to check. 187 */ 188 if (((desc & DESC_MASK) == TABLE_DESC) && 189 (level < XLAT_TABLE_LEVEL_MAX)) { 190 /* 191 * Do not print any PA for a table descriptor, 192 * as it doesn't directly map physical memory 193 * but instead points to the next translation 194 * table in the translation table walk. 195 */ 196 printf("%sVA:0x%lx size:0x%zx\n", 197 level_spacers[level], 198 table_idx_va, level_size); 199 200 uintptr_t addr_inner = desc & TABLE_ADDR_MASK; 201 202 xlat_tables_print_internal(ctx, table_idx_va, 203 (uint64_t *)addr_inner, 204 XLAT_TABLE_ENTRIES, level + 1U); 205 } else { 206 printf("%sVA:0x%lx PA:0x%" PRIx64 " size:0x%zx ", 207 level_spacers[level], table_idx_va, 208 (uint64_t)(desc & TABLE_ADDR_MASK), 209 level_size); 210 xlat_desc_print(ctx, desc); 211 printf("\n"); 212 } 213 } 214 215 table_idx++; 216 table_idx_va += level_size; 217 } 218 219 if (invalid_row_count > 1) { 220 printf(invalid_descriptors_ommited, 221 level_spacers[level], invalid_row_count - 1); 222 } 223 } 224 225 void xlat_tables_print(xlat_ctx_t *ctx) 226 { 227 const char *xlat_regime_str; 228 int used_page_tables; 229 230 if (ctx->xlat_regime == EL1_EL0_REGIME) { 231 xlat_regime_str = "1&0"; 232 } else if (ctx->xlat_regime == EL2_REGIME) { 233 xlat_regime_str = "2"; 234 } else { 235 assert(ctx->xlat_regime == EL3_REGIME); 236 xlat_regime_str = "3"; 237 } 238 VERBOSE("Translation tables state:\n"); 239 VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str); 240 VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address); 241 VERBOSE(" Max allowed VA: 0x%lx\n", ctx->va_max_address); 242 VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa); 243 VERBOSE(" Max mapped VA: 0x%lx\n", ctx->max_va); 244 245 VERBOSE(" Initial lookup level: %u\n", ctx->base_level); 246 VERBOSE(" Entries @initial lookup level: %u\n", 247 ctx->base_table_entries); 248 249 #if PLAT_XLAT_TABLES_DYNAMIC 250 used_page_tables = 0; 251 for (int i = 0; i < ctx->tables_num; ++i) { 252 if (ctx->tables_mapped_regions[i] != 0) 253 ++used_page_tables; 254 } 255 #else 256 used_page_tables = ctx->next_table; 257 #endif 258 VERBOSE(" Used %d sub-tables out of %d (spare: %d)\n", 259 used_page_tables, ctx->tables_num, 260 ctx->tables_num - used_page_tables); 261 262 xlat_tables_print_internal(ctx, 0U, ctx->base_table, 263 ctx->base_table_entries, ctx->base_level); 264 } 265 266 #endif /* ifndef LOG_DEBUG */ 267 268 /* 269 * Do a translation table walk to find the block or page descriptor that maps 270 * virtual_addr. 271 * 272 * On success, return the address of the descriptor within the translation 273 * table. Its lookup level is stored in '*out_level'. 274 * On error, return NULL. 275 * 276 * xlat_table_base 277 * Base address for the initial lookup level. 278 * xlat_table_base_entries 279 * Number of entries in the translation table for the initial lookup level. 280 * virt_addr_space_size 281 * Size in bytes of the virtual address space. 282 */ 283 static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr, 284 uint64_t *xlat_table_base, 285 unsigned int xlat_table_base_entries, 286 unsigned long long virt_addr_space_size, 287 unsigned int *out_level) 288 { 289 unsigned int start_level; 290 uint64_t *table; 291 unsigned int entries; 292 293 start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size); 294 295 table = xlat_table_base; 296 entries = xlat_table_base_entries; 297 298 for (unsigned int level = start_level; 299 level <= XLAT_TABLE_LEVEL_MAX; 300 ++level) { 301 uint64_t idx, desc, desc_type; 302 303 idx = XLAT_TABLE_IDX(virtual_addr, level); 304 if (idx >= entries) { 305 WARN("Missing xlat table entry at address 0x%lx\n", 306 virtual_addr); 307 return NULL; 308 } 309 310 desc = table[idx]; 311 desc_type = desc & DESC_MASK; 312 313 if (desc_type == INVALID_DESC) { 314 VERBOSE("Invalid entry (memory not mapped)\n"); 315 return NULL; 316 } 317 318 if (level == XLAT_TABLE_LEVEL_MAX) { 319 /* 320 * Only page descriptors allowed at the final lookup 321 * level. 322 */ 323 assert(desc_type == PAGE_DESC); 324 *out_level = level; 325 return &table[idx]; 326 } 327 328 if (desc_type == BLOCK_DESC) { 329 *out_level = level; 330 return &table[idx]; 331 } 332 333 assert(desc_type == TABLE_DESC); 334 table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK); 335 entries = XLAT_TABLE_ENTRIES; 336 } 337 338 /* 339 * This shouldn't be reached, the translation table walk should end at 340 * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop. 341 */ 342 assert(false); 343 344 return NULL; 345 } 346 347 348 static int xlat_get_mem_attributes_internal(const xlat_ctx_t *ctx, 349 uintptr_t base_va, uint32_t *attributes, uint64_t **table_entry, 350 unsigned long long *addr_pa, unsigned int *table_level) 351 { 352 uint64_t *entry; 353 uint64_t desc; 354 unsigned int level; 355 unsigned long long virt_addr_space_size; 356 357 /* 358 * Sanity-check arguments. 359 */ 360 assert(ctx != NULL); 361 assert(ctx->initialized); 362 assert((ctx->xlat_regime == EL1_EL0_REGIME) || 363 (ctx->xlat_regime == EL2_REGIME) || 364 (ctx->xlat_regime == EL3_REGIME)); 365 366 virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1ULL; 367 assert(virt_addr_space_size > 0U); 368 369 entry = find_xlat_table_entry(base_va, 370 ctx->base_table, 371 ctx->base_table_entries, 372 virt_addr_space_size, 373 &level); 374 if (entry == NULL) { 375 WARN("Address 0x%lx is not mapped.\n", base_va); 376 return -EINVAL; 377 } 378 379 if (addr_pa != NULL) { 380 *addr_pa = *entry & TABLE_ADDR_MASK; 381 } 382 383 if (table_entry != NULL) { 384 *table_entry = entry; 385 } 386 387 if (table_level != NULL) { 388 *table_level = level; 389 } 390 391 desc = *entry; 392 393 #ifdef LOG_DEBUG 394 VERBOSE("Attributes: "); 395 xlat_desc_print(ctx, desc); 396 printf("\n"); 397 #endif 398 399 assert(attributes != NULL); 400 *attributes = 0U; 401 402 uint64_t attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; 403 404 if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) { 405 *attributes |= MT_MEMORY; 406 } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) { 407 *attributes |= MT_NON_CACHEABLE; 408 } else { 409 assert(attr_index == ATTR_DEVICE_INDEX); 410 *attributes |= MT_DEVICE; 411 } 412 413 uint64_t ap2_bit = (desc >> AP2_SHIFT) & 1U; 414 415 if (ap2_bit == AP2_RW) 416 *attributes |= MT_RW; 417 418 if (ctx->xlat_regime == EL1_EL0_REGIME) { 419 uint64_t ap1_bit = (desc >> AP1_SHIFT) & 1U; 420 421 if (ap1_bit == AP1_ACCESS_UNPRIVILEGED) 422 *attributes |= MT_USER; 423 } 424 425 uint64_t ns_bit = (desc >> NS_SHIFT) & 1ULL; 426 427 #if ENABLE_RME 428 uint64_t nse_bit = (desc >> NSE_SHIFT) & 1ULL; 429 uint32_t sec_state = (uint32_t)(ns_bit | (nse_bit << 1ULL)); 430 431 /* 432 * ========================================================= 433 * NSE NS | Output PA space 434 * ========================================================= 435 * 0 0 | Secure (if S-EL2 is present, else invalid) 436 * 0 1 | Non-secure 437 * 1 0 | Root 438 * 1 1 | Realm 439 *========================================================== 440 */ 441 switch (sec_state) { 442 case 0U: 443 /* 444 * We expect to get Secure mapping on an RME system only if 445 * S-EL2 is enabled. 446 * Hence panic() if we hit the case without EEL2 being enabled. 447 */ 448 if ((read_scr_el3() & SCR_EEL2_BIT) == 0ULL) { 449 ERROR("A secure descriptor is not supported when" 450 "FEAT_RME is implemented and FEAT_SEL2 is" 451 "not enabled\n"); 452 panic(); 453 } else { 454 *attributes |= MT_SECURE; 455 } 456 break; 457 case 1U: 458 *attributes |= MT_NS; 459 break; 460 case 2U: 461 *attributes |= MT_ROOT; 462 break; 463 case 3U: 464 *attributes |= MT_REALM; 465 break; 466 default: 467 /* unreachable code */ 468 assert(false); 469 break; 470 } 471 #else /* !ENABLE_RME */ 472 if (ns_bit == 1ULL) { 473 *attributes |= MT_NS; 474 } else { 475 *attributes |= MT_SECURE; 476 } 477 #endif /* ENABLE_RME */ 478 479 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime); 480 481 if ((desc & xn_mask) == xn_mask) { 482 *attributes |= MT_EXECUTE_NEVER; 483 } else { 484 assert((desc & xn_mask) == 0U); 485 } 486 487 return 0; 488 } 489 490 491 int xlat_get_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va, 492 uint32_t *attr, unsigned int *table_level) 493 { 494 return xlat_get_mem_attributes_internal(ctx, base_va, attr, 495 NULL, NULL, table_level); 496 } 497 498 499 int xlat_change_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va, 500 size_t size, uint32_t attr) 501 { 502 /* Note: This implementation isn't optimized. */ 503 504 assert(ctx != NULL); 505 assert(ctx->initialized); 506 507 unsigned long long virt_addr_space_size = 508 (unsigned long long)ctx->va_max_address + 1U; 509 assert(virt_addr_space_size > 0U); 510 511 if (!IS_PAGE_ALIGNED(base_va)) { 512 WARN("%s: Address 0x%lx is not aligned on a page boundary.\n", 513 __func__, base_va); 514 return -EINVAL; 515 } 516 517 if (size == 0U) { 518 WARN("%s: Size is 0.\n", __func__); 519 return -EINVAL; 520 } 521 522 if ((size % PAGE_SIZE) != 0U) { 523 WARN("%s: Size 0x%zx is not a multiple of a page size.\n", 524 __func__, size); 525 return -EINVAL; 526 } 527 528 if (((attr & MT_EXECUTE_NEVER) == 0U) && ((attr & MT_RW) != 0U)) { 529 WARN("%s: Mapping memory as read-write and executable not allowed.\n", 530 __func__); 531 return -EINVAL; 532 } 533 534 size_t pages_count = size / PAGE_SIZE; 535 536 VERBOSE("Changing memory attributes of %zu pages starting from address 0x%lx...\n", 537 pages_count, base_va); 538 539 uintptr_t base_va_original = base_va; 540 541 /* 542 * Sanity checks. 543 */ 544 for (unsigned int i = 0U; i < pages_count; ++i) { 545 const uint64_t *entry; 546 uint64_t desc, attr_index; 547 unsigned int level; 548 549 entry = find_xlat_table_entry(base_va, 550 ctx->base_table, 551 ctx->base_table_entries, 552 virt_addr_space_size, 553 &level); 554 if (entry == NULL) { 555 WARN("Address 0x%lx is not mapped.\n", base_va); 556 return -EINVAL; 557 } 558 559 desc = *entry; 560 561 /* 562 * Check that all the required pages are mapped at page 563 * granularity. 564 */ 565 if (((desc & DESC_MASK) != PAGE_DESC) || 566 (level != XLAT_TABLE_LEVEL_MAX)) { 567 WARN("Address 0x%lx is not mapped at the right granularity.\n", 568 base_va); 569 WARN("Granularity is 0x%lx, should be 0x%lx.\n", 570 XLAT_BLOCK_SIZE(level), PAGE_SIZE); 571 return -EINVAL; 572 } 573 574 /* 575 * If the region type is device, it shouldn't be executable. 576 */ 577 attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK; 578 if (attr_index == ATTR_DEVICE_INDEX) { 579 if ((attr & MT_EXECUTE_NEVER) == 0U) { 580 WARN("Setting device memory as executable at address 0x%lx.", 581 base_va); 582 return -EINVAL; 583 } 584 } 585 586 base_va += PAGE_SIZE; 587 } 588 589 /* Restore original value. */ 590 base_va = base_va_original; 591 592 for (unsigned int i = 0U; i < pages_count; ++i) { 593 594 uint32_t old_attr = 0U, new_attr; 595 uint64_t *entry = NULL; 596 unsigned int level = 0U; 597 unsigned long long addr_pa = 0ULL; 598 599 (void) xlat_get_mem_attributes_internal(ctx, base_va, &old_attr, 600 &entry, &addr_pa, &level); 601 602 /* 603 * From attr, only MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and 604 * MT_USER/MT_PRIVILEGED are taken into account. Any other 605 * information is ignored. 606 */ 607 608 /* Clean the old attributes so that they can be rebuilt. */ 609 new_attr = old_attr & ~(MT_RW | MT_EXECUTE_NEVER | MT_USER); 610 611 /* 612 * Update attributes, but filter out the ones this function 613 * isn't allowed to change. 614 */ 615 new_attr |= attr & (MT_RW | MT_EXECUTE_NEVER | MT_USER); 616 617 /* 618 * The break-before-make sequence requires writing an invalid 619 * descriptor and making sure that the system sees the change 620 * before writing the new descriptor. 621 */ 622 *entry = INVALID_DESC; 623 #if !HW_ASSISTED_COHERENCY 624 dccvac((uintptr_t)entry); 625 #endif 626 /* Invalidate any cached copy of this mapping in the TLBs. */ 627 xlat_arch_tlbi_va(base_va, ctx->xlat_regime); 628 629 /* Ensure completion of the invalidation. */ 630 xlat_arch_tlbi_va_sync(); 631 632 /* Write new descriptor */ 633 *entry = xlat_desc(ctx, new_attr, addr_pa, level); 634 #if !HW_ASSISTED_COHERENCY 635 dccvac((uintptr_t)entry); 636 #endif 637 base_va += PAGE_SIZE; 638 } 639 640 /* Ensure that the last descriptor written is seen by the system. */ 641 dsbish(); 642 643 return 0; 644 } 645