1 /*
2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <errno.h>
9 #include <inttypes.h>
10 #include <stdbool.h>
11 #include <stdint.h>
12 #include <stdio.h>
13
14 #include <platform_def.h>
15
16 #include <arch_features.h>
17 #include <arch_helpers.h>
18 #include <common/debug.h>
19 #include <lib/utils_def.h>
20 #include <lib/xlat_tables/xlat_tables_defs.h>
21 #include <lib/xlat_tables/xlat_tables_v2.h>
22
23 #include "xlat_tables_private.h"
24
25 /* Uncomment, when xlat mmap details prints required*/
26 //#define LOG_DEBUG
27
28 #ifndef LOG_DEBUG
xlat_mmap_print(__unused const mmap_region_t * mmap)29 void xlat_mmap_print(__unused const mmap_region_t *mmap)
30 {
31 /* Empty */
32 }
33
xlat_tables_print(__unused xlat_ctx_t * ctx)34 void xlat_tables_print(__unused xlat_ctx_t *ctx)
35 {
36 /* Empty */
37 }
38
39 #else /* ifndef LOG_DEBUG */
40
xlat_mmap_print(const mmap_region_t * mmap)41 void xlat_mmap_print(const mmap_region_t *mmap)
42 {
43 printf("mmap:\n");
44 const mmap_region_t *mm = mmap;
45
46 while (mm->size != 0U) {
47 printf(" VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x granularity:0x%zx\n",
48 mm->base_va, mm->base_pa, mm->size, mm->attr,
49 mm->granularity);
50 ++mm;
51 };
52 printf("\n");
53 }
54
55 /* Print the attributes of the specified block descriptor. */
xlat_desc_print(const xlat_ctx_t * ctx,uint64_t desc)56 static void xlat_desc_print(const xlat_ctx_t *ctx, uint64_t desc)
57 {
58 uint64_t mem_type_index = ATTR_INDEX_GET(desc);
59 int xlat_regime = ctx->xlat_regime;
60
61 if (mem_type_index == ATTR_IWBWA_OWBWA_NTR_INDEX) {
62 printf("MEM");
63 } else if (mem_type_index == ATTR_NON_CACHEABLE_INDEX) {
64 printf("NC");
65 } else {
66 assert(mem_type_index == ATTR_DEVICE_INDEX);
67 printf("DEV");
68 }
69
70 if ((xlat_regime == EL3_REGIME) || (xlat_regime == EL2_REGIME)) {
71 /* For EL3 and EL2 only check the AP[2] and XN bits. */
72 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW");
73 printf(((desc & UPPER_ATTRS(XN)) != 0ULL) ? "-XN" : "-EXEC");
74 } else {
75 assert(xlat_regime == EL1_EL0_REGIME);
76 /*
77 * For EL0 and EL1:
78 * - In AArch64 PXN and UXN can be set independently but in
79 * AArch32 there is no UXN (XN affects both privilege levels).
80 * For consistency, we set them simultaneously in both cases.
81 * - RO and RW permissions must be the same in EL1 and EL0. If
82 * EL0 can access that memory region, so can EL1, with the
83 * same permissions.
84 */
85 #if ENABLE_ASSERTIONS
86 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(EL1_EL0_REGIME);
87 uint64_t xn_perm = desc & xn_mask;
88
89 assert((xn_perm == xn_mask) || (xn_perm == 0ULL));
90 #endif
91 printf(((desc & LOWER_ATTRS(AP_RO)) != 0ULL) ? "-RO" : "-RW");
92 /* Only check one of PXN and UXN, the other one is the same. */
93 printf(((desc & UPPER_ATTRS(PXN)) != 0ULL) ? "-XN" : "-EXEC");
94 /*
95 * Privileged regions can only be accessed from EL1, user
96 * regions can be accessed from EL1 and EL0.
97 */
98 printf(((desc & LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED)) != 0ULL)
99 ? "-USER" : "-PRIV");
100 }
101
102 if (is_feat_rme_supported()) {
103 /*
104 * When RME is enabled xlat library do not support any
105 * translation regimes other than EL3 stage1.
106 */
107 assert(ctx->xlat_regime == EL3_REGIME);
108
109 switch (desc & LOWER_ATTRS(EL3_S1_NSE | NS)) {
110 case LOWER_ATTRS(NS):
111 printf("-NS");
112 break;
113 case LOWER_ATTRS(EL3_S1_NSE):
114 printf("-RT");
115 break;
116 case LOWER_ATTRS(EL3_S1_NSE | NS):
117 #if ENABLE_RMM
118 printf("-RL");
119 #else
120 assert(false);
121 #endif
122 break;
123 default:
124 /*
125 * Secure PAS is supported only when SEL2 is
126 * implemented
127 */
128 assert(is_feat_sel2_supported());
129 printf("-S");
130 break;
131 }
132 } else {
133 /*
134 * If RME not supported, desc should not have NSE bit set in
135 * EL3 stage 1.
136 */
137 if (ctx->xlat_regime == EL3_REGIME) {
138 assert((LOWER_ATTRS(EL3_S1_NSE) & desc) == 0);
139 }
140
141 printf(((LOWER_ATTRS(NS) & desc) != 0ULL) ? "-NS" : "-S");
142 }
143
144 #ifdef __aarch64__
145 /* Check Guarded Page bit */
146 if ((desc & GP) != 0ULL) {
147 printf("-GP");
148 }
149 #endif /* __aarch64__ */
150 }
151
152 static const char * const level_spacers[] = {
153 "[LV0] ",
154 " [LV1] ",
155 " [LV2] ",
156 " [LV3] "
157 };
158
159 static const char *invalid_descriptors_ommited =
160 "%s(%d invalid descriptors omitted)\n";
161
162 /*
163 * Recursive function that reads the translation tables passed as an argument
164 * and prints their status.
165 */
xlat_tables_print_internal(xlat_ctx_t * ctx,uintptr_t table_base_va,const uint64_t * table_base,unsigned int table_entries,unsigned int level)166 static void xlat_tables_print_internal(xlat_ctx_t *ctx, uintptr_t table_base_va,
167 const uint64_t *table_base, unsigned int table_entries,
168 unsigned int level)
169 {
170 assert(level <= XLAT_TABLE_LEVEL_MAX);
171
172 uint64_t desc;
173 uintptr_t table_idx_va = table_base_va;
174 unsigned int table_idx = 0U;
175 size_t level_size = XLAT_BLOCK_SIZE(level);
176
177 /*
178 * Keep track of how many invalid descriptors are counted in a row.
179 * Whenever multiple invalid descriptors are found, only the first one
180 * is printed, and a line is added to inform about how many descriptors
181 * have been omitted.
182 */
183 int invalid_row_count = 0;
184
185 while (table_idx < table_entries) {
186
187 desc = table_base[table_idx];
188
189 if ((desc & DESC_MASK) == INVALID_DESC) {
190
191 if (invalid_row_count == 0) {
192 printf("%sVA:0x%lx size:0x%zx\n",
193 level_spacers[level],
194 table_idx_va, level_size);
195 }
196 invalid_row_count++;
197
198 } else {
199
200 if (invalid_row_count > 1) {
201 printf(invalid_descriptors_ommited,
202 level_spacers[level],
203 invalid_row_count - 1);
204 }
205 invalid_row_count = 0;
206
207 /*
208 * Check if this is a table or a block. Tables are only
209 * allowed in levels other than 3, but DESC_PAGE has the
210 * same value as DESC_TABLE, so we need to check.
211 */
212 if (((desc & DESC_MASK) == TABLE_DESC) &&
213 (level < XLAT_TABLE_LEVEL_MAX)) {
214 /*
215 * Do not print any PA for a table descriptor,
216 * as it doesn't directly map physical memory
217 * but instead points to the next translation
218 * table in the translation table walk.
219 */
220 printf("%sVA:0x%lx size:0x%zx\n",
221 level_spacers[level],
222 table_idx_va, level_size);
223
224 uintptr_t addr_inner = desc & TABLE_ADDR_MASK;
225
226 xlat_tables_print_internal(ctx, table_idx_va,
227 (uint64_t *)addr_inner,
228 XLAT_TABLE_ENTRIES, level + 1U);
229 } else {
230 printf("%sVA:0x%lx PA:0x%" PRIx64 " size:0x%zx ",
231 level_spacers[level], table_idx_va,
232 (uint64_t)(desc & TABLE_ADDR_MASK),
233 level_size);
234 xlat_desc_print(ctx, desc);
235 printf("\n");
236 }
237 }
238
239 table_idx++;
240 table_idx_va += level_size;
241 }
242
243 if (invalid_row_count > 1) {
244 printf(invalid_descriptors_ommited,
245 level_spacers[level], invalid_row_count - 1);
246 }
247 }
248
xlat_tables_print(xlat_ctx_t * ctx)249 void xlat_tables_print(xlat_ctx_t *ctx)
250 {
251 const char *xlat_regime_str;
252 int used_page_tables;
253
254 if (ctx->xlat_regime == EL1_EL0_REGIME) {
255 xlat_regime_str = "1&0";
256 } else if (ctx->xlat_regime == EL2_REGIME) {
257 xlat_regime_str = "2";
258 } else {
259 assert(ctx->xlat_regime == EL3_REGIME);
260 xlat_regime_str = "3";
261 }
262 VERBOSE("Translation tables state:\n");
263 VERBOSE(" Xlat regime: EL%s\n", xlat_regime_str);
264 VERBOSE(" Max allowed PA: 0x%llx\n", ctx->pa_max_address);
265 VERBOSE(" Max allowed VA: 0x%lx\n", ctx->va_max_address);
266 VERBOSE(" Max mapped PA: 0x%llx\n", ctx->max_pa);
267 VERBOSE(" Max mapped VA: 0x%lx\n", ctx->max_va);
268
269 VERBOSE(" Initial lookup level: %u\n", ctx->base_level);
270 VERBOSE(" Entries @initial lookup level: %u\n",
271 ctx->base_table_entries);
272
273 #if PLAT_XLAT_TABLES_DYNAMIC
274 used_page_tables = 0;
275 for (int i = 0; i < ctx->tables_num; ++i) {
276 if (ctx->tables_mapped_regions[i] != 0)
277 ++used_page_tables;
278 }
279 #else
280 used_page_tables = ctx->next_table;
281 #endif
282 VERBOSE(" Used %d sub-tables out of %d (spare: %d)\n",
283 used_page_tables, ctx->tables_num,
284 ctx->tables_num - used_page_tables);
285
286 xlat_tables_print_internal(ctx, 0U, ctx->base_table,
287 ctx->base_table_entries, ctx->base_level);
288 }
289
290 #endif /* ifndef LOG_DEBUG */
291
292 /*
293 * Do a translation table walk to find the block or page descriptor that maps
294 * virtual_addr.
295 *
296 * On success, return the address of the descriptor within the translation
297 * table. Its lookup level is stored in '*out_level'.
298 * On error, return NULL.
299 *
300 * xlat_table_base
301 * Base address for the initial lookup level.
302 * xlat_table_base_entries
303 * Number of entries in the translation table for the initial lookup level.
304 * virt_addr_space_size
305 * Size in bytes of the virtual address space.
306 */
find_xlat_table_entry(uintptr_t virtual_addr,uint64_t * xlat_table_base,unsigned int xlat_table_base_entries,unsigned long long virt_addr_space_size,unsigned int * out_level)307 static uint64_t *find_xlat_table_entry(uintptr_t virtual_addr,
308 uint64_t *xlat_table_base,
309 unsigned int xlat_table_base_entries,
310 unsigned long long virt_addr_space_size,
311 unsigned int *out_level)
312 {
313 unsigned int start_level;
314 uint64_t *table;
315 unsigned int entries;
316
317 start_level = GET_XLAT_TABLE_LEVEL_BASE(virt_addr_space_size);
318
319 table = xlat_table_base;
320 entries = xlat_table_base_entries;
321
322 for (unsigned int level = start_level;
323 level <= XLAT_TABLE_LEVEL_MAX;
324 ++level) {
325 uint64_t idx, desc, desc_type;
326
327 idx = XLAT_TABLE_IDX(virtual_addr, level);
328 if (idx >= entries) {
329 WARN("Missing xlat table entry at address 0x%lx\n",
330 virtual_addr);
331 return NULL;
332 }
333
334 desc = table[idx];
335 desc_type = desc & DESC_MASK;
336
337 if (desc_type == INVALID_DESC) {
338 VERBOSE("Invalid entry (memory not mapped)\n");
339 return NULL;
340 }
341
342 if (level == XLAT_TABLE_LEVEL_MAX) {
343 /*
344 * Only page descriptors allowed at the final lookup
345 * level.
346 */
347 assert(desc_type == PAGE_DESC);
348 *out_level = level;
349 return &table[idx];
350 }
351
352 if (desc_type == BLOCK_DESC) {
353 *out_level = level;
354 return &table[idx];
355 }
356
357 assert(desc_type == TABLE_DESC);
358 table = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
359 entries = XLAT_TABLE_ENTRIES;
360 }
361
362 /*
363 * This shouldn't be reached, the translation table walk should end at
364 * most at level XLAT_TABLE_LEVEL_MAX and return from inside the loop.
365 */
366 assert(false);
367
368 return NULL;
369 }
370
371
xlat_get_mem_attributes_internal(const xlat_ctx_t * ctx,uintptr_t base_va,uint32_t * attributes,uint64_t ** table_entry,unsigned long long * addr_pa,unsigned int * table_level)372 static int xlat_get_mem_attributes_internal(const xlat_ctx_t *ctx,
373 uintptr_t base_va, uint32_t *attributes, uint64_t **table_entry,
374 unsigned long long *addr_pa, unsigned int *table_level)
375 {
376 uint64_t *entry;
377 uint64_t desc;
378 unsigned int level;
379 unsigned long long virt_addr_space_size;
380
381 /*
382 * Sanity-check arguments.
383 */
384 assert(ctx != NULL);
385 assert(ctx->initialized);
386 assert((ctx->xlat_regime == EL1_EL0_REGIME) ||
387 (ctx->xlat_regime == EL2_REGIME) ||
388 (ctx->xlat_regime == EL3_REGIME));
389
390 virt_addr_space_size = (unsigned long long)ctx->va_max_address + 1ULL;
391 assert(virt_addr_space_size > 0U);
392
393 entry = find_xlat_table_entry(base_va,
394 ctx->base_table,
395 ctx->base_table_entries,
396 virt_addr_space_size,
397 &level);
398 if (entry == NULL) {
399 WARN("Address 0x%lx is not mapped.\n", base_va);
400 return -EINVAL;
401 }
402
403 if (addr_pa != NULL) {
404 *addr_pa = *entry & TABLE_ADDR_MASK;
405 }
406
407 if (table_entry != NULL) {
408 *table_entry = entry;
409 }
410
411 if (table_level != NULL) {
412 *table_level = level;
413 }
414
415 desc = *entry;
416
417 #ifdef LOG_DEBUG
418 VERBOSE("Attributes: ");
419 xlat_desc_print(ctx, desc);
420 printf("\n");
421 #endif
422
423 assert(attributes != NULL);
424 *attributes = 0U;
425
426 uint64_t attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK;
427
428 if (attr_index == ATTR_IWBWA_OWBWA_NTR_INDEX) {
429 *attributes |= MT_MEMORY;
430 } else if (attr_index == ATTR_NON_CACHEABLE_INDEX) {
431 *attributes |= MT_NON_CACHEABLE;
432 } else {
433 assert(attr_index == ATTR_DEVICE_INDEX);
434 *attributes |= MT_DEVICE;
435 }
436
437 uint64_t ap2_bit = (desc >> AP2_SHIFT) & 1U;
438
439 if (ap2_bit == AP2_RW)
440 *attributes |= MT_RW;
441
442 if (ctx->xlat_regime == EL1_EL0_REGIME) {
443 uint64_t ap1_bit = (desc >> AP1_SHIFT) & 1U;
444
445 if (ap1_bit == AP1_ACCESS_UNPRIVILEGED)
446 *attributes |= MT_USER;
447 }
448
449 uint64_t xn_mask = xlat_arch_regime_get_xn_desc(ctx->xlat_regime);
450
451 if ((desc & xn_mask) == xn_mask) {
452 *attributes |= MT_EXECUTE_NEVER;
453 } else {
454 assert((desc & xn_mask) == 0U);
455 }
456
457 return 0;
458 }
459
460
xlat_get_mem_attributes_ctx(const xlat_ctx_t * ctx,uintptr_t base_va,uint32_t * attr,unsigned int * table_level)461 int xlat_get_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va,
462 uint32_t *attr, unsigned int *table_level)
463 {
464 return xlat_get_mem_attributes_internal(ctx, base_va, attr,
465 NULL, NULL, table_level);
466 }
467
468 /*
469 * From argument 'attr', only flags MT_RO/MT_RW, MT_EXECUTE/MT_EXECUTE_NEVER and
470 * MT_USER/MT_PRIVILEGED are taken into account. Any other flags like memory
471 * type, memory security state information are ignored.
472 */
xlat_change_mem_attributes_ctx(const xlat_ctx_t * ctx,uintptr_t base_va,size_t size,uint32_t attr)473 int xlat_change_mem_attributes_ctx(const xlat_ctx_t *ctx, uintptr_t base_va,
474 size_t size, uint32_t attr)
475 {
476 /* Note: This implementation isn't optimized. */
477
478 assert(ctx != NULL);
479 assert(ctx->initialized);
480
481 unsigned long long virt_addr_space_size =
482 (unsigned long long)ctx->va_max_address + 1U;
483 assert(virt_addr_space_size > 0U);
484
485 if (!IS_PAGE_ALIGNED(base_va)) {
486 WARN("%s: Address 0x%lx is not aligned on a page boundary.\n",
487 __func__, base_va);
488 return -EINVAL;
489 }
490
491 if (size == 0U) {
492 WARN("%s: Size is 0.\n", __func__);
493 return -EINVAL;
494 }
495
496 if ((size % PAGE_SIZE) != 0U) {
497 WARN("%s: Size 0x%zx is not a multiple of a page size.\n",
498 __func__, size);
499 return -EINVAL;
500 }
501
502 if (((attr & MT_EXECUTE_NEVER) == 0U) && ((attr & MT_RW) != 0U)) {
503 WARN("%s: Mapping memory as read-write and executable not allowed.\n",
504 __func__);
505 return -EINVAL;
506 }
507
508 size_t pages_count = size / PAGE_SIZE;
509
510 VERBOSE("Changing memory attributes of %zu pages starting from address 0x%lx...\n",
511 pages_count, base_va);
512
513 uintptr_t base_va_original = base_va;
514
515 /*
516 * Sanity checks.
517 */
518 for (unsigned int i = 0U; i < pages_count; ++i) {
519 const uint64_t *entry;
520 uint64_t desc, attr_index;
521 unsigned int level;
522
523 entry = find_xlat_table_entry(base_va,
524 ctx->base_table,
525 ctx->base_table_entries,
526 virt_addr_space_size,
527 &level);
528 if (entry == NULL) {
529 WARN("Address 0x%lx is not mapped.\n", base_va);
530 return -EINVAL;
531 }
532
533 desc = *entry;
534
535 /*
536 * Check that all the required pages are mapped at page
537 * granularity.
538 */
539 if (((desc & DESC_MASK) != PAGE_DESC) ||
540 (level != XLAT_TABLE_LEVEL_MAX)) {
541 WARN("Address 0x%lx is not mapped at the right granularity.\n",
542 base_va);
543 WARN("Granularity is 0x%lx, should be 0x%lx.\n",
544 XLAT_BLOCK_SIZE(level), PAGE_SIZE);
545 return -EINVAL;
546 }
547
548 /*
549 * If the region type is device, it shouldn't be executable.
550 */
551 attr_index = (desc >> ATTR_INDEX_SHIFT) & ATTR_INDEX_MASK;
552 if (attr_index == ATTR_DEVICE_INDEX) {
553 if ((attr & MT_EXECUTE_NEVER) == 0U) {
554 WARN("Setting device memory as executable at address 0x%lx.",
555 base_va);
556 return -EINVAL;
557 }
558 }
559
560 base_va += PAGE_SIZE;
561 }
562
563 /* Restore original value. */
564 base_va = base_va_original;
565
566 for (unsigned int i = 0U; i < pages_count; ++i) {
567
568 uint32_t old_attr = 0U, new_attr;
569 uint64_t *entry = NULL;
570 uint64_t new_desc, old_desc;
571 unsigned int level = 0U;
572 unsigned long long addr_pa = 0ULL;
573
574 (void) xlat_get_mem_attributes_internal(ctx, base_va, &old_attr,
575 &entry, &addr_pa, &level);
576
577 old_desc = *entry;
578
579 /* Clean the old attributes so that they can be rebuilt. */
580 new_attr = old_attr & ~(MT_RW | MT_EXECUTE_NEVER | MT_USER);
581
582 /*
583 * Update attributes, but filter out the ones this function
584 * isn't allowed to change.
585 */
586 new_attr |= attr & (MT_RW | MT_EXECUTE_NEVER | MT_USER);
587
588 /* Create new descriptor */
589 new_desc = xlat_desc(ctx, new_attr, addr_pa, level);
590
591 /* Retain NS bit */
592 new_desc |= old_desc & LOWER_ATTRS(NS);
593
594 /* Retain NSE bit */
595 if (is_feat_rme_supported() &&
596 (ctx->xlat_regime == EL3_REGIME)) {
597 new_desc |= old_desc & LOWER_ATTRS(EL3_S1_NSE);
598 }
599
600 /*
601 * The break-before-make sequence requires writing an invalid
602 * descriptor and making sure that the system sees the change
603 * before writing the new descriptor.
604 */
605 *entry = INVALID_DESC;
606 #if !HW_ASSISTED_COHERENCY
607 dccvac((uintptr_t)entry);
608 #endif
609 /* Invalidate any cached copy of this mapping in the TLBs. */
610 xlat_arch_tlbi_va(base_va, ctx->xlat_regime);
611
612 /* Ensure completion of the invalidation. */
613 xlat_arch_tlbi_va_sync();
614
615 /* Write new descriptor */
616 *entry = new_desc;
617
618 #if !HW_ASSISTED_COHERENCY
619 dccvac((uintptr_t)entry);
620 #endif
621 base_va += PAGE_SIZE;
622 }
623
624 /* Ensure that the last descriptor written is seen by the system. */
625 dsbish();
626
627 return 0;
628 }
629