1 /*
2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <errno.h>
9 #include <stdbool.h>
10 #include <stdint.h>
11 #include <string.h>
12
13 #include <platform_def.h>
14
15 #include <arch_features.h>
16 #include <arch_helpers.h>
17 #include <common/debug.h>
18 #include <lib/utils.h>
19 #include <lib/utils_def.h>
20 #include <lib/xlat_tables/xlat_tables_defs.h>
21 #include <lib/xlat_tables/xlat_tables_v2.h>
22
23 #include "xlat_tables_private.h"
24
25 /* Helper function that cleans the data cache only if it is enabled. */
xlat_clean_dcache_range(uintptr_t addr,size_t size)26 static inline __attribute__((unused)) void xlat_clean_dcache_range(uintptr_t addr, size_t size)
27 {
28 if (is_dcache_enabled()) {
29 clean_dcache_range(addr, size);
30 }
31 }
32
33 #if PLAT_XLAT_TABLES_DYNAMIC
34
35 /*
36 * The following functions assume that they will be called using subtables only.
37 * The base table can't be unmapped, so it is not needed to do any special
38 * handling for it.
39 */
40
41 /*
42 * Returns the index of the array corresponding to the specified translation
43 * table.
44 */
xlat_table_get_index(const xlat_ctx_t * ctx,const uint64_t * table)45 static int xlat_table_get_index(const xlat_ctx_t *ctx, const uint64_t *table)
46 {
47 for (int i = 0; i < ctx->tables_num; i++)
48 if (ctx->tables[i] == table)
49 return i;
50
51 /*
52 * Maybe we were asked to get the index of the base level table, which
53 * should never happen.
54 */
55 assert(false);
56
57 return -1;
58 }
59
60 /* Returns a pointer to an empty translation table. */
xlat_table_get_empty(const xlat_ctx_t * ctx)61 static uint64_t *xlat_table_get_empty(const xlat_ctx_t *ctx)
62 {
63 for (int i = 0; i < ctx->tables_num; i++)
64 if (ctx->tables_mapped_regions[i] == 0)
65 return ctx->tables[i];
66
67 return NULL;
68 }
69
70 /* Increments region count for a given table. */
xlat_table_inc_regions_count(const xlat_ctx_t * ctx,const uint64_t * table)71 static void xlat_table_inc_regions_count(const xlat_ctx_t *ctx,
72 const uint64_t *table)
73 {
74 int idx = xlat_table_get_index(ctx, table);
75
76 ctx->tables_mapped_regions[idx]++;
77 }
78
79 /* Decrements region count for a given table. */
xlat_table_dec_regions_count(const xlat_ctx_t * ctx,const uint64_t * table)80 static void xlat_table_dec_regions_count(const xlat_ctx_t *ctx,
81 const uint64_t *table)
82 {
83 int idx = xlat_table_get_index(ctx, table);
84
85 ctx->tables_mapped_regions[idx]--;
86 }
87
88 /* Returns 0 if the specified table isn't empty, otherwise 1. */
xlat_table_is_empty(const xlat_ctx_t * ctx,const uint64_t * table)89 static bool xlat_table_is_empty(const xlat_ctx_t *ctx, const uint64_t *table)
90 {
91 return ctx->tables_mapped_regions[xlat_table_get_index(ctx, table)] == 0;
92 }
93
94 #else /* PLAT_XLAT_TABLES_DYNAMIC */
95
96 /* Returns a pointer to the first empty translation table. */
xlat_table_get_empty(xlat_ctx_t * ctx)97 static uint64_t *xlat_table_get_empty(xlat_ctx_t *ctx)
98 {
99 assert(ctx->next_table < ctx->tables_num);
100
101 return ctx->tables[ctx->next_table++];
102 }
103
104 #endif /* PLAT_XLAT_TABLES_DYNAMIC */
105
106 /*
107 * Returns a block/page table descriptor for the given level and attributes.
108 */
xlat_desc(const xlat_ctx_t * ctx,uint32_t attr,unsigned long long addr_pa,unsigned int level)109 uint64_t xlat_desc(const xlat_ctx_t *ctx, uint32_t attr,
110 unsigned long long addr_pa, unsigned int level)
111 {
112 uint64_t desc;
113 uint32_t mem_type;
114 uint32_t shareability_type;
115
116 /* Make sure that the granularity is fine enough to map this address. */
117 assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0U);
118
119 desc = addr_pa;
120 /*
121 * There are different translation table descriptors for level 3 and the
122 * rest.
123 */
124 desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC;
125 /*
126 * Always set the access flag, as this library assumes access flag
127 * faults aren't managed.
128 */
129 desc |= LOWER_ATTRS(ACCESS_FLAG);
130
131 /* Determine the physical address space this region belongs to. */
132 desc |= xlat_arch_get_pas(attr);
133
134 /*
135 * Deduce other fields of the descriptor based on the MT_RW memory
136 * region attributes.
137 */
138 desc |= ((attr & MT_RW) != 0U) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO);
139
140 /*
141 * Do not allow unprivileged access when the mapping is for a privileged
142 * EL. For translation regimes that do not have mappings for access for
143 * lower exception levels, set AP[2] to AP_NO_ACCESS_UNPRIVILEGED.
144 */
145 if (ctx->xlat_regime == EL1_EL0_REGIME) {
146 if ((attr & MT_USER) != 0U) {
147 /* EL0 mapping requested, so we give User access */
148 desc |= LOWER_ATTRS(AP_ACCESS_UNPRIVILEGED);
149 } else {
150 /* EL1 mapping requested, no User access granted */
151 desc |= LOWER_ATTRS(AP_NO_ACCESS_UNPRIVILEGED);
152 }
153 } else {
154 assert((ctx->xlat_regime == EL2_REGIME) ||
155 (ctx->xlat_regime == EL3_REGIME));
156 desc |= LOWER_ATTRS(AP_ONE_VA_RANGE_RES1);
157 }
158
159 /*
160 * Deduce shareability domain and executability of the memory region
161 * from the memory type of the attributes (MT_TYPE).
162 *
163 * Data accesses to device memory and non-cacheable normal memory are
164 * coherent for all observers in the system, and correspondingly are
165 * always treated as being Outer Shareable. Therefore, for these 2 types
166 * of memory, it is not strictly needed to set the shareability field
167 * in the translation tables.
168 */
169 mem_type = MT_TYPE(attr);
170 if (mem_type == MT_DEVICE) {
171 desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH);
172 /*
173 * Always map device memory as execute-never.
174 * This is to avoid the possibility of a speculative instruction
175 * fetch, which could be an issue if this memory region
176 * corresponds to a read-sensitive peripheral.
177 */
178 desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime);
179
180 } else { /* Normal memory */
181 /*
182 * Always map read-write normal memory as execute-never.
183 * This library assumes that it is used by software that does
184 * not self-modify its code, therefore R/W memory is reserved
185 * for data storage, which must not be executable.
186 *
187 * Note that setting the XN bit here is for consistency only.
188 * The function that enables the MMU sets the SCTLR_ELx.WXN bit,
189 * which makes any writable memory region to be treated as
190 * execute-never, regardless of the value of the XN bit in the
191 * translation table.
192 *
193 * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER
194 * attribute to figure out the value of the XN bit. The actual
195 * XN bit(s) to set in the descriptor depends on the context's
196 * translation regime and the policy applied in
197 * xlat_arch_regime_get_xn_desc().
198 */
199 if (((attr & MT_RW) != 0U) || ((attr & MT_EXECUTE_NEVER) != 0U)) {
200 desc |= xlat_arch_regime_get_xn_desc(ctx->xlat_regime);
201 }
202
203 shareability_type = MT_SHAREABILITY(attr);
204 if (mem_type == MT_MEMORY) {
205 desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX);
206 if (shareability_type == MT_SHAREABILITY_NSH) {
207 desc |= LOWER_ATTRS(NSH);
208 } else if (shareability_type == MT_SHAREABILITY_OSH) {
209 desc |= LOWER_ATTRS(OSH);
210 } else {
211 desc |= LOWER_ATTRS(ISH);
212 }
213
214 /* Set GP bit for block and page code entries
215 * if BTI mechanism is implemented.
216 */
217 if (is_feat_bti_supported() &&
218 ((attr & (MT_TYPE_MASK | MT_RW |
219 MT_EXECUTE_NEVER)) == MT_CODE)) {
220 desc |= GP;
221 }
222 } else {
223 assert(mem_type == MT_NON_CACHEABLE);
224 desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH);
225 }
226 }
227
228 return desc;
229 }
230
231 /*
232 * Enumeration of actions that can be made when mapping table entries depending
233 * on the previous value in that entry and information about the region being
234 * mapped.
235 */
236 typedef enum {
237
238 /* Do nothing */
239 ACTION_NONE,
240
241 /* Write a block (or page, if in level 3) entry. */
242 ACTION_WRITE_BLOCK_ENTRY,
243
244 /*
245 * Create a new table and write a table entry pointing to it. Recurse
246 * into it for further processing.
247 */
248 ACTION_CREATE_NEW_TABLE,
249
250 /*
251 * There is a table descriptor in this entry, read it and recurse into
252 * that table for further processing.
253 */
254 ACTION_RECURSE_INTO_TABLE,
255
256 } action_t;
257
258 /*
259 * Function that returns the first VA of the table affected by the specified
260 * mmap region.
261 */
xlat_tables_find_start_va(mmap_region_t * mm,const uintptr_t table_base_va,const unsigned int level)262 static uintptr_t xlat_tables_find_start_va(mmap_region_t *mm,
263 const uintptr_t table_base_va,
264 const unsigned int level)
265 {
266 uintptr_t table_idx_va;
267
268 if (mm->base_va > table_base_va) {
269 /* Find the first index of the table affected by the region. */
270 table_idx_va = mm->base_va & ~XLAT_BLOCK_MASK(level);
271 } else {
272 /* Start from the beginning of the table. */
273 table_idx_va = table_base_va;
274 }
275
276 return table_idx_va;
277 }
278
279 /*
280 * Function that returns table index for the given VA and level arguments.
281 */
xlat_tables_va_to_index(const uintptr_t table_base_va,const uintptr_t va,const unsigned int level)282 static inline unsigned int xlat_tables_va_to_index(const uintptr_t table_base_va,
283 const uintptr_t va,
284 const unsigned int level)
285 {
286 return (unsigned int)((va - table_base_va) >> XLAT_ADDR_SHIFT(level));
287 }
288
289 #if PLAT_XLAT_TABLES_DYNAMIC
290
291 /*
292 * From the given arguments, it decides which action to take when unmapping the
293 * specified region.
294 */
xlat_tables_unmap_region_action(const mmap_region_t * mm,const uintptr_t table_idx_va,const uintptr_t table_idx_end_va,const unsigned int level,const uint64_t desc_type)295 static action_t xlat_tables_unmap_region_action(const mmap_region_t *mm,
296 const uintptr_t table_idx_va, const uintptr_t table_idx_end_va,
297 const unsigned int level, const uint64_t desc_type)
298 {
299 action_t action;
300 uintptr_t region_end_va = mm->base_va + mm->size - 1U;
301
302 if ((mm->base_va <= table_idx_va) &&
303 (region_end_va >= table_idx_end_va)) {
304 /* Region covers all block */
305
306 if (level == 3U) {
307 /*
308 * Last level, only page descriptors allowed,
309 * erase it.
310 */
311 assert(desc_type == PAGE_DESC);
312
313 action = ACTION_WRITE_BLOCK_ENTRY;
314 } else {
315 /*
316 * Other levels can have table descriptors. If
317 * so, recurse into it and erase descriptors
318 * inside it as needed. If there is a block
319 * descriptor, just erase it. If an invalid
320 * descriptor is found, this table isn't
321 * actually mapped, which shouldn't happen.
322 */
323 if (desc_type == TABLE_DESC) {
324 action = ACTION_RECURSE_INTO_TABLE;
325 } else {
326 assert(desc_type == BLOCK_DESC);
327 action = ACTION_WRITE_BLOCK_ENTRY;
328 }
329 }
330
331 } else if ((mm->base_va <= table_idx_end_va) ||
332 (region_end_va >= table_idx_va)) {
333 /*
334 * Region partially covers block.
335 *
336 * It can't happen in level 3.
337 *
338 * There must be a table descriptor here, if not there
339 * was a problem when mapping the region.
340 */
341 assert(level < 3U);
342 assert(desc_type == TABLE_DESC);
343
344 action = ACTION_RECURSE_INTO_TABLE;
345 } else {
346 /* The region doesn't cover the block at all */
347 action = ACTION_NONE;
348 }
349
350 return action;
351 }
352 /*
353 * Recursive function that writes to the translation tables and unmaps the
354 * specified region.
355 */
xlat_tables_unmap_region(xlat_ctx_t * ctx,mmap_region_t * mm,const uintptr_t table_base_va,uint64_t * const table_base,const unsigned int table_entries,const unsigned int level)356 static void xlat_tables_unmap_region(xlat_ctx_t *ctx, mmap_region_t *mm,
357 const uintptr_t table_base_va,
358 uint64_t *const table_base,
359 const unsigned int table_entries,
360 const unsigned int level)
361 {
362 assert((level >= ctx->base_level) && (level <= XLAT_TABLE_LEVEL_MAX));
363
364 uint64_t *subtable;
365 uint64_t desc;
366
367 uintptr_t table_idx_va;
368 uintptr_t table_idx_end_va; /* End VA of this entry */
369
370 uintptr_t region_end_va = mm->base_va + mm->size - 1U;
371
372 unsigned int table_idx;
373
374 table_idx_va = xlat_tables_find_start_va(mm, table_base_va, level);
375 table_idx = xlat_tables_va_to_index(table_base_va, table_idx_va, level);
376
377 while (table_idx < table_entries) {
378
379 table_idx_end_va = table_idx_va + XLAT_BLOCK_SIZE(level) - 1U;
380
381 desc = table_base[table_idx];
382 uint64_t desc_type = desc & DESC_MASK;
383
384 action_t action = xlat_tables_unmap_region_action(mm,
385 table_idx_va, table_idx_end_va, level,
386 desc_type);
387
388 if (action == ACTION_WRITE_BLOCK_ENTRY) {
389
390 table_base[table_idx] = INVALID_DESC;
391 xlat_arch_tlbi_va(table_idx_va, ctx->xlat_regime);
392
393 } else if (action == ACTION_RECURSE_INTO_TABLE) {
394
395 subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
396
397 /* Recurse to write into subtable */
398 xlat_tables_unmap_region(ctx, mm, table_idx_va,
399 subtable, XLAT_TABLE_ENTRIES,
400 level + 1U);
401 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
402 xlat_clean_dcache_range((uintptr_t)subtable,
403 XLAT_TABLE_ENTRIES * sizeof(uint64_t));
404 #endif
405 /*
406 * If the subtable is now empty, remove its reference.
407 */
408 if (xlat_table_is_empty(ctx, subtable)) {
409 table_base[table_idx] = INVALID_DESC;
410 xlat_arch_tlbi_va(table_idx_va,
411 ctx->xlat_regime);
412 }
413
414 } else {
415 assert(action == ACTION_NONE);
416 }
417
418 table_idx++;
419 table_idx_va += XLAT_BLOCK_SIZE(level);
420
421 /* If reached the end of the region, exit */
422 if (region_end_va <= table_idx_va)
423 break;
424 }
425
426 if (level > ctx->base_level)
427 xlat_table_dec_regions_count(ctx, table_base);
428 }
429
430 #endif /* PLAT_XLAT_TABLES_DYNAMIC */
431
432 /*
433 * From the given arguments, it decides which action to take when mapping the
434 * specified region.
435 */
xlat_tables_map_region_action(const mmap_region_t * mm,unsigned int desc_type,unsigned long long dest_pa,uintptr_t table_entry_base_va,unsigned int level)436 static action_t xlat_tables_map_region_action(const mmap_region_t *mm,
437 unsigned int desc_type, unsigned long long dest_pa,
438 uintptr_t table_entry_base_va, unsigned int level)
439 {
440 uintptr_t mm_end_va = mm->base_va + mm->size - 1U;
441 uintptr_t table_entry_end_va =
442 table_entry_base_va + XLAT_BLOCK_SIZE(level) - 1U;
443
444 /*
445 * The descriptor types allowed depend on the current table level.
446 */
447
448 if ((mm->base_va <= table_entry_base_va) &&
449 (mm_end_va >= table_entry_end_va)) {
450
451 /*
452 * Table entry is covered by region
453 * --------------------------------
454 *
455 * This means that this table entry can describe the whole
456 * translation with this granularity in principle.
457 */
458
459 if (level == 3U) {
460 /*
461 * Last level, only page descriptors are allowed.
462 */
463 if (desc_type == PAGE_DESC) {
464 /*
465 * There's another region mapped here, don't
466 * overwrite.
467 */
468 return ACTION_NONE;
469 } else {
470 assert(desc_type == INVALID_DESC);
471 return ACTION_WRITE_BLOCK_ENTRY;
472 }
473
474 } else {
475
476 /*
477 * Other levels. Table descriptors are allowed. Block
478 * descriptors too, but they have some limitations.
479 */
480
481 if (desc_type == TABLE_DESC) {
482 /* There's already a table, recurse into it. */
483 return ACTION_RECURSE_INTO_TABLE;
484
485 } else if (desc_type == INVALID_DESC) {
486 /*
487 * There's nothing mapped here, create a new
488 * entry.
489 *
490 * Check if the destination granularity allows
491 * us to use a block descriptor or we need a
492 * finer table for it.
493 *
494 * Also, check if the current level allows block
495 * descriptors. If not, create a table instead.
496 */
497 if (((dest_pa & XLAT_BLOCK_MASK(level)) != 0U)
498 || (level < MIN_LVL_BLOCK_DESC) ||
499 (mm->granularity < XLAT_BLOCK_SIZE(level))) {
500 return ACTION_CREATE_NEW_TABLE;
501 } else {
502 return ACTION_WRITE_BLOCK_ENTRY;
503 }
504
505 } else {
506 /*
507 * There's another region mapped here, don't
508 * overwrite.
509 */
510 assert(desc_type == BLOCK_DESC);
511
512 return ACTION_NONE;
513 }
514 }
515
516 } else if ((mm->base_va <= table_entry_end_va) ||
517 (mm_end_va >= table_entry_base_va)) {
518
519 /*
520 * Region partially covers table entry
521 * -----------------------------------
522 *
523 * This means that this table entry can't describe the whole
524 * translation, a finer table is needed.
525
526 * There cannot be partial block overlaps in level 3. If that
527 * happens, some of the preliminary checks when adding the
528 * mmap region failed to detect that PA and VA must at least be
529 * aligned to PAGE_SIZE.
530 */
531 assert(level < 3U);
532
533 if (desc_type == INVALID_DESC) {
534 /*
535 * The block is not fully covered by the region. Create
536 * a new table, recurse into it and try to map the
537 * region with finer granularity.
538 */
539 return ACTION_CREATE_NEW_TABLE;
540
541 } else {
542 assert(desc_type == TABLE_DESC);
543 /*
544 * The block is not fully covered by the region, but
545 * there is already a table here. Recurse into it and
546 * try to map with finer granularity.
547 *
548 * PAGE_DESC for level 3 has the same value as
549 * TABLE_DESC, but this code can't run on a level 3
550 * table because there can't be overlaps in level 3.
551 */
552 return ACTION_RECURSE_INTO_TABLE;
553 }
554 } else {
555
556 /*
557 * This table entry is outside of the region specified in the
558 * arguments, don't write anything to it.
559 */
560 return ACTION_NONE;
561 }
562 }
563
564 /*
565 * Recursive function that writes to the translation tables and maps the
566 * specified region. On success, it returns the VA of the last byte that was
567 * successfully mapped. On error, it returns the VA of the next entry that
568 * should have been mapped.
569 */
xlat_tables_map_region(xlat_ctx_t * ctx,mmap_region_t * mm,uintptr_t table_base_va,uint64_t * const table_base,unsigned int table_entries,unsigned int level)570 static uintptr_t xlat_tables_map_region(xlat_ctx_t *ctx, mmap_region_t *mm,
571 uintptr_t table_base_va,
572 uint64_t *const table_base,
573 unsigned int table_entries,
574 unsigned int level)
575 {
576 assert((level >= ctx->base_level) && (level <= XLAT_TABLE_LEVEL_MAX));
577
578 uintptr_t mm_end_va = mm->base_va + mm->size - 1U;
579
580 uintptr_t table_idx_va;
581 unsigned long long table_idx_pa;
582
583 uint64_t *subtable;
584 uint64_t desc;
585
586 unsigned int table_idx;
587
588 table_idx_va = xlat_tables_find_start_va(mm, table_base_va, level);
589 table_idx = xlat_tables_va_to_index(table_base_va, table_idx_va, level);
590
591 #if PLAT_XLAT_TABLES_DYNAMIC
592 if (level > ctx->base_level)
593 xlat_table_inc_regions_count(ctx, table_base);
594 #endif
595
596 while (table_idx < table_entries) {
597
598 desc = table_base[table_idx];
599
600 table_idx_pa = mm->base_pa + table_idx_va - mm->base_va;
601
602 action_t action = xlat_tables_map_region_action(mm,
603 (uint32_t)(desc & DESC_MASK), table_idx_pa,
604 table_idx_va, level);
605
606 if (action == ACTION_WRITE_BLOCK_ENTRY) {
607
608 table_base[table_idx] =
609 xlat_desc(ctx, (uint32_t)mm->attr, table_idx_pa,
610 level);
611
612 } else if (action == ACTION_CREATE_NEW_TABLE) {
613 uintptr_t end_va;
614
615 subtable = xlat_table_get_empty(ctx);
616 if (subtable == NULL) {
617 /* Not enough free tables to map this region */
618 return table_idx_va;
619 }
620
621 /* Point to new subtable from this one. */
622 table_base[table_idx] =
623 TABLE_DESC | (uintptr_t)subtable;
624
625 /* Recurse to write into subtable */
626 end_va = xlat_tables_map_region(ctx, mm, table_idx_va,
627 subtable, XLAT_TABLE_ENTRIES,
628 level + 1U);
629 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
630 xlat_clean_dcache_range((uintptr_t)subtable,
631 XLAT_TABLE_ENTRIES * sizeof(uint64_t));
632 #endif
633 if (end_va !=
634 (table_idx_va + XLAT_BLOCK_SIZE(level) - 1U)) {
635 return end_va;
636 }
637
638 } else if (action == ACTION_RECURSE_INTO_TABLE) {
639 uintptr_t end_va;
640
641 subtable = (uint64_t *)(uintptr_t)(desc & TABLE_ADDR_MASK);
642 /* Recurse to write into subtable */
643 end_va = xlat_tables_map_region(ctx, mm, table_idx_va,
644 subtable, XLAT_TABLE_ENTRIES,
645 level + 1U);
646 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
647 xlat_clean_dcache_range((uintptr_t)subtable,
648 XLAT_TABLE_ENTRIES * sizeof(uint64_t));
649 #endif
650 if (end_va !=
651 (table_idx_va + XLAT_BLOCK_SIZE(level) - 1U)) {
652 return end_va;
653 }
654
655 } else {
656
657 assert(action == ACTION_NONE);
658
659 }
660
661 table_idx++;
662 table_idx_va += XLAT_BLOCK_SIZE(level);
663
664 /* If reached the end of the region, exit */
665 if (mm_end_va <= table_idx_va) {
666 break;
667 }
668 }
669
670 return table_idx_va - 1U;
671 }
672
673 /*
674 * Function that verifies that a region can be mapped.
675 * Returns:
676 * 0: Success, the mapping is allowed.
677 * EINVAL: Invalid values were used as arguments.
678 * ERANGE: The memory limits were surpassed.
679 * ENOMEM: There is not enough memory in the mmap array.
680 * EPERM: Region overlaps another one in an invalid way.
681 */
mmap_add_region_check(const xlat_ctx_t * ctx,const mmap_region_t * mm)682 static int mmap_add_region_check(const xlat_ctx_t *ctx, const mmap_region_t *mm)
683 {
684 unsigned long long base_pa = mm->base_pa;
685 uintptr_t base_va = mm->base_va;
686 size_t size = mm->size;
687 size_t granularity = mm->granularity;
688
689 unsigned long long end_pa = base_pa + size - 1U;
690 uintptr_t end_va = base_va + size - 1U;
691
692 if (!IS_PAGE_ALIGNED(base_pa) || !IS_PAGE_ALIGNED(base_va) ||
693 !IS_PAGE_ALIGNED(size)) {
694 return -EINVAL;
695 }
696
697 if ((granularity != XLAT_BLOCK_SIZE(1U)) &&
698 (granularity != XLAT_BLOCK_SIZE(2U)) &&
699 (granularity != XLAT_BLOCK_SIZE(3U))) {
700 return -EINVAL;
701 }
702
703 /* Check for overflows */
704 if ((base_pa > end_pa) || (base_va > end_va)) {
705 return -ERANGE;
706 }
707
708 if (end_va > ctx->va_max_address) {
709 return -ERANGE;
710 }
711
712 if (end_pa > ctx->pa_max_address) {
713 return -ERANGE;
714 }
715
716 /* Check that there is space in the ctx->mmap array */
717 if (ctx->mmap[ctx->mmap_num - 1].size != 0U) {
718 return -ENOMEM;
719 }
720
721 /* Check for PAs and VAs overlaps with all other regions */
722 for (const mmap_region_t *mm_cursor = ctx->mmap;
723 mm_cursor->size != 0U; ++mm_cursor) {
724
725 uintptr_t mm_cursor_end_va = mm_cursor->base_va
726 + mm_cursor->size - 1U;
727
728 /*
729 * Check if one of the regions is completely inside the other
730 * one.
731 */
732 bool fully_overlapped_va =
733 ((base_va >= mm_cursor->base_va) &&
734 (end_va <= mm_cursor_end_va)) ||
735 ((mm_cursor->base_va >= base_va) &&
736 (mm_cursor_end_va <= end_va));
737
738 /*
739 * Full VA overlaps are only allowed if both regions are
740 * identity mapped (zero offset) or have the same VA to PA
741 * offset. Also, make sure that it's not the exact same area.
742 * This can only be done with static regions.
743 */
744 if (fully_overlapped_va) {
745
746 #if PLAT_XLAT_TABLES_DYNAMIC
747 if (((mm->attr & MT_DYNAMIC) != 0U) ||
748 ((mm_cursor->attr & MT_DYNAMIC) != 0U)) {
749 return -EPERM;
750 }
751 #endif /* PLAT_XLAT_TABLES_DYNAMIC */
752 if ((mm_cursor->base_va - mm_cursor->base_pa) !=
753 (base_va - base_pa)) {
754 return -EPERM;
755 }
756
757 if ((base_va == mm_cursor->base_va) &&
758 (size == mm_cursor->size)) {
759 return -EPERM;
760 }
761
762 } else {
763 /*
764 * If the regions do not have fully overlapping VAs,
765 * then they must have fully separated VAs and PAs.
766 * Partial overlaps are not allowed
767 */
768
769 unsigned long long mm_cursor_end_pa =
770 mm_cursor->base_pa + mm_cursor->size - 1U;
771
772 bool separated_pa = (end_pa < mm_cursor->base_pa) ||
773 (base_pa > mm_cursor_end_pa);
774 bool separated_va = (end_va < mm_cursor->base_va) ||
775 (base_va > mm_cursor_end_va);
776
777 if (!separated_va || !separated_pa) {
778 return -EPERM;
779 }
780 }
781 }
782
783 return 0;
784 }
785
mmap_add_region_ctx(xlat_ctx_t * ctx,const mmap_region_t * mm)786 void mmap_add_region_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm)
787 {
788 mmap_region_t *mm_cursor = ctx->mmap, *mm_destination;
789 const mmap_region_t *mm_end = ctx->mmap + ctx->mmap_num;
790 const mmap_region_t *mm_last;
791 unsigned long long end_pa = mm->base_pa + mm->size - 1U;
792 uintptr_t end_va = mm->base_va + mm->size - 1U;
793 int ret;
794
795 /* Ignore empty regions */
796 if (mm->size == 0U) {
797 return;
798 }
799
800 /* Static regions must be added before initializing the xlat tables. */
801 assert(!ctx->initialized);
802
803 ret = mmap_add_region_check(ctx, mm);
804 if (ret != 0) {
805 ERROR("mmap_add_region_check() failed. error %d\n", ret);
806 assert(false);
807 return;
808 }
809
810 /*
811 * Find correct place in mmap to insert new region.
812 *
813 * 1 - Lower region VA end first.
814 * 2 - Smaller region size first.
815 *
816 * VA 0 0xFF
817 *
818 * 1st |------|
819 * 2nd |------------|
820 * 3rd |------|
821 * 4th |---|
822 * 5th |---|
823 * 6th |----------|
824 * 7th |-------------------------------------|
825 *
826 * This is required for overlapping regions only. It simplifies adding
827 * regions with the loop in xlat_tables_init_internal because the outer
828 * ones won't overwrite block or page descriptors of regions added
829 * previously.
830 *
831 * Overlapping is only allowed for static regions.
832 */
833
834 while (((mm_cursor->base_va + mm_cursor->size - 1U) < end_va)
835 && (mm_cursor->size != 0U)) {
836 ++mm_cursor;
837 }
838
839 while (((mm_cursor->base_va + mm_cursor->size - 1U) == end_va) &&
840 (mm_cursor->size != 0U) && (mm_cursor->size < mm->size)) {
841 ++mm_cursor;
842 }
843
844 /*
845 * Find the last entry marker in the mmap
846 */
847 mm_last = ctx->mmap;
848 while ((mm_last->size != 0U) && (mm_last < mm_end)) {
849 ++mm_last;
850 }
851
852 /*
853 * Check if we have enough space in the memory mapping table.
854 * This shouldn't happen as we have checked in mmap_add_region_check
855 * that there is free space.
856 */
857 assert(mm_last->size == 0U);
858
859 /* Make room for new region by moving other regions up by one place */
860 mm_destination = mm_cursor + 1;
861 (void)memmove(mm_destination, mm_cursor,
862 (uintptr_t)mm_last - (uintptr_t)mm_cursor);
863
864 /*
865 * Check we haven't lost the empty sentinel from the end of the array.
866 * This shouldn't happen as we have checked in mmap_add_region_check
867 * that there is free space.
868 */
869 assert(mm_end->size == 0U);
870
871 *mm_cursor = *mm;
872
873 if (end_pa > ctx->max_pa) {
874 ctx->max_pa = end_pa;
875 }
876
877 if (end_va > ctx->max_va) {
878 ctx->max_va = end_va;
879 }
880 }
881
882 /*
883 * Determine the table level closest to the initial lookup level that
884 * can describe this translation. Then, align base VA to the next block
885 * at the determined level.
886 */
mmap_alloc_va_align_ctx(xlat_ctx_t * ctx,mmap_region_t * mm)887 static void mmap_alloc_va_align_ctx(xlat_ctx_t *ctx, mmap_region_t *mm)
888 {
889 /*
890 * By or'ing the size and base PA the alignment will be the one
891 * corresponding to the smallest boundary of the two of them.
892 *
893 * There are three different cases. For example (for 4 KiB page size):
894 *
895 * +--------------+------------------++--------------+
896 * | PA alignment | Size multiple of || VA alignment |
897 * +--------------+------------------++--------------+
898 * | 2 MiB | 2 MiB || 2 MiB | (1)
899 * | 2 MiB | 4 KiB || 4 KiB | (2)
900 * | 4 KiB | 2 MiB || 4 KiB | (3)
901 * +--------------+------------------++--------------+
902 *
903 * - In (1), it is possible to take advantage of the alignment of the PA
904 * and the size of the region to use a level 2 translation table
905 * instead of a level 3 one.
906 *
907 * - In (2), the size is smaller than a block entry of level 2, so it is
908 * needed to use a level 3 table to describe the region or the library
909 * will map more memory than the desired one.
910 *
911 * - In (3), even though the region has the size of one level 2 block
912 * entry, it isn't possible to describe the translation with a level 2
913 * block entry because of the alignment of the base PA.
914 *
915 * Only bits 47:21 of a level 2 block descriptor are used by the MMU,
916 * bits 20:0 of the resulting address are 0 in this case. Because of
917 * this, the PA generated as result of this translation is aligned to
918 * 2 MiB. The PA that was requested to be mapped is aligned to 4 KiB,
919 * though, which means that the resulting translation is incorrect.
920 * The only way to prevent this is by using a finer granularity.
921 */
922 unsigned long long align_check;
923
924 align_check = mm->base_pa | (unsigned long long)mm->size;
925
926 /*
927 * Assume it is always aligned to level 3. There's no need to check that
928 * level because its block size is PAGE_SIZE. The checks to verify that
929 * the addresses and size are aligned to PAGE_SIZE are inside
930 * mmap_add_region.
931 */
932 for (unsigned int level = ctx->base_level; level <= 2U; ++level) {
933
934 if ((align_check & XLAT_BLOCK_MASK(level)) != 0U) {
935 continue;
936 }
937
938 mm->base_va = round_up(mm->base_va, XLAT_BLOCK_SIZE(level));
939 return;
940 }
941 }
942
mmap_add_region_alloc_va_ctx(xlat_ctx_t * ctx,mmap_region_t * mm)943 void mmap_add_region_alloc_va_ctx(xlat_ctx_t *ctx, mmap_region_t *mm)
944 {
945 mm->base_va = ctx->max_va + 1UL;
946
947 assert(mm->size > 0U);
948
949 mmap_alloc_va_align_ctx(ctx, mm);
950
951 /* Detect overflows. More checks are done in mmap_add_region_check(). */
952 assert(mm->base_va > ctx->max_va);
953
954 mmap_add_region_ctx(ctx, mm);
955 }
956
mmap_add_ctx(xlat_ctx_t * ctx,const mmap_region_t * mm)957 void mmap_add_ctx(xlat_ctx_t *ctx, const mmap_region_t *mm)
958 {
959 const mmap_region_t *mm_cursor = mm;
960
961 while (mm_cursor->granularity != 0U) {
962 mmap_add_region_ctx(ctx, mm_cursor);
963 mm_cursor++;
964 }
965 }
966
967 #if PLAT_XLAT_TABLES_DYNAMIC
968
mmap_add_dynamic_region_ctx(xlat_ctx_t * ctx,mmap_region_t * mm)969 int mmap_add_dynamic_region_ctx(xlat_ctx_t *ctx, mmap_region_t *mm)
970 {
971 mmap_region_t *mm_cursor = ctx->mmap;
972 const mmap_region_t *mm_last = mm_cursor + ctx->mmap_num;
973 unsigned long long end_pa = mm->base_pa + mm->size - 1U;
974 uintptr_t end_va = mm->base_va + mm->size - 1U;
975 int ret;
976
977 /* Nothing to do */
978 if (mm->size == 0U)
979 return 0;
980
981 /* Now this region is a dynamic one */
982 mm->attr |= MT_DYNAMIC;
983
984 ret = mmap_add_region_check(ctx, mm);
985 if (ret != 0)
986 return ret;
987
988 /*
989 * Find the adequate entry in the mmap array in the same way done for
990 * static regions in mmap_add_region_ctx().
991 */
992
993 while (((mm_cursor->base_va + mm_cursor->size - 1U) < end_va)
994 && (mm_cursor->size != 0U)) {
995 ++mm_cursor;
996 }
997
998 while (((mm_cursor->base_va + mm_cursor->size - 1U) == end_va) &&
999 (mm_cursor->size != 0U) && (mm_cursor->size < mm->size)) {
1000 ++mm_cursor;
1001 }
1002
1003 /* Make room for new region by moving other regions up by one place */
1004 (void)memmove(mm_cursor + 1U, mm_cursor,
1005 (uintptr_t)mm_last - (uintptr_t)mm_cursor);
1006
1007 /*
1008 * Check we haven't lost the empty sentinel from the end of the array.
1009 * This shouldn't happen as we have checked in mmap_add_region_check
1010 * that there is free space.
1011 */
1012 assert(mm_last->size == 0U);
1013
1014 *mm_cursor = *mm;
1015
1016 /*
1017 * Update the translation tables if the xlat tables are initialized. If
1018 * not, this region will be mapped when they are initialized.
1019 */
1020 if (ctx->initialized) {
1021 end_va = xlat_tables_map_region(ctx, mm_cursor,
1022 0U, ctx->base_table, ctx->base_table_entries,
1023 ctx->base_level);
1024 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
1025 xlat_clean_dcache_range((uintptr_t)ctx->base_table,
1026 ctx->base_table_entries * sizeof(uint64_t));
1027 #endif
1028 /* Failed to map, remove mmap entry, unmap and return error. */
1029 if (end_va != (mm_cursor->base_va + mm_cursor->size - 1U)) {
1030 (void)memmove(mm_cursor, mm_cursor + 1U,
1031 (uintptr_t)mm_last - (uintptr_t)mm_cursor);
1032
1033 /*
1034 * Check if the mapping function actually managed to map
1035 * anything. If not, just return now.
1036 */
1037 if (mm->base_va >= end_va)
1038 return -ENOMEM;
1039
1040 /*
1041 * Something went wrong after mapping some table
1042 * entries, undo every change done up to this point.
1043 */
1044 mmap_region_t unmap_mm = {
1045 .base_pa = 0U,
1046 .base_va = mm->base_va,
1047 .size = end_va - mm->base_va,
1048 .attr = 0U
1049 };
1050 xlat_tables_unmap_region(ctx, &unmap_mm, 0U,
1051 ctx->base_table, ctx->base_table_entries,
1052 ctx->base_level);
1053 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
1054 xlat_clean_dcache_range((uintptr_t)ctx->base_table,
1055 ctx->base_table_entries * sizeof(uint64_t));
1056 #endif
1057 return -ENOMEM;
1058 }
1059
1060 /*
1061 * Make sure that all entries are written to the memory. There
1062 * is no need to invalidate entries when mapping dynamic regions
1063 * because new table/block/page descriptors only replace old
1064 * invalid descriptors, that aren't TLB cached.
1065 */
1066 dsbishst();
1067 }
1068
1069 if (end_pa > ctx->max_pa)
1070 ctx->max_pa = end_pa;
1071 if (end_va > ctx->max_va)
1072 ctx->max_va = end_va;
1073
1074 return 0;
1075 }
1076
mmap_add_dynamic_region_alloc_va_ctx(xlat_ctx_t * ctx,mmap_region_t * mm)1077 int mmap_add_dynamic_region_alloc_va_ctx(xlat_ctx_t *ctx, mmap_region_t *mm)
1078 {
1079 mm->base_va = ctx->max_va + 1UL;
1080
1081 if (mm->size == 0U)
1082 return 0;
1083
1084 mmap_alloc_va_align_ctx(ctx, mm);
1085
1086 /* Detect overflows. More checks are done in mmap_add_region_check(). */
1087 if (mm->base_va < ctx->max_va) {
1088 return -ENOMEM;
1089 }
1090
1091 return mmap_add_dynamic_region_ctx(ctx, mm);
1092 }
1093
1094 /*
1095 * Removes the region with given base Virtual Address and size from the given
1096 * context.
1097 *
1098 * Returns:
1099 * 0: Success.
1100 * EINVAL: Invalid values were used as arguments (region not found).
1101 * EPERM: Tried to remove a static region.
1102 */
mmap_remove_dynamic_region_ctx(xlat_ctx_t * ctx,uintptr_t base_va,size_t size)1103 int mmap_remove_dynamic_region_ctx(xlat_ctx_t *ctx, uintptr_t base_va,
1104 size_t size)
1105 {
1106 mmap_region_t *mm = ctx->mmap;
1107 const mmap_region_t *mm_last = mm + ctx->mmap_num;
1108 int update_max_va_needed = 0;
1109 int update_max_pa_needed = 0;
1110
1111 /* Check sanity of mmap array. */
1112 assert(mm[ctx->mmap_num].size == 0U);
1113
1114 while (mm->size != 0U) {
1115 if ((mm->base_va == base_va) && (mm->size == size))
1116 break;
1117 ++mm;
1118 }
1119
1120 /* Check that the region was found */
1121 if (mm->size == 0U)
1122 return -EINVAL;
1123
1124 /* If the region is static it can't be removed */
1125 if ((mm->attr & MT_DYNAMIC) == 0U)
1126 return -EPERM;
1127
1128 /* Check if this region is using the top VAs or PAs. */
1129 if ((mm->base_va + mm->size - 1U) == ctx->max_va)
1130 update_max_va_needed = 1;
1131 if ((mm->base_pa + mm->size - 1U) == ctx->max_pa)
1132 update_max_pa_needed = 1;
1133
1134 /* Update the translation tables if needed */
1135 if (ctx->initialized) {
1136 xlat_tables_unmap_region(ctx, mm, 0U, ctx->base_table,
1137 ctx->base_table_entries,
1138 ctx->base_level);
1139 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
1140 xlat_clean_dcache_range((uintptr_t)ctx->base_table,
1141 ctx->base_table_entries * sizeof(uint64_t));
1142 #endif
1143 xlat_arch_tlbi_va_sync();
1144 }
1145
1146 /* Remove this region by moving the rest down by one place. */
1147 (void)memmove(mm, mm + 1U, (uintptr_t)mm_last - (uintptr_t)mm);
1148
1149 /* Check if we need to update the max VAs and PAs */
1150 if (update_max_va_needed == 1) {
1151 ctx->max_va = 0U;
1152 mm = ctx->mmap;
1153 while (mm->size != 0U) {
1154 if ((mm->base_va + mm->size - 1U) > ctx->max_va)
1155 ctx->max_va = mm->base_va + mm->size - 1U;
1156 ++mm;
1157 }
1158 }
1159
1160 if (update_max_pa_needed == 1) {
1161 ctx->max_pa = 0U;
1162 mm = ctx->mmap;
1163 while (mm->size != 0U) {
1164 if ((mm->base_pa + mm->size - 1U) > ctx->max_pa)
1165 ctx->max_pa = mm->base_pa + mm->size - 1U;
1166 ++mm;
1167 }
1168 }
1169
1170 return 0;
1171 }
1172
xlat_setup_dynamic_ctx(xlat_ctx_t * ctx,unsigned long long pa_max,uintptr_t va_max,struct mmap_region * mmap,unsigned int mmap_num,uint64_t ** tables,unsigned int tables_num,uint64_t * base_table,int xlat_regime,int * mapped_regions)1173 void xlat_setup_dynamic_ctx(xlat_ctx_t *ctx, unsigned long long pa_max,
1174 uintptr_t va_max, struct mmap_region *mmap,
1175 unsigned int mmap_num, uint64_t **tables,
1176 unsigned int tables_num, uint64_t *base_table,
1177 int xlat_regime, int *mapped_regions)
1178 {
1179 ctx->xlat_regime = xlat_regime;
1180
1181 ctx->pa_max_address = pa_max;
1182 ctx->va_max_address = va_max;
1183
1184 ctx->mmap = mmap;
1185 ctx->mmap_num = mmap_num;
1186 memset(ctx->mmap, 0, sizeof(struct mmap_region) * mmap_num);
1187
1188 ctx->tables = (void *) tables;
1189 ctx->tables_num = tables_num;
1190
1191 uintptr_t va_space_size = va_max + 1;
1192 ctx->base_level = GET_XLAT_TABLE_LEVEL_BASE(va_space_size);
1193 ctx->base_table = base_table;
1194 ctx->base_table_entries = GET_NUM_BASE_LEVEL_ENTRIES(va_space_size);
1195
1196 ctx->tables_mapped_regions = mapped_regions;
1197
1198 ctx->max_pa = 0;
1199 ctx->max_va = 0;
1200 ctx->initialized = 0;
1201 }
1202
1203 #endif /* PLAT_XLAT_TABLES_DYNAMIC */
1204
init_xlat_tables_ctx(xlat_ctx_t * ctx)1205 void __init init_xlat_tables_ctx(xlat_ctx_t *ctx)
1206 {
1207 assert(ctx != NULL);
1208 assert(!ctx->initialized);
1209 assert((ctx->xlat_regime == EL3_REGIME) ||
1210 (ctx->xlat_regime == EL2_REGIME) ||
1211 (ctx->xlat_regime == EL1_EL0_REGIME));
1212 assert(!is_mmu_enabled_ctx(ctx));
1213
1214 mmap_region_t *mm = ctx->mmap;
1215
1216 assert(ctx->va_max_address >=
1217 (xlat_get_min_virt_addr_space_size() - 1U));
1218 assert(ctx->va_max_address <= (MAX_VIRT_ADDR_SPACE_SIZE - 1U));
1219 assert(IS_POWER_OF_TWO(ctx->va_max_address + 1U));
1220
1221 xlat_mmap_print(mm);
1222
1223 /* All tables must be zeroed before mapping any region. */
1224 zeromem(ctx->base_table, ctx->base_table_entries * sizeof(uint64_t));
1225
1226 #if PLAT_XLAT_TABLES_DYNAMIC
1227 zeromem(ctx->tables_mapped_regions, ctx->tables_num * sizeof(uint32_t));
1228 #endif
1229 for (int i = 0; i < ctx->tables_num; i++) {
1230 zeromem(ctx->tables[i], XLAT_TABLE_ENTRIES * sizeof(uint64_t));
1231 }
1232
1233 while (mm->size != 0U) {
1234 uintptr_t end_va = xlat_tables_map_region(ctx, mm, 0U,
1235 ctx->base_table, ctx->base_table_entries,
1236 ctx->base_level);
1237 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
1238 xlat_clean_dcache_range((uintptr_t)ctx->base_table,
1239 ctx->base_table_entries * sizeof(uint64_t));
1240 #endif
1241 if (end_va != (mm->base_va + mm->size - 1U)) {
1242 ERROR("Not enough memory to map region:\n"
1243 " VA:0x%lx PA:0x%llx size:0x%zx attr:0x%x\n",
1244 mm->base_va, mm->base_pa, mm->size, mm->attr);
1245 panic();
1246 }
1247
1248 mm++;
1249 }
1250
1251 assert(ctx->pa_max_address <= xlat_arch_get_max_supported_pa());
1252 assert(ctx->max_va <= ctx->va_max_address);
1253 assert(ctx->max_pa <= ctx->pa_max_address);
1254
1255 ctx->initialized = true;
1256
1257 xlat_tables_print(ctx);
1258 }
1259