1 /* 2 * Copyright (c) 2016-2017, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <arch.h> 8 #include <arch_helpers.h> 9 #include <assert.h> 10 #include <cassert.h> 11 #include <common_def.h> 12 #include <debug.h> 13 #include <platform_def.h> 14 #include <string.h> 15 #include <types.h> 16 #include <utils.h> 17 #include <xlat_tables.h> 18 #include "xlat_tables_private.h" 19 20 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 21 #define LVL0_SPACER "" 22 #define LVL1_SPACER " " 23 #define LVL2_SPACER " " 24 #define LVL3_SPACER " " 25 #define get_level_spacer(level) \ 26 (((level) == 0) ? LVL0_SPACER : \ 27 (((level) == 1) ? LVL1_SPACER : \ 28 (((level) == 2) ? LVL2_SPACER : LVL3_SPACER))) 29 #define debug_print(...) tf_printf(__VA_ARGS__) 30 #else 31 #define debug_print(...) ((void)0) 32 #endif 33 34 #define UNSET_DESC ~0ull 35 36 static uint64_t xlat_tables[MAX_XLAT_TABLES][XLAT_TABLE_ENTRIES] 37 __aligned(XLAT_TABLE_SIZE) __section("xlat_table"); 38 39 static unsigned next_xlat; 40 static unsigned long long xlat_max_pa; 41 static uintptr_t xlat_max_va; 42 43 /* 44 * Array of all memory regions stored in order of ascending base address. 45 * The list is terminated by the first entry with size == 0. 46 */ 47 static mmap_region_t mmap[MAX_MMAP_REGIONS + 1]; 48 49 50 void print_mmap(void) 51 { 52 #if LOG_LEVEL >= LOG_LEVEL_VERBOSE 53 debug_print("mmap:\n"); 54 mmap_region_t *mm = mmap; 55 while (mm->size) { 56 debug_print(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", 57 (void *)mm->base_va, mm->base_pa, 58 mm->size, mm->attr); 59 ++mm; 60 }; 61 debug_print("\n"); 62 #endif 63 } 64 65 void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, 66 size_t size, mmap_attr_t attr) 67 { 68 mmap_region_t *mm = mmap; 69 mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1; 70 unsigned long long end_pa = base_pa + size - 1; 71 uintptr_t end_va = base_va + size - 1; 72 73 assert(IS_PAGE_ALIGNED(base_pa)); 74 assert(IS_PAGE_ALIGNED(base_va)); 75 assert(IS_PAGE_ALIGNED(size)); 76 77 if (!size) 78 return; 79 80 assert(base_pa < end_pa); /* Check for overflows */ 81 assert(base_va < end_va); 82 83 assert((base_va + (uintptr_t)size - (uintptr_t)1) <= 84 (PLAT_VIRT_ADDR_SPACE_SIZE - 1)); 85 assert((base_pa + (unsigned long long)size - 1ULL) <= 86 (PLAT_PHY_ADDR_SPACE_SIZE - 1)); 87 88 #if ENABLE_ASSERTIONS 89 90 /* Check for PAs and VAs overlaps with all other regions */ 91 for (mm = mmap; mm->size; ++mm) { 92 93 uintptr_t mm_end_va = mm->base_va + mm->size - 1; 94 95 /* 96 * Check if one of the regions is completely inside the other 97 * one. 98 */ 99 int fully_overlapped_va = 100 ((base_va >= mm->base_va) && (end_va <= mm_end_va)) || 101 ((mm->base_va >= base_va) && (mm_end_va <= end_va)); 102 103 /* 104 * Full VA overlaps are only allowed if both regions are 105 * identity mapped (zero offset) or have the same VA to PA 106 * offset. Also, make sure that it's not the exact same area. 107 */ 108 if (fully_overlapped_va) { 109 assert((mm->base_va - mm->base_pa) == 110 (base_va - base_pa)); 111 assert((base_va != mm->base_va) || (size != mm->size)); 112 } else { 113 /* 114 * If the regions do not have fully overlapping VAs, 115 * then they must have fully separated VAs and PAs. 116 * Partial overlaps are not allowed 117 */ 118 119 unsigned long long mm_end_pa = 120 mm->base_pa + mm->size - 1; 121 122 int separated_pa = 123 (end_pa < mm->base_pa) || (base_pa > mm_end_pa); 124 int separated_va = 125 (end_va < mm->base_va) || (base_va > mm_end_va); 126 127 assert(separated_va && separated_pa); 128 } 129 } 130 131 mm = mmap; /* Restore pointer to the start of the array */ 132 133 #endif /* ENABLE_ASSERTIONS */ 134 135 /* Find correct place in mmap to insert new region */ 136 while (mm->base_va < base_va && mm->size) 137 ++mm; 138 139 /* 140 * If a section is contained inside another one with the same base 141 * address, it must be placed after the one it is contained in: 142 * 143 * 1st |-----------------------| 144 * 2nd |------------| 145 * 3rd |------| 146 * 147 * This is required for mmap_region_attr() to get the attributes of the 148 * small region correctly. 149 */ 150 while ((mm->base_va == base_va) && (mm->size > size)) 151 ++mm; 152 153 /* Make room for new region by moving other regions up by one place */ 154 memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm); 155 156 /* Check we haven't lost the empty sentinal from the end of the array */ 157 assert(mm_last->size == 0); 158 159 mm->base_pa = base_pa; 160 mm->base_va = base_va; 161 mm->size = size; 162 mm->attr = attr; 163 164 if (end_pa > xlat_max_pa) 165 xlat_max_pa = end_pa; 166 if (end_va > xlat_max_va) 167 xlat_max_va = end_va; 168 } 169 170 void mmap_add(const mmap_region_t *mm) 171 { 172 while (mm->size) { 173 mmap_add_region(mm->base_pa, mm->base_va, mm->size, mm->attr); 174 ++mm; 175 } 176 } 177 178 static uint64_t mmap_desc(mmap_attr_t attr, unsigned long long addr_pa, 179 int level) 180 { 181 uint64_t desc; 182 int mem_type; 183 184 /* Make sure that the granularity is fine enough to map this address. */ 185 assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0); 186 187 desc = addr_pa; 188 /* 189 * There are different translation table descriptors for level 3 and the 190 * rest. 191 */ 192 desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; 193 desc |= (attr & MT_NS) ? LOWER_ATTRS(NS) : 0; 194 desc |= (attr & MT_RW) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); 195 desc |= LOWER_ATTRS(ACCESS_FLAG); 196 197 /* 198 * Deduce shareability domain and executability of the memory region 199 * from the memory type. 200 * 201 * Data accesses to device memory and non-cacheable normal memory are 202 * coherent for all observers in the system, and correspondingly are 203 * always treated as being Outer Shareable. Therefore, for these 2 types 204 * of memory, it is not strictly needed to set the shareability field 205 * in the translation tables. 206 */ 207 mem_type = MT_TYPE(attr); 208 if (mem_type == MT_DEVICE) { 209 desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); 210 /* 211 * Always map device memory as execute-never. 212 * This is to avoid the possibility of a speculative instruction 213 * fetch, which could be an issue if this memory region 214 * corresponds to a read-sensitive peripheral. 215 */ 216 desc |= UPPER_ATTRS(XN); 217 } else { /* Normal memory */ 218 /* 219 * Always map read-write normal memory as execute-never. 220 * (Trusted Firmware doesn't self-modify its code, therefore 221 * R/W memory is reserved for data storage, which must not be 222 * executable.) 223 * Note that setting the XN bit here is for consistency only. 224 * The enable_mmu_elx() function sets the SCTLR_EL3.WXN bit, 225 * which makes any writable memory region to be treated as 226 * execute-never, regardless of the value of the XN bit in the 227 * translation table. 228 * 229 * For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER 230 * attribute to figure out the value of the XN bit. 231 */ 232 if ((attr & MT_RW) || (attr & MT_EXECUTE_NEVER)) 233 desc |= UPPER_ATTRS(XN); 234 235 if (mem_type == MT_MEMORY) { 236 desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); 237 } else { 238 assert(mem_type == MT_NON_CACHEABLE); 239 desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); 240 } 241 } 242 243 debug_print((mem_type == MT_MEMORY) ? "MEM" : 244 ((mem_type == MT_NON_CACHEABLE) ? "NC" : "DEV")); 245 debug_print(attr & MT_RW ? "-RW" : "-RO"); 246 debug_print(attr & MT_NS ? "-NS" : "-S"); 247 debug_print(attr & MT_EXECUTE_NEVER ? "-XN" : "-EXEC"); 248 return desc; 249 } 250 251 /* 252 * Returns attributes of area at `base_va` with size `size`. It returns the 253 * attributes of the innermost region that contains it. If there are partial 254 * overlaps, it returns -1, as a smaller size is needed. 255 */ 256 static mmap_attr_t mmap_region_attr(mmap_region_t *mm, uintptr_t base_va, 257 size_t size) 258 { 259 /* Don't assume that the area is contained in the first region */ 260 mmap_attr_t attr = -1; 261 262 /* 263 * Get attributes from last (innermost) region that contains the 264 * requested area. Don't stop as soon as one region doesn't contain it 265 * because there may be other internal regions that contain this area: 266 * 267 * |-----------------------------1-----------------------------| 268 * |----2----| |-------3-------| |----5----| 269 * |--4--| 270 * 271 * |---| <- Area we want the attributes of. 272 * 273 * In this example, the area is contained in regions 1, 3 and 4 but not 274 * in region 2. The loop shouldn't stop at region 2 as inner regions 275 * have priority over outer regions, it should stop at region 5. 276 */ 277 for (;; ++mm) { 278 279 if (!mm->size) 280 return attr; /* Reached end of list */ 281 282 if (mm->base_va > base_va + size - 1) 283 return attr; /* Next region is after area so end */ 284 285 if (mm->base_va + mm->size - 1 < base_va) 286 continue; /* Next region has already been overtaken */ 287 288 if (mm->attr == attr) 289 continue; /* Region doesn't override attribs so skip */ 290 291 if (mm->base_va > base_va || 292 mm->base_va + mm->size - 1 < base_va + size - 1) 293 return -1; /* Region doesn't fully cover our area */ 294 295 attr = mm->attr; 296 } 297 } 298 299 static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm, 300 uintptr_t base_va, 301 uint64_t *table, 302 int level) 303 { 304 assert(level >= XLAT_TABLE_LEVEL_MIN && level <= XLAT_TABLE_LEVEL_MAX); 305 306 unsigned int level_size_shift = 307 L0_XLAT_ADDRESS_SHIFT - level * XLAT_TABLE_ENTRIES_SHIFT; 308 u_register_t level_size = (u_register_t)1 << level_size_shift; 309 u_register_t level_index_mask = 310 ((u_register_t)XLAT_TABLE_ENTRIES_MASK) << level_size_shift; 311 312 debug_print("New xlat table:\n"); 313 314 do { 315 uint64_t desc = UNSET_DESC; 316 317 if (!mm->size) { 318 /* Done mapping regions; finish zeroing the table */ 319 desc = INVALID_DESC; 320 } else if (mm->base_va + mm->size - 1 < base_va) { 321 /* This area is after the region so get next region */ 322 ++mm; 323 continue; 324 } 325 326 debug_print("%s VA:%p size:0x%llx ", get_level_spacer(level), 327 (void *)base_va, (unsigned long long)level_size); 328 329 if (mm->base_va > base_va + level_size - 1) { 330 /* Next region is after this area. Nothing to map yet */ 331 desc = INVALID_DESC; 332 /* Make sure that the current level allows block descriptors */ 333 } else if (level >= XLAT_BLOCK_LEVEL_MIN) { 334 /* 335 * Try to get attributes of this area. It will fail if 336 * there are partially overlapping regions. On success, 337 * it will return the innermost region's attributes. 338 */ 339 mmap_attr_t attr = mmap_region_attr(mm, base_va, 340 level_size); 341 if (attr >= 0) { 342 desc = mmap_desc(attr, 343 base_va - mm->base_va + mm->base_pa, 344 level); 345 } 346 } 347 348 if (desc == UNSET_DESC) { 349 /* Area not covered by a region so need finer table */ 350 uint64_t *new_table = xlat_tables[next_xlat++]; 351 assert(next_xlat <= MAX_XLAT_TABLES); 352 desc = TABLE_DESC | (uintptr_t)new_table; 353 354 /* Recurse to fill in new table */ 355 mm = init_xlation_table_inner(mm, base_va, 356 new_table, level+1); 357 } 358 359 debug_print("\n"); 360 361 *table++ = desc; 362 base_va += level_size; 363 } while ((base_va & level_index_mask) && 364 (base_va - 1 < PLAT_VIRT_ADDR_SPACE_SIZE - 1)); 365 366 return mm; 367 } 368 369 void init_xlation_table(uintptr_t base_va, uint64_t *table, 370 int level, uintptr_t *max_va, 371 unsigned long long *max_pa) 372 { 373 374 init_xlation_table_inner(mmap, base_va, table, level); 375 *max_va = xlat_max_va; 376 *max_pa = xlat_max_pa; 377 } 378