1 /* 2 * (C) Copyright 2013 3 * David Feng <fenghua@phytium.com.cn> 4 * 5 * SPDX-License-Identifier: GPL-2.0+ 6 */ 7 8 #include <common.h> 9 #include <asm/system.h> 10 #include <asm/armv8/mmu.h> 11 12 DECLARE_GLOBAL_DATA_PTR; 13 14 #ifndef CONFIG_SYS_DCACHE_OFF 15 16 #ifdef CONFIG_SYS_FULL_VA 17 static void set_ptl1_entry(u64 index, u64 ptl2_entry) 18 { 19 u64 *pgd = (u64 *)gd->arch.tlb_addr; 20 u64 value; 21 22 value = ptl2_entry | PTL1_TYPE_TABLE; 23 pgd[index] = value; 24 } 25 26 static void set_ptl2_block(u64 ptl1, u64 bfn, u64 address, u64 memory_attrs) 27 { 28 u64 *pmd = (u64 *)ptl1; 29 u64 value; 30 31 value = address | PTL2_TYPE_BLOCK | PTL2_BLOCK_AF; 32 value |= memory_attrs; 33 pmd[bfn] = value; 34 } 35 36 static struct mm_region mem_map[] = CONFIG_SYS_MEM_MAP; 37 38 #define PTL1_ENTRIES CONFIG_SYS_PTL1_ENTRIES 39 #define PTL2_ENTRIES CONFIG_SYS_PTL2_ENTRIES 40 41 static u64 get_tcr(int el, u64 *pips, u64 *pva_bits) 42 { 43 u64 max_addr = 0; 44 u64 ips, va_bits; 45 u64 tcr; 46 int i; 47 48 /* Find the largest address we need to support */ 49 for (i = 0; i < ARRAY_SIZE(mem_map); i++) 50 max_addr = max(max_addr, mem_map[i].base + mem_map[i].size); 51 52 /* Calculate the maximum physical (and thus virtual) address */ 53 if (max_addr > (1ULL << 44)) { 54 ips = 5; 55 va_bits = 48; 56 } else if (max_addr > (1ULL << 42)) { 57 ips = 4; 58 va_bits = 44; 59 } else if (max_addr > (1ULL << 40)) { 60 ips = 3; 61 va_bits = 42; 62 } else if (max_addr > (1ULL << 36)) { 63 ips = 2; 64 va_bits = 40; 65 } else if (max_addr > (1ULL << 32)) { 66 ips = 1; 67 va_bits = 36; 68 } else { 69 ips = 0; 70 va_bits = 32; 71 } 72 73 if (el == 1) { 74 tcr = TCR_EL1_RSVD | (ips << 32); 75 } else if (el == 2) { 76 tcr = TCR_EL2_RSVD | (ips << 16); 77 } else { 78 tcr = TCR_EL3_RSVD | (ips << 16); 79 } 80 81 /* PTWs cacheable, inner/outer WBWA and inner shareable */ 82 tcr |= TCR_TG0_64K | TCR_SHARED_INNER | TCR_ORGN_WBWA | TCR_IRGN_WBWA; 83 tcr |= TCR_T0SZ(VA_BITS); 84 85 if (pips) 86 *pips = ips; 87 if (pva_bits) 88 *pva_bits = va_bits; 89 90 return tcr; 91 } 92 93 static void setup_pgtables(void) 94 { 95 int l1_e, l2_e; 96 unsigned long pmd = 0; 97 unsigned long address; 98 99 /* Setup the PMD pointers */ 100 for (l1_e = 0; l1_e < CONFIG_SYS_MEM_MAP_SIZE; l1_e++) { 101 gd->arch.pmd_addr[l1_e] = gd->arch.tlb_addr + 102 PTL1_ENTRIES * sizeof(u64); 103 gd->arch.pmd_addr[l1_e] += PTL2_ENTRIES * sizeof(u64) * l1_e; 104 gd->arch.pmd_addr[l1_e] = ALIGN(gd->arch.pmd_addr[l1_e], 105 0x10000UL); 106 } 107 108 /* Setup the page tables */ 109 for (l1_e = 0; l1_e < PTL1_ENTRIES; l1_e++) { 110 if (mem_map[pmd].base == 111 (uintptr_t)l1_e << PTL2_BITS) { 112 set_ptl1_entry(l1_e, gd->arch.pmd_addr[pmd]); 113 114 for (l2_e = 0; l2_e < PTL2_ENTRIES; l2_e++) { 115 address = mem_map[pmd].base 116 + (uintptr_t)l2_e * BLOCK_SIZE; 117 set_ptl2_block(gd->arch.pmd_addr[pmd], l2_e, 118 address, mem_map[pmd].attrs); 119 } 120 121 pmd++; 122 } else { 123 set_ptl1_entry(l1_e, 0); 124 } 125 } 126 } 127 128 #else 129 130 inline void set_pgtable_section(u64 *page_table, u64 index, u64 section, 131 u64 memory_type, u64 attribute) 132 { 133 u64 value; 134 135 value = section | PMD_TYPE_SECT | PMD_SECT_AF; 136 value |= PMD_ATTRINDX(memory_type); 137 value |= attribute; 138 page_table[index] = value; 139 } 140 141 inline void set_pgtable_table(u64 *page_table, u64 index, u64 *table_addr) 142 { 143 u64 value; 144 145 value = (u64)table_addr | PMD_TYPE_TABLE; 146 page_table[index] = value; 147 } 148 #endif 149 150 /* to activate the MMU we need to set up virtual memory */ 151 __weak void mmu_setup(void) 152 { 153 #ifndef CONFIG_SYS_FULL_VA 154 bd_t *bd = gd->bd; 155 u64 *page_table = (u64 *)gd->arch.tlb_addr, i, j; 156 #endif 157 int el; 158 159 #ifdef CONFIG_SYS_FULL_VA 160 unsigned long coreid = read_mpidr() & CONFIG_COREID_MASK; 161 162 /* Set up page tables only on BSP */ 163 if (coreid == BSP_COREID) 164 setup_pgtables(); 165 166 el = current_el(); 167 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, get_tcr(el, NULL, NULL), 168 MEMORY_ATTRIBUTES); 169 #else 170 /* Setup an identity-mapping for all spaces */ 171 for (i = 0; i < (PGTABLE_SIZE >> 3); i++) { 172 set_pgtable_section(page_table, i, i << SECTION_SHIFT, 173 MT_DEVICE_NGNRNE, PMD_SECT_NON_SHARE); 174 } 175 176 /* Setup an identity-mapping for all RAM space */ 177 for (i = 0; i < CONFIG_NR_DRAM_BANKS; i++) { 178 ulong start = bd->bi_dram[i].start; 179 ulong end = bd->bi_dram[i].start + bd->bi_dram[i].size; 180 for (j = start >> SECTION_SHIFT; 181 j < end >> SECTION_SHIFT; j++) { 182 set_pgtable_section(page_table, j, j << SECTION_SHIFT, 183 MT_NORMAL, PMD_SECT_NON_SHARE); 184 } 185 } 186 187 /* load TTBR0 */ 188 el = current_el(); 189 if (el == 1) { 190 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, 191 TCR_EL1_RSVD | TCR_FLAGS | TCR_EL1_IPS_BITS, 192 MEMORY_ATTRIBUTES); 193 } else if (el == 2) { 194 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, 195 TCR_EL2_RSVD | TCR_FLAGS | TCR_EL2_IPS_BITS, 196 MEMORY_ATTRIBUTES); 197 } else { 198 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, 199 TCR_EL3_RSVD | TCR_FLAGS | TCR_EL3_IPS_BITS, 200 MEMORY_ATTRIBUTES); 201 } 202 #endif 203 204 /* enable the mmu */ 205 set_sctlr(get_sctlr() | CR_M); 206 } 207 208 /* 209 * Performs a invalidation of the entire data cache at all levels 210 */ 211 void invalidate_dcache_all(void) 212 { 213 __asm_invalidate_dcache_all(); 214 } 215 216 /* 217 * Performs a clean & invalidation of the entire data cache at all levels. 218 * This function needs to be inline to avoid using stack. 219 * __asm_flush_l3_cache return status of timeout 220 */ 221 inline void flush_dcache_all(void) 222 { 223 int ret; 224 225 __asm_flush_dcache_all(); 226 ret = __asm_flush_l3_cache(); 227 if (ret) 228 debug("flushing dcache returns 0x%x\n", ret); 229 else 230 debug("flushing dcache successfully.\n"); 231 } 232 233 /* 234 * Invalidates range in all levels of D-cache/unified cache 235 */ 236 void invalidate_dcache_range(unsigned long start, unsigned long stop) 237 { 238 __asm_flush_dcache_range(start, stop); 239 } 240 241 /* 242 * Flush range(clean & invalidate) from all levels of D-cache/unified cache 243 */ 244 void flush_dcache_range(unsigned long start, unsigned long stop) 245 { 246 __asm_flush_dcache_range(start, stop); 247 } 248 249 void dcache_enable(void) 250 { 251 /* The data cache is not active unless the mmu is enabled */ 252 if (!(get_sctlr() & CR_M)) { 253 invalidate_dcache_all(); 254 __asm_invalidate_tlb_all(); 255 mmu_setup(); 256 } 257 258 set_sctlr(get_sctlr() | CR_C); 259 } 260 261 void dcache_disable(void) 262 { 263 uint32_t sctlr; 264 265 sctlr = get_sctlr(); 266 267 /* if cache isn't enabled no need to disable */ 268 if (!(sctlr & CR_C)) 269 return; 270 271 set_sctlr(sctlr & ~(CR_C|CR_M)); 272 273 flush_dcache_all(); 274 __asm_invalidate_tlb_all(); 275 } 276 277 int dcache_status(void) 278 { 279 return (get_sctlr() & CR_C) != 0; 280 } 281 282 u64 *__weak arch_get_page_table(void) { 283 puts("No page table offset defined\n"); 284 285 return NULL; 286 } 287 288 #ifndef CONFIG_SYS_FULL_VA 289 void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size, 290 enum dcache_option option) 291 { 292 u64 *page_table = arch_get_page_table(); 293 u64 upto, end; 294 295 if (page_table == NULL) 296 return; 297 298 end = ALIGN(start + size, (1 << MMU_SECTION_SHIFT)) >> 299 MMU_SECTION_SHIFT; 300 start = start >> MMU_SECTION_SHIFT; 301 for (upto = start; upto < end; upto++) { 302 page_table[upto] &= ~PMD_ATTRINDX_MASK; 303 page_table[upto] |= PMD_ATTRINDX(option); 304 } 305 asm volatile("dsb sy"); 306 __asm_invalidate_tlb_all(); 307 asm volatile("dsb sy"); 308 asm volatile("isb"); 309 start = start << MMU_SECTION_SHIFT; 310 end = end << MMU_SECTION_SHIFT; 311 flush_dcache_range(start, end); 312 asm volatile("dsb sy"); 313 } 314 #endif 315 316 #else /* CONFIG_SYS_DCACHE_OFF */ 317 318 void invalidate_dcache_all(void) 319 { 320 } 321 322 void flush_dcache_all(void) 323 { 324 } 325 326 void dcache_enable(void) 327 { 328 } 329 330 void dcache_disable(void) 331 { 332 } 333 334 int dcache_status(void) 335 { 336 return 0; 337 } 338 339 void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size, 340 enum dcache_option option) 341 { 342 } 343 344 #endif /* CONFIG_SYS_DCACHE_OFF */ 345 346 #ifndef CONFIG_SYS_ICACHE_OFF 347 348 void icache_enable(void) 349 { 350 __asm_invalidate_icache_all(); 351 set_sctlr(get_sctlr() | CR_I); 352 } 353 354 void icache_disable(void) 355 { 356 set_sctlr(get_sctlr() & ~CR_I); 357 } 358 359 int icache_status(void) 360 { 361 return (get_sctlr() & CR_I) != 0; 362 } 363 364 void invalidate_icache_all(void) 365 { 366 __asm_invalidate_icache_all(); 367 } 368 369 #else /* CONFIG_SYS_ICACHE_OFF */ 370 371 void icache_enable(void) 372 { 373 } 374 375 void icache_disable(void) 376 { 377 } 378 379 int icache_status(void) 380 { 381 return 0; 382 } 383 384 void invalidate_icache_all(void) 385 { 386 } 387 388 #endif /* CONFIG_SYS_ICACHE_OFF */ 389 390 /* 391 * Enable dCache & iCache, whether cache is actually enabled 392 * depend on CONFIG_SYS_DCACHE_OFF and CONFIG_SYS_ICACHE_OFF 393 */ 394 void __weak enable_caches(void) 395 { 396 icache_enable(); 397 dcache_enable(); 398 } 399