1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Based on arch/arm/mm/proc.S 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * Author: Catalin Marinas <catalin.marinas@arm.com> 8 */ 9 10#include <linux/init.h> 11#include <linux/linkage.h> 12#include <linux/pgtable.h> 13#include <asm/assembler.h> 14#include <asm/asm-offsets.h> 15#include <asm/asm_pointer_auth.h> 16#include <asm/hwcap.h> 17#include <asm/pgtable-hwdef.h> 18#include <asm/cpufeature.h> 19#include <asm/alternative.h> 20#include <asm/smp.h> 21#include <asm/sysreg.h> 22 23#ifdef CONFIG_ARM64_64K_PAGES 24#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K 25#elif defined(CONFIG_ARM64_16K_PAGES) 26#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K 27#else /* CONFIG_ARM64_4K_PAGES */ 28#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K 29#endif 30 31#ifdef CONFIG_RANDOMIZE_BASE 32#define TCR_KASLR_FLAGS TCR_NFD1 33#else 34#define TCR_KASLR_FLAGS 0 35#endif 36 37#define TCR_SMP_FLAGS TCR_SHARED 38 39/* PTWs cacheable, inner/outer WBWA */ 40#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA 41 42#ifdef CONFIG_KASAN_SW_TAGS 43#define TCR_KASAN_SW_FLAGS TCR_TBI1 | TCR_TBID1 44#else 45#define TCR_KASAN_SW_FLAGS 0 46#endif 47 48#ifdef CONFIG_KASAN_HW_TAGS 49#define TCR_MTE_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1 50#else 51/* 52 * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on 53 * TBI being enabled at EL1. 54 */ 55#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1 56#endif 57 58/* 59 * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and 60 * changed during __cpu_setup to Normal Tagged if the system supports MTE. 61 */ 62#define MAIR_EL1_SET \ 63 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ 64 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ 65 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \ 66 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ 67 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ 68 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) | \ 69 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED) | \ 70 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_iNC_oWB, MT_NORMAL_iNC_oWB)) 71 72#ifdef CONFIG_CPU_PM 73/** 74 * cpu_do_suspend - save CPU registers context 75 * 76 * x0: virtual address of context pointer 77 * 78 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. 79 */ 80SYM_FUNC_START(cpu_do_suspend) 81 mrs x2, tpidr_el0 82 mrs x3, tpidrro_el0 83 mrs x4, contextidr_el1 84 mrs x5, osdlr_el1 85 mrs x6, cpacr_el1 86 mrs x7, tcr_el1 87 mrs x8, vbar_el1 88 mrs x9, mdscr_el1 89 mrs x10, oslsr_el1 90 mrs x11, sctlr_el1 91alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 92 mrs x12, tpidr_el1 93alternative_else 94 mrs x12, tpidr_el2 95alternative_endif 96 mrs x13, sp_el0 97 stp x2, x3, [x0] 98 stp x4, x5, [x0, #16] 99 stp x6, x7, [x0, #32] 100 stp x8, x9, [x0, #48] 101 stp x10, x11, [x0, #64] 102 stp x12, x13, [x0, #80] 103 /* 104 * Save x18 as it may be used as a platform register, e.g. by shadow 105 * call stack. 106 */ 107 str x18, [x0, #96] 108 ret 109SYM_FUNC_END(cpu_do_suspend) 110 111/** 112 * cpu_do_resume - restore CPU register context 113 * 114 * x0: Address of context pointer 115 */ 116 .pushsection ".idmap.text", "awx" 117SYM_FUNC_START(cpu_do_resume) 118 ldp x2, x3, [x0] 119 ldp x4, x5, [x0, #16] 120 ldp x6, x8, [x0, #32] 121 ldp x9, x10, [x0, #48] 122 ldp x11, x12, [x0, #64] 123 ldp x13, x14, [x0, #80] 124 /* 125 * Restore x18, as it may be used as a platform register, and clear 126 * the buffer to minimize the risk of exposure when used for shadow 127 * call stack. 128 */ 129 ldr x18, [x0, #96] 130 str xzr, [x0, #96] 131 msr tpidr_el0, x2 132 msr tpidrro_el0, x3 133 msr contextidr_el1, x4 134 msr cpacr_el1, x6 135 136 /* Don't change t0sz here, mask those bits when restoring */ 137 mrs x7, tcr_el1 138 bfi x8, x7, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH 139 140 msr tcr_el1, x8 141 msr vbar_el1, x9 142 143 /* 144 * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking 145 * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug 146 * exception. Mask them until local_daif_restore() in cpu_suspend() 147 * resets them. 148 */ 149 disable_daif 150 msr mdscr_el1, x10 151 152 msr sctlr_el1, x12 153alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 154 msr tpidr_el1, x13 155alternative_else 156 msr tpidr_el2, x13 157alternative_endif 158 msr sp_el0, x14 159 /* 160 * Restore oslsr_el1 by writing oslar_el1 161 */ 162 msr osdlr_el1, x5 163 ubfx x11, x11, #1, #1 164 msr oslar_el1, x11 165 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 166 reset_amuserenr_el0 x0 // Disable AMU access from EL0 167 168alternative_if ARM64_HAS_RAS_EXTN 169 msr_s SYS_DISR_EL1, xzr 170alternative_else_nop_endif 171 172 ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 173 isb 174 ret 175SYM_FUNC_END(cpu_do_resume) 176 .popsection 177#endif 178 179 .pushsection ".idmap.text", "awx" 180 181.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 182 adrp \tmp1, reserved_pg_dir 183 phys_to_ttbr \tmp2, \tmp1 184 offset_ttbr1 \tmp2, \tmp1 185 msr ttbr1_el1, \tmp2 186 isb 187 tlbi vmalle1 188 dsb nsh 189 isb 190.endm 191 192/* 193 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) 194 * 195 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be 196 * called by anything else. It can only be executed from a TTBR0 mapping. 197 */ 198SYM_FUNC_START(idmap_cpu_replace_ttbr1) 199 save_and_disable_daif flags=x2 200 201 __idmap_cpu_set_reserved_ttbr1 x1, x3 202 203 offset_ttbr1 x0, x3 204 msr ttbr1_el1, x0 205 isb 206 207 restore_daif x2 208 209 ret 210SYM_FUNC_END(idmap_cpu_replace_ttbr1) 211 .popsection 212 213#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 214 .pushsection ".idmap.text", "awx" 215 216 .macro __idmap_kpti_get_pgtable_ent, type 217 dc cvac, cur_\()\type\()p // Ensure any existing dirty 218 dmb sy // lines are written back before 219 ldr \type, [cur_\()\type\()p] // loading the entry 220 tbz \type, #0, skip_\()\type // Skip invalid and 221 tbnz \type, #11, skip_\()\type // non-global entries 222 .endm 223 224 .macro __idmap_kpti_put_pgtable_ent_ng, type 225 orr \type, \type, #PTE_NG // Same bit for blocks and pages 226 str \type, [cur_\()\type\()p] // Update the entry and ensure 227 dmb sy // that it is visible to all 228 dc civac, cur_\()\type\()p // CPUs. 229 .endm 230 231/* 232 * void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper) 233 * 234 * Called exactly once from stop_machine context by each CPU found during boot. 235 */ 236__idmap_kpti_flag: 237 .long 1 238SYM_FUNC_START(idmap_kpti_install_ng_mappings) 239 cpu .req w0 240 num_cpus .req w1 241 swapper_pa .req x2 242 swapper_ttb .req x3 243 flag_ptr .req x4 244 cur_pgdp .req x5 245 end_pgdp .req x6 246 pgd .req x7 247 cur_pudp .req x8 248 end_pudp .req x9 249 pud .req x10 250 cur_pmdp .req x11 251 end_pmdp .req x12 252 pmd .req x13 253 cur_ptep .req x14 254 end_ptep .req x15 255 pte .req x16 256 257 mrs swapper_ttb, ttbr1_el1 258 restore_ttbr1 swapper_ttb 259 adr flag_ptr, __idmap_kpti_flag 260 261 cbnz cpu, __idmap_kpti_secondary 262 263 /* We're the boot CPU. Wait for the others to catch up */ 264 sevl 2651: wfe 266 ldaxr w17, [flag_ptr] 267 eor w17, w17, num_cpus 268 cbnz w17, 1b 269 270 /* We need to walk swapper, so turn off the MMU. */ 271 pre_disable_mmu_workaround 272 mrs x17, sctlr_el1 273 bic x17, x17, #SCTLR_ELx_M 274 msr sctlr_el1, x17 275 isb 276 277 /* Everybody is enjoying the idmap, so we can rewrite swapper. */ 278 /* PGD */ 279 mov cur_pgdp, swapper_pa 280 add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8) 281do_pgd: __idmap_kpti_get_pgtable_ent pgd 282 tbnz pgd, #1, walk_puds 283next_pgd: 284 __idmap_kpti_put_pgtable_ent_ng pgd 285skip_pgd: 286 add cur_pgdp, cur_pgdp, #8 287 cmp cur_pgdp, end_pgdp 288 b.ne do_pgd 289 290 /* Publish the updated tables and nuke all the TLBs */ 291 dsb sy 292 tlbi vmalle1is 293 dsb ish 294 isb 295 296 /* We're done: fire up the MMU again */ 297 mrs x17, sctlr_el1 298 orr x17, x17, #SCTLR_ELx_M 299 set_sctlr_el1 x17 300 301 /* Set the flag to zero to indicate that we're all done */ 302 str wzr, [flag_ptr] 303 ret 304 305 /* PUD */ 306walk_puds: 307 .if CONFIG_PGTABLE_LEVELS > 3 308 pte_to_phys cur_pudp, pgd 309 add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8) 310do_pud: __idmap_kpti_get_pgtable_ent pud 311 tbnz pud, #1, walk_pmds 312next_pud: 313 __idmap_kpti_put_pgtable_ent_ng pud 314skip_pud: 315 add cur_pudp, cur_pudp, 8 316 cmp cur_pudp, end_pudp 317 b.ne do_pud 318 b next_pgd 319 .else /* CONFIG_PGTABLE_LEVELS <= 3 */ 320 mov pud, pgd 321 b walk_pmds 322next_pud: 323 b next_pgd 324 .endif 325 326 /* PMD */ 327walk_pmds: 328 .if CONFIG_PGTABLE_LEVELS > 2 329 pte_to_phys cur_pmdp, pud 330 add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8) 331do_pmd: __idmap_kpti_get_pgtable_ent pmd 332 tbnz pmd, #1, walk_ptes 333next_pmd: 334 __idmap_kpti_put_pgtable_ent_ng pmd 335skip_pmd: 336 add cur_pmdp, cur_pmdp, #8 337 cmp cur_pmdp, end_pmdp 338 b.ne do_pmd 339 b next_pud 340 .else /* CONFIG_PGTABLE_LEVELS <= 2 */ 341 mov pmd, pud 342 b walk_ptes 343next_pmd: 344 b next_pud 345 .endif 346 347 /* PTE */ 348walk_ptes: 349 pte_to_phys cur_ptep, pmd 350 add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8) 351do_pte: __idmap_kpti_get_pgtable_ent pte 352 __idmap_kpti_put_pgtable_ent_ng pte 353skip_pte: 354 add cur_ptep, cur_ptep, #8 355 cmp cur_ptep, end_ptep 356 b.ne do_pte 357 b next_pmd 358 359 .unreq cpu 360 .unreq num_cpus 361 .unreq swapper_pa 362 .unreq cur_pgdp 363 .unreq end_pgdp 364 .unreq pgd 365 .unreq cur_pudp 366 .unreq end_pudp 367 .unreq pud 368 .unreq cur_pmdp 369 .unreq end_pmdp 370 .unreq pmd 371 .unreq cur_ptep 372 .unreq end_ptep 373 .unreq pte 374 375 /* Secondary CPUs end up here */ 376__idmap_kpti_secondary: 377 /* Uninstall swapper before surgery begins */ 378 __idmap_cpu_set_reserved_ttbr1 x16, x17 379 380 /* Increment the flag to let the boot CPU we're ready */ 3811: ldxr w16, [flag_ptr] 382 add w16, w16, #1 383 stxr w17, w16, [flag_ptr] 384 cbnz w17, 1b 385 386 /* Wait for the boot CPU to finish messing around with swapper */ 387 sevl 3881: wfe 389 ldxr w16, [flag_ptr] 390 cbnz w16, 1b 391 392 /* All done, act like nothing happened */ 393 offset_ttbr1 swapper_ttb, x16 394 msr ttbr1_el1, swapper_ttb 395 isb 396 ret 397 398 .unreq swapper_ttb 399 .unreq flag_ptr 400SYM_FUNC_END(idmap_kpti_install_ng_mappings) 401 .popsection 402#endif 403 404/* 405 * __cpu_setup 406 * 407 * Initialise the processor for turning the MMU on. 408 * 409 * Output: 410 * Return in x0 the value of the SCTLR_EL1 register. 411 */ 412 .pushsection ".idmap.text", "awx" 413SYM_FUNC_START(__cpu_setup) 414 tlbi vmalle1 // Invalidate local TLB 415 dsb nsh 416 417 mov x1, #3 << 20 418 msr cpacr_el1, x1 // Enable FP/ASIMD 419 mov x1, #1 << 12 // Reset mdscr_el1 and disable 420 msr mdscr_el1, x1 // access to the DCC from EL0 421 isb // Unmask debug exceptions now, 422 enable_dbg // since this is per-cpu 423 reset_pmuserenr_el0 x1 // Disable PMU access from EL0 424 reset_amuserenr_el0 x1 // Disable AMU access from EL0 425 426 /* 427 * Memory region attributes 428 */ 429 mov_q x5, MAIR_EL1_SET 430#ifdef CONFIG_ARM64_MTE 431 mte_tcr .req x20 432 433 mov mte_tcr, #0 434 435 /* 436 * Update MAIR_EL1, GCR_EL1 and TFSR*_EL1 if MTE is supported 437 * (ID_AA64PFR1_EL1[11:8] > 1). 438 */ 439 mrs x10, ID_AA64PFR1_EL1 440 ubfx x10, x10, #ID_AA64PFR1_MTE_SHIFT, #4 441 cmp x10, #ID_AA64PFR1_MTE 442 b.lt 1f 443 444 /* Normal Tagged memory type at the corresponding MAIR index */ 445 mov x10, #MAIR_ATTR_NORMAL_TAGGED 446 bfi x5, x10, #(8 * MT_NORMAL_TAGGED), #8 447 448 mov x10, #KERNEL_GCR_EL1 449 msr_s SYS_GCR_EL1, x10 450 451 /* 452 * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then 453 * RGSR_EL1.SEED must be non-zero for IRG to produce 454 * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we 455 * must initialize it. 456 */ 457 mrs x10, CNTVCT_EL0 458 ands x10, x10, #SYS_RGSR_EL1_SEED_MASK 459 csinc x10, x10, xzr, ne 460 lsl x10, x10, #SYS_RGSR_EL1_SEED_SHIFT 461 msr_s SYS_RGSR_EL1, x10 462 463 /* clear any pending tag check faults in TFSR*_EL1 */ 464 msr_s SYS_TFSR_EL1, xzr 465 msr_s SYS_TFSRE0_EL1, xzr 466 467 /* set the TCR_EL1 bits */ 468 mov_q mte_tcr, TCR_MTE_FLAGS 4691: 470#endif 471 msr mair_el1, x5 472 /* 473 * Set/prepare TCR and TTBR. TCR_EL1.T1SZ gets further 474 * adjusted if the kernel is compiled with 52bit VA support. 475 */ 476 mov_q x10, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ 477 TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ 478 TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS 479#ifdef CONFIG_ARM64_MTE 480 orr x10, x10, mte_tcr 481 .unreq mte_tcr 482#endif 483 tcr_clear_errata_bits x10, x9, x5 484 485#ifdef CONFIG_ARM64_VA_BITS_52 486 ldr_l x9, vabits_actual 487 sub x9, xzr, x9 488 add x9, x9, #64 489 tcr_set_t1sz x10, x9 490#else 491 ldr_l x9, idmap_t0sz 492#endif 493 tcr_set_t0sz x10, x9 494 495 /* 496 * Set the IPS bits in TCR_EL1. 497 */ 498 tcr_compute_pa_size x10, #TCR_IPS_SHIFT, x5, x6 499#ifdef CONFIG_ARM64_HW_AFDBM 500 /* 501 * Enable hardware update of the Access Flags bit. 502 * Hardware dirty bit management is enabled later, 503 * via capabilities. 504 */ 505 mrs x9, ID_AA64MMFR1_EL1 506 and x9, x9, #0xf 507 cbz x9, 1f 508 orr x10, x10, #TCR_HA // hardware Access flag update 5091: 510#endif /* CONFIG_ARM64_HW_AFDBM */ 511 msr tcr_el1, x10 512 /* 513 * Prepare SCTLR 514 */ 515 mov_q x0, INIT_SCTLR_EL1_MMU_ON 516 ret // return to head.S 517SYM_FUNC_END(__cpu_setup) 518