1/* 2 * Copyright (c) 2017-2020, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <neoverse_n1.h> 10#include <cpuamu.h> 11#include <cpu_macros.S> 12#include <context.h> 13 14/* Hardware handled coherency */ 15#if HW_ASSISTED_COHERENCY == 0 16#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 17#endif 18 19/* 64-bit only core */ 20#if CTX_INCLUDE_AARCH32_REGS == 1 21#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 22#endif 23 24 .global neoverse_n1_errata_ic_trap_handler 25 26/* -------------------------------------------------- 27 * Errata Workaround for Neoverse N1 Erratum 1043202. 28 * This applies to revision r0p0 and r1p0 of Neoverse N1. 29 * Inputs: 30 * x0: variant[4:7] and revision[0:3] of current cpu. 31 * Shall clobber: x0-x17 32 * -------------------------------------------------- 33 */ 34func errata_n1_1043202_wa 35 /* Compare x0 against revision r1p0 */ 36 mov x17, x30 37 bl check_errata_1043202 38 cbz x0, 1f 39 40 /* Apply instruction patching sequence */ 41 ldr x0, =0x0 42 msr CPUPSELR_EL3, x0 43 ldr x0, =0xF3BF8F2F 44 msr CPUPOR_EL3, x0 45 ldr x0, =0xFFFFFFFF 46 msr CPUPMR_EL3, x0 47 ldr x0, =0x800200071 48 msr CPUPCR_EL3, x0 49 isb 501: 51 ret x17 52endfunc errata_n1_1043202_wa 53 54func check_errata_1043202 55 /* Applies to r0p0 and r1p0 */ 56 mov x1, #0x10 57 b cpu_rev_var_ls 58endfunc check_errata_1043202 59 60/* -------------------------------------------------- 61 * Disable speculative loads if Neoverse N1 supports 62 * SSBS. 63 * 64 * Shall clobber: x0. 65 * -------------------------------------------------- 66 */ 67func neoverse_n1_disable_speculative_loads 68 /* Check if the PE implements SSBS */ 69 mrs x0, id_aa64pfr1_el1 70 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 71 b.eq 1f 72 73 /* Disable speculative loads */ 74 msr SSBS, xzr 75 761: 77 ret 78endfunc neoverse_n1_disable_speculative_loads 79 80/* -------------------------------------------------- 81 * Errata Workaround for Neoverse N1 Errata #1073348 82 * This applies to revision r0p0 and r1p0 of Neoverse N1. 83 * Inputs: 84 * x0: variant[4:7] and revision[0:3] of current cpu. 85 * Shall clobber: x0-x17 86 * -------------------------------------------------- 87 */ 88func errata_n1_1073348_wa 89 /* Compare x0 against revision r1p0 */ 90 mov x17, x30 91 bl check_errata_1073348 92 cbz x0, 1f 93 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 94 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 95 msr NEOVERSE_N1_CPUACTLR_EL1, x1 961: 97 ret x17 98endfunc errata_n1_1073348_wa 99 100func check_errata_1073348 101 /* Applies to r0p0 and r1p0 */ 102 mov x1, #0x10 103 b cpu_rev_var_ls 104endfunc check_errata_1073348 105 106/* -------------------------------------------------- 107 * Errata Workaround for Neoverse N1 Errata #1130799 108 * This applies to revision <=r2p0 of Neoverse N1. 109 * Inputs: 110 * x0: variant[4:7] and revision[0:3] of current cpu. 111 * Shall clobber: x0-x17 112 * -------------------------------------------------- 113 */ 114func errata_n1_1130799_wa 115 /* Compare x0 against revision r2p0 */ 116 mov x17, x30 117 bl check_errata_1130799 118 cbz x0, 1f 119 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 120 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 121 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1221: 123 ret x17 124endfunc errata_n1_1130799_wa 125 126func check_errata_1130799 127 /* Applies to <=r2p0 */ 128 mov x1, #0x20 129 b cpu_rev_var_ls 130endfunc check_errata_1130799 131 132/* -------------------------------------------------- 133 * Errata Workaround for Neoverse N1 Errata #1165347 134 * This applies to revision <=r2p0 of Neoverse N1. 135 * Inputs: 136 * x0: variant[4:7] and revision[0:3] of current cpu. 137 * Shall clobber: x0-x17 138 * -------------------------------------------------- 139 */ 140func errata_n1_1165347_wa 141 /* Compare x0 against revision r2p0 */ 142 mov x17, x30 143 bl check_errata_1165347 144 cbz x0, 1f 145 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 146 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 147 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 148 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1491: 150 ret x17 151endfunc errata_n1_1165347_wa 152 153func check_errata_1165347 154 /* Applies to <=r2p0 */ 155 mov x1, #0x20 156 b cpu_rev_var_ls 157endfunc check_errata_1165347 158 159/* -------------------------------------------------- 160 * Errata Workaround for Neoverse N1 Errata #1207823 161 * This applies to revision <=r2p0 of Neoverse N1. 162 * Inputs: 163 * x0: variant[4:7] and revision[0:3] of current cpu. 164 * Shall clobber: x0-x17 165 * -------------------------------------------------- 166 */ 167func errata_n1_1207823_wa 168 /* Compare x0 against revision r2p0 */ 169 mov x17, x30 170 bl check_errata_1207823 171 cbz x0, 1f 172 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 173 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 174 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1751: 176 ret x17 177endfunc errata_n1_1207823_wa 178 179func check_errata_1207823 180 /* Applies to <=r2p0 */ 181 mov x1, #0x20 182 b cpu_rev_var_ls 183endfunc check_errata_1207823 184 185/* -------------------------------------------------- 186 * Errata Workaround for Neoverse N1 Errata #1220197 187 * This applies to revision <=r2p0 of Neoverse N1. 188 * Inputs: 189 * x0: variant[4:7] and revision[0:3] of current cpu. 190 * Shall clobber: x0-x17 191 * -------------------------------------------------- 192 */ 193func errata_n1_1220197_wa 194 /* Compare x0 against revision r2p0 */ 195 mov x17, x30 196 bl check_errata_1220197 197 cbz x0, 1f 198 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 199 orr x1, x1, NEOVERSE_N1_WS_THR_L2_MASK 200 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2011: 202 ret x17 203endfunc errata_n1_1220197_wa 204 205func check_errata_1220197 206 /* Applies to <=r2p0 */ 207 mov x1, #0x20 208 b cpu_rev_var_ls 209endfunc check_errata_1220197 210 211/* -------------------------------------------------- 212 * Errata Workaround for Neoverse N1 Errata #1257314 213 * This applies to revision <=r3p0 of Neoverse N1. 214 * Inputs: 215 * x0: variant[4:7] and revision[0:3] of current cpu. 216 * Shall clobber: x0-x17 217 * -------------------------------------------------- 218 */ 219func errata_n1_1257314_wa 220 /* Compare x0 against revision r3p0 */ 221 mov x17, x30 222 bl check_errata_1257314 223 cbz x0, 1f 224 mrs x1, NEOVERSE_N1_CPUACTLR3_EL1 225 orr x1, x1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 226 msr NEOVERSE_N1_CPUACTLR3_EL1, x1 2271: 228 ret x17 229endfunc errata_n1_1257314_wa 230 231func check_errata_1257314 232 /* Applies to <=r3p0 */ 233 mov x1, #0x30 234 b cpu_rev_var_ls 235endfunc check_errata_1257314 236 237/* -------------------------------------------------- 238 * Errata Workaround for Neoverse N1 Errata #1262606 239 * This applies to revision <=r3p0 of Neoverse N1. 240 * Inputs: 241 * x0: variant[4:7] and revision[0:3] of current cpu. 242 * Shall clobber: x0-x17 243 * -------------------------------------------------- 244 */ 245func errata_n1_1262606_wa 246 /* Compare x0 against revision r3p0 */ 247 mov x17, x30 248 bl check_errata_1262606 249 cbz x0, 1f 250 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 251 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 252 msr NEOVERSE_N1_CPUACTLR_EL1, x1 2531: 254 ret x17 255endfunc errata_n1_1262606_wa 256 257func check_errata_1262606 258 /* Applies to <=r3p0 */ 259 mov x1, #0x30 260 b cpu_rev_var_ls 261endfunc check_errata_1262606 262 263/* -------------------------------------------------- 264 * Errata Workaround for Neoverse N1 Errata #1262888 265 * This applies to revision <=r3p0 of Neoverse N1. 266 * Inputs: 267 * x0: variant[4:7] and revision[0:3] of current cpu. 268 * Shall clobber: x0-x17 269 * -------------------------------------------------- 270 */ 271func errata_n1_1262888_wa 272 /* Compare x0 against revision r3p0 */ 273 mov x17, x30 274 bl check_errata_1262888 275 cbz x0, 1f 276 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 277 orr x1, x1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 278 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2791: 280 ret x17 281endfunc errata_n1_1262888_wa 282 283func check_errata_1262888 284 /* Applies to <=r3p0 */ 285 mov x1, #0x30 286 b cpu_rev_var_ls 287endfunc check_errata_1262888 288 289/* -------------------------------------------------- 290 * Errata Workaround for Neoverse N1 Errata #1275112 291 * This applies to revision <=r3p0 of Neoverse N1. 292 * Inputs: 293 * x0: variant[4:7] and revision[0:3] of current cpu. 294 * Shall clobber: x0-x17 295 * -------------------------------------------------- 296 */ 297func errata_n1_1275112_wa 298 /* Compare x0 against revision r3p0 */ 299 mov x17, x30 300 bl check_errata_1275112 301 cbz x0, 1f 302 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 303 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 304 msr NEOVERSE_N1_CPUACTLR_EL1, x1 3051: 306 ret x17 307endfunc errata_n1_1275112_wa 308 309func check_errata_1275112 310 /* Applies to <=r3p0 */ 311 mov x1, #0x30 312 b cpu_rev_var_ls 313endfunc check_errata_1275112 314 315/* -------------------------------------------------- 316 * Errata Workaround for Neoverse N1 Erratum 1315703. 317 * This applies to revision <= r3p0 of Neoverse N1. 318 * Inputs: 319 * x0: variant[4:7] and revision[0:3] of current cpu. 320 * Shall clobber: x0-x17 321 * -------------------------------------------------- 322 */ 323func errata_n1_1315703_wa 324 /* Compare x0 against revision r3p1 */ 325 mov x17, x30 326 bl check_errata_1315703 327 cbz x0, 1f 328 329 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 330 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 331 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 332 3331: 334 ret x17 335endfunc errata_n1_1315703_wa 336 337func check_errata_1315703 338 /* Applies to everything <= r3p0. */ 339 mov x1, #0x30 340 b cpu_rev_var_ls 341endfunc check_errata_1315703 342 343/* -------------------------------------------------- 344 * Errata Workaround for Neoverse N1 Erratum 1542419. 345 * This applies to revisions r3p0 - r4p0 of Neoverse N1 346 * Inputs: 347 * x0: variant[4:7] and revision[0:3] of current cpu. 348 * Shall clobber: x0-x17 349 * -------------------------------------------------- 350 */ 351func errata_n1_1542419_wa 352 /* Compare x0 against revision r3p0 and r4p0 */ 353 mov x17, x30 354 bl check_errata_1542419 355 cbz x0, 1f 356 357 /* Apply instruction patching sequence */ 358 ldr x0, =0x0 359 msr CPUPSELR_EL3, x0 360 ldr x0, =0xEE670D35 361 msr CPUPOR_EL3, x0 362 ldr x0, =0xFFFF0FFF 363 msr CPUPMR_EL3, x0 364 ldr x0, =0x08000020007D 365 msr CPUPCR_EL3, x0 366 isb 3671: 368 ret x17 369endfunc errata_n1_1542419_wa 370 371func check_errata_1542419 372 /* Applies to everything r3p0 - r4p0. */ 373 mov x1, #0x30 374 mov x2, #0x40 375 b cpu_rev_var_range 376endfunc check_errata_1542419 377 378func neoverse_n1_reset_func 379 mov x19, x30 380 381 bl neoverse_n1_disable_speculative_loads 382 383 /* Forces all cacheable atomic instructions to be near */ 384 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 385 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 386 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 387 isb 388 389 bl cpu_get_rev_var 390 mov x18, x0 391 392#if ERRATA_N1_1043202 393 mov x0, x18 394 bl errata_n1_1043202_wa 395#endif 396 397#if ERRATA_N1_1073348 398 mov x0, x18 399 bl errata_n1_1073348_wa 400#endif 401 402#if ERRATA_N1_1130799 403 mov x0, x18 404 bl errata_n1_1130799_wa 405#endif 406 407#if ERRATA_N1_1165347 408 mov x0, x18 409 bl errata_n1_1165347_wa 410#endif 411 412#if ERRATA_N1_1207823 413 mov x0, x18 414 bl errata_n1_1207823_wa 415#endif 416 417#if ERRATA_N1_1220197 418 mov x0, x18 419 bl errata_n1_1220197_wa 420#endif 421 422#if ERRATA_N1_1257314 423 mov x0, x18 424 bl errata_n1_1257314_wa 425#endif 426 427#if ERRATA_N1_1262606 428 mov x0, x18 429 bl errata_n1_1262606_wa 430#endif 431 432#if ERRATA_N1_1262888 433 mov x0, x18 434 bl errata_n1_1262888_wa 435#endif 436 437#if ERRATA_N1_1275112 438 mov x0, x18 439 bl errata_n1_1275112_wa 440#endif 441 442#if ERRATA_N1_1315703 443 mov x0, x18 444 bl errata_n1_1315703_wa 445#endif 446 447#if ERRATA_N1_1542419 448 mov x0, x18 449 bl errata_n1_1542419_wa 450#endif 451 452#if ENABLE_AMU 453 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 454 mrs x0, actlr_el3 455 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 456 msr actlr_el3, x0 457 458 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 459 mrs x0, actlr_el2 460 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 461 msr actlr_el2, x0 462 463 /* Enable group0 counters */ 464 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 465 msr CPUAMCNTENSET_EL0, x0 466#endif 467 468#if NEOVERSE_N1_EXTERNAL_LLC 469 /* Some system may have External LLC, core needs to be made aware */ 470 mrs x0, NEOVERSE_N1_CPUECTLR_EL1 471 orr x0, x0, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 472 msr NEOVERSE_N1_CPUECTLR_EL1, x0 473#endif 474 475#if ERRATA_DSU_936184 476 bl errata_dsu_936184_wa 477#endif 478 479 isb 480 ret x19 481endfunc neoverse_n1_reset_func 482 483 /* --------------------------------------------- 484 * HW will do the cache maintenance while powering down 485 * --------------------------------------------- 486 */ 487func neoverse_n1_core_pwr_dwn 488 /* --------------------------------------------- 489 * Enable CPU power down bit in power control register 490 * --------------------------------------------- 491 */ 492 mrs x0, NEOVERSE_N1_CPUPWRCTLR_EL1 493 orr x0, x0, #NEOVERSE_N1_CORE_PWRDN_EN_MASK 494 msr NEOVERSE_N1_CPUPWRCTLR_EL1, x0 495 isb 496 ret 497endfunc neoverse_n1_core_pwr_dwn 498 499#if REPORT_ERRATA 500/* 501 * Errata printing function for Neoverse N1. Must follow AAPCS. 502 */ 503func neoverse_n1_errata_report 504 stp x8, x30, [sp, #-16]! 505 506 bl cpu_get_rev_var 507 mov x8, x0 508 509 /* 510 * Report all errata. The revision-variant information is passed to 511 * checking functions of each errata. 512 */ 513 report_errata ERRATA_N1_1043202, neoverse_n1, 1043202 514 report_errata ERRATA_N1_1073348, neoverse_n1, 1073348 515 report_errata ERRATA_N1_1130799, neoverse_n1, 1130799 516 report_errata ERRATA_N1_1165347, neoverse_n1, 1165347 517 report_errata ERRATA_N1_1207823, neoverse_n1, 1207823 518 report_errata ERRATA_N1_1220197, neoverse_n1, 1220197 519 report_errata ERRATA_N1_1257314, neoverse_n1, 1257314 520 report_errata ERRATA_N1_1262606, neoverse_n1, 1262606 521 report_errata ERRATA_N1_1262888, neoverse_n1, 1262888 522 report_errata ERRATA_N1_1275112, neoverse_n1, 1275112 523 report_errata ERRATA_N1_1315703, neoverse_n1, 1315703 524 report_errata ERRATA_N1_1542419, neoverse_n1, 1542419 525 report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184 526 527 ldp x8, x30, [sp], #16 528 ret 529endfunc neoverse_n1_errata_report 530#endif 531 532/* 533 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 534 * inner-shareable invalidation to an arbitrary address followed by a DSB. 535 * 536 * x1: Exception Syndrome 537 */ 538func neoverse_n1_errata_ic_trap_handler 539 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 540 b.ne 1f 541 tlbi vae3is, xzr 542 dsb sy 543 544 # Skip the IC instruction itself 545 mrs x3, elr_el3 546 add x3, x3, #4 547 msr elr_el3, x3 548 549 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 550 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 551 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 552 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 553 554#if IMAGE_BL31 && RAS_EXTENSION 555 /* 556 * Issue Error Synchronization Barrier to synchronize SErrors before 557 * exiting EL3. We're running with EAs unmasked, so any synchronized 558 * errors would be taken immediately; therefore no need to inspect 559 * DISR_EL1 register. 560 */ 561 esb 562#endif 563 exception_return 5641: 565 ret 566endfunc neoverse_n1_errata_ic_trap_handler 567 568 /* --------------------------------------------- 569 * This function provides neoverse_n1 specific 570 * register information for crash reporting. 571 * It needs to return with x6 pointing to 572 * a list of register names in ascii and 573 * x8 - x15 having values of registers to be 574 * reported. 575 * --------------------------------------------- 576 */ 577.section .rodata.neoverse_n1_regs, "aS" 578neoverse_n1_regs: /* The ascii list of register names to be reported */ 579 .asciz "cpuectlr_el1", "" 580 581func neoverse_n1_cpu_reg_dump 582 adr x6, neoverse_n1_regs 583 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 584 ret 585endfunc neoverse_n1_cpu_reg_dump 586 587declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 588 neoverse_n1_reset_func, \ 589 neoverse_n1_errata_ic_trap_handler, \ 590 neoverse_n1_core_pwr_dwn 591