1/* 2 * Copyright (c) 2017-2020, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <cpuamu.h> 10#include <cpu_macros.S> 11#include <context.h> 12#include <neoverse_n1.h> 13 14/* Hardware handled coherency */ 15#if HW_ASSISTED_COHERENCY == 0 16#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 17#endif 18 19/* 64-bit only core */ 20#if CTX_INCLUDE_AARCH32_REGS == 1 21#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 22#endif 23 24 .global neoverse_n1_errata_ic_trap_handler 25 26/* -------------------------------------------------- 27 * Errata Workaround for Neoverse N1 Erratum 1043202. 28 * This applies to revision r0p0 and r1p0 of Neoverse N1. 29 * Inputs: 30 * x0: variant[4:7] and revision[0:3] of current cpu. 31 * Shall clobber: x0-x17 32 * -------------------------------------------------- 33 */ 34func errata_n1_1043202_wa 35 /* Compare x0 against revision r1p0 */ 36 mov x17, x30 37 bl check_errata_1043202 38 cbz x0, 1f 39 40 /* Apply instruction patching sequence */ 41 ldr x0, =0x0 42 msr CPUPSELR_EL3, x0 43 ldr x0, =0xF3BF8F2F 44 msr CPUPOR_EL3, x0 45 ldr x0, =0xFFFFFFFF 46 msr CPUPMR_EL3, x0 47 ldr x0, =0x800200071 48 msr CPUPCR_EL3, x0 49 isb 501: 51 ret x17 52endfunc errata_n1_1043202_wa 53 54func check_errata_1043202 55 /* Applies to r0p0 and r1p0 */ 56 mov x1, #0x10 57 b cpu_rev_var_ls 58endfunc check_errata_1043202 59 60/* -------------------------------------------------- 61 * Disable speculative loads if Neoverse N1 supports 62 * SSBS. 63 * 64 * Shall clobber: x0. 65 * -------------------------------------------------- 66 */ 67func neoverse_n1_disable_speculative_loads 68 /* Check if the PE implements SSBS */ 69 mrs x0, id_aa64pfr1_el1 70 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 71 b.eq 1f 72 73 /* Disable speculative loads */ 74 msr SSBS, xzr 75 761: 77 ret 78endfunc neoverse_n1_disable_speculative_loads 79 80/* -------------------------------------------------- 81 * Errata Workaround for Neoverse N1 Errata #1073348 82 * This applies to revision r0p0 and r1p0 of Neoverse N1. 83 * Inputs: 84 * x0: variant[4:7] and revision[0:3] of current cpu. 85 * Shall clobber: x0-x17 86 * -------------------------------------------------- 87 */ 88func errata_n1_1073348_wa 89 /* Compare x0 against revision r1p0 */ 90 mov x17, x30 91 bl check_errata_1073348 92 cbz x0, 1f 93 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 94 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 95 msr NEOVERSE_N1_CPUACTLR_EL1, x1 961: 97 ret x17 98endfunc errata_n1_1073348_wa 99 100func check_errata_1073348 101 /* Applies to r0p0 and r1p0 */ 102 mov x1, #0x10 103 b cpu_rev_var_ls 104endfunc check_errata_1073348 105 106/* -------------------------------------------------- 107 * Errata Workaround for Neoverse N1 Errata #1130799 108 * This applies to revision <=r2p0 of Neoverse N1. 109 * Inputs: 110 * x0: variant[4:7] and revision[0:3] of current cpu. 111 * Shall clobber: x0-x17 112 * -------------------------------------------------- 113 */ 114func errata_n1_1130799_wa 115 /* Compare x0 against revision r2p0 */ 116 mov x17, x30 117 bl check_errata_1130799 118 cbz x0, 1f 119 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 120 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 121 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1221: 123 ret x17 124endfunc errata_n1_1130799_wa 125 126func check_errata_1130799 127 /* Applies to <=r2p0 */ 128 mov x1, #0x20 129 b cpu_rev_var_ls 130endfunc check_errata_1130799 131 132/* -------------------------------------------------- 133 * Errata Workaround for Neoverse N1 Errata #1165347 134 * This applies to revision <=r2p0 of Neoverse N1. 135 * Inputs: 136 * x0: variant[4:7] and revision[0:3] of current cpu. 137 * Shall clobber: x0-x17 138 * -------------------------------------------------- 139 */ 140func errata_n1_1165347_wa 141 /* Compare x0 against revision r2p0 */ 142 mov x17, x30 143 bl check_errata_1165347 144 cbz x0, 1f 145 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 146 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 147 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 148 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1491: 150 ret x17 151endfunc errata_n1_1165347_wa 152 153func check_errata_1165347 154 /* Applies to <=r2p0 */ 155 mov x1, #0x20 156 b cpu_rev_var_ls 157endfunc check_errata_1165347 158 159/* -------------------------------------------------- 160 * Errata Workaround for Neoverse N1 Errata #1207823 161 * This applies to revision <=r2p0 of Neoverse N1. 162 * Inputs: 163 * x0: variant[4:7] and revision[0:3] of current cpu. 164 * Shall clobber: x0-x17 165 * -------------------------------------------------- 166 */ 167func errata_n1_1207823_wa 168 /* Compare x0 against revision r2p0 */ 169 mov x17, x30 170 bl check_errata_1207823 171 cbz x0, 1f 172 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 173 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 174 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1751: 176 ret x17 177endfunc errata_n1_1207823_wa 178 179func check_errata_1207823 180 /* Applies to <=r2p0 */ 181 mov x1, #0x20 182 b cpu_rev_var_ls 183endfunc check_errata_1207823 184 185/* -------------------------------------------------- 186 * Errata Workaround for Neoverse N1 Errata #1220197 187 * This applies to revision <=r2p0 of Neoverse N1. 188 * Inputs: 189 * x0: variant[4:7] and revision[0:3] of current cpu. 190 * Shall clobber: x0-x17 191 * -------------------------------------------------- 192 */ 193func errata_n1_1220197_wa 194 /* Compare x0 against revision r2p0 */ 195 mov x17, x30 196 bl check_errata_1220197 197 cbz x0, 1f 198 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 199 orr x1, x1, NEOVERSE_N1_WS_THR_L2_MASK 200 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2011: 202 ret x17 203endfunc errata_n1_1220197_wa 204 205func check_errata_1220197 206 /* Applies to <=r2p0 */ 207 mov x1, #0x20 208 b cpu_rev_var_ls 209endfunc check_errata_1220197 210 211/* -------------------------------------------------- 212 * Errata Workaround for Neoverse N1 Errata #1257314 213 * This applies to revision <=r3p0 of Neoverse N1. 214 * Inputs: 215 * x0: variant[4:7] and revision[0:3] of current cpu. 216 * Shall clobber: x0-x17 217 * -------------------------------------------------- 218 */ 219func errata_n1_1257314_wa 220 /* Compare x0 against revision r3p0 */ 221 mov x17, x30 222 bl check_errata_1257314 223 cbz x0, 1f 224 mrs x1, NEOVERSE_N1_CPUACTLR3_EL1 225 orr x1, x1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 226 msr NEOVERSE_N1_CPUACTLR3_EL1, x1 2271: 228 ret x17 229endfunc errata_n1_1257314_wa 230 231func check_errata_1257314 232 /* Applies to <=r3p0 */ 233 mov x1, #0x30 234 b cpu_rev_var_ls 235endfunc check_errata_1257314 236 237/* -------------------------------------------------- 238 * Errata Workaround for Neoverse N1 Errata #1262606 239 * This applies to revision <=r3p0 of Neoverse N1. 240 * Inputs: 241 * x0: variant[4:7] and revision[0:3] of current cpu. 242 * Shall clobber: x0-x17 243 * -------------------------------------------------- 244 */ 245func errata_n1_1262606_wa 246 /* Compare x0 against revision r3p0 */ 247 mov x17, x30 248 bl check_errata_1262606 249 cbz x0, 1f 250 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 251 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 252 msr NEOVERSE_N1_CPUACTLR_EL1, x1 2531: 254 ret x17 255endfunc errata_n1_1262606_wa 256 257func check_errata_1262606 258 /* Applies to <=r3p0 */ 259 mov x1, #0x30 260 b cpu_rev_var_ls 261endfunc check_errata_1262606 262 263/* -------------------------------------------------- 264 * Errata Workaround for Neoverse N1 Errata #1262888 265 * This applies to revision <=r3p0 of Neoverse N1. 266 * Inputs: 267 * x0: variant[4:7] and revision[0:3] of current cpu. 268 * Shall clobber: x0-x17 269 * -------------------------------------------------- 270 */ 271func errata_n1_1262888_wa 272 /* Compare x0 against revision r3p0 */ 273 mov x17, x30 274 bl check_errata_1262888 275 cbz x0, 1f 276 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 277 orr x1, x1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 278 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2791: 280 ret x17 281endfunc errata_n1_1262888_wa 282 283func check_errata_1262888 284 /* Applies to <=r3p0 */ 285 mov x1, #0x30 286 b cpu_rev_var_ls 287endfunc check_errata_1262888 288 289/* -------------------------------------------------- 290 * Errata Workaround for Neoverse N1 Errata #1275112 291 * This applies to revision <=r3p0 of Neoverse N1. 292 * Inputs: 293 * x0: variant[4:7] and revision[0:3] of current cpu. 294 * Shall clobber: x0-x17 295 * -------------------------------------------------- 296 */ 297func errata_n1_1275112_wa 298 /* Compare x0 against revision r3p0 */ 299 mov x17, x30 300 bl check_errata_1275112 301 cbz x0, 1f 302 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 303 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 304 msr NEOVERSE_N1_CPUACTLR_EL1, x1 3051: 306 ret x17 307endfunc errata_n1_1275112_wa 308 309func check_errata_1275112 310 /* Applies to <=r3p0 */ 311 mov x1, #0x30 312 b cpu_rev_var_ls 313endfunc check_errata_1275112 314 315/* -------------------------------------------------- 316 * Errata Workaround for Neoverse N1 Erratum 1315703. 317 * This applies to revision <= r3p0 of Neoverse N1. 318 * Inputs: 319 * x0: variant[4:7] and revision[0:3] of current cpu. 320 * Shall clobber: x0-x17 321 * -------------------------------------------------- 322 */ 323func errata_n1_1315703_wa 324 /* Compare x0 against revision r3p1 */ 325 mov x17, x30 326 bl check_errata_1315703 327 cbz x0, 1f 328 329 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 330 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 331 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 332 3331: 334 ret x17 335endfunc errata_n1_1315703_wa 336 337func check_errata_1315703 338 /* Applies to everything <= r3p0. */ 339 mov x1, #0x30 340 b cpu_rev_var_ls 341endfunc check_errata_1315703 342 343/* -------------------------------------------------- 344 * Errata Workaround for Neoverse N1 Erratum 1542419. 345 * This applies to revisions r3p0 - r4p0 of Neoverse N1 346 * Inputs: 347 * x0: variant[4:7] and revision[0:3] of current cpu. 348 * Shall clobber: x0-x17 349 * -------------------------------------------------- 350 */ 351func errata_n1_1542419_wa 352 /* Compare x0 against revision r3p0 and r4p0 */ 353 mov x17, x30 354 bl check_errata_1542419 355 cbz x0, 1f 356 357 /* Apply instruction patching sequence */ 358 ldr x0, =0x0 359 msr CPUPSELR_EL3, x0 360 ldr x0, =0xEE670D35 361 msr CPUPOR_EL3, x0 362 ldr x0, =0xFFFF0FFF 363 msr CPUPMR_EL3, x0 364 ldr x0, =0x08000020007D 365 msr CPUPCR_EL3, x0 366 isb 3671: 368 ret x17 369endfunc errata_n1_1542419_wa 370 371func check_errata_1542419 372 /* Applies to everything r3p0 - r4p0. */ 373 mov x1, #0x30 374 mov x2, #0x40 375 b cpu_rev_var_range 376endfunc check_errata_1542419 377 378 /* -------------------------------------------------- 379 * Errata Workaround for Neoverse N1 Errata #1868343. 380 * This applies to revision <= r4p0 of Neoverse N1. 381 * This workaround is the same as the workaround for 382 * errata 1262606 and 1275112 but applies to a wider 383 * revision range. 384 * Inputs: 385 * x0: variant[4:7] and revision[0:3] of current cpu. 386 * Shall clobber: x0-x17 387 * -------------------------------------------------- 388 */ 389func errata_n1_1868343_wa 390 /* 391 * Compare x0 against revision r4p0 392 */ 393 mov x17, x30 394 bl check_errata_1868343 395 cbz x0, 1f 396 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 397 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 398 msr NEOVERSE_N1_CPUACTLR_EL1, x1 399 isb 4001: 401 ret x17 402endfunc errata_n1_1868343_wa 403 404func check_errata_1868343 405 /* Applies to everything <= r4p0 */ 406 mov x1, #0x40 407 b cpu_rev_var_ls 408endfunc check_errata_1868343 409 410func neoverse_n1_reset_func 411 mov x19, x30 412 413 bl neoverse_n1_disable_speculative_loads 414 415 /* Forces all cacheable atomic instructions to be near */ 416 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 417 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 418 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 419 isb 420 421 bl cpu_get_rev_var 422 mov x18, x0 423 424#if ERRATA_N1_1043202 425 mov x0, x18 426 bl errata_n1_1043202_wa 427#endif 428 429#if ERRATA_N1_1073348 430 mov x0, x18 431 bl errata_n1_1073348_wa 432#endif 433 434#if ERRATA_N1_1130799 435 mov x0, x18 436 bl errata_n1_1130799_wa 437#endif 438 439#if ERRATA_N1_1165347 440 mov x0, x18 441 bl errata_n1_1165347_wa 442#endif 443 444#if ERRATA_N1_1207823 445 mov x0, x18 446 bl errata_n1_1207823_wa 447#endif 448 449#if ERRATA_N1_1220197 450 mov x0, x18 451 bl errata_n1_1220197_wa 452#endif 453 454#if ERRATA_N1_1257314 455 mov x0, x18 456 bl errata_n1_1257314_wa 457#endif 458 459#if ERRATA_N1_1262606 460 mov x0, x18 461 bl errata_n1_1262606_wa 462#endif 463 464#if ERRATA_N1_1262888 465 mov x0, x18 466 bl errata_n1_1262888_wa 467#endif 468 469#if ERRATA_N1_1275112 470 mov x0, x18 471 bl errata_n1_1275112_wa 472#endif 473 474#if ERRATA_N1_1315703 475 mov x0, x18 476 bl errata_n1_1315703_wa 477#endif 478 479#if ERRATA_N1_1542419 480 mov x0, x18 481 bl errata_n1_1542419_wa 482#endif 483 484#if ERRATA_N1_1868343 485 mov x0, x18 486 bl errata_n1_1868343_wa 487#endif 488 489#if ENABLE_AMU 490 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 491 mrs x0, actlr_el3 492 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 493 msr actlr_el3, x0 494 495 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 496 mrs x0, actlr_el2 497 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 498 msr actlr_el2, x0 499 500 /* Enable group0 counters */ 501 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 502 msr CPUAMCNTENSET_EL0, x0 503#endif 504 505#if NEOVERSE_Nx_EXTERNAL_LLC 506 /* Some system may have External LLC, core needs to be made aware */ 507 mrs x0, NEOVERSE_N1_CPUECTLR_EL1 508 orr x0, x0, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 509 msr NEOVERSE_N1_CPUECTLR_EL1, x0 510#endif 511 512#if ERRATA_DSU_936184 513 bl errata_dsu_936184_wa 514#endif 515 516 isb 517 ret x19 518endfunc neoverse_n1_reset_func 519 520 /* --------------------------------------------- 521 * HW will do the cache maintenance while powering down 522 * --------------------------------------------- 523 */ 524func neoverse_n1_core_pwr_dwn 525 /* --------------------------------------------- 526 * Enable CPU power down bit in power control register 527 * --------------------------------------------- 528 */ 529 mrs x0, NEOVERSE_N1_CPUPWRCTLR_EL1 530 orr x0, x0, #NEOVERSE_N1_CORE_PWRDN_EN_MASK 531 msr NEOVERSE_N1_CPUPWRCTLR_EL1, x0 532 isb 533 ret 534endfunc neoverse_n1_core_pwr_dwn 535 536#if REPORT_ERRATA 537/* 538 * Errata printing function for Neoverse N1. Must follow AAPCS. 539 */ 540func neoverse_n1_errata_report 541 stp x8, x30, [sp, #-16]! 542 543 bl cpu_get_rev_var 544 mov x8, x0 545 546 /* 547 * Report all errata. The revision-variant information is passed to 548 * checking functions of each errata. 549 */ 550 report_errata ERRATA_N1_1043202, neoverse_n1, 1043202 551 report_errata ERRATA_N1_1073348, neoverse_n1, 1073348 552 report_errata ERRATA_N1_1130799, neoverse_n1, 1130799 553 report_errata ERRATA_N1_1165347, neoverse_n1, 1165347 554 report_errata ERRATA_N1_1207823, neoverse_n1, 1207823 555 report_errata ERRATA_N1_1220197, neoverse_n1, 1220197 556 report_errata ERRATA_N1_1257314, neoverse_n1, 1257314 557 report_errata ERRATA_N1_1262606, neoverse_n1, 1262606 558 report_errata ERRATA_N1_1262888, neoverse_n1, 1262888 559 report_errata ERRATA_N1_1275112, neoverse_n1, 1275112 560 report_errata ERRATA_N1_1315703, neoverse_n1, 1315703 561 report_errata ERRATA_N1_1542419, neoverse_n1, 1542419 562 report_errata ERRATA_N1_1868343, neoverse_n1, 1868343 563 report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184 564 565 ldp x8, x30, [sp], #16 566 ret 567endfunc neoverse_n1_errata_report 568#endif 569 570/* 571 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 572 * inner-shareable invalidation to an arbitrary address followed by a DSB. 573 * 574 * x1: Exception Syndrome 575 */ 576func neoverse_n1_errata_ic_trap_handler 577 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 578 b.ne 1f 579 tlbi vae3is, xzr 580 dsb sy 581 582 # Skip the IC instruction itself 583 mrs x3, elr_el3 584 add x3, x3, #4 585 msr elr_el3, x3 586 587 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 588 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 589 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 590 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 591 592#if IMAGE_BL31 && RAS_EXTENSION 593 /* 594 * Issue Error Synchronization Barrier to synchronize SErrors before 595 * exiting EL3. We're running with EAs unmasked, so any synchronized 596 * errors would be taken immediately; therefore no need to inspect 597 * DISR_EL1 register. 598 */ 599 esb 600#endif 601 exception_return 6021: 603 ret 604endfunc neoverse_n1_errata_ic_trap_handler 605 606 /* --------------------------------------------- 607 * This function provides neoverse_n1 specific 608 * register information for crash reporting. 609 * It needs to return with x6 pointing to 610 * a list of register names in ascii and 611 * x8 - x15 having values of registers to be 612 * reported. 613 * --------------------------------------------- 614 */ 615.section .rodata.neoverse_n1_regs, "aS" 616neoverse_n1_regs: /* The ascii list of register names to be reported */ 617 .asciz "cpuectlr_el1", "" 618 619func neoverse_n1_cpu_reg_dump 620 adr x6, neoverse_n1_regs 621 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 622 ret 623endfunc neoverse_n1_cpu_reg_dump 624 625declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 626 neoverse_n1_reset_func, \ 627 neoverse_n1_errata_ic_trap_handler, \ 628 neoverse_n1_core_pwr_dwn 629