1/* 2 * Copyright (c) 2017-2020, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <neoverse_n1.h> 10#include <cpuamu.h> 11#include <cpu_macros.S> 12#include <context.h> 13 14/* Hardware handled coherency */ 15#if HW_ASSISTED_COHERENCY == 0 16#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 17#endif 18 19/* 64-bit only core */ 20#if CTX_INCLUDE_AARCH32_REGS == 1 21#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 22#endif 23 24 .global neoverse_n1_errata_ic_trap_handler 25 .global is_scu_present_in_dsu 26 27/* 28 * Check DSU is configured with SCU and L3 unit 29 * 1-> SCU present 30 * 0-> SCU not present 31 */ 32func is_scu_present_in_dsu 33 mrs x0, CPUCFR_EL1 34 ubfx x0, x0, #SCU_SHIFT, #1 35 eor x0, x0, #1 36 ret 37endfunc is_scu_present_in_dsu 38 39/* -------------------------------------------------- 40 * Errata Workaround for Neoverse N1 Erratum 1043202. 41 * This applies to revision r0p0 and r1p0 of Neoverse N1. 42 * Inputs: 43 * x0: variant[4:7] and revision[0:3] of current cpu. 44 * Shall clobber: x0-x17 45 * -------------------------------------------------- 46 */ 47func errata_n1_1043202_wa 48 /* Compare x0 against revision r1p0 */ 49 mov x17, x30 50 bl check_errata_1043202 51 cbz x0, 1f 52 53 /* Apply instruction patching sequence */ 54 ldr x0, =0x0 55 msr CPUPSELR_EL3, x0 56 ldr x0, =0xF3BF8F2F 57 msr CPUPOR_EL3, x0 58 ldr x0, =0xFFFFFFFF 59 msr CPUPMR_EL3, x0 60 ldr x0, =0x800200071 61 msr CPUPCR_EL3, x0 62 isb 631: 64 ret x17 65endfunc errata_n1_1043202_wa 66 67func check_errata_1043202 68 /* Applies to r0p0 and r1p0 */ 69 mov x1, #0x10 70 b cpu_rev_var_ls 71endfunc check_errata_1043202 72 73/* -------------------------------------------------- 74 * Disable speculative loads if Neoverse N1 supports 75 * SSBS. 76 * 77 * Shall clobber: x0. 78 * -------------------------------------------------- 79 */ 80func neoverse_n1_disable_speculative_loads 81 /* Check if the PE implements SSBS */ 82 mrs x0, id_aa64pfr1_el1 83 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 84 b.eq 1f 85 86 /* Disable speculative loads */ 87 msr SSBS, xzr 88 891: 90 ret 91endfunc neoverse_n1_disable_speculative_loads 92 93/* -------------------------------------------------- 94 * Errata Workaround for Neoverse N1 Errata #1073348 95 * This applies to revision r0p0 and r1p0 of Neoverse N1. 96 * Inputs: 97 * x0: variant[4:7] and revision[0:3] of current cpu. 98 * Shall clobber: x0-x17 99 * -------------------------------------------------- 100 */ 101func errata_n1_1073348_wa 102 /* Compare x0 against revision r1p0 */ 103 mov x17, x30 104 bl check_errata_1073348 105 cbz x0, 1f 106 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 107 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 108 msr NEOVERSE_N1_CPUACTLR_EL1, x1 1091: 110 ret x17 111endfunc errata_n1_1073348_wa 112 113func check_errata_1073348 114 /* Applies to r0p0 and r1p0 */ 115 mov x1, #0x10 116 b cpu_rev_var_ls 117endfunc check_errata_1073348 118 119/* -------------------------------------------------- 120 * Errata Workaround for Neoverse N1 Errata #1130799 121 * This applies to revision <=r2p0 of Neoverse N1. 122 * Inputs: 123 * x0: variant[4:7] and revision[0:3] of current cpu. 124 * Shall clobber: x0-x17 125 * -------------------------------------------------- 126 */ 127func errata_n1_1130799_wa 128 /* Compare x0 against revision r2p0 */ 129 mov x17, x30 130 bl check_errata_1130799 131 cbz x0, 1f 132 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 133 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 134 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1351: 136 ret x17 137endfunc errata_n1_1130799_wa 138 139func check_errata_1130799 140 /* Applies to <=r2p0 */ 141 mov x1, #0x20 142 b cpu_rev_var_ls 143endfunc check_errata_1130799 144 145/* -------------------------------------------------- 146 * Errata Workaround for Neoverse N1 Errata #1165347 147 * This applies to revision <=r2p0 of Neoverse N1. 148 * Inputs: 149 * x0: variant[4:7] and revision[0:3] of current cpu. 150 * Shall clobber: x0-x17 151 * -------------------------------------------------- 152 */ 153func errata_n1_1165347_wa 154 /* Compare x0 against revision r2p0 */ 155 mov x17, x30 156 bl check_errata_1165347 157 cbz x0, 1f 158 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 159 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 160 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 161 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1621: 163 ret x17 164endfunc errata_n1_1165347_wa 165 166func check_errata_1165347 167 /* Applies to <=r2p0 */ 168 mov x1, #0x20 169 b cpu_rev_var_ls 170endfunc check_errata_1165347 171 172/* -------------------------------------------------- 173 * Errata Workaround for Neoverse N1 Errata #1207823 174 * This applies to revision <=r2p0 of Neoverse N1. 175 * Inputs: 176 * x0: variant[4:7] and revision[0:3] of current cpu. 177 * Shall clobber: x0-x17 178 * -------------------------------------------------- 179 */ 180func errata_n1_1207823_wa 181 /* Compare x0 against revision r2p0 */ 182 mov x17, x30 183 bl check_errata_1207823 184 cbz x0, 1f 185 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 186 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 187 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1881: 189 ret x17 190endfunc errata_n1_1207823_wa 191 192func check_errata_1207823 193 /* Applies to <=r2p0 */ 194 mov x1, #0x20 195 b cpu_rev_var_ls 196endfunc check_errata_1207823 197 198/* -------------------------------------------------- 199 * Errata Workaround for Neoverse N1 Errata #1220197 200 * This applies to revision <=r2p0 of Neoverse N1. 201 * Inputs: 202 * x0: variant[4:7] and revision[0:3] of current cpu. 203 * Shall clobber: x0-x17 204 * -------------------------------------------------- 205 */ 206func errata_n1_1220197_wa 207 /* Compare x0 against revision r2p0 */ 208 mov x17, x30 209 bl check_errata_1220197 210 cbz x0, 1f 211 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 212 orr x1, x1, NEOVERSE_N1_WS_THR_L2_MASK 213 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2141: 215 ret x17 216endfunc errata_n1_1220197_wa 217 218func check_errata_1220197 219 /* Applies to <=r2p0 */ 220 mov x1, #0x20 221 b cpu_rev_var_ls 222endfunc check_errata_1220197 223 224/* -------------------------------------------------- 225 * Errata Workaround for Neoverse N1 Errata #1257314 226 * This applies to revision <=r3p0 of Neoverse N1. 227 * Inputs: 228 * x0: variant[4:7] and revision[0:3] of current cpu. 229 * Shall clobber: x0-x17 230 * -------------------------------------------------- 231 */ 232func errata_n1_1257314_wa 233 /* Compare x0 against revision r3p0 */ 234 mov x17, x30 235 bl check_errata_1257314 236 cbz x0, 1f 237 mrs x1, NEOVERSE_N1_CPUACTLR3_EL1 238 orr x1, x1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 239 msr NEOVERSE_N1_CPUACTLR3_EL1, x1 2401: 241 ret x17 242endfunc errata_n1_1257314_wa 243 244func check_errata_1257314 245 /* Applies to <=r3p0 */ 246 mov x1, #0x30 247 b cpu_rev_var_ls 248endfunc check_errata_1257314 249 250/* -------------------------------------------------- 251 * Errata Workaround for Neoverse N1 Errata #1262606 252 * This applies to revision <=r3p0 of Neoverse N1. 253 * Inputs: 254 * x0: variant[4:7] and revision[0:3] of current cpu. 255 * Shall clobber: x0-x17 256 * -------------------------------------------------- 257 */ 258func errata_n1_1262606_wa 259 /* Compare x0 against revision r3p0 */ 260 mov x17, x30 261 bl check_errata_1262606 262 cbz x0, 1f 263 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 264 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 265 msr NEOVERSE_N1_CPUACTLR_EL1, x1 2661: 267 ret x17 268endfunc errata_n1_1262606_wa 269 270func check_errata_1262606 271 /* Applies to <=r3p0 */ 272 mov x1, #0x30 273 b cpu_rev_var_ls 274endfunc check_errata_1262606 275 276/* -------------------------------------------------- 277 * Errata Workaround for Neoverse N1 Errata #1262888 278 * This applies to revision <=r3p0 of Neoverse N1. 279 * Inputs: 280 * x0: variant[4:7] and revision[0:3] of current cpu. 281 * Shall clobber: x0-x17 282 * -------------------------------------------------- 283 */ 284func errata_n1_1262888_wa 285 /* Compare x0 against revision r3p0 */ 286 mov x17, x30 287 bl check_errata_1262888 288 cbz x0, 1f 289 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 290 orr x1, x1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 291 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2921: 293 ret x17 294endfunc errata_n1_1262888_wa 295 296func check_errata_1262888 297 /* Applies to <=r3p0 */ 298 mov x1, #0x30 299 b cpu_rev_var_ls 300endfunc check_errata_1262888 301 302/* -------------------------------------------------- 303 * Errata Workaround for Neoverse N1 Errata #1275112 304 * This applies to revision <=r3p0 of Neoverse N1. 305 * Inputs: 306 * x0: variant[4:7] and revision[0:3] of current cpu. 307 * Shall clobber: x0-x17 308 * -------------------------------------------------- 309 */ 310func errata_n1_1275112_wa 311 /* Compare x0 against revision r3p0 */ 312 mov x17, x30 313 bl check_errata_1275112 314 cbz x0, 1f 315 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 316 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 317 msr NEOVERSE_N1_CPUACTLR_EL1, x1 3181: 319 ret x17 320endfunc errata_n1_1275112_wa 321 322func check_errata_1275112 323 /* Applies to <=r3p0 */ 324 mov x1, #0x30 325 b cpu_rev_var_ls 326endfunc check_errata_1275112 327 328/* -------------------------------------------------- 329 * Errata Workaround for Neoverse N1 Erratum 1315703. 330 * This applies to revision <= r3p0 of Neoverse N1. 331 * Inputs: 332 * x0: variant[4:7] and revision[0:3] of current cpu. 333 * Shall clobber: x0-x17 334 * -------------------------------------------------- 335 */ 336func errata_n1_1315703_wa 337 /* Compare x0 against revision r3p1 */ 338 mov x17, x30 339 bl check_errata_1315703 340 cbz x0, 1f 341 342 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 343 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 344 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 345 3461: 347 ret x17 348endfunc errata_n1_1315703_wa 349 350func check_errata_1315703 351 /* Applies to everything <= r3p0. */ 352 mov x1, #0x30 353 b cpu_rev_var_ls 354endfunc check_errata_1315703 355 356/* -------------------------------------------------- 357 * Errata Workaround for Neoverse N1 Erratum 1542419. 358 * This applies to revisions r3p0 - r4p0 of Neoverse N1 359 * Inputs: 360 * x0: variant[4:7] and revision[0:3] of current cpu. 361 * Shall clobber: x0-x17 362 * -------------------------------------------------- 363 */ 364func errata_n1_1542419_wa 365 /* Compare x0 against revision r3p0 and r4p0 */ 366 mov x17, x30 367 bl check_errata_1542419 368 cbz x0, 1f 369 370 /* Apply instruction patching sequence */ 371 ldr x0, =0x0 372 msr CPUPSELR_EL3, x0 373 ldr x0, =0xEE670D35 374 msr CPUPOR_EL3, x0 375 ldr x0, =0xFFFF0FFF 376 msr CPUPMR_EL3, x0 377 ldr x0, =0x08000020007D 378 msr CPUPCR_EL3, x0 379 isb 3801: 381 ret x17 382endfunc errata_n1_1542419_wa 383 384func check_errata_1542419 385 /* Applies to everything r3p0 - r4p0. */ 386 mov x1, #0x30 387 mov x2, #0x40 388 b cpu_rev_var_range 389endfunc check_errata_1542419 390 391 /* -------------------------------------------------- 392 * Errata Workaround for Neoverse N1 Errata #1868343. 393 * This applies to revision <= r4p0 of Neoverse N1. 394 * This workaround is the same as the workaround for 395 * errata 1262606 and 1275112 but applies to a wider 396 * revision range. 397 * Inputs: 398 * x0: variant[4:7] and revision[0:3] of current cpu. 399 * Shall clobber: x0-x17 400 * -------------------------------------------------- 401 */ 402func errata_n1_1868343_wa 403 /* 404 * Compare x0 against revision r4p0 405 */ 406 mov x17, x30 407 bl check_errata_1868343 408 cbz x0, 1f 409 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 410 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 411 msr NEOVERSE_N1_CPUACTLR_EL1, x1 412 isb 4131: 414 ret x17 415endfunc errata_n1_1868343_wa 416 417func check_errata_1868343 418 /* Applies to everything <= r4p0 */ 419 mov x1, #0x40 420 b cpu_rev_var_ls 421endfunc check_errata_1868343 422 423func neoverse_n1_reset_func 424 mov x19, x30 425 426 bl neoverse_n1_disable_speculative_loads 427 428 /* Forces all cacheable atomic instructions to be near */ 429 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 430 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 431 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 432 isb 433 434 bl cpu_get_rev_var 435 mov x18, x0 436 437#if ERRATA_N1_1043202 438 mov x0, x18 439 bl errata_n1_1043202_wa 440#endif 441 442#if ERRATA_N1_1073348 443 mov x0, x18 444 bl errata_n1_1073348_wa 445#endif 446 447#if ERRATA_N1_1130799 448 mov x0, x18 449 bl errata_n1_1130799_wa 450#endif 451 452#if ERRATA_N1_1165347 453 mov x0, x18 454 bl errata_n1_1165347_wa 455#endif 456 457#if ERRATA_N1_1207823 458 mov x0, x18 459 bl errata_n1_1207823_wa 460#endif 461 462#if ERRATA_N1_1220197 463 mov x0, x18 464 bl errata_n1_1220197_wa 465#endif 466 467#if ERRATA_N1_1257314 468 mov x0, x18 469 bl errata_n1_1257314_wa 470#endif 471 472#if ERRATA_N1_1262606 473 mov x0, x18 474 bl errata_n1_1262606_wa 475#endif 476 477#if ERRATA_N1_1262888 478 mov x0, x18 479 bl errata_n1_1262888_wa 480#endif 481 482#if ERRATA_N1_1275112 483 mov x0, x18 484 bl errata_n1_1275112_wa 485#endif 486 487#if ERRATA_N1_1315703 488 mov x0, x18 489 bl errata_n1_1315703_wa 490#endif 491 492#if ERRATA_N1_1542419 493 mov x0, x18 494 bl errata_n1_1542419_wa 495#endif 496 497#if ERRATA_N1_1868343 498 mov x0, x18 499 bl errata_n1_1868343_wa 500#endif 501 502#if ENABLE_AMU 503 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 504 mrs x0, actlr_el3 505 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 506 msr actlr_el3, x0 507 508 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 509 mrs x0, actlr_el2 510 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 511 msr actlr_el2, x0 512 513 /* Enable group0 counters */ 514 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 515 msr CPUAMCNTENSET_EL0, x0 516#endif 517 518#if NEOVERSE_N1_EXTERNAL_LLC 519 /* Some system may have External LLC, core needs to be made aware */ 520 mrs x0, NEOVERSE_N1_CPUECTLR_EL1 521 orr x0, x0, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 522 msr NEOVERSE_N1_CPUECTLR_EL1, x0 523#endif 524 525#if ERRATA_DSU_936184 526 bl errata_dsu_936184_wa 527#endif 528 529 isb 530 ret x19 531endfunc neoverse_n1_reset_func 532 533 /* --------------------------------------------- 534 * HW will do the cache maintenance while powering down 535 * --------------------------------------------- 536 */ 537func neoverse_n1_core_pwr_dwn 538 /* --------------------------------------------- 539 * Enable CPU power down bit in power control register 540 * --------------------------------------------- 541 */ 542 mrs x0, NEOVERSE_N1_CPUPWRCTLR_EL1 543 orr x0, x0, #NEOVERSE_N1_CORE_PWRDN_EN_MASK 544 msr NEOVERSE_N1_CPUPWRCTLR_EL1, x0 545 isb 546 ret 547endfunc neoverse_n1_core_pwr_dwn 548 549#if REPORT_ERRATA 550/* 551 * Errata printing function for Neoverse N1. Must follow AAPCS. 552 */ 553func neoverse_n1_errata_report 554 stp x8, x30, [sp, #-16]! 555 556 bl cpu_get_rev_var 557 mov x8, x0 558 559 /* 560 * Report all errata. The revision-variant information is passed to 561 * checking functions of each errata. 562 */ 563 report_errata ERRATA_N1_1043202, neoverse_n1, 1043202 564 report_errata ERRATA_N1_1073348, neoverse_n1, 1073348 565 report_errata ERRATA_N1_1130799, neoverse_n1, 1130799 566 report_errata ERRATA_N1_1165347, neoverse_n1, 1165347 567 report_errata ERRATA_N1_1207823, neoverse_n1, 1207823 568 report_errata ERRATA_N1_1220197, neoverse_n1, 1220197 569 report_errata ERRATA_N1_1257314, neoverse_n1, 1257314 570 report_errata ERRATA_N1_1262606, neoverse_n1, 1262606 571 report_errata ERRATA_N1_1262888, neoverse_n1, 1262888 572 report_errata ERRATA_N1_1275112, neoverse_n1, 1275112 573 report_errata ERRATA_N1_1315703, neoverse_n1, 1315703 574 report_errata ERRATA_N1_1542419, neoverse_n1, 1542419 575 report_errata ERRATA_N1_1868343, neoverse_n1, 1868343 576 report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184 577 578 ldp x8, x30, [sp], #16 579 ret 580endfunc neoverse_n1_errata_report 581#endif 582 583/* 584 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 585 * inner-shareable invalidation to an arbitrary address followed by a DSB. 586 * 587 * x1: Exception Syndrome 588 */ 589func neoverse_n1_errata_ic_trap_handler 590 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 591 b.ne 1f 592 tlbi vae3is, xzr 593 dsb sy 594 595 # Skip the IC instruction itself 596 mrs x3, elr_el3 597 add x3, x3, #4 598 msr elr_el3, x3 599 600 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 601 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 602 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 603 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 604 605#if IMAGE_BL31 && RAS_EXTENSION 606 /* 607 * Issue Error Synchronization Barrier to synchronize SErrors before 608 * exiting EL3. We're running with EAs unmasked, so any synchronized 609 * errors would be taken immediately; therefore no need to inspect 610 * DISR_EL1 register. 611 */ 612 esb 613#endif 614 exception_return 6151: 616 ret 617endfunc neoverse_n1_errata_ic_trap_handler 618 619 /* --------------------------------------------- 620 * This function provides neoverse_n1 specific 621 * register information for crash reporting. 622 * It needs to return with x6 pointing to 623 * a list of register names in ascii and 624 * x8 - x15 having values of registers to be 625 * reported. 626 * --------------------------------------------- 627 */ 628.section .rodata.neoverse_n1_regs, "aS" 629neoverse_n1_regs: /* The ascii list of register names to be reported */ 630 .asciz "cpuectlr_el1", "" 631 632func neoverse_n1_cpu_reg_dump 633 adr x6, neoverse_n1_regs 634 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 635 ret 636endfunc neoverse_n1_cpu_reg_dump 637 638declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 639 neoverse_n1_reset_func, \ 640 neoverse_n1_errata_ic_trap_handler, \ 641 neoverse_n1_core_pwr_dwn 642