1/* 2 * Copyright (c) 2017-2020, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <neoverse_n1.h> 10#include <cpuamu.h> 11#include <cpu_macros.S> 12#include <context.h> 13 14/* Hardware handled coherency */ 15#if HW_ASSISTED_COHERENCY == 0 16#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 17#endif 18 19/* 64-bit only core */ 20#if CTX_INCLUDE_AARCH32_REGS == 1 21#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 22#endif 23 24 .global neoverse_n1_errata_ic_trap_handler 25 26/* -------------------------------------------------- 27 * Errata Workaround for Neoverse N1 Erratum 1043202. 28 * This applies to revision r0p0 and r1p0 of Neoverse N1. 29 * Inputs: 30 * x0: variant[4:7] and revision[0:3] of current cpu. 31 * Shall clobber: x0-x17 32 * -------------------------------------------------- 33 */ 34func errata_n1_1043202_wa 35 /* Compare x0 against revision r1p0 */ 36 mov x17, x30 37 bl check_errata_1043202 38 cbz x0, 1f 39 40 /* Apply instruction patching sequence */ 41 ldr x0, =0x0 42 msr CPUPSELR_EL3, x0 43 ldr x0, =0xF3BF8F2F 44 msr CPUPOR_EL3, x0 45 ldr x0, =0xFFFFFFFF 46 msr CPUPMR_EL3, x0 47 ldr x0, =0x800200071 48 msr CPUPCR_EL3, x0 49 isb 501: 51 ret x17 52endfunc errata_n1_1043202_wa 53 54func check_errata_1043202 55 /* Applies to r0p0 and r1p0 */ 56 mov x1, #0x10 57 b cpu_rev_var_ls 58endfunc check_errata_1043202 59 60/* -------------------------------------------------- 61 * Disable speculative loads if Neoverse N1 supports 62 * SSBS. 63 * 64 * Shall clobber: x0. 65 * -------------------------------------------------- 66 */ 67func neoverse_n1_disable_speculative_loads 68 /* Check if the PE implements SSBS */ 69 mrs x0, id_aa64pfr1_el1 70 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 71 b.eq 1f 72 73 /* Disable speculative loads */ 74 msr SSBS, xzr 75 761: 77 ret 78endfunc neoverse_n1_disable_speculative_loads 79 80/* -------------------------------------------------- 81 * Errata Workaround for Neoverse N1 Errata #1073348 82 * This applies to revision r0p0 and r1p0 of Neoverse N1. 83 * Inputs: 84 * x0: variant[4:7] and revision[0:3] of current cpu. 85 * Shall clobber: x0-x17 86 * -------------------------------------------------- 87 */ 88func errata_n1_1073348_wa 89 /* Compare x0 against revision r1p0 */ 90 mov x17, x30 91 bl check_errata_1073348 92 cbz x0, 1f 93 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 94 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 95 msr NEOVERSE_N1_CPUACTLR_EL1, x1 961: 97 ret x17 98endfunc errata_n1_1073348_wa 99 100func check_errata_1073348 101 /* Applies to r0p0 and r1p0 */ 102 mov x1, #0x10 103 b cpu_rev_var_ls 104endfunc check_errata_1073348 105 106/* -------------------------------------------------- 107 * Errata Workaround for Neoverse N1 Errata #1130799 108 * This applies to revision <=r2p0 of Neoverse N1. 109 * Inputs: 110 * x0: variant[4:7] and revision[0:3] of current cpu. 111 * Shall clobber: x0-x17 112 * -------------------------------------------------- 113 */ 114func errata_n1_1130799_wa 115 /* Compare x0 against revision r2p0 */ 116 mov x17, x30 117 bl check_errata_1130799 118 cbz x0, 1f 119 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 120 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 121 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1221: 123 ret x17 124endfunc errata_n1_1130799_wa 125 126func check_errata_1130799 127 /* Applies to <=r2p0 */ 128 mov x1, #0x20 129 b cpu_rev_var_ls 130endfunc check_errata_1130799 131 132/* -------------------------------------------------- 133 * Errata Workaround for Neoverse N1 Errata #1165347 134 * This applies to revision <=r2p0 of Neoverse N1. 135 * Inputs: 136 * x0: variant[4:7] and revision[0:3] of current cpu. 137 * Shall clobber: x0-x17 138 * -------------------------------------------------- 139 */ 140func errata_n1_1165347_wa 141 /* Compare x0 against revision r2p0 */ 142 mov x17, x30 143 bl check_errata_1165347 144 cbz x0, 1f 145 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 146 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 147 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 148 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1491: 150 ret x17 151endfunc errata_n1_1165347_wa 152 153func check_errata_1165347 154 /* Applies to <=r2p0 */ 155 mov x1, #0x20 156 b cpu_rev_var_ls 157endfunc check_errata_1165347 158 159/* -------------------------------------------------- 160 * Errata Workaround for Neoverse N1 Errata #1207823 161 * This applies to revision <=r2p0 of Neoverse N1. 162 * Inputs: 163 * x0: variant[4:7] and revision[0:3] of current cpu. 164 * Shall clobber: x0-x17 165 * -------------------------------------------------- 166 */ 167func errata_n1_1207823_wa 168 /* Compare x0 against revision r2p0 */ 169 mov x17, x30 170 bl check_errata_1207823 171 cbz x0, 1f 172 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 173 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 174 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1751: 176 ret x17 177endfunc errata_n1_1207823_wa 178 179func check_errata_1207823 180 /* Applies to <=r2p0 */ 181 mov x1, #0x20 182 b cpu_rev_var_ls 183endfunc check_errata_1207823 184 185/* -------------------------------------------------- 186 * Errata Workaround for Neoverse N1 Errata #1220197 187 * This applies to revision <=r2p0 of Neoverse N1. 188 * Inputs: 189 * x0: variant[4:7] and revision[0:3] of current cpu. 190 * Shall clobber: x0-x17 191 * -------------------------------------------------- 192 */ 193func errata_n1_1220197_wa 194 /* Compare x0 against revision r2p0 */ 195 mov x17, x30 196 bl check_errata_1220197 197 cbz x0, 1f 198 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 199 orr x1, x1, NEOVERSE_N1_WS_THR_L2_MASK 200 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2011: 202 ret x17 203endfunc errata_n1_1220197_wa 204 205func check_errata_1220197 206 /* Applies to <=r2p0 */ 207 mov x1, #0x20 208 b cpu_rev_var_ls 209endfunc check_errata_1220197 210 211/* -------------------------------------------------- 212 * Errata Workaround for Neoverse N1 Errata #1257314 213 * This applies to revision <=r3p0 of Neoverse N1. 214 * Inputs: 215 * x0: variant[4:7] and revision[0:3] of current cpu. 216 * Shall clobber: x0-x17 217 * -------------------------------------------------- 218 */ 219func errata_n1_1257314_wa 220 /* Compare x0 against revision r3p0 */ 221 mov x17, x30 222 bl check_errata_1257314 223 cbz x0, 1f 224 mrs x1, NEOVERSE_N1_CPUACTLR3_EL1 225 orr x1, x1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 226 msr NEOVERSE_N1_CPUACTLR3_EL1, x1 2271: 228 ret x17 229endfunc errata_n1_1257314_wa 230 231func check_errata_1257314 232 /* Applies to <=r3p0 */ 233 mov x1, #0x30 234 b cpu_rev_var_ls 235endfunc check_errata_1257314 236 237/* -------------------------------------------------- 238 * Errata Workaround for Neoverse N1 Errata #1262606 239 * This applies to revision <=r3p0 of Neoverse N1. 240 * Inputs: 241 * x0: variant[4:7] and revision[0:3] of current cpu. 242 * Shall clobber: x0-x17 243 * -------------------------------------------------- 244 */ 245func errata_n1_1262606_wa 246 /* Compare x0 against revision r3p0 */ 247 mov x17, x30 248 bl check_errata_1262606 249 cbz x0, 1f 250 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 251 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 252 msr NEOVERSE_N1_CPUACTLR_EL1, x1 2531: 254 ret x17 255endfunc errata_n1_1262606_wa 256 257func check_errata_1262606 258 /* Applies to <=r3p0 */ 259 mov x1, #0x30 260 b cpu_rev_var_ls 261endfunc check_errata_1262606 262 263/* -------------------------------------------------- 264 * Errata Workaround for Neoverse N1 Errata #1262888 265 * This applies to revision <=r3p0 of Neoverse N1. 266 * Inputs: 267 * x0: variant[4:7] and revision[0:3] of current cpu. 268 * Shall clobber: x0-x17 269 * -------------------------------------------------- 270 */ 271func errata_n1_1262888_wa 272 /* Compare x0 against revision r3p0 */ 273 mov x17, x30 274 bl check_errata_1262888 275 cbz x0, 1f 276 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 277 orr x1, x1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 278 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2791: 280 ret x17 281endfunc errata_n1_1262888_wa 282 283func check_errata_1262888 284 /* Applies to <=r3p0 */ 285 mov x1, #0x30 286 b cpu_rev_var_ls 287endfunc check_errata_1262888 288 289/* -------------------------------------------------- 290 * Errata Workaround for Neoverse N1 Errata #1275112 291 * This applies to revision <=r3p0 of Neoverse N1. 292 * Inputs: 293 * x0: variant[4:7] and revision[0:3] of current cpu. 294 * Shall clobber: x0-x17 295 * -------------------------------------------------- 296 */ 297func errata_n1_1275112_wa 298 /* Compare x0 against revision r3p0 */ 299 mov x17, x30 300 bl check_errata_1275112 301 cbz x0, 1f 302 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 303 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 304 msr NEOVERSE_N1_CPUACTLR_EL1, x1 3051: 306 ret x17 307endfunc errata_n1_1275112_wa 308 309func check_errata_1275112 310 /* Applies to <=r3p0 */ 311 mov x1, #0x30 312 b cpu_rev_var_ls 313endfunc check_errata_1275112 314 315/* -------------------------------------------------- 316 * Errata Workaround for Neoverse N1 Erratum 1315703. 317 * This applies to revision <= r3p0 of Neoverse N1. 318 * Inputs: 319 * x0: variant[4:7] and revision[0:3] of current cpu. 320 * Shall clobber: x0-x17 321 * -------------------------------------------------- 322 */ 323func errata_n1_1315703_wa 324 /* Compare x0 against revision r3p1 */ 325 mov x17, x30 326 bl check_errata_1315703 327 cbz x0, 1f 328 329 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 330 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 331 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 332 3331: 334 ret x17 335endfunc errata_n1_1315703_wa 336 337func check_errata_1315703 338 /* Applies to everything <= r3p0. */ 339 mov x1, #0x30 340 b cpu_rev_var_ls 341endfunc check_errata_1315703 342 343/* -------------------------------------------------- 344 * Errata Workaround for Neoverse N1 Erratum 1542419. 345 * This applies to revisions r3p0 - r4p0 of Neoverse N1 346 * Inputs: 347 * x0: variant[4:7] and revision[0:3] of current cpu. 348 * Shall clobber: x0-x17 349 * -------------------------------------------------- 350 */ 351func errata_n1_1542419_wa 352 /* Compare x0 against revision r3p0 and r4p0 */ 353 mov x17, x30 354 bl check_errata_1542419 355 cbz x0, 1f 356 357 /* Apply instruction patching sequence */ 358 ldr x0, =0x0 359 msr CPUPSELR_EL3, x0 360 ldr x0, =0xEE670D35 361 msr CPUPOR_EL3, x0 362 ldr x0, =0xFFFF0FFF 363 msr CPUPMR_EL3, x0 364 ldr x0, =0x08000020007D 365 msr CPUPCR_EL3, x0 366 isb 3671: 368 ret x17 369endfunc errata_n1_1542419_wa 370 371func check_errata_1542419 372 /* Applies to everything r3p0 - r4p0. */ 373 mov x1, #0x30 374 mov x2, #0x40 375 b cpu_rev_var_range 376endfunc check_errata_1542419 377 378/* -------------------------------------------------- 379 * Errata Workaround for Neoverse N1 Erratum 1800710. 380 * This applies to revisions <= r4p0 of Neoverse N1 381 * Inputs: 382 * x0: variant[4:7] and revision[0:3] of current cpu. 383 * Shall clobber: x0-x17 384 * -------------------------------------------------- 385 */ 386func errata_n1_1800710_wa 387 /* Compare x0 against revision <= r4p0 */ 388 mov x17, x30 389 bl check_errata_1800710 390 cbz x0, 1f 391 392 /* Disable allocation of splintered pages in the L2 TLB */ 393 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 394 orr x1, x1, NEOVERSE_N1_CPUECTLR_EL1_BIT_53 395 msr NEOVERSE_N1_CPUECTLR_EL1, x1 396 isb 3971: 398 ret x17 399endfunc errata_n1_1800710_wa 400 401func check_errata_1800710 402 /* Applies to everything <= r4p0 */ 403 mov x1, #0x40 404 b cpu_rev_var_ls 405endfunc check_errata_1800710 406 407func neoverse_n1_reset_func 408 mov x19, x30 409 410 bl neoverse_n1_disable_speculative_loads 411 412 /* Forces all cacheable atomic instructions to be near */ 413 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 414 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 415 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 416 isb 417 418 bl cpu_get_rev_var 419 mov x18, x0 420 421#if ERRATA_N1_1043202 422 mov x0, x18 423 bl errata_n1_1043202_wa 424#endif 425 426#if ERRATA_N1_1073348 427 mov x0, x18 428 bl errata_n1_1073348_wa 429#endif 430 431#if ERRATA_N1_1130799 432 mov x0, x18 433 bl errata_n1_1130799_wa 434#endif 435 436#if ERRATA_N1_1165347 437 mov x0, x18 438 bl errata_n1_1165347_wa 439#endif 440 441#if ERRATA_N1_1207823 442 mov x0, x18 443 bl errata_n1_1207823_wa 444#endif 445 446#if ERRATA_N1_1220197 447 mov x0, x18 448 bl errata_n1_1220197_wa 449#endif 450 451#if ERRATA_N1_1257314 452 mov x0, x18 453 bl errata_n1_1257314_wa 454#endif 455 456#if ERRATA_N1_1262606 457 mov x0, x18 458 bl errata_n1_1262606_wa 459#endif 460 461#if ERRATA_N1_1262888 462 mov x0, x18 463 bl errata_n1_1262888_wa 464#endif 465 466#if ERRATA_N1_1275112 467 mov x0, x18 468 bl errata_n1_1275112_wa 469#endif 470 471#if ERRATA_N1_1315703 472 mov x0, x18 473 bl errata_n1_1315703_wa 474#endif 475 476#if ERRATA_N1_1542419 477 mov x0, x18 478 bl errata_n1_1542419_wa 479#endif 480 481#if ERRATA_N1_1800710 482 mov x0, x18 483 bl errata_n1_1800710_wa 484#endif 485 486#if ENABLE_AMU 487 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 488 mrs x0, actlr_el3 489 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 490 msr actlr_el3, x0 491 492 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 493 mrs x0, actlr_el2 494 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 495 msr actlr_el2, x0 496 497 /* Enable group0 counters */ 498 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 499 msr CPUAMCNTENSET_EL0, x0 500#endif 501 502#if NEOVERSE_N1_EXTERNAL_LLC 503 /* Some system may have External LLC, core needs to be made aware */ 504 mrs x0, NEOVERSE_N1_CPUECTLR_EL1 505 orr x0, x0, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 506 msr NEOVERSE_N1_CPUECTLR_EL1, x0 507#endif 508 509#if ERRATA_DSU_936184 510 bl errata_dsu_936184_wa 511#endif 512 513 isb 514 ret x19 515endfunc neoverse_n1_reset_func 516 517 /* --------------------------------------------- 518 * HW will do the cache maintenance while powering down 519 * --------------------------------------------- 520 */ 521func neoverse_n1_core_pwr_dwn 522 /* --------------------------------------------- 523 * Enable CPU power down bit in power control register 524 * --------------------------------------------- 525 */ 526 mrs x0, NEOVERSE_N1_CPUPWRCTLR_EL1 527 orr x0, x0, #NEOVERSE_N1_CORE_PWRDN_EN_MASK 528 msr NEOVERSE_N1_CPUPWRCTLR_EL1, x0 529 isb 530 ret 531endfunc neoverse_n1_core_pwr_dwn 532 533#if REPORT_ERRATA 534/* 535 * Errata printing function for Neoverse N1. Must follow AAPCS. 536 */ 537func neoverse_n1_errata_report 538 stp x8, x30, [sp, #-16]! 539 540 bl cpu_get_rev_var 541 mov x8, x0 542 543 /* 544 * Report all errata. The revision-variant information is passed to 545 * checking functions of each errata. 546 */ 547 report_errata ERRATA_N1_1043202, neoverse_n1, 1043202 548 report_errata ERRATA_N1_1073348, neoverse_n1, 1073348 549 report_errata ERRATA_N1_1130799, neoverse_n1, 1130799 550 report_errata ERRATA_N1_1165347, neoverse_n1, 1165347 551 report_errata ERRATA_N1_1207823, neoverse_n1, 1207823 552 report_errata ERRATA_N1_1220197, neoverse_n1, 1220197 553 report_errata ERRATA_N1_1257314, neoverse_n1, 1257314 554 report_errata ERRATA_N1_1262606, neoverse_n1, 1262606 555 report_errata ERRATA_N1_1262888, neoverse_n1, 1262888 556 report_errata ERRATA_N1_1275112, neoverse_n1, 1275112 557 report_errata ERRATA_N1_1315703, neoverse_n1, 1315703 558 report_errata ERRATA_N1_1542419, neoverse_n1, 1542419 559 report_errata ERRATA_N1_1800710, neoverse_n1, 1800710 560 report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184 561 562 ldp x8, x30, [sp], #16 563 ret 564endfunc neoverse_n1_errata_report 565#endif 566 567/* 568 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 569 * inner-shareable invalidation to an arbitrary address followed by a DSB. 570 * 571 * x1: Exception Syndrome 572 */ 573func neoverse_n1_errata_ic_trap_handler 574 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 575 b.ne 1f 576 tlbi vae3is, xzr 577 dsb sy 578 579 # Skip the IC instruction itself 580 mrs x3, elr_el3 581 add x3, x3, #4 582 msr elr_el3, x3 583 584 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 585 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 586 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 587 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 588 589#if IMAGE_BL31 && RAS_EXTENSION 590 /* 591 * Issue Error Synchronization Barrier to synchronize SErrors before 592 * exiting EL3. We're running with EAs unmasked, so any synchronized 593 * errors would be taken immediately; therefore no need to inspect 594 * DISR_EL1 register. 595 */ 596 esb 597#endif 598 exception_return 5991: 600 ret 601endfunc neoverse_n1_errata_ic_trap_handler 602 603 /* --------------------------------------------- 604 * This function provides neoverse_n1 specific 605 * register information for crash reporting. 606 * It needs to return with x6 pointing to 607 * a list of register names in ascii and 608 * x8 - x15 having values of registers to be 609 * reported. 610 * --------------------------------------------- 611 */ 612.section .rodata.neoverse_n1_regs, "aS" 613neoverse_n1_regs: /* The ascii list of register names to be reported */ 614 .asciz "cpuectlr_el1", "" 615 616func neoverse_n1_cpu_reg_dump 617 adr x6, neoverse_n1_regs 618 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 619 ret 620endfunc neoverse_n1_cpu_reg_dump 621 622declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 623 neoverse_n1_reset_func, \ 624 neoverse_n1_errata_ic_trap_handler, \ 625 neoverse_n1_core_pwr_dwn 626