1/* 2 * Copyright (c) 2017-2020, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <neoverse_n1.h> 10#include <cpuamu.h> 11#include <cpu_macros.S> 12#include <context.h> 13 14/* Hardware handled coherency */ 15#if HW_ASSISTED_COHERENCY == 0 16#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 17#endif 18 19/* 64-bit only core */ 20#if CTX_INCLUDE_AARCH32_REGS == 1 21#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 22#endif 23 24 .global neoverse_n1_errata_ic_trap_handler 25 .global is_scu_present_in_dsu 26 27/* 28 * Check DSU is configured with SCU and L3 unit 29 * 1-> SCU present 30 * 0-> SCU not present 31 */ 32func is_scu_present_in_dsu 33 mrs x0, CPUCFR_EL1 34 ubfx x0, x0, #SCU_SHIFT, #1 35 eor x0, x0, #1 36 ret 37endfunc is_scu_present_in_dsu 38 39/* -------------------------------------------------- 40 * Errata Workaround for Neoverse N1 Erratum 1043202. 41 * This applies to revision r0p0 and r1p0 of Neoverse N1. 42 * Inputs: 43 * x0: variant[4:7] and revision[0:3] of current cpu. 44 * Shall clobber: x0-x17 45 * -------------------------------------------------- 46 */ 47func errata_n1_1043202_wa 48 /* Compare x0 against revision r1p0 */ 49 mov x17, x30 50 bl check_errata_1043202 51 cbz x0, 1f 52 53 /* Apply instruction patching sequence */ 54 ldr x0, =0x0 55 msr CPUPSELR_EL3, x0 56 ldr x0, =0xF3BF8F2F 57 msr CPUPOR_EL3, x0 58 ldr x0, =0xFFFFFFFF 59 msr CPUPMR_EL3, x0 60 ldr x0, =0x800200071 61 msr CPUPCR_EL3, x0 62 isb 631: 64 ret x17 65endfunc errata_n1_1043202_wa 66 67func check_errata_1043202 68 /* Applies to r0p0 and r1p0 */ 69 mov x1, #0x10 70 b cpu_rev_var_ls 71endfunc check_errata_1043202 72 73/* -------------------------------------------------- 74 * Disable speculative loads if Neoverse N1 supports 75 * SSBS. 76 * 77 * Shall clobber: x0. 78 * -------------------------------------------------- 79 */ 80func neoverse_n1_disable_speculative_loads 81 /* Check if the PE implements SSBS */ 82 mrs x0, id_aa64pfr1_el1 83 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 84 b.eq 1f 85 86 /* Disable speculative loads */ 87 msr SSBS, xzr 88 891: 90 ret 91endfunc neoverse_n1_disable_speculative_loads 92 93/* -------------------------------------------------- 94 * Errata Workaround for Neoverse N1 Errata #1073348 95 * This applies to revision r0p0 and r1p0 of Neoverse N1. 96 * Inputs: 97 * x0: variant[4:7] and revision[0:3] of current cpu. 98 * Shall clobber: x0-x17 99 * -------------------------------------------------- 100 */ 101func errata_n1_1073348_wa 102 /* Compare x0 against revision r1p0 */ 103 mov x17, x30 104 bl check_errata_1073348 105 cbz x0, 1f 106 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 107 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 108 msr NEOVERSE_N1_CPUACTLR_EL1, x1 1091: 110 ret x17 111endfunc errata_n1_1073348_wa 112 113func check_errata_1073348 114 /* Applies to r0p0 and r1p0 */ 115 mov x1, #0x10 116 b cpu_rev_var_ls 117endfunc check_errata_1073348 118 119/* -------------------------------------------------- 120 * Errata Workaround for Neoverse N1 Errata #1130799 121 * This applies to revision <=r2p0 of Neoverse N1. 122 * Inputs: 123 * x0: variant[4:7] and revision[0:3] of current cpu. 124 * Shall clobber: x0-x17 125 * -------------------------------------------------- 126 */ 127func errata_n1_1130799_wa 128 /* Compare x0 against revision r2p0 */ 129 mov x17, x30 130 bl check_errata_1130799 131 cbz x0, 1f 132 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 133 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 134 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1351: 136 ret x17 137endfunc errata_n1_1130799_wa 138 139func check_errata_1130799 140 /* Applies to <=r2p0 */ 141 mov x1, #0x20 142 b cpu_rev_var_ls 143endfunc check_errata_1130799 144 145/* -------------------------------------------------- 146 * Errata Workaround for Neoverse N1 Errata #1165347 147 * This applies to revision <=r2p0 of Neoverse N1. 148 * Inputs: 149 * x0: variant[4:7] and revision[0:3] of current cpu. 150 * Shall clobber: x0-x17 151 * -------------------------------------------------- 152 */ 153func errata_n1_1165347_wa 154 /* Compare x0 against revision r2p0 */ 155 mov x17, x30 156 bl check_errata_1165347 157 cbz x0, 1f 158 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 159 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 160 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 161 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1621: 163 ret x17 164endfunc errata_n1_1165347_wa 165 166func check_errata_1165347 167 /* Applies to <=r2p0 */ 168 mov x1, #0x20 169 b cpu_rev_var_ls 170endfunc check_errata_1165347 171 172/* -------------------------------------------------- 173 * Errata Workaround for Neoverse N1 Errata #1207823 174 * This applies to revision <=r2p0 of Neoverse N1. 175 * Inputs: 176 * x0: variant[4:7] and revision[0:3] of current cpu. 177 * Shall clobber: x0-x17 178 * -------------------------------------------------- 179 */ 180func errata_n1_1207823_wa 181 /* Compare x0 against revision r2p0 */ 182 mov x17, x30 183 bl check_errata_1207823 184 cbz x0, 1f 185 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 186 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 187 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1881: 189 ret x17 190endfunc errata_n1_1207823_wa 191 192func check_errata_1207823 193 /* Applies to <=r2p0 */ 194 mov x1, #0x20 195 b cpu_rev_var_ls 196endfunc check_errata_1207823 197 198/* -------------------------------------------------- 199 * Errata Workaround for Neoverse N1 Errata #1220197 200 * This applies to revision <=r2p0 of Neoverse N1. 201 * Inputs: 202 * x0: variant[4:7] and revision[0:3] of current cpu. 203 * Shall clobber: x0-x17 204 * -------------------------------------------------- 205 */ 206func errata_n1_1220197_wa 207 /* Compare x0 against revision r2p0 */ 208 mov x17, x30 209 bl check_errata_1220197 210 cbz x0, 1f 211 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 212 orr x1, x1, NEOVERSE_N1_WS_THR_L2_MASK 213 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2141: 215 ret x17 216endfunc errata_n1_1220197_wa 217 218func check_errata_1220197 219 /* Applies to <=r2p0 */ 220 mov x1, #0x20 221 b cpu_rev_var_ls 222endfunc check_errata_1220197 223 224/* -------------------------------------------------- 225 * Errata Workaround for Neoverse N1 Errata #1257314 226 * This applies to revision <=r3p0 of Neoverse N1. 227 * Inputs: 228 * x0: variant[4:7] and revision[0:3] of current cpu. 229 * Shall clobber: x0-x17 230 * -------------------------------------------------- 231 */ 232func errata_n1_1257314_wa 233 /* Compare x0 against revision r3p0 */ 234 mov x17, x30 235 bl check_errata_1257314 236 cbz x0, 1f 237 mrs x1, NEOVERSE_N1_CPUACTLR3_EL1 238 orr x1, x1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 239 msr NEOVERSE_N1_CPUACTLR3_EL1, x1 2401: 241 ret x17 242endfunc errata_n1_1257314_wa 243 244func check_errata_1257314 245 /* Applies to <=r3p0 */ 246 mov x1, #0x30 247 b cpu_rev_var_ls 248endfunc check_errata_1257314 249 250/* -------------------------------------------------- 251 * Errata Workaround for Neoverse N1 Errata #1262606 252 * This applies to revision <=r3p0 of Neoverse N1. 253 * Inputs: 254 * x0: variant[4:7] and revision[0:3] of current cpu. 255 * Shall clobber: x0-x17 256 * -------------------------------------------------- 257 */ 258func errata_n1_1262606_wa 259 /* Compare x0 against revision r3p0 */ 260 mov x17, x30 261 bl check_errata_1262606 262 cbz x0, 1f 263 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 264 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 265 msr NEOVERSE_N1_CPUACTLR_EL1, x1 2661: 267 ret x17 268endfunc errata_n1_1262606_wa 269 270func check_errata_1262606 271 /* Applies to <=r3p0 */ 272 mov x1, #0x30 273 b cpu_rev_var_ls 274endfunc check_errata_1262606 275 276/* -------------------------------------------------- 277 * Errata Workaround for Neoverse N1 Errata #1262888 278 * This applies to revision <=r3p0 of Neoverse N1. 279 * Inputs: 280 * x0: variant[4:7] and revision[0:3] of current cpu. 281 * Shall clobber: x0-x17 282 * -------------------------------------------------- 283 */ 284func errata_n1_1262888_wa 285 /* Compare x0 against revision r3p0 */ 286 mov x17, x30 287 bl check_errata_1262888 288 cbz x0, 1f 289 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 290 orr x1, x1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 291 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2921: 293 ret x17 294endfunc errata_n1_1262888_wa 295 296func check_errata_1262888 297 /* Applies to <=r3p0 */ 298 mov x1, #0x30 299 b cpu_rev_var_ls 300endfunc check_errata_1262888 301 302/* -------------------------------------------------- 303 * Errata Workaround for Neoverse N1 Errata #1275112 304 * This applies to revision <=r3p0 of Neoverse N1. 305 * Inputs: 306 * x0: variant[4:7] and revision[0:3] of current cpu. 307 * Shall clobber: x0-x17 308 * -------------------------------------------------- 309 */ 310func errata_n1_1275112_wa 311 /* Compare x0 against revision r3p0 */ 312 mov x17, x30 313 bl check_errata_1275112 314 cbz x0, 1f 315 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 316 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 317 msr NEOVERSE_N1_CPUACTLR_EL1, x1 3181: 319 ret x17 320endfunc errata_n1_1275112_wa 321 322func check_errata_1275112 323 /* Applies to <=r3p0 */ 324 mov x1, #0x30 325 b cpu_rev_var_ls 326endfunc check_errata_1275112 327 328/* -------------------------------------------------- 329 * Errata Workaround for Neoverse N1 Erratum 1315703. 330 * This applies to revision <= r3p0 of Neoverse N1. 331 * Inputs: 332 * x0: variant[4:7] and revision[0:3] of current cpu. 333 * Shall clobber: x0-x17 334 * -------------------------------------------------- 335 */ 336func errata_n1_1315703_wa 337 /* Compare x0 against revision r3p1 */ 338 mov x17, x30 339 bl check_errata_1315703 340 cbz x0, 1f 341 342 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 343 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 344 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 345 3461: 347 ret x17 348endfunc errata_n1_1315703_wa 349 350func check_errata_1315703 351 /* Applies to everything <= r3p0. */ 352 mov x1, #0x30 353 b cpu_rev_var_ls 354endfunc check_errata_1315703 355 356/* -------------------------------------------------- 357 * Errata Workaround for Neoverse N1 Erratum 1542419. 358 * This applies to revisions r3p0 - r4p0 of Neoverse N1 359 * Inputs: 360 * x0: variant[4:7] and revision[0:3] of current cpu. 361 * Shall clobber: x0-x17 362 * -------------------------------------------------- 363 */ 364func errata_n1_1542419_wa 365 /* Compare x0 against revision r3p0 and r4p0 */ 366 mov x17, x30 367 bl check_errata_1542419 368 cbz x0, 1f 369 370 /* Apply instruction patching sequence */ 371 ldr x0, =0x0 372 msr CPUPSELR_EL3, x0 373 ldr x0, =0xEE670D35 374 msr CPUPOR_EL3, x0 375 ldr x0, =0xFFFF0FFF 376 msr CPUPMR_EL3, x0 377 ldr x0, =0x08000020007D 378 msr CPUPCR_EL3, x0 379 isb 3801: 381 ret x17 382endfunc errata_n1_1542419_wa 383 384func check_errata_1542419 385 /* Applies to everything r3p0 - r4p0. */ 386 mov x1, #0x30 387 mov x2, #0x40 388 b cpu_rev_var_range 389endfunc check_errata_1542419 390 391func neoverse_n1_reset_func 392 mov x19, x30 393 394 bl neoverse_n1_disable_speculative_loads 395 396 /* Forces all cacheable atomic instructions to be near */ 397 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 398 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 399 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 400 isb 401 402 bl cpu_get_rev_var 403 mov x18, x0 404 405#if ERRATA_N1_1043202 406 mov x0, x18 407 bl errata_n1_1043202_wa 408#endif 409 410#if ERRATA_N1_1073348 411 mov x0, x18 412 bl errata_n1_1073348_wa 413#endif 414 415#if ERRATA_N1_1130799 416 mov x0, x18 417 bl errata_n1_1130799_wa 418#endif 419 420#if ERRATA_N1_1165347 421 mov x0, x18 422 bl errata_n1_1165347_wa 423#endif 424 425#if ERRATA_N1_1207823 426 mov x0, x18 427 bl errata_n1_1207823_wa 428#endif 429 430#if ERRATA_N1_1220197 431 mov x0, x18 432 bl errata_n1_1220197_wa 433#endif 434 435#if ERRATA_N1_1257314 436 mov x0, x18 437 bl errata_n1_1257314_wa 438#endif 439 440#if ERRATA_N1_1262606 441 mov x0, x18 442 bl errata_n1_1262606_wa 443#endif 444 445#if ERRATA_N1_1262888 446 mov x0, x18 447 bl errata_n1_1262888_wa 448#endif 449 450#if ERRATA_N1_1275112 451 mov x0, x18 452 bl errata_n1_1275112_wa 453#endif 454 455#if ERRATA_N1_1315703 456 mov x0, x18 457 bl errata_n1_1315703_wa 458#endif 459 460#if ERRATA_N1_1542419 461 mov x0, x18 462 bl errata_n1_1542419_wa 463#endif 464 465#if ENABLE_AMU 466 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 467 mrs x0, actlr_el3 468 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 469 msr actlr_el3, x0 470 471 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 472 mrs x0, actlr_el2 473 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 474 msr actlr_el2, x0 475 476 /* Enable group0 counters */ 477 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 478 msr CPUAMCNTENSET_EL0, x0 479#endif 480 481#if NEOVERSE_N1_EXTERNAL_LLC 482 /* Some system may have External LLC, core needs to be made aware */ 483 mrs x0, NEOVERSE_N1_CPUECTLR_EL1 484 orr x0, x0, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 485 msr NEOVERSE_N1_CPUECTLR_EL1, x0 486#endif 487 488#if ERRATA_DSU_936184 489 bl errata_dsu_936184_wa 490#endif 491 492 isb 493 ret x19 494endfunc neoverse_n1_reset_func 495 496 /* --------------------------------------------- 497 * HW will do the cache maintenance while powering down 498 * --------------------------------------------- 499 */ 500func neoverse_n1_core_pwr_dwn 501 /* --------------------------------------------- 502 * Enable CPU power down bit in power control register 503 * --------------------------------------------- 504 */ 505 mrs x0, NEOVERSE_N1_CPUPWRCTLR_EL1 506 orr x0, x0, #NEOVERSE_N1_CORE_PWRDN_EN_MASK 507 msr NEOVERSE_N1_CPUPWRCTLR_EL1, x0 508 isb 509 ret 510endfunc neoverse_n1_core_pwr_dwn 511 512#if REPORT_ERRATA 513/* 514 * Errata printing function for Neoverse N1. Must follow AAPCS. 515 */ 516func neoverse_n1_errata_report 517 stp x8, x30, [sp, #-16]! 518 519 bl cpu_get_rev_var 520 mov x8, x0 521 522 /* 523 * Report all errata. The revision-variant information is passed to 524 * checking functions of each errata. 525 */ 526 report_errata ERRATA_N1_1043202, neoverse_n1, 1043202 527 report_errata ERRATA_N1_1073348, neoverse_n1, 1073348 528 report_errata ERRATA_N1_1130799, neoverse_n1, 1130799 529 report_errata ERRATA_N1_1165347, neoverse_n1, 1165347 530 report_errata ERRATA_N1_1207823, neoverse_n1, 1207823 531 report_errata ERRATA_N1_1220197, neoverse_n1, 1220197 532 report_errata ERRATA_N1_1257314, neoverse_n1, 1257314 533 report_errata ERRATA_N1_1262606, neoverse_n1, 1262606 534 report_errata ERRATA_N1_1262888, neoverse_n1, 1262888 535 report_errata ERRATA_N1_1275112, neoverse_n1, 1275112 536 report_errata ERRATA_N1_1315703, neoverse_n1, 1315703 537 report_errata ERRATA_N1_1542419, neoverse_n1, 1542419 538 report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184 539 540 ldp x8, x30, [sp], #16 541 ret 542endfunc neoverse_n1_errata_report 543#endif 544 545/* 546 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 547 * inner-shareable invalidation to an arbitrary address followed by a DSB. 548 * 549 * x1: Exception Syndrome 550 */ 551func neoverse_n1_errata_ic_trap_handler 552 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 553 b.ne 1f 554 tlbi vae3is, xzr 555 dsb sy 556 557 # Skip the IC instruction itself 558 mrs x3, elr_el3 559 add x3, x3, #4 560 msr elr_el3, x3 561 562 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 563 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 564 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 565 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 566 567#if IMAGE_BL31 && RAS_EXTENSION 568 /* 569 * Issue Error Synchronization Barrier to synchronize SErrors before 570 * exiting EL3. We're running with EAs unmasked, so any synchronized 571 * errors would be taken immediately; therefore no need to inspect 572 * DISR_EL1 register. 573 */ 574 esb 575#endif 576 exception_return 5771: 578 ret 579endfunc neoverse_n1_errata_ic_trap_handler 580 581 /* --------------------------------------------- 582 * This function provides neoverse_n1 specific 583 * register information for crash reporting. 584 * It needs to return with x6 pointing to 585 * a list of register names in ascii and 586 * x8 - x15 having values of registers to be 587 * reported. 588 * --------------------------------------------- 589 */ 590.section .rodata.neoverse_n1_regs, "aS" 591neoverse_n1_regs: /* The ascii list of register names to be reported */ 592 .asciz "cpuectlr_el1", "" 593 594func neoverse_n1_cpu_reg_dump 595 adr x6, neoverse_n1_regs 596 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 597 ret 598endfunc neoverse_n1_cpu_reg_dump 599 600declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 601 neoverse_n1_reset_func, \ 602 neoverse_n1_errata_ic_trap_handler, \ 603 neoverse_n1_core_pwr_dwn 604