1/* 2 * Copyright (c) 2017-2019, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <neoverse_n1.h> 10#include <cpuamu.h> 11#include <cpu_macros.S> 12#include <context.h> 13 14/* Hardware handled coherency */ 15#if HW_ASSISTED_COHERENCY == 0 16#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 17#endif 18 19/* 64-bit only core */ 20#if CTX_INCLUDE_AARCH32_REGS == 1 21#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 22#endif 23 24#if ERRATA_N1_IC_TRAP 25 .global neoverse_n1_errata_ic_trap_handler 26#endif 27 28/* -------------------------------------------------- 29 * Errata Workaround for Neoverse N1 Erratum 1043202. 30 * This applies to revision r0p0 and r1p0 of Neoverse N1. 31 * Inputs: 32 * x0: variant[4:7] and revision[0:3] of current cpu. 33 * Shall clobber: x0-x17 34 * -------------------------------------------------- 35 */ 36func errata_n1_1043202_wa 37 /* Compare x0 against revision r1p0 */ 38 mov x17, x30 39 bl check_errata_1043202 40 cbz x0, 1f 41 42 /* Apply instruction patching sequence */ 43 ldr x0, =0x0 44 msr CPUPSELR_EL3, x0 45 ldr x0, =0xF3BF8F2F 46 msr CPUPOR_EL3, x0 47 ldr x0, =0xFFFFFFFF 48 msr CPUPMR_EL3, x0 49 ldr x0, =0x800200071 50 msr CPUPCR_EL3, x0 51 isb 521: 53 ret x17 54endfunc errata_n1_1043202_wa 55 56func check_errata_1043202 57 /* Applies to r0p0 and r1p0 */ 58 mov x1, #0x10 59 b cpu_rev_var_ls 60endfunc check_errata_1043202 61 62/* -------------------------------------------------- 63 * Disable speculative loads if Neoverse N1 supports 64 * SSBS. 65 * 66 * Shall clobber: x0. 67 * -------------------------------------------------- 68 */ 69func neoverse_n1_disable_speculative_loads 70 /* Check if the PE implements SSBS */ 71 mrs x0, id_aa64pfr1_el1 72 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 73 b.eq 1f 74 75 /* Disable speculative loads */ 76 msr SSBS, xzr 77 781: 79 ret 80endfunc neoverse_n1_disable_speculative_loads 81 82/* -------------------------------------------------- 83 * Errata Workaround for Neoverse N1 Errata #1073348 84 * This applies to revision r0p0 and r1p0 of Neoverse N1. 85 * Inputs: 86 * x0: variant[4:7] and revision[0:3] of current cpu. 87 * Shall clobber: x0-x17 88 * -------------------------------------------------- 89 */ 90func errata_n1_1073348_wa 91 /* Compare x0 against revision r1p0 */ 92 mov x17, x30 93 bl check_errata_1073348 94 cbz x0, 1f 95 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 96 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 97 msr NEOVERSE_N1_CPUACTLR_EL1, x1 981: 99 ret x17 100endfunc errata_n1_1073348_wa 101 102func check_errata_1073348 103 /* Applies to r0p0 and r1p0 */ 104 mov x1, #0x10 105 b cpu_rev_var_ls 106endfunc check_errata_1073348 107 108/* -------------------------------------------------- 109 * Errata Workaround for Neoverse N1 Errata #1130799 110 * This applies to revision <=r2p0 of Neoverse N1. 111 * Inputs: 112 * x0: variant[4:7] and revision[0:3] of current cpu. 113 * Shall clobber: x0-x17 114 * -------------------------------------------------- 115 */ 116func errata_n1_1130799_wa 117 /* Compare x0 against revision r2p0 */ 118 mov x17, x30 119 bl check_errata_1130799 120 cbz x0, 1f 121 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 122 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 123 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1241: 125 ret x17 126endfunc errata_n1_1130799_wa 127 128func check_errata_1130799 129 /* Applies to <=r2p0 */ 130 mov x1, #0x20 131 b cpu_rev_var_ls 132endfunc check_errata_1130799 133 134/* -------------------------------------------------- 135 * Errata Workaround for Neoverse N1 Errata #1165347 136 * This applies to revision <=r2p0 of Neoverse N1. 137 * Inputs: 138 * x0: variant[4:7] and revision[0:3] of current cpu. 139 * Shall clobber: x0-x17 140 * -------------------------------------------------- 141 */ 142func errata_n1_1165347_wa 143 /* Compare x0 against revision r2p0 */ 144 mov x17, x30 145 bl check_errata_1165347 146 cbz x0, 1f 147 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 148 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 149 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 150 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1511: 152 ret x17 153endfunc errata_n1_1165347_wa 154 155func check_errata_1165347 156 /* Applies to <=r2p0 */ 157 mov x1, #0x20 158 b cpu_rev_var_ls 159endfunc check_errata_1165347 160 161/* -------------------------------------------------- 162 * Errata Workaround for Neoverse N1 Errata #1207823 163 * This applies to revision <=r2p0 of Neoverse N1. 164 * Inputs: 165 * x0: variant[4:7] and revision[0:3] of current cpu. 166 * Shall clobber: x0-x17 167 * -------------------------------------------------- 168 */ 169func errata_n1_1207823_wa 170 /* Compare x0 against revision r2p0 */ 171 mov x17, x30 172 bl check_errata_1207823 173 cbz x0, 1f 174 mrs x1, NEOVERSE_N1_CPUACTLR2_EL1 175 orr x1, x1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 176 msr NEOVERSE_N1_CPUACTLR2_EL1, x1 1771: 178 ret x17 179endfunc errata_n1_1207823_wa 180 181func check_errata_1207823 182 /* Applies to <=r2p0 */ 183 mov x1, #0x20 184 b cpu_rev_var_ls 185endfunc check_errata_1207823 186 187/* -------------------------------------------------- 188 * Errata Workaround for Neoverse N1 Errata #1220197 189 * This applies to revision <=r2p0 of Neoverse N1. 190 * Inputs: 191 * x0: variant[4:7] and revision[0:3] of current cpu. 192 * Shall clobber: x0-x17 193 * -------------------------------------------------- 194 */ 195func errata_n1_1220197_wa 196 /* Compare x0 against revision r2p0 */ 197 mov x17, x30 198 bl check_errata_1220197 199 cbz x0, 1f 200 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 201 orr x1, x1, NEOVERSE_N1_WS_THR_L2_MASK 202 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2031: 204 ret x17 205endfunc errata_n1_1220197_wa 206 207func check_errata_1220197 208 /* Applies to <=r2p0 */ 209 mov x1, #0x20 210 b cpu_rev_var_ls 211endfunc check_errata_1220197 212 213/* -------------------------------------------------- 214 * Errata Workaround for Neoverse N1 Errata #1257314 215 * This applies to revision <=r3p0 of Neoverse N1. 216 * Inputs: 217 * x0: variant[4:7] and revision[0:3] of current cpu. 218 * Shall clobber: x0-x17 219 * -------------------------------------------------- 220 */ 221func errata_n1_1257314_wa 222 /* Compare x0 against revision r3p0 */ 223 mov x17, x30 224 bl check_errata_1257314 225 cbz x0, 1f 226 mrs x1, NEOVERSE_N1_CPUACTLR3_EL1 227 orr x1, x1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 228 msr NEOVERSE_N1_CPUACTLR3_EL1, x1 2291: 230 ret x17 231endfunc errata_n1_1257314_wa 232 233func check_errata_1257314 234 /* Applies to <=r3p0 */ 235 mov x1, #0x30 236 b cpu_rev_var_ls 237endfunc check_errata_1257314 238 239/* -------------------------------------------------- 240 * Errata Workaround for Neoverse N1 Errata #1262606 241 * This applies to revision <=r3p0 of Neoverse N1. 242 * Inputs: 243 * x0: variant[4:7] and revision[0:3] of current cpu. 244 * Shall clobber: x0-x17 245 * -------------------------------------------------- 246 */ 247func errata_n1_1262606_wa 248 /* Compare x0 against revision r3p0 */ 249 mov x17, x30 250 bl check_errata_1262606 251 cbz x0, 1f 252 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 253 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 254 msr NEOVERSE_N1_CPUACTLR_EL1, x1 2551: 256 ret x17 257endfunc errata_n1_1262606_wa 258 259func check_errata_1262606 260 /* Applies to <=r3p0 */ 261 mov x1, #0x30 262 b cpu_rev_var_ls 263endfunc check_errata_1262606 264 265/* -------------------------------------------------- 266 * Errata Workaround for Neoverse N1 Errata #1262888 267 * This applies to revision <=r3p0 of Neoverse N1. 268 * Inputs: 269 * x0: variant[4:7] and revision[0:3] of current cpu. 270 * Shall clobber: x0-x17 271 * -------------------------------------------------- 272 */ 273func errata_n1_1262888_wa 274 /* Compare x0 against revision r3p0 */ 275 mov x17, x30 276 bl check_errata_1262888 277 cbz x0, 1f 278 mrs x1, NEOVERSE_N1_CPUECTLR_EL1 279 orr x1, x1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 280 msr NEOVERSE_N1_CPUECTLR_EL1, x1 2811: 282 ret x17 283endfunc errata_n1_1262888_wa 284 285func check_errata_1262888 286 /* Applies to <=r3p0 */ 287 mov x1, #0x30 288 b cpu_rev_var_ls 289endfunc check_errata_1262888 290 291/* -------------------------------------------------- 292 * Errata Workaround for Neoverse N1 Errata #1275112 293 * This applies to revision <=r3p0 of Neoverse N1. 294 * Inputs: 295 * x0: variant[4:7] and revision[0:3] of current cpu. 296 * Shall clobber: x0-x17 297 * -------------------------------------------------- 298 */ 299func errata_n1_1275112_wa 300 /* Compare x0 against revision r3p0 */ 301 mov x17, x30 302 bl check_errata_1275112 303 cbz x0, 1f 304 mrs x1, NEOVERSE_N1_CPUACTLR_EL1 305 orr x1, x1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 306 msr NEOVERSE_N1_CPUACTLR_EL1, x1 3071: 308 ret x17 309endfunc errata_n1_1275112_wa 310 311func check_errata_1275112 312 /* Applies to <=r3p0 */ 313 mov x1, #0x30 314 b cpu_rev_var_ls 315endfunc check_errata_1275112 316 317/* -------------------------------------------------- 318 * Errata Workaround for Neoverse N1 Erratum 1315703. 319 * This applies to revision <= r3p0 of Neoverse N1. 320 * Inputs: 321 * x0: variant[4:7] and revision[0:3] of current cpu. 322 * Shall clobber: x0-x17 323 * -------------------------------------------------- 324 */ 325func errata_n1_1315703_wa 326 /* Compare x0 against revision r3p1 */ 327 mov x17, x30 328 bl check_errata_1315703 329 cbz x0, 1f 330 331 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 332 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 333 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 334 3351: 336 ret x17 337endfunc errata_n1_1315703_wa 338 339func check_errata_1315703 340 /* Applies to everything <= r3p0. */ 341 mov x1, #0x30 342 b cpu_rev_var_ls 343endfunc check_errata_1315703 344 345/* -------------------------------------------------- 346 * Errata Workaround for Neoverse N1 Erratum 1542419. 347 * This applies to revisions r3p0 - r4p0 of Neoverse N1 348 * Inputs: 349 * x0: variant[4:7] and revision[0:3] of current cpu. 350 * Shall clobber: x0-x17 351 * -------------------------------------------------- 352 */ 353func errata_n1_1542419_wa 354 /* Compare x0 against revision r3p0 and r4p0 */ 355 mov x17, x30 356 bl check_errata_1542419 357 cbz x0, 1f 358 359 /* Apply instruction patching sequence */ 360 ldr x0, =0x0 361 msr CPUPSELR_EL3, x0 362 ldr x0, =0xEE670D35 363 msr CPUPOR_EL3, x0 364 ldr x0, =0xFFFF0FFF 365 msr CPUPMR_EL3, x0 366 ldr x0, =0x08000020007D 367 msr CPUPCR_EL3, x0 368 isb 3691: 370 ret x17 371endfunc errata_n1_1542419_wa 372 373func check_errata_1542419 374 /* Applies to everything r3p0 - r4p0. */ 375 mov x1, #0x30 376 mov x2, #0x40 377 b cpu_rev_var_range 378endfunc check_errata_1542419 379 380func neoverse_n1_reset_func 381 mov x19, x30 382 383 bl neoverse_n1_disable_speculative_loads 384 385 /* Forces all cacheable atomic instructions to be near */ 386 mrs x0, NEOVERSE_N1_CPUACTLR2_EL1 387 orr x0, x0, #NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 388 msr NEOVERSE_N1_CPUACTLR2_EL1, x0 389 isb 390 391 bl cpu_get_rev_var 392 mov x18, x0 393 394#if ERRATA_N1_1043202 395 mov x0, x18 396 bl errata_n1_1043202_wa 397#endif 398 399#if ERRATA_N1_1073348 400 mov x0, x18 401 bl errata_n1_1073348_wa 402#endif 403 404#if ERRATA_N1_1130799 405 mov x0, x18 406 bl errata_n1_1130799_wa 407#endif 408 409#if ERRATA_N1_1165347 410 mov x0, x18 411 bl errata_n1_1165347_wa 412#endif 413 414#if ERRATA_N1_1207823 415 mov x0, x18 416 bl errata_n1_1207823_wa 417#endif 418 419#if ERRATA_N1_1220197 420 mov x0, x18 421 bl errata_n1_1220197_wa 422#endif 423 424#if ERRATA_N1_1257314 425 mov x0, x18 426 bl errata_n1_1257314_wa 427#endif 428 429#if ERRATA_N1_1262606 430 mov x0, x18 431 bl errata_n1_1262606_wa 432#endif 433 434#if ERRATA_N1_1262888 435 mov x0, x18 436 bl errata_n1_1262888_wa 437#endif 438 439#if ERRATA_N1_1275112 440 mov x0, x18 441 bl errata_n1_1275112_wa 442#endif 443 444#if ERRATA_N1_1315703 445 mov x0, x18 446 bl errata_n1_1315703_wa 447#endif 448 449#if ERRATA_N1_1542419 450 mov x0, x18 451 bl errata_n1_1542419_wa 452#endif 453 454#if ENABLE_AMU 455 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 456 mrs x0, actlr_el3 457 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 458 msr actlr_el3, x0 459 460 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 461 mrs x0, actlr_el2 462 orr x0, x0, #NEOVERSE_N1_ACTLR_AMEN_BIT 463 msr actlr_el2, x0 464 465 /* Enable group0 counters */ 466 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 467 msr CPUAMCNTENSET_EL0, x0 468#endif 469 470#if ERRATA_DSU_936184 471 bl errata_dsu_936184_wa 472#endif 473 474 isb 475 ret x19 476endfunc neoverse_n1_reset_func 477 478 /* --------------------------------------------- 479 * HW will do the cache maintenance while powering down 480 * --------------------------------------------- 481 */ 482func neoverse_n1_core_pwr_dwn 483 /* --------------------------------------------- 484 * Enable CPU power down bit in power control register 485 * --------------------------------------------- 486 */ 487 mrs x0, NEOVERSE_N1_CPUPWRCTLR_EL1 488 orr x0, x0, #NEOVERSE_N1_CORE_PWRDN_EN_MASK 489 msr NEOVERSE_N1_CPUPWRCTLR_EL1, x0 490 isb 491 ret 492endfunc neoverse_n1_core_pwr_dwn 493 494#if REPORT_ERRATA 495/* 496 * Errata printing function for Neoverse N1. Must follow AAPCS. 497 */ 498func neoverse_n1_errata_report 499 stp x8, x30, [sp, #-16]! 500 501 bl cpu_get_rev_var 502 mov x8, x0 503 504 /* 505 * Report all errata. The revision-variant information is passed to 506 * checking functions of each errata. 507 */ 508 report_errata ERRATA_N1_1043202, neoverse_n1, 1043202 509 report_errata ERRATA_N1_1073348, neoverse_n1, 1073348 510 report_errata ERRATA_N1_1130799, neoverse_n1, 1130799 511 report_errata ERRATA_N1_1165347, neoverse_n1, 1165347 512 report_errata ERRATA_N1_1207823, neoverse_n1, 1207823 513 report_errata ERRATA_N1_1220197, neoverse_n1, 1220197 514 report_errata ERRATA_N1_1257314, neoverse_n1, 1257314 515 report_errata ERRATA_N1_1262606, neoverse_n1, 1262606 516 report_errata ERRATA_N1_1262888, neoverse_n1, 1262888 517 report_errata ERRATA_N1_1275112, neoverse_n1, 1275112 518 report_errata ERRATA_N1_1315703, neoverse_n1, 1315703 519 report_errata ERRATA_N1_1542419, neoverse_n1, 1542419 520 report_errata ERRATA_DSU_936184, neoverse_n1, dsu_936184 521 522 ldp x8, x30, [sp], #16 523 ret 524endfunc neoverse_n1_errata_report 525#endif 526 527/* 528 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 529 * inner-shareable invalidation to an arbitrary address followed by a DSB. 530 * 531 * x1: Exception Syndrome 532 */ 533func neoverse_n1_errata_ic_trap_handler 534 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 535 b.ne 1f 536 tlbi vae3is, xzr 537 dsb sy 538 539 # Skip the IC instruction itself 540 mrs x3, elr_el3 541 add x3, x3, #4 542 msr elr_el3, x3 543 544 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 545 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 546 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 547 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 548 549#if IMAGE_BL31 && RAS_EXTENSION 550 /* 551 * Issue Error Synchronization Barrier to synchronize SErrors before 552 * exiting EL3. We're running with EAs unmasked, so any synchronized 553 * errors would be taken immediately; therefore no need to inspect 554 * DISR_EL1 register. 555 */ 556 esb 557#endif 558 eret 5591: 560 ret 561endfunc neoverse_n1_errata_ic_trap_handler 562 563 /* --------------------------------------------- 564 * This function provides neoverse_n1 specific 565 * register information for crash reporting. 566 * It needs to return with x6 pointing to 567 * a list of register names in ascii and 568 * x8 - x15 having values of registers to be 569 * reported. 570 * --------------------------------------------- 571 */ 572.section .rodata.neoverse_n1_regs, "aS" 573neoverse_n1_regs: /* The ascii list of register names to be reported */ 574 .asciz "cpuectlr_el1", "" 575 576func neoverse_n1_cpu_reg_dump 577 adr x6, neoverse_n1_regs 578 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 579 ret 580endfunc neoverse_n1_cpu_reg_dump 581 582declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 583 neoverse_n1_reset_func, \ 584 neoverse_n1_errata_ic_trap_handler, \ 585 neoverse_n1_core_pwr_dwn 586