1/* 2 * Copyright (c) 2014-2019, ARM Limited and Contributors. All rights reserved. 3 * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. 4 * 5 * SPDX-License-Identifier: BSD-3-Clause 6 */ 7#include <arch.h> 8#include <asm_macros.S> 9#include <assert_macros.S> 10#include <common/bl_common.h> 11#include <common/debug.h> 12#include <cortex_a57.h> 13#include <cpu_macros.S> 14#include <plat_macros.S> 15 16 /* --------------------------------------------- 17 * Disable L1 data cache and unified L2 cache 18 * --------------------------------------------- 19 */ 20func cortex_a57_disable_dcache 21 mrs x1, sctlr_el3 22 bic x1, x1, #SCTLR_C_BIT 23 msr sctlr_el3, x1 24 isb 25 ret 26endfunc cortex_a57_disable_dcache 27 28 /* --------------------------------------------- 29 * Disable all types of L2 prefetches. 30 * --------------------------------------------- 31 */ 32func cortex_a57_disable_l2_prefetch 33 mrs x0, CORTEX_A57_ECTLR_EL1 34 orr x0, x0, #CORTEX_A57_ECTLR_DIS_TWD_ACC_PFTCH_BIT 35 mov x1, #CORTEX_A57_ECTLR_L2_IPFTCH_DIST_MASK 36 orr x1, x1, #CORTEX_A57_ECTLR_L2_DPFTCH_DIST_MASK 37 bic x0, x0, x1 38 msr CORTEX_A57_ECTLR_EL1, x0 39 isb 40 dsb ish 41 ret 42endfunc cortex_a57_disable_l2_prefetch 43 44 /* --------------------------------------------- 45 * Disable intra-cluster coherency 46 * --------------------------------------------- 47 */ 48func cortex_a57_disable_smp 49 mrs x0, CORTEX_A57_ECTLR_EL1 50 bic x0, x0, #CORTEX_A57_ECTLR_SMP_BIT 51 msr CORTEX_A57_ECTLR_EL1, x0 52 ret 53endfunc cortex_a57_disable_smp 54 55 /* --------------------------------------------- 56 * Disable debug interfaces 57 * --------------------------------------------- 58 */ 59func cortex_a57_disable_ext_debug 60 mov x0, #1 61 msr osdlr_el1, x0 62 isb 63#if ERRATA_A57_817169 64 /* 65 * Invalidate any TLB address 66 */ 67 mov x0, #0 68 tlbi vae3, x0 69#endif 70 dsb sy 71 ret 72endfunc cortex_a57_disable_ext_debug 73 74 /* -------------------------------------------------- 75 * Errata Workaround for Cortex A57 Errata #806969. 76 * This applies only to revision r0p0 of Cortex A57. 77 * Inputs: 78 * x0: variant[4:7] and revision[0:3] of current cpu. 79 * Shall clobber: x0-x17 80 * -------------------------------------------------- 81 */ 82func errata_a57_806969_wa 83 /* 84 * Compare x0 against revision r0p0 85 */ 86 mov x17, x30 87 bl check_errata_806969 88 cbz x0, 1f 89 mrs x1, CORTEX_A57_CPUACTLR_EL1 90 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA 91 msr CORTEX_A57_CPUACTLR_EL1, x1 921: 93 ret x17 94endfunc errata_a57_806969_wa 95 96func check_errata_806969 97 mov x1, #0x00 98 b cpu_rev_var_ls 99endfunc check_errata_806969 100 101 /* --------------------------------------------------- 102 * Errata Workaround for Cortex A57 Errata #813419. 103 * This applies only to revision r0p0 of Cortex A57. 104 * --------------------------------------------------- 105 */ 106func check_errata_813419 107 /* 108 * Even though this is only needed for revision r0p0, it 109 * is always applied due to limitations of the current 110 * errata framework. 111 */ 112 mov x0, #ERRATA_APPLIES 113 ret 114endfunc check_errata_813419 115 116 /* --------------------------------------------------- 117 * Errata Workaround for Cortex A57 Errata #813420. 118 * This applies only to revision r0p0 of Cortex A57. 119 * Inputs: 120 * x0: variant[4:7] and revision[0:3] of current cpu. 121 * Shall clobber: x0-x17 122 * --------------------------------------------------- 123 */ 124func errata_a57_813420_wa 125 /* 126 * Compare x0 against revision r0p0 127 */ 128 mov x17, x30 129 bl check_errata_813420 130 cbz x0, 1f 131 mrs x1, CORTEX_A57_CPUACTLR_EL1 132 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DCC_AS_DCCI 133 msr CORTEX_A57_CPUACTLR_EL1, x1 1341: 135 ret x17 136endfunc errata_a57_813420_wa 137 138func check_errata_813420 139 mov x1, #0x00 140 b cpu_rev_var_ls 141endfunc check_errata_813420 142 143 /* --------------------------------------------------- 144 * Errata Workaround for Cortex A57 Errata #814670. 145 * This applies only to revision r0p0 of Cortex A57. 146 * Inputs: 147 * x0: variant[4:7] and revision[0:3] of current cpu. 148 * Shall clobber: x0-x17 149 * --------------------------------------------------- 150 */ 151func errata_a57_814670_wa 152 /* 153 * Compare x0 against revision r0p0 154 */ 155 mov x17, x30 156 bl check_errata_814670 157 cbz x0, 1f 158 mrs x1, CORTEX_A57_CPUACTLR_EL1 159 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_DMB_NULLIFICATION 160 msr CORTEX_A57_CPUACTLR_EL1, x1 161 isb 1621: 163 ret x17 164endfunc errata_a57_814670_wa 165 166func check_errata_814670 167 mov x1, #0x00 168 b cpu_rev_var_ls 169endfunc check_errata_814670 170 171 /* ---------------------------------------------------- 172 * Errata Workaround for Cortex A57 Errata #817169. 173 * This applies only to revision <= r0p1 of Cortex A57. 174 * ---------------------------------------------------- 175 */ 176func check_errata_817169 177 /* 178 * Even though this is only needed for revision <= r0p1, it 179 * is always applied because of the low cost of the workaround. 180 */ 181 mov x0, #ERRATA_APPLIES 182 ret 183endfunc check_errata_817169 184 185 /* -------------------------------------------------------------------- 186 * Disable the over-read from the LDNP instruction. 187 * 188 * This applies to all revisions <= r1p2. The performance degradation 189 * observed with LDNP/STNP has been fixed on r1p3 and onwards. 190 * 191 * Inputs: 192 * x0: variant[4:7] and revision[0:3] of current cpu. 193 * Shall clobber: x0-x17 194 * --------------------------------------------------------------------- 195 */ 196func a57_disable_ldnp_overread 197 /* 198 * Compare x0 against revision r1p2 199 */ 200 mov x17, x30 201 bl check_errata_disable_ldnp_overread 202 cbz x0, 1f 203 mrs x1, CORTEX_A57_CPUACTLR_EL1 204 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_OVERREAD 205 msr CORTEX_A57_CPUACTLR_EL1, x1 2061: 207 ret x17 208endfunc a57_disable_ldnp_overread 209 210func check_errata_disable_ldnp_overread 211 mov x1, #0x12 212 b cpu_rev_var_ls 213endfunc check_errata_disable_ldnp_overread 214 215 /* --------------------------------------------------- 216 * Errata Workaround for Cortex A57 Errata #826974. 217 * This applies only to revision <= r1p1 of Cortex A57. 218 * Inputs: 219 * x0: variant[4:7] and revision[0:3] of current cpu. 220 * Shall clobber: x0-x17 221 * --------------------------------------------------- 222 */ 223func errata_a57_826974_wa 224 /* 225 * Compare x0 against revision r1p1 226 */ 227 mov x17, x30 228 bl check_errata_826974 229 cbz x0, 1f 230 mrs x1, CORTEX_A57_CPUACTLR_EL1 231 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_DMB 232 msr CORTEX_A57_CPUACTLR_EL1, x1 2331: 234 ret x17 235endfunc errata_a57_826974_wa 236 237func check_errata_826974 238 mov x1, #0x11 239 b cpu_rev_var_ls 240endfunc check_errata_826974 241 242 /* --------------------------------------------------- 243 * Errata Workaround for Cortex A57 Errata #826977. 244 * This applies only to revision <= r1p1 of Cortex A57. 245 * Inputs: 246 * x0: variant[4:7] and revision[0:3] of current cpu. 247 * Shall clobber: x0-x17 248 * --------------------------------------------------- 249 */ 250func errata_a57_826977_wa 251 /* 252 * Compare x0 against revision r1p1 253 */ 254 mov x17, x30 255 bl check_errata_826977 256 cbz x0, 1f 257 mrs x1, CORTEX_A57_CPUACTLR_EL1 258 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_GRE_NGRE_AS_NGNRE 259 msr CORTEX_A57_CPUACTLR_EL1, x1 2601: 261 ret x17 262endfunc errata_a57_826977_wa 263 264func check_errata_826977 265 mov x1, #0x11 266 b cpu_rev_var_ls 267endfunc check_errata_826977 268 269 /* --------------------------------------------------- 270 * Errata Workaround for Cortex A57 Errata #828024. 271 * This applies only to revision <= r1p1 of Cortex A57. 272 * Inputs: 273 * x0: variant[4:7] and revision[0:3] of current cpu. 274 * Shall clobber: x0-x17 275 * --------------------------------------------------- 276 */ 277func errata_a57_828024_wa 278 /* 279 * Compare x0 against revision r1p1 280 */ 281 mov x17, x30 282 bl check_errata_828024 283 cbz x0, 1f 284 mrs x1, CORTEX_A57_CPUACTLR_EL1 285 /* 286 * Setting the relevant bits in CPUACTLR_EL1 has to be done in 2 287 * instructions here because the resulting bitmask doesn't fit in a 288 * 16-bit value so it cannot be encoded in a single instruction. 289 */ 290 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA 291 orr x1, x1, #(CORTEX_A57_CPUACTLR_EL1_DIS_L1_STREAMING | \ 292 CORTEX_A57_CPUACTLR_EL1_DIS_STREAMING) 293 msr CORTEX_A57_CPUACTLR_EL1, x1 2941: 295 ret x17 296endfunc errata_a57_828024_wa 297 298func check_errata_828024 299 mov x1, #0x11 300 b cpu_rev_var_ls 301endfunc check_errata_828024 302 303 /* --------------------------------------------------- 304 * Errata Workaround for Cortex A57 Errata #829520. 305 * This applies only to revision <= r1p2 of Cortex A57. 306 * Inputs: 307 * x0: variant[4:7] and revision[0:3] of current cpu. 308 * Shall clobber: x0-x17 309 * --------------------------------------------------- 310 */ 311func errata_a57_829520_wa 312 /* 313 * Compare x0 against revision r1p2 314 */ 315 mov x17, x30 316 bl check_errata_829520 317 cbz x0, 1f 318 mrs x1, CORTEX_A57_CPUACTLR_EL1 319 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_INDIRECT_PREDICTOR 320 msr CORTEX_A57_CPUACTLR_EL1, x1 3211: 322 ret x17 323endfunc errata_a57_829520_wa 324 325func check_errata_829520 326 mov x1, #0x12 327 b cpu_rev_var_ls 328endfunc check_errata_829520 329 330 /* --------------------------------------------------- 331 * Errata Workaround for Cortex A57 Errata #833471. 332 * This applies only to revision <= r1p2 of Cortex A57. 333 * Inputs: 334 * x0: variant[4:7] and revision[0:3] of current cpu. 335 * Shall clobber: x0-x17 336 * --------------------------------------------------- 337 */ 338func errata_a57_833471_wa 339 /* 340 * Compare x0 against revision r1p2 341 */ 342 mov x17, x30 343 bl check_errata_833471 344 cbz x0, 1f 345 mrs x1, CORTEX_A57_CPUACTLR_EL1 346 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_FORCE_FPSCR_FLUSH 347 msr CORTEX_A57_CPUACTLR_EL1, x1 3481: 349 ret x17 350endfunc errata_a57_833471_wa 351 352func check_errata_833471 353 mov x1, #0x12 354 b cpu_rev_var_ls 355endfunc check_errata_833471 356 357 /* -------------------------------------------------- 358 * Errata Workaround for Cortex A57 Errata #859972. 359 * This applies only to revision <= r1p3 of Cortex A57. 360 * Inputs: 361 * x0: variant[4:7] and revision[0:3] of current cpu. 362 * Shall clobber: 363 * -------------------------------------------------- 364 */ 365func errata_a57_859972_wa 366 mov x17, x30 367 bl check_errata_859972 368 cbz x0, 1f 369 mrs x1, CORTEX_A57_CPUACTLR_EL1 370 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_INSTR_PREFETCH 371 msr CORTEX_A57_CPUACTLR_EL1, x1 3721: 373 ret x17 374endfunc errata_a57_859972_wa 375 376func check_errata_859972 377 mov x1, #0x13 378 b cpu_rev_var_ls 379endfunc check_errata_859972 380 381func check_errata_cve_2017_5715 382#if WORKAROUND_CVE_2017_5715 383 mov x0, #ERRATA_APPLIES 384#else 385 mov x0, #ERRATA_MISSING 386#endif 387 ret 388endfunc check_errata_cve_2017_5715 389 390func check_errata_cve_2018_3639 391#if WORKAROUND_CVE_2018_3639 392 mov x0, #ERRATA_APPLIES 393#else 394 mov x0, #ERRATA_MISSING 395#endif 396 ret 397endfunc check_errata_cve_2018_3639 398 399 /* ------------------------------------------------- 400 * The CPU Ops reset function for Cortex-A57. 401 * Shall clobber: x0-x19 402 * ------------------------------------------------- 403 */ 404func cortex_a57_reset_func 405 mov x19, x30 406 bl cpu_get_rev_var 407 mov x18, x0 408 409#if ERRATA_A57_806969 410 mov x0, x18 411 bl errata_a57_806969_wa 412#endif 413 414#if ERRATA_A57_813420 415 mov x0, x18 416 bl errata_a57_813420_wa 417#endif 418 419#if ERRATA_A57_814670 420 mov x0, x18 421 bl errata_a57_814670_wa 422#endif 423 424#if A57_DISABLE_NON_TEMPORAL_HINT 425 mov x0, x18 426 bl a57_disable_ldnp_overread 427#endif 428 429#if ERRATA_A57_826974 430 mov x0, x18 431 bl errata_a57_826974_wa 432#endif 433 434#if ERRATA_A57_826977 435 mov x0, x18 436 bl errata_a57_826977_wa 437#endif 438 439#if ERRATA_A57_828024 440 mov x0, x18 441 bl errata_a57_828024_wa 442#endif 443 444#if ERRATA_A57_829520 445 mov x0, x18 446 bl errata_a57_829520_wa 447#endif 448 449#if ERRATA_A57_833471 450 mov x0, x18 451 bl errata_a57_833471_wa 452#endif 453 454#if ERRATA_A57_859972 455 mov x0, x18 456 bl errata_a57_859972_wa 457#endif 458 459#if IMAGE_BL31 && WORKAROUND_CVE_2017_5715 460 adr x0, wa_cve_2017_5715_mmu_vbar 461 msr vbar_el3, x0 462 /* isb will be performed before returning from this function */ 463#endif 464 465#if WORKAROUND_CVE_2018_3639 466 mrs x0, CORTEX_A57_CPUACTLR_EL1 467 orr x0, x0, #CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_STORE 468 msr CORTEX_A57_CPUACTLR_EL1, x0 469 isb 470 dsb sy 471#endif 472 473#if A57_ENABLE_NONCACHEABLE_LOAD_FWD 474 /* --------------------------------------------- 475 * Enable higher performance non-cacheable load 476 * forwarding 477 * --------------------------------------------- 478 */ 479 mrs x0, CORTEX_A57_CPUACTLR_EL1 480 orr x0, x0, #CORTEX_A57_CPUACTLR_EL1_EN_NC_LOAD_FWD 481 msr CORTEX_A57_CPUACTLR_EL1, x0 482#endif 483 484 /* --------------------------------------------- 485 * Enable the SMP bit. 486 * --------------------------------------------- 487 */ 488 mrs x0, CORTEX_A57_ECTLR_EL1 489 orr x0, x0, #CORTEX_A57_ECTLR_SMP_BIT 490 msr CORTEX_A57_ECTLR_EL1, x0 491 isb 492 ret x19 493endfunc cortex_a57_reset_func 494 495 /* ---------------------------------------------------- 496 * The CPU Ops core power down function for Cortex-A57. 497 * ---------------------------------------------------- 498 */ 499func cortex_a57_core_pwr_dwn 500 mov x18, x30 501 502 /* --------------------------------------------- 503 * Turn off caches. 504 * --------------------------------------------- 505 */ 506 bl cortex_a57_disable_dcache 507 508 /* --------------------------------------------- 509 * Disable the L2 prefetches. 510 * --------------------------------------------- 511 */ 512 bl cortex_a57_disable_l2_prefetch 513 514 /* --------------------------------------------- 515 * Flush L1 caches. 516 * --------------------------------------------- 517 */ 518 mov x0, #DCCISW 519 bl dcsw_op_level1 520 521 /* --------------------------------------------- 522 * Come out of intra cluster coherency 523 * --------------------------------------------- 524 */ 525 bl cortex_a57_disable_smp 526 527 /* --------------------------------------------- 528 * Force the debug interfaces to be quiescent 529 * --------------------------------------------- 530 */ 531 mov x30, x18 532 b cortex_a57_disable_ext_debug 533endfunc cortex_a57_core_pwr_dwn 534 535 /* ------------------------------------------------------- 536 * The CPU Ops cluster power down function for Cortex-A57. 537 * ------------------------------------------------------- 538 */ 539func cortex_a57_cluster_pwr_dwn 540 mov x18, x30 541 542 /* --------------------------------------------- 543 * Turn off caches. 544 * --------------------------------------------- 545 */ 546 bl cortex_a57_disable_dcache 547 548 /* --------------------------------------------- 549 * Disable the L2 prefetches. 550 * --------------------------------------------- 551 */ 552 bl cortex_a57_disable_l2_prefetch 553 554#if !SKIP_A57_L1_FLUSH_PWR_DWN 555 /* ------------------------------------------------- 556 * Flush the L1 caches. 557 * ------------------------------------------------- 558 */ 559 mov x0, #DCCISW 560 bl dcsw_op_level1 561#endif 562 /* --------------------------------------------- 563 * Disable the optional ACP. 564 * --------------------------------------------- 565 */ 566 bl plat_disable_acp 567 568 /* ------------------------------------------------- 569 * Flush the L2 caches. 570 * ------------------------------------------------- 571 */ 572 mov x0, #DCCISW 573 bl dcsw_op_level2 574 575 /* --------------------------------------------- 576 * Come out of intra cluster coherency 577 * --------------------------------------------- 578 */ 579 bl cortex_a57_disable_smp 580 581 /* --------------------------------------------- 582 * Force the debug interfaces to be quiescent 583 * --------------------------------------------- 584 */ 585 mov x30, x18 586 b cortex_a57_disable_ext_debug 587endfunc cortex_a57_cluster_pwr_dwn 588 589#if REPORT_ERRATA 590/* 591 * Errata printing function for Cortex A57. Must follow AAPCS. 592 */ 593func cortex_a57_errata_report 594 stp x8, x30, [sp, #-16]! 595 596 bl cpu_get_rev_var 597 mov x8, x0 598 599 /* 600 * Report all errata. The revision-variant information is passed to 601 * checking functions of each errata. 602 */ 603 report_errata ERRATA_A57_806969, cortex_a57, 806969 604 report_errata ERRATA_A57_813419, cortex_a57, 813419 605 report_errata ERRATA_A57_813420, cortex_a57, 813420 606 report_errata ERRATA_A57_814670, cortex_a57, 814670 607 report_errata ERRATA_A57_817169, cortex_a57, 817169 608 report_errata A57_DISABLE_NON_TEMPORAL_HINT, cortex_a57, \ 609 disable_ldnp_overread 610 report_errata ERRATA_A57_826974, cortex_a57, 826974 611 report_errata ERRATA_A57_826977, cortex_a57, 826977 612 report_errata ERRATA_A57_828024, cortex_a57, 828024 613 report_errata ERRATA_A57_829520, cortex_a57, 829520 614 report_errata ERRATA_A57_833471, cortex_a57, 833471 615 report_errata ERRATA_A57_859972, cortex_a57, 859972 616 report_errata WORKAROUND_CVE_2017_5715, cortex_a57, cve_2017_5715 617 report_errata WORKAROUND_CVE_2018_3639, cortex_a57, cve_2018_3639 618 619 ldp x8, x30, [sp], #16 620 ret 621endfunc cortex_a57_errata_report 622#endif 623 624 /* --------------------------------------------- 625 * This function provides cortex_a57 specific 626 * register information for crash reporting. 627 * It needs to return with x6 pointing to 628 * a list of register names in ascii and 629 * x8 - x15 having values of registers to be 630 * reported. 631 * --------------------------------------------- 632 */ 633.section .rodata.cortex_a57_regs, "aS" 634cortex_a57_regs: /* The ascii list of register names to be reported */ 635 .asciz "cpuectlr_el1", "cpumerrsr_el1", "l2merrsr_el1", "" 636 637func cortex_a57_cpu_reg_dump 638 adr x6, cortex_a57_regs 639 mrs x8, CORTEX_A57_ECTLR_EL1 640 mrs x9, CORTEX_A57_MERRSR_EL1 641 mrs x10, CORTEX_A57_L2MERRSR_EL1 642 ret 643endfunc cortex_a57_cpu_reg_dump 644 645declare_cpu_ops_wa cortex_a57, CORTEX_A57_MIDR, \ 646 cortex_a57_reset_func, \ 647 check_errata_cve_2017_5715, \ 648 CPU_NO_EXTRA2_FUNC, \ 649 cortex_a57_core_pwr_dwn, \ 650 cortex_a57_cluster_pwr_dwn 651