1/* 2 * Copyright (c) 2014-2022, ARM Limited and Contributors. All rights reserved. 3 * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. 4 * 5 * SPDX-License-Identifier: BSD-3-Clause 6 */ 7#include <arch.h> 8#include <asm_macros.S> 9#include <assert_macros.S> 10#include <common/bl_common.h> 11#include <common/debug.h> 12#include <cortex_a57.h> 13#include <cpu_macros.S> 14#include <plat_macros.S> 15 16 /* --------------------------------------------- 17 * Disable L1 data cache and unified L2 cache 18 * --------------------------------------------- 19 */ 20func cortex_a57_disable_dcache 21 mrs x1, sctlr_el3 22 bic x1, x1, #SCTLR_C_BIT 23 msr sctlr_el3, x1 24 isb 25 ret 26endfunc cortex_a57_disable_dcache 27 28 /* --------------------------------------------- 29 * Disable all types of L2 prefetches. 30 * --------------------------------------------- 31 */ 32func cortex_a57_disable_l2_prefetch 33 mrs x0, CORTEX_A57_ECTLR_EL1 34 orr x0, x0, #CORTEX_A57_ECTLR_DIS_TWD_ACC_PFTCH_BIT 35 mov x1, #CORTEX_A57_ECTLR_L2_IPFTCH_DIST_MASK 36 orr x1, x1, #CORTEX_A57_ECTLR_L2_DPFTCH_DIST_MASK 37 bic x0, x0, x1 38 msr CORTEX_A57_ECTLR_EL1, x0 39 isb 40 dsb ish 41 ret 42endfunc cortex_a57_disable_l2_prefetch 43 44 /* --------------------------------------------- 45 * Disable intra-cluster coherency 46 * --------------------------------------------- 47 */ 48func cortex_a57_disable_smp 49 mrs x0, CORTEX_A57_ECTLR_EL1 50 bic x0, x0, #CORTEX_A57_ECTLR_SMP_BIT 51 msr CORTEX_A57_ECTLR_EL1, x0 52 ret 53endfunc cortex_a57_disable_smp 54 55 /* --------------------------------------------- 56 * Disable debug interfaces 57 * --------------------------------------------- 58 */ 59func cortex_a57_disable_ext_debug 60 mov x0, #1 61 msr osdlr_el1, x0 62 isb 63#if ERRATA_A57_817169 64 /* 65 * Invalidate any TLB address 66 */ 67 mov x0, #0 68 tlbi vae3, x0 69#endif 70 dsb sy 71 ret 72endfunc cortex_a57_disable_ext_debug 73 74 /* -------------------------------------------------------------------- 75 * Disable the over-read from the LDNP instruction. 76 * 77 * This applies to all revisions <= r1p2. The performance degradation 78 * observed with LDNP/STNP has been fixed on r1p3 and onwards. 79 * 80 * Inputs: 81 * x0: variant[4:7] and revision[0:3] of current cpu. 82 * Shall clobber: x0-x17 83 * --------------------------------------------------------------------- 84 */ 85func a57_disable_ldnp_overread 86 /* 87 * Compare x0 against revision r1p2 88 */ 89 mov x17, x30 90 bl check_errata_disable_ldnp_overread 91 cbz x0, 1f 92 mrs x1, CORTEX_A57_CPUACTLR_EL1 93 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_OVERREAD 94 msr CORTEX_A57_CPUACTLR_EL1, x1 951: 96 ret x17 97endfunc a57_disable_ldnp_overread 98 99func check_errata_disable_ldnp_overread 100 mov x1, #0x12 101 b cpu_rev_var_ls 102endfunc check_errata_disable_ldnp_overread 103 104 /* -------------------------------------------------- 105 * Errata Workaround for Cortex A57 Errata #806969. 106 * This applies only to revision r0p0 of Cortex A57. 107 * Inputs: 108 * x0: variant[4:7] and revision[0:3] of current cpu. 109 * Shall clobber: x0-x17 110 * -------------------------------------------------- 111 */ 112func errata_a57_806969_wa 113 /* 114 * Compare x0 against revision r0p0 115 */ 116 mov x17, x30 117 bl check_errata_806969 118 cbz x0, 1f 119 mrs x1, CORTEX_A57_CPUACTLR_EL1 120 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA 121 msr CORTEX_A57_CPUACTLR_EL1, x1 1221: 123 ret x17 124endfunc errata_a57_806969_wa 125 126func check_errata_806969 127 mov x1, #0x00 128 b cpu_rev_var_ls 129endfunc check_errata_806969 130 131 /* --------------------------------------------------- 132 * Errata Workaround for Cortex A57 Errata #813419. 133 * This applies only to revision r0p0 of Cortex A57. 134 * --------------------------------------------------- 135 */ 136func check_errata_813419 137 /* 138 * Even though this is only needed for revision r0p0, it 139 * is always applied due to limitations of the current 140 * errata framework. 141 */ 142 mov x0, #ERRATA_APPLIES 143 ret 144endfunc check_errata_813419 145 146 /* --------------------------------------------------- 147 * Errata Workaround for Cortex A57 Errata #813420. 148 * This applies only to revision r0p0 of Cortex A57. 149 * Inputs: 150 * x0: variant[4:7] and revision[0:3] of current cpu. 151 * Shall clobber: x0-x17 152 * --------------------------------------------------- 153 */ 154func errata_a57_813420_wa 155 /* 156 * Compare x0 against revision r0p0 157 */ 158 mov x17, x30 159 bl check_errata_813420 160 cbz x0, 1f 161 mrs x1, CORTEX_A57_CPUACTLR_EL1 162 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DCC_AS_DCCI 163 msr CORTEX_A57_CPUACTLR_EL1, x1 1641: 165 ret x17 166endfunc errata_a57_813420_wa 167 168func check_errata_813420 169 mov x1, #0x00 170 b cpu_rev_var_ls 171endfunc check_errata_813420 172 173 /* --------------------------------------------------- 174 * Errata Workaround for Cortex A57 Errata #814670. 175 * This applies only to revision r0p0 of Cortex A57. 176 * Inputs: 177 * x0: variant[4:7] and revision[0:3] of current cpu. 178 * Shall clobber: x0-x17 179 * --------------------------------------------------- 180 */ 181func errata_a57_814670_wa 182 /* 183 * Compare x0 against revision r0p0 184 */ 185 mov x17, x30 186 bl check_errata_814670 187 cbz x0, 1f 188 mrs x1, CORTEX_A57_CPUACTLR_EL1 189 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_DMB_NULLIFICATION 190 msr CORTEX_A57_CPUACTLR_EL1, x1 191 isb 1921: 193 ret x17 194endfunc errata_a57_814670_wa 195 196func check_errata_814670 197 mov x1, #0x00 198 b cpu_rev_var_ls 199endfunc check_errata_814670 200 201 /* ---------------------------------------------------- 202 * Errata Workaround for Cortex A57 Errata #817169. 203 * This applies only to revision <= r0p1 of Cortex A57. 204 * ---------------------------------------------------- 205 */ 206func check_errata_817169 207 /* 208 * Even though this is only needed for revision <= r0p1, it 209 * is always applied because of the low cost of the workaround. 210 */ 211 mov x0, #ERRATA_APPLIES 212 ret 213endfunc check_errata_817169 214 215 /* --------------------------------------------------- 216 * Errata Workaround for Cortex A57 Errata #826974. 217 * This applies only to revision <= r1p1 of Cortex A57. 218 * Inputs: 219 * x0: variant[4:7] and revision[0:3] of current cpu. 220 * Shall clobber: x0-x17 221 * --------------------------------------------------- 222 */ 223func errata_a57_826974_wa 224 /* 225 * Compare x0 against revision r1p1 226 */ 227 mov x17, x30 228 bl check_errata_826974 229 cbz x0, 1f 230 mrs x1, CORTEX_A57_CPUACTLR_EL1 231 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_DMB 232 msr CORTEX_A57_CPUACTLR_EL1, x1 2331: 234 ret x17 235endfunc errata_a57_826974_wa 236 237func check_errata_826974 238 mov x1, #0x11 239 b cpu_rev_var_ls 240endfunc check_errata_826974 241 242 /* --------------------------------------------------- 243 * Errata Workaround for Cortex A57 Errata #826977. 244 * This applies only to revision <= r1p1 of Cortex A57. 245 * Inputs: 246 * x0: variant[4:7] and revision[0:3] of current cpu. 247 * Shall clobber: x0-x17 248 * --------------------------------------------------- 249 */ 250func errata_a57_826977_wa 251 /* 252 * Compare x0 against revision r1p1 253 */ 254 mov x17, x30 255 bl check_errata_826977 256 cbz x0, 1f 257 mrs x1, CORTEX_A57_CPUACTLR_EL1 258 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_GRE_NGRE_AS_NGNRE 259 msr CORTEX_A57_CPUACTLR_EL1, x1 2601: 261 ret x17 262endfunc errata_a57_826977_wa 263 264func check_errata_826977 265 mov x1, #0x11 266 b cpu_rev_var_ls 267endfunc check_errata_826977 268 269 /* --------------------------------------------------- 270 * Errata Workaround for Cortex A57 Errata #828024. 271 * This applies only to revision <= r1p1 of Cortex A57. 272 * Inputs: 273 * x0: variant[4:7] and revision[0:3] of current cpu. 274 * Shall clobber: x0-x17 275 * --------------------------------------------------- 276 */ 277func errata_a57_828024_wa 278 /* 279 * Compare x0 against revision r1p1 280 */ 281 mov x17, x30 282 bl check_errata_828024 283 cbz x0, 1f 284 mrs x1, CORTEX_A57_CPUACTLR_EL1 285 /* 286 * Setting the relevant bits in CPUACTLR_EL1 has to be done in 2 287 * instructions here because the resulting bitmask doesn't fit in a 288 * 16-bit value so it cannot be encoded in a single instruction. 289 */ 290 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA 291 orr x1, x1, #(CORTEX_A57_CPUACTLR_EL1_DIS_L1_STREAMING | \ 292 CORTEX_A57_CPUACTLR_EL1_DIS_STREAMING) 293 msr CORTEX_A57_CPUACTLR_EL1, x1 2941: 295 ret x17 296endfunc errata_a57_828024_wa 297 298func check_errata_828024 299 mov x1, #0x11 300 b cpu_rev_var_ls 301endfunc check_errata_828024 302 303 /* --------------------------------------------------- 304 * Errata Workaround for Cortex A57 Errata #829520. 305 * This applies only to revision <= r1p2 of Cortex A57. 306 * Inputs: 307 * x0: variant[4:7] and revision[0:3] of current cpu. 308 * Shall clobber: x0-x17 309 * --------------------------------------------------- 310 */ 311func errata_a57_829520_wa 312 /* 313 * Compare x0 against revision r1p2 314 */ 315 mov x17, x30 316 bl check_errata_829520 317 cbz x0, 1f 318 mrs x1, CORTEX_A57_CPUACTLR_EL1 319 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_INDIRECT_PREDICTOR 320 msr CORTEX_A57_CPUACTLR_EL1, x1 3211: 322 ret x17 323endfunc errata_a57_829520_wa 324 325func check_errata_829520 326 mov x1, #0x12 327 b cpu_rev_var_ls 328endfunc check_errata_829520 329 330 /* --------------------------------------------------- 331 * Errata Workaround for Cortex A57 Errata #833471. 332 * This applies only to revision <= r1p2 of Cortex A57. 333 * Inputs: 334 * x0: variant[4:7] and revision[0:3] of current cpu. 335 * Shall clobber: x0-x17 336 * --------------------------------------------------- 337 */ 338func errata_a57_833471_wa 339 /* 340 * Compare x0 against revision r1p2 341 */ 342 mov x17, x30 343 bl check_errata_833471 344 cbz x0, 1f 345 mrs x1, CORTEX_A57_CPUACTLR_EL1 346 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_FORCE_FPSCR_FLUSH 347 msr CORTEX_A57_CPUACTLR_EL1, x1 3481: 349 ret x17 350endfunc errata_a57_833471_wa 351 352func check_errata_833471 353 mov x1, #0x12 354 b cpu_rev_var_ls 355endfunc check_errata_833471 356 357 /* -------------------------------------------------- 358 * Errata Workaround for Cortex A57 Errata #859972. 359 * This applies only to revision <= r1p3 of Cortex A57. 360 * Inputs: 361 * x0: variant[4:7] and revision[0:3] of current cpu. 362 * Shall clobber: 363 * -------------------------------------------------- 364 */ 365func errata_a57_859972_wa 366 mov x17, x30 367 bl check_errata_859972 368 cbz x0, 1f 369 mrs x1, CORTEX_A57_CPUACTLR_EL1 370 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_INSTR_PREFETCH 371 msr CORTEX_A57_CPUACTLR_EL1, x1 3721: 373 ret x17 374endfunc errata_a57_859972_wa 375 376func check_errata_859972 377 mov x1, #0x13 378 b cpu_rev_var_ls 379endfunc check_errata_859972 380 381 /* -------------------------------------------------- 382 * Errata workaround for Cortex A57 Errata #1319537. 383 * This applies to all revisions of Cortex A57. 384 * -------------------------------------------------- 385 */ 386func check_errata_1319537 387#if ERRATA_A57_1319537 388 mov x0, #ERRATA_APPLIES 389#else 390 mov x0, #ERRATA_MISSING 391#endif 392 ret 393endfunc check_errata_1319537 394 395func check_errata_cve_2017_5715 396#if WORKAROUND_CVE_2017_5715 397 mov x0, #ERRATA_APPLIES 398#else 399 mov x0, #ERRATA_MISSING 400#endif 401 ret 402endfunc check_errata_cve_2017_5715 403 404func check_errata_cve_2018_3639 405#if WORKAROUND_CVE_2018_3639 406 mov x0, #ERRATA_APPLIES 407#else 408 mov x0, #ERRATA_MISSING 409#endif 410 ret 411endfunc check_errata_cve_2018_3639 412 413func check_errata_cve_2022_23960 414#if WORKAROUND_CVE_2022_23960 415 mov x0, #ERRATA_APPLIES 416#else 417 mov x0, #ERRATA_MISSING 418#endif 419 ret 420endfunc check_errata_cve_2022_23960 421 422 /* ------------------------------------------------- 423 * The CPU Ops reset function for Cortex-A57. 424 * Shall clobber: x0-x19 425 * ------------------------------------------------- 426 */ 427func cortex_a57_reset_func 428 mov x19, x30 429 bl cpu_get_rev_var 430 mov x18, x0 431 432#if ERRATA_A57_806969 433 mov x0, x18 434 bl errata_a57_806969_wa 435#endif 436 437#if ERRATA_A57_813420 438 mov x0, x18 439 bl errata_a57_813420_wa 440#endif 441 442#if ERRATA_A57_814670 443 mov x0, x18 444 bl errata_a57_814670_wa 445#endif 446 447#if A57_DISABLE_NON_TEMPORAL_HINT 448 mov x0, x18 449 bl a57_disable_ldnp_overread 450#endif 451 452#if ERRATA_A57_826974 453 mov x0, x18 454 bl errata_a57_826974_wa 455#endif 456 457#if ERRATA_A57_826977 458 mov x0, x18 459 bl errata_a57_826977_wa 460#endif 461 462#if ERRATA_A57_828024 463 mov x0, x18 464 bl errata_a57_828024_wa 465#endif 466 467#if ERRATA_A57_829520 468 mov x0, x18 469 bl errata_a57_829520_wa 470#endif 471 472#if ERRATA_A57_833471 473 mov x0, x18 474 bl errata_a57_833471_wa 475#endif 476 477#if ERRATA_A57_859972 478 mov x0, x18 479 bl errata_a57_859972_wa 480#endif 481 482#if IMAGE_BL31 && ( WORKAROUND_CVE_2017_5715 || WORKAROUND_CVE_2022_23960 ) 483 /* --------------------------------------------------------------- 484 * Override vector table & enable existing workaround if either of 485 * the build flags are enabled 486 * --------------------------------------------------------------- 487 */ 488 adr x0, wa_cve_2017_5715_mmu_vbar 489 msr vbar_el3, x0 490 /* isb will be performed before returning from this function */ 491#endif 492 493#if WORKAROUND_CVE_2018_3639 494 mrs x0, CORTEX_A57_CPUACTLR_EL1 495 orr x0, x0, #CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_STORE 496 msr CORTEX_A57_CPUACTLR_EL1, x0 497 isb 498 dsb sy 499#endif 500 501#if A57_ENABLE_NONCACHEABLE_LOAD_FWD 502 /* --------------------------------------------- 503 * Enable higher performance non-cacheable load 504 * forwarding 505 * --------------------------------------------- 506 */ 507 mrs x0, CORTEX_A57_CPUACTLR_EL1 508 orr x0, x0, #CORTEX_A57_CPUACTLR_EL1_EN_NC_LOAD_FWD 509 msr CORTEX_A57_CPUACTLR_EL1, x0 510#endif 511 512 /* --------------------------------------------- 513 * Enable the SMP bit. 514 * --------------------------------------------- 515 */ 516 mrs x0, CORTEX_A57_ECTLR_EL1 517 orr x0, x0, #CORTEX_A57_ECTLR_SMP_BIT 518 msr CORTEX_A57_ECTLR_EL1, x0 519 isb 520 ret x19 521endfunc cortex_a57_reset_func 522 523func check_smccc_arch_workaround_3 524 mov x0, #ERRATA_APPLIES 525 ret 526endfunc check_smccc_arch_workaround_3 527 528 /* ---------------------------------------------------- 529 * The CPU Ops core power down function for Cortex-A57. 530 * ---------------------------------------------------- 531 */ 532func cortex_a57_core_pwr_dwn 533 mov x18, x30 534 535 /* --------------------------------------------- 536 * Turn off caches. 537 * --------------------------------------------- 538 */ 539 bl cortex_a57_disable_dcache 540 541 /* --------------------------------------------- 542 * Disable the L2 prefetches. 543 * --------------------------------------------- 544 */ 545 bl cortex_a57_disable_l2_prefetch 546 547 /* --------------------------------------------- 548 * Flush L1 caches. 549 * --------------------------------------------- 550 */ 551 mov x0, #DCCISW 552 bl dcsw_op_level1 553 554 /* --------------------------------------------- 555 * Come out of intra cluster coherency 556 * --------------------------------------------- 557 */ 558 bl cortex_a57_disable_smp 559 560 /* --------------------------------------------- 561 * Force the debug interfaces to be quiescent 562 * --------------------------------------------- 563 */ 564 mov x30, x18 565 b cortex_a57_disable_ext_debug 566endfunc cortex_a57_core_pwr_dwn 567 568 /* ------------------------------------------------------- 569 * The CPU Ops cluster power down function for Cortex-A57. 570 * ------------------------------------------------------- 571 */ 572func cortex_a57_cluster_pwr_dwn 573 mov x18, x30 574 575 /* --------------------------------------------- 576 * Turn off caches. 577 * --------------------------------------------- 578 */ 579 bl cortex_a57_disable_dcache 580 581 /* --------------------------------------------- 582 * Disable the L2 prefetches. 583 * --------------------------------------------- 584 */ 585 bl cortex_a57_disable_l2_prefetch 586 587#if !SKIP_A57_L1_FLUSH_PWR_DWN 588 /* ------------------------------------------------- 589 * Flush the L1 caches. 590 * ------------------------------------------------- 591 */ 592 mov x0, #DCCISW 593 bl dcsw_op_level1 594#endif 595 /* --------------------------------------------- 596 * Disable the optional ACP. 597 * --------------------------------------------- 598 */ 599 bl plat_disable_acp 600 601 /* ------------------------------------------------- 602 * Flush the L2 caches. 603 * ------------------------------------------------- 604 */ 605 mov x0, #DCCISW 606 bl dcsw_op_level2 607 608 /* --------------------------------------------- 609 * Come out of intra cluster coherency 610 * --------------------------------------------- 611 */ 612 bl cortex_a57_disable_smp 613 614 /* --------------------------------------------- 615 * Force the debug interfaces to be quiescent 616 * --------------------------------------------- 617 */ 618 mov x30, x18 619 b cortex_a57_disable_ext_debug 620endfunc cortex_a57_cluster_pwr_dwn 621 622#if REPORT_ERRATA 623/* 624 * Errata printing function for Cortex A57. Must follow AAPCS. 625 */ 626func cortex_a57_errata_report 627 stp x8, x30, [sp, #-16]! 628 629 bl cpu_get_rev_var 630 mov x8, x0 631 632 /* 633 * Report all errata. The revision-variant information is passed to 634 * checking functions of each errata. 635 */ 636 report_errata ERRATA_A57_806969, cortex_a57, 806969 637 report_errata ERRATA_A57_813419, cortex_a57, 813419 638 report_errata ERRATA_A57_813420, cortex_a57, 813420 639 report_errata ERRATA_A57_814670, cortex_a57, 814670 640 report_errata ERRATA_A57_817169, cortex_a57, 817169 641 report_errata A57_DISABLE_NON_TEMPORAL_HINT, cortex_a57, \ 642 disable_ldnp_overread 643 report_errata ERRATA_A57_826974, cortex_a57, 826974 644 report_errata ERRATA_A57_826977, cortex_a57, 826977 645 report_errata ERRATA_A57_828024, cortex_a57, 828024 646 report_errata ERRATA_A57_829520, cortex_a57, 829520 647 report_errata ERRATA_A57_833471, cortex_a57, 833471 648 report_errata ERRATA_A57_859972, cortex_a57, 859972 649 report_errata ERRATA_A57_1319537, cortex_a57, 1319537 650 report_errata WORKAROUND_CVE_2017_5715, cortex_a57, cve_2017_5715 651 report_errata WORKAROUND_CVE_2018_3639, cortex_a57, cve_2018_3639 652 report_errata WORKAROUND_CVE_2022_23960, cortex_a57, cve_2022_23960 653 654 ldp x8, x30, [sp], #16 655 ret 656endfunc cortex_a57_errata_report 657#endif 658 659 /* --------------------------------------------- 660 * This function provides cortex_a57 specific 661 * register information for crash reporting. 662 * It needs to return with x6 pointing to 663 * a list of register names in ascii and 664 * x8 - x15 having values of registers to be 665 * reported. 666 * --------------------------------------------- 667 */ 668.section .rodata.cortex_a57_regs, "aS" 669cortex_a57_regs: /* The ascii list of register names to be reported */ 670 .asciz "cpuectlr_el1", "cpumerrsr_el1", "l2merrsr_el1", "" 671 672func cortex_a57_cpu_reg_dump 673 adr x6, cortex_a57_regs 674 mrs x8, CORTEX_A57_ECTLR_EL1 675 mrs x9, CORTEX_A57_MERRSR_EL1 676 mrs x10, CORTEX_A57_L2MERRSR_EL1 677 ret 678endfunc cortex_a57_cpu_reg_dump 679 680declare_cpu_ops_wa cortex_a57, CORTEX_A57_MIDR, \ 681 cortex_a57_reset_func, \ 682 check_errata_cve_2017_5715, \ 683 CPU_NO_EXTRA2_FUNC, \ 684 check_smccc_arch_workaround_3, \ 685 cortex_a57_core_pwr_dwn, \ 686 cortex_a57_cluster_pwr_dwn 687