13a8c55f6SVarun Wadekar/* 2*15702f28SBoyan Karatotev * Copyright (c) 2015-2023, Arm Limited and Contributors. All rights reserved. 3b2ed9989SVarun Wadekar * Copyright (c) 2020-2022, NVIDIA Corporation. All rights reserved. 43a8c55f6SVarun Wadekar * 582cb2c1aSdp-arm * SPDX-License-Identifier: BSD-3-Clause 63a8c55f6SVarun Wadekar */ 73a8c55f6SVarun Wadekar 83a8c55f6SVarun Wadekar#include <arch.h> 93a8c55f6SVarun Wadekar#include <asm_macros.S> 103a8c55f6SVarun Wadekar#include <assert_macros.S> 11b0301467SVarun Wadekar#include <context.h> 123a8c55f6SVarun Wadekar#include <denver.h> 133a8c55f6SVarun Wadekar#include <cpu_macros.S> 143a8c55f6SVarun Wadekar#include <plat_macros.S> 153a8c55f6SVarun Wadekar 16b0301467SVarun Wadekar /* ------------------------------------------------- 17b0301467SVarun Wadekar * CVE-2017-5715 mitigation 18b0301467SVarun Wadekar * 19b0301467SVarun Wadekar * Flush the indirect branch predictor and RSB on 20b0301467SVarun Wadekar * entry to EL3 by issuing a newly added instruction 21b0301467SVarun Wadekar * for Denver CPUs. 22b0301467SVarun Wadekar * 23b0301467SVarun Wadekar * To achieve this without performing any branch 24b0301467SVarun Wadekar * instruction, a per-cpu vbar is installed which 25b0301467SVarun Wadekar * executes the workaround and then branches off to 26b0301467SVarun Wadekar * the corresponding vector entry in the main vector 27b0301467SVarun Wadekar * table. 28b0301467SVarun Wadekar * ------------------------------------------------- 29b0301467SVarun Wadekar */ 30b0301467SVarun Wadekarvector_base workaround_bpflush_runtime_exceptions 31b0301467SVarun Wadekar 32b0301467SVarun Wadekar .macro apply_workaround 33b0301467SVarun Wadekar stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 34b0301467SVarun Wadekar 35c5c1af0dSVarun Wadekar /* Disable cycle counter when event counting is prohibited */ 36c5c1af0dSVarun Wadekar mrs x1, pmcr_el0 37c5c1af0dSVarun Wadekar orr x0, x1, #PMCR_EL0_DP_BIT 38c5c1af0dSVarun Wadekar msr pmcr_el0, x0 39c5c1af0dSVarun Wadekar isb 40c5c1af0dSVarun Wadekar 41b0301467SVarun Wadekar /* ------------------------------------------------- 42b0301467SVarun Wadekar * A new write-only system register where a write of 43b0301467SVarun Wadekar * 1 to bit 0 will cause the indirect branch predictor 44b0301467SVarun Wadekar * and RSB to be flushed. 45b0301467SVarun Wadekar * 46b0301467SVarun Wadekar * A write of 0 to bit 0 will be ignored. A write of 47b0301467SVarun Wadekar * 1 to any other bit will cause an MCA. 48b0301467SVarun Wadekar * ------------------------------------------------- 49b0301467SVarun Wadekar */ 50b0301467SVarun Wadekar mov x0, #1 51b0301467SVarun Wadekar msr s3_0_c15_c0_6, x0 52b0301467SVarun Wadekar isb 53b0301467SVarun Wadekar 54b0301467SVarun Wadekar ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 55b0301467SVarun Wadekar .endm 56b0301467SVarun Wadekar 57b0301467SVarun Wadekar /* --------------------------------------------------------------------- 58b0301467SVarun Wadekar * Current EL with SP_EL0 : 0x0 - 0x200 59b0301467SVarun Wadekar * --------------------------------------------------------------------- 60b0301467SVarun Wadekar */ 61b0301467SVarun Wadekarvector_entry workaround_bpflush_sync_exception_sp_el0 62b0301467SVarun Wadekar b sync_exception_sp_el0 63a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_sync_exception_sp_el0 64b0301467SVarun Wadekar 65b0301467SVarun Wadekarvector_entry workaround_bpflush_irq_sp_el0 66b0301467SVarun Wadekar b irq_sp_el0 67a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_irq_sp_el0 68b0301467SVarun Wadekar 69b0301467SVarun Wadekarvector_entry workaround_bpflush_fiq_sp_el0 70b0301467SVarun Wadekar b fiq_sp_el0 71a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_fiq_sp_el0 72b0301467SVarun Wadekar 73b0301467SVarun Wadekarvector_entry workaround_bpflush_serror_sp_el0 74b0301467SVarun Wadekar b serror_sp_el0 75a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_serror_sp_el0 76b0301467SVarun Wadekar 77b0301467SVarun Wadekar /* --------------------------------------------------------------------- 78b0301467SVarun Wadekar * Current EL with SP_ELx: 0x200 - 0x400 79b0301467SVarun Wadekar * --------------------------------------------------------------------- 80b0301467SVarun Wadekar */ 81b0301467SVarun Wadekarvector_entry workaround_bpflush_sync_exception_sp_elx 82b0301467SVarun Wadekar b sync_exception_sp_elx 83a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_sync_exception_sp_elx 84b0301467SVarun Wadekar 85b0301467SVarun Wadekarvector_entry workaround_bpflush_irq_sp_elx 86b0301467SVarun Wadekar b irq_sp_elx 87a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_irq_sp_elx 88b0301467SVarun Wadekar 89b0301467SVarun Wadekarvector_entry workaround_bpflush_fiq_sp_elx 90b0301467SVarun Wadekar b fiq_sp_elx 91a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_fiq_sp_elx 92b0301467SVarun Wadekar 93b0301467SVarun Wadekarvector_entry workaround_bpflush_serror_sp_elx 94b0301467SVarun Wadekar b serror_sp_elx 95a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_serror_sp_elx 96b0301467SVarun Wadekar 97b0301467SVarun Wadekar /* --------------------------------------------------------------------- 98b0301467SVarun Wadekar * Lower EL using AArch64 : 0x400 - 0x600 99b0301467SVarun Wadekar * --------------------------------------------------------------------- 100b0301467SVarun Wadekar */ 101b0301467SVarun Wadekarvector_entry workaround_bpflush_sync_exception_aarch64 102b0301467SVarun Wadekar apply_workaround 103b0301467SVarun Wadekar b sync_exception_aarch64 104a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_sync_exception_aarch64 105b0301467SVarun Wadekar 106b0301467SVarun Wadekarvector_entry workaround_bpflush_irq_aarch64 107b0301467SVarun Wadekar apply_workaround 108b0301467SVarun Wadekar b irq_aarch64 109a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_irq_aarch64 110b0301467SVarun Wadekar 111b0301467SVarun Wadekarvector_entry workaround_bpflush_fiq_aarch64 112b0301467SVarun Wadekar apply_workaround 113b0301467SVarun Wadekar b fiq_aarch64 114a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_fiq_aarch64 115b0301467SVarun Wadekar 116b0301467SVarun Wadekarvector_entry workaround_bpflush_serror_aarch64 117b0301467SVarun Wadekar apply_workaround 118b0301467SVarun Wadekar b serror_aarch64 119a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_serror_aarch64 120b0301467SVarun Wadekar 121b0301467SVarun Wadekar /* --------------------------------------------------------------------- 122b0301467SVarun Wadekar * Lower EL using AArch32 : 0x600 - 0x800 123b0301467SVarun Wadekar * --------------------------------------------------------------------- 124b0301467SVarun Wadekar */ 125b0301467SVarun Wadekarvector_entry workaround_bpflush_sync_exception_aarch32 126b0301467SVarun Wadekar apply_workaround 127b0301467SVarun Wadekar b sync_exception_aarch32 128a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_sync_exception_aarch32 129b0301467SVarun Wadekar 130b0301467SVarun Wadekarvector_entry workaround_bpflush_irq_aarch32 131b0301467SVarun Wadekar apply_workaround 132b0301467SVarun Wadekar b irq_aarch32 133a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_irq_aarch32 134b0301467SVarun Wadekar 135b0301467SVarun Wadekarvector_entry workaround_bpflush_fiq_aarch32 136b0301467SVarun Wadekar apply_workaround 137b0301467SVarun Wadekar b fiq_aarch32 138a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_fiq_aarch32 139b0301467SVarun Wadekar 140b0301467SVarun Wadekarvector_entry workaround_bpflush_serror_aarch32 141b0301467SVarun Wadekar apply_workaround 142b0301467SVarun Wadekar b serror_aarch32 143a9203edaSRoberto Vargasend_vector_entry workaround_bpflush_serror_aarch32 144b0301467SVarun Wadekar 1459f1c5dd1SVarun Wadekar .global denver_disable_dco 1469f1c5dd1SVarun Wadekar 1473a8c55f6SVarun Wadekar /* --------------------------------------------- 1483a8c55f6SVarun Wadekar * Disable debug interfaces 1493a8c55f6SVarun Wadekar * --------------------------------------------- 1503a8c55f6SVarun Wadekar */ 1513a8c55f6SVarun Wadekarfunc denver_disable_ext_debug 1523a8c55f6SVarun Wadekar mov x0, #1 1533a8c55f6SVarun Wadekar msr osdlr_el1, x0 1543a8c55f6SVarun Wadekar isb 1553a8c55f6SVarun Wadekar dsb sy 1563a8c55f6SVarun Wadekar ret 1573a8c55f6SVarun Wadekarendfunc denver_disable_ext_debug 1583a8c55f6SVarun Wadekar 1593a8c55f6SVarun Wadekar /* ---------------------------------------------------- 1603a8c55f6SVarun Wadekar * Enable dynamic code optimizer (DCO) 1613a8c55f6SVarun Wadekar * ---------------------------------------------------- 1623a8c55f6SVarun Wadekar */ 1633a8c55f6SVarun Wadekarfunc denver_enable_dco 1645f902752SVarun Wadekar /* DCO is not supported on PN5 and later */ 1655f902752SVarun Wadekar mrs x1, midr_el1 1665f902752SVarun Wadekar mov_imm x2, DENVER_MIDR_PN4 1675f902752SVarun Wadekar cmp x1, x2 1685f902752SVarun Wadekar b.hi 1f 1695f902752SVarun Wadekar 170e6c0da15SKalyani Chidambaram mov x18, x30 1711593cae4SVarun Wadekar bl plat_my_core_pos 1723a8c55f6SVarun Wadekar mov x1, #1 1733a8c55f6SVarun Wadekar lsl x1, x1, x0 1743a8c55f6SVarun Wadekar msr s3_0_c15_c0_2, x1 175e6c0da15SKalyani Chidambaram mov x30, x18 1765f902752SVarun Wadekar1: ret 1773a8c55f6SVarun Wadekarendfunc denver_enable_dco 1783a8c55f6SVarun Wadekar 1793a8c55f6SVarun Wadekar /* ---------------------------------------------------- 1803a8c55f6SVarun Wadekar * Disable dynamic code optimizer (DCO) 1813a8c55f6SVarun Wadekar * ---------------------------------------------------- 1823a8c55f6SVarun Wadekar */ 1833a8c55f6SVarun Wadekarfunc denver_disable_dco 1845f902752SVarun Wadekar /* DCO is not supported on PN5 and later */ 1855f902752SVarun Wadekar mrs x1, midr_el1 1865f902752SVarun Wadekar mov_imm x2, DENVER_MIDR_PN4 1875f902752SVarun Wadekar cmp x1, x2 1885f902752SVarun Wadekar b.hi 2f 1891593cae4SVarun Wadekar 1903a8c55f6SVarun Wadekar /* turn off background work */ 1915f902752SVarun Wadekar mov x18, x30 1921593cae4SVarun Wadekar bl plat_my_core_pos 1933a8c55f6SVarun Wadekar mov x1, #1 1943a8c55f6SVarun Wadekar lsl x1, x1, x0 1953a8c55f6SVarun Wadekar lsl x2, x1, #16 1963a8c55f6SVarun Wadekar msr s3_0_c15_c0_2, x2 1973a8c55f6SVarun Wadekar isb 1983a8c55f6SVarun Wadekar 1993a8c55f6SVarun Wadekar /* wait till the background work turns off */ 2003a8c55f6SVarun Wadekar1: mrs x2, s3_0_c15_c0_2 2013a8c55f6SVarun Wadekar lsr x2, x2, #32 2023a8c55f6SVarun Wadekar and w2, w2, 0xFFFF 2033a8c55f6SVarun Wadekar and x2, x2, x1 2043a8c55f6SVarun Wadekar cbnz x2, 1b 2053a8c55f6SVarun Wadekar 206e6c0da15SKalyani Chidambaram mov x30, x18 2075f902752SVarun Wadekar2: ret 2083a8c55f6SVarun Wadekarendfunc denver_disable_dco 2093a8c55f6SVarun Wadekar 210*15702f28SBoyan Karatotevworkaround_reset_start denver, CVE(2017, 5715), WORKAROUND_CVE_2017_5715 211*15702f28SBoyan Karatotev#if IMAGE_BL31 212*15702f28SBoyan Karatotev adr x1, workaround_bpflush_runtime_exceptions 213*15702f28SBoyan Karatotev msr vbar_el3, x1 214*15702f28SBoyan Karatotev#endif 215*15702f28SBoyan Karatotevworkaround_reset_end denver, CVE(2017, 5715) 216*15702f28SBoyan Karatotev 217*15702f28SBoyan Karatotevcheck_erratum_custom_start denver, CVE(2017, 5715) 21883353962SVarun Wadekar mov x0, #ERRATA_MISSING 21983353962SVarun Wadekar#if WORKAROUND_CVE_2017_5715 22083353962SVarun Wadekar /* 22183353962SVarun Wadekar * Check if the CPU supports the special instruction 22283353962SVarun Wadekar * required to flush the indirect branch predictor and 22383353962SVarun Wadekar * RSB. Support for this operation can be determined by 22483353962SVarun Wadekar * comparing bits 19:16 of ID_AFR0_EL1 with 0b0001. 22583353962SVarun Wadekar */ 22683353962SVarun Wadekar mrs x1, id_afr0_el1 22783353962SVarun Wadekar mov x2, #0x10000 22883353962SVarun Wadekar and x1, x1, x2 22983353962SVarun Wadekar cbz x1, 1f 23083353962SVarun Wadekar mov x0, #ERRATA_APPLIES 23183353962SVarun Wadekar1: 23283353962SVarun Wadekar#endif 23383353962SVarun Wadekar ret 234*15702f28SBoyan Karatotevcheck_erratum_custom_end denver, CVE(2017, 5715) 23583353962SVarun Wadekar 236*15702f28SBoyan Karatotevworkaround_reset_start denver, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 2376cf8d65fSVarun Wadekar /* 2386cf8d65fSVarun Wadekar * Denver CPUs with DENVER_MIDR_PN3 or earlier, use different 2396cf8d65fSVarun Wadekar * bits in the ACTLR_EL3 register to disable speculative 2406cf8d65fSVarun Wadekar * store buffer and memory disambiguation. 2416cf8d65fSVarun Wadekar */ 2426cf8d65fSVarun Wadekar mrs x0, midr_el1 2436cf8d65fSVarun Wadekar mov_imm x1, DENVER_MIDR_PN4 2446cf8d65fSVarun Wadekar cmp x0, x1 2456cf8d65fSVarun Wadekar mrs x0, actlr_el3 2466cf8d65fSVarun Wadekar mov x1, #(DENVER_CPU_DIS_MD_EL3 | DENVER_CPU_DIS_SSB_EL3) 2476cf8d65fSVarun Wadekar mov x2, #(DENVER_PN4_CPU_DIS_MD_EL3 | DENVER_PN4_CPU_DIS_SSB_EL3) 2486cf8d65fSVarun Wadekar csel x3, x1, x2, ne 2496cf8d65fSVarun Wadekar orr x0, x0, x3 2506cf8d65fSVarun Wadekar msr actlr_el3, x0 2516cf8d65fSVarun Wadekar isb 2526cf8d65fSVarun Wadekar dsb sy 253*15702f28SBoyan Karatotevworkaround_reset_end denver, CVE(2018, 3639) 2546cf8d65fSVarun Wadekar 255*15702f28SBoyan Karatotevcheck_erratum_chosen denver, CVE(2018, 3639), WORKAROUND_CVE_2018_3639 256*15702f28SBoyan Karatotev 257*15702f28SBoyan Karatotevcpu_reset_func_start denver 2583a8c55f6SVarun Wadekar /* ---------------------------------------------------- 259cf3ed0dcSVarun Wadekar * Reset ACTLR.PMSTATE to C1 state 260cf3ed0dcSVarun Wadekar * ---------------------------------------------------- 261cf3ed0dcSVarun Wadekar */ 262cf3ed0dcSVarun Wadekar mrs x0, actlr_el1 263cf3ed0dcSVarun Wadekar bic x0, x0, #DENVER_CPU_PMSTATE_MASK 264cf3ed0dcSVarun Wadekar orr x0, x0, #DENVER_CPU_PMSTATE_C1 265cf3ed0dcSVarun Wadekar msr actlr_el1, x0 266cf3ed0dcSVarun Wadekar 267cf3ed0dcSVarun Wadekar /* ---------------------------------------------------- 2683a8c55f6SVarun Wadekar * Enable dynamic code optimizer (DCO) 2693a8c55f6SVarun Wadekar * ---------------------------------------------------- 2703a8c55f6SVarun Wadekar */ 2713a8c55f6SVarun Wadekar bl denver_enable_dco 272*15702f28SBoyan Karatotevcpu_reset_func_end denver 2733a8c55f6SVarun Wadekar 2743a8c55f6SVarun Wadekar /* ---------------------------------------------------- 2753a8c55f6SVarun Wadekar * The CPU Ops core power down function for Denver. 2763a8c55f6SVarun Wadekar * ---------------------------------------------------- 2773a8c55f6SVarun Wadekar */ 2783a8c55f6SVarun Wadekarfunc denver_core_pwr_dwn 2793a8c55f6SVarun Wadekar 2803a8c55f6SVarun Wadekar mov x19, x30 2813a8c55f6SVarun Wadekar 2823a8c55f6SVarun Wadekar /* --------------------------------------------- 2833a8c55f6SVarun Wadekar * Force the debug interfaces to be quiescent 2843a8c55f6SVarun Wadekar * --------------------------------------------- 2853a8c55f6SVarun Wadekar */ 2863a8c55f6SVarun Wadekar bl denver_disable_ext_debug 2873a8c55f6SVarun Wadekar 2883a8c55f6SVarun Wadekar ret x19 2893a8c55f6SVarun Wadekarendfunc denver_core_pwr_dwn 2903a8c55f6SVarun Wadekar 2913a8c55f6SVarun Wadekar /* ------------------------------------------------------- 2923a8c55f6SVarun Wadekar * The CPU Ops cluster power down function for Denver. 2933a8c55f6SVarun Wadekar * ------------------------------------------------------- 2943a8c55f6SVarun Wadekar */ 2953a8c55f6SVarun Wadekarfunc denver_cluster_pwr_dwn 2963a8c55f6SVarun Wadekar ret 2973a8c55f6SVarun Wadekarendfunc denver_cluster_pwr_dwn 2983a8c55f6SVarun Wadekar 299*15702f28SBoyan Karatoteverrata_report_shim denver 30083353962SVarun Wadekar 3013a8c55f6SVarun Wadekar /* --------------------------------------------- 3023a8c55f6SVarun Wadekar * This function provides Denver specific 3033a8c55f6SVarun Wadekar * register information for crash reporting. 3043a8c55f6SVarun Wadekar * It needs to return with x6 pointing to 3053a8c55f6SVarun Wadekar * a list of register names in ascii and 3063a8c55f6SVarun Wadekar * x8 - x15 having values of registers to be 3073a8c55f6SVarun Wadekar * reported. 3083a8c55f6SVarun Wadekar * --------------------------------------------- 3093a8c55f6SVarun Wadekar */ 3103a8c55f6SVarun Wadekar.section .rodata.denver_regs, "aS" 3113a8c55f6SVarun Wadekardenver_regs: /* The ascii list of register names to be reported */ 3123a8c55f6SVarun Wadekar .asciz "actlr_el1", "" 3133a8c55f6SVarun Wadekar 3143a8c55f6SVarun Wadekarfunc denver_cpu_reg_dump 3153a8c55f6SVarun Wadekar adr x6, denver_regs 3163a8c55f6SVarun Wadekar mrs x8, ACTLR_EL1 3173a8c55f6SVarun Wadekar ret 3183a8c55f6SVarun Wadekarendfunc denver_cpu_reg_dump 3193a8c55f6SVarun Wadekar 3209b624a7dSVarun Wadekar/* macro to declare cpu_ops for Denver SKUs */ 3219b624a7dSVarun Wadekar.macro denver_cpu_ops_wa midr 3229b624a7dSVarun Wadekar declare_cpu_ops_wa denver, \midr, \ 323e956e228SVarun Wadekar denver_reset_func, \ 324*15702f28SBoyan Karatotev check_erratum_denver_5715, \ 32583353962SVarun Wadekar CPU_NO_EXTRA2_FUNC, \ 326b2ed9989SVarun Wadekar CPU_NO_EXTRA3_FUNC, \ 327e956e228SVarun Wadekar denver_core_pwr_dwn, \ 328e956e228SVarun Wadekar denver_cluster_pwr_dwn 3299b624a7dSVarun Wadekar.endm 330e956e228SVarun Wadekar 3319b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN0 3329b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN1 3339b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN2 3349b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN3 3359b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN4 3369b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN5 3379b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN6 3389b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN7 3399b624a7dSVarun Wadekardenver_cpu_ops_wa DENVER_MIDR_PN8 340c6d25c00SHemant Nigamdenver_cpu_ops_wa DENVER_MIDR_PN9 341