1/* 2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <cpuamu.h> 10#include <cpu_macros.S> 11#include <dsu_macros.S> 12#include <neoverse_n1.h> 13#include "wa_cve_2022_23960_bhb_vector.S" 14 15/* Hardware handled coherency */ 16#if HW_ASSISTED_COHERENCY == 0 17#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 18#endif 19 20/* 64-bit only core */ 21#if CTX_INCLUDE_AARCH32_REGS == 1 22#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 23#endif 24 25 .global neoverse_n1_errata_ic_trap_handler 26 27#if WORKAROUND_CVE_2022_23960 28 wa_cve_2022_23960_bhb_vector_table NEOVERSE_N1_BHB_LOOP_COUNT, neoverse_n1 29#endif /* WORKAROUND_CVE_2022_23960 */ 30 31cpu_reset_prologue neoverse_n1 32 33workaround_reset_start neoverse_n1, ERRATUM(936184), ERRATA_DSU_936184 34 errata_dsu_936184_wa_impl 35workaround_reset_end neoverse_n1, ERRATUM(936184) 36 37check_erratum_custom_start neoverse_n1, ERRATUM(936184) 38 branch_if_scu_not_present 2f /* label 1 is used in the macro */ 39 check_errata_dsu_936184_impl 40 2: 41 ret 42check_erratum_custom_end neoverse_n1, ERRATUM(936184) 43 44workaround_reset_start neoverse_n1, ERRATUM(1043202), ERRATA_N1_1043202 45 /* Apply instruction patching sequence */ 46 ldr x0, =0x0 47 msr CPUPSELR_EL3, x0 48 ldr x0, =0xF3BF8F2F 49 msr CPUPOR_EL3, x0 50 ldr x0, =0xFFFFFFFF 51 msr CPUPMR_EL3, x0 52 ldr x0, =0x800200071 53 msr CPUPCR_EL3, x0 54workaround_reset_end neoverse_n1, ERRATUM(1043202) 55 56check_erratum_ls neoverse_n1, ERRATUM(1043202), CPU_REV(1, 0) 57 58workaround_reset_start neoverse_n1, ERRATUM(1073348), ERRATA_N1_1073348 59 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 60workaround_reset_end neoverse_n1, ERRATUM(1073348) 61 62check_erratum_ls neoverse_n1, ERRATUM(1073348), CPU_REV(1, 0) 63 64workaround_reset_start neoverse_n1, ERRATUM(1130799), ERRATA_N1_1130799 65 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 66workaround_reset_end neoverse_n1, ERRATUM(1130799) 67 68check_erratum_ls neoverse_n1, ERRATUM(1130799), CPU_REV(2, 0) 69 70workaround_reset_start neoverse_n1, ERRATUM(1165347), ERRATA_N1_1165347 71 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 72 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 73workaround_reset_end neoverse_n1, ERRATUM(1165347) 74 75check_erratum_ls neoverse_n1, ERRATUM(1165347), CPU_REV(2, 0) 76 77workaround_reset_start neoverse_n1, ERRATUM(1207823), ERRATA_N1_1207823 78 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 79workaround_reset_end neoverse_n1, ERRATUM(1207823) 80 81check_erratum_ls neoverse_n1, ERRATUM(1207823), CPU_REV(2, 0) 82 83workaround_reset_start neoverse_n1, ERRATUM(1220197), ERRATA_N1_1220197 84 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_WS_THR_L2_MASK 85workaround_reset_end neoverse_n1, ERRATUM(1220197) 86 87check_erratum_ls neoverse_n1, ERRATUM(1220197), CPU_REV(2, 0) 88 89workaround_reset_start neoverse_n1, ERRATUM(1257314), ERRATA_N1_1257314 90 sysreg_bit_set NEOVERSE_N1_CPUACTLR3_EL1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 91workaround_reset_end neoverse_n1, ERRATUM(1257314) 92 93check_erratum_ls neoverse_n1, ERRATUM(1257314), CPU_REV(3, 0) 94 95workaround_reset_start neoverse_n1, ERRATUM(1262606), ERRATA_N1_1262606 96 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 97workaround_reset_end neoverse_n1, ERRATUM(1262606) 98 99check_erratum_ls neoverse_n1, ERRATUM(1262606), CPU_REV(3, 0) 100 101workaround_reset_start neoverse_n1, ERRATUM(1262888), ERRATA_N1_1262888 102 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 103workaround_reset_end neoverse_n1, ERRATUM(1262888) 104 105check_erratum_ls neoverse_n1, ERRATUM(1262888), CPU_REV(3, 0) 106 107workaround_reset_start neoverse_n1, ERRATUM(1275112), ERRATA_N1_1275112 108 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 109workaround_reset_end neoverse_n1, ERRATUM(1275112) 110 111check_erratum_ls neoverse_n1, ERRATUM(1275112), CPU_REV(3, 0) 112 113workaround_reset_start neoverse_n1, ERRATUM(1315703), ERRATA_N1_1315703 114 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 115workaround_reset_end neoverse_n1, ERRATUM(1315703) 116 117check_erratum_ls neoverse_n1, ERRATUM(1315703), CPU_REV(3, 0) 118 119workaround_reset_start neoverse_n1, ERRATUM(1542419), ERRATA_N1_1542419, SPLIT_WA 120 /* Apply instruction patching sequence */ 121 ldr x0, =0x0 122 msr CPUPSELR_EL3, x0 123 ldr x0, =0xEE670D35 124 msr CPUPOR_EL3, x0 125 ldr x0, =0xFFFF0FFF 126 msr CPUPMR_EL3, x0 127 ldr x0, =0x08000020007D 128 msr CPUPCR_EL3, x0 129 isb 130workaround_reset_end neoverse_n1, ERRATUM(1542419) 131 132check_erratum_range neoverse_n1, ERRATUM(1542419), CPU_REV(3, 0), CPU_REV(4, 0) 133 134workaround_reset_start neoverse_n1, ERRATUM(1868343), ERRATA_N1_1868343 135 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 136workaround_reset_end neoverse_n1, ERRATUM(1868343) 137 138check_erratum_ls neoverse_n1, ERRATUM(1868343), CPU_REV(4, 0) 139 140workaround_reset_start neoverse_n1, ERRATUM(1946160), ERRATA_N1_1946160 141 mov x0, #3 142 msr S3_6_C15_C8_0, x0 143 ldr x0, =0x10E3900002 144 msr S3_6_C15_C8_2, x0 145 ldr x0, =0x10FFF00083 146 msr S3_6_C15_C8_3, x0 147 ldr x0, =0x2001003FF 148 msr S3_6_C15_C8_1, x0 149 mov x0, #4 150 msr S3_6_C15_C8_0, x0 151 ldr x0, =0x10E3800082 152 msr S3_6_C15_C8_2, x0 153 ldr x0, =0x10FFF00083 154 msr S3_6_C15_C8_3, x0 155 ldr x0, =0x2001003FF 156 msr S3_6_C15_C8_1, x0 157 mov x0, #5 158 msr S3_6_C15_C8_0, x0 159 ldr x0, =0x10E3800200 160 msr S3_6_C15_C8_2, x0 161 ldr x0, =0x10FFF003E0 162 msr S3_6_C15_C8_3, x0 163 ldr x0, =0x2001003FF 164 msr S3_6_C15_C8_1, x0 165 isb 166workaround_reset_end neoverse_n1, ERRATUM(1946160) 167 168check_erratum_range neoverse_n1, ERRATUM(1946160), CPU_REV(3, 0), CPU_REV(4, 1) 169 170workaround_runtime_start neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102 171 /* dsb before isb of power down sequence */ 172 dsb sy 173workaround_runtime_end neoverse_n1, ERRATUM(2743102) 174 175check_erratum_ls neoverse_n1, ERRATUM(2743102), CPU_REV(4, 1) 176 177workaround_reset_start neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 178#if IMAGE_BL31 179 /* 180 * The Neoverse-N1 generic vectors are overridden to apply errata 181 * mitigation on exception entry from lower ELs. 182 */ 183 override_vector_table wa_cve_vbar_neoverse_n1 184#endif /* IMAGE_BL31 */ 185workaround_reset_end neoverse_n1, CVE(2022, 23960) 186 187check_erratum_chosen neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 188 189/* -------------------------------------------------- 190 * Disable speculative loads if Neoverse N1 supports 191 * SSBS. 192 * 193 * Shall clobber: x0. 194 * -------------------------------------------------- 195 */ 196func neoverse_n1_disable_speculative_loads 197 /* Check if the PE implements SSBS */ 198 mrs x0, id_aa64pfr1_el1 199 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 200 b.eq 1f 201 202 /* Disable speculative loads */ 203 msr SSBS, xzr 204 2051: 206 ret 207endfunc neoverse_n1_disable_speculative_loads 208 209cpu_reset_func_start neoverse_n1 210 bl neoverse_n1_disable_speculative_loads 211 212 /* Forces all cacheable atomic instructions to be near */ 213 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 214 isb 215 216#if ENABLE_FEAT_AMU 217 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 218 sysreg_bit_set actlr_el3, NEOVERSE_N1_ACTLR_AMEN_BIT 219 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 220 sysreg_bit_set actlr_el2, NEOVERSE_N1_ACTLR_AMEN_BIT 221 /* Enable group0 counters */ 222 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 223 msr CPUAMCNTENSET_EL0, x0 224#endif 225 226#if NEOVERSE_Nx_EXTERNAL_LLC 227 /* Some system may have External LLC, core needs to be made aware */ 228 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 229#endif 230cpu_reset_func_end neoverse_n1 231 232 /* --------------------------------------------- 233 * HW will do the cache maintenance while powering down 234 * --------------------------------------------- 235 */ 236func neoverse_n1_core_pwr_dwn 237 /* --------------------------------------------- 238 * Enable CPU power down bit in power control register 239 * --------------------------------------------- 240 */ 241 sysreg_bit_set NEOVERSE_N1_CPUPWRCTLR_EL1, NEOVERSE_N1_CORE_PWRDN_EN_MASK 242 243 apply_erratum neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102, NO_GET_CPU_REV 244 245 isb 246 ret 247endfunc neoverse_n1_core_pwr_dwn 248 249/* 250 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 251 * inner-shareable invalidation to an arbitrary address followed by a DSB. 252 * 253 * x1: Exception Syndrome 254 */ 255func neoverse_n1_errata_ic_trap_handler 256 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 257 b.ne 1f 258 tlbi vae3is, xzr 259 dsb sy 260 261 # Skip the IC instruction itself 262 mrs x3, elr_el3 263 add x3, x3, #4 264 msr elr_el3, x3 265 266 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 267 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 268 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 269 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 270 271 exception_return 2721: 273 ret 274endfunc neoverse_n1_errata_ic_trap_handler 275 276 /* --------------------------------------------- 277 * This function provides neoverse_n1 specific 278 * register information for crash reporting. 279 * It needs to return with x6 pointing to 280 * a list of register names in ascii and 281 * x8 - x15 having values of registers to be 282 * reported. 283 * --------------------------------------------- 284 */ 285.section .rodata.neoverse_n1_regs, "aS" 286neoverse_n1_regs: /* The ascii list of register names to be reported */ 287 .asciz "cpuectlr_el1", "" 288 289func neoverse_n1_cpu_reg_dump 290 adr x6, neoverse_n1_regs 291 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 292 ret 293endfunc neoverse_n1_cpu_reg_dump 294 295declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 296 neoverse_n1_reset_func, \ 297 neoverse_n1_errata_ic_trap_handler, \ 298 neoverse_n1_core_pwr_dwn 299