1/* 2 * Copyright (c) 2017-2026, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <cpuamu.h> 10#include <cpu_macros.S> 11#include <dsu_macros.S> 12#include <neoverse_n1.h> 13#include "wa_cve_2022_23960_bhb_vector.S" 14 15/* Hardware handled coherency */ 16#if HW_ASSISTED_COHERENCY == 0 17#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 18#endif 19 20/* 64-bit only core */ 21#if CTX_INCLUDE_AARCH32_REGS == 1 22#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 23#endif 24 25 .global neoverse_n1_errata_ic_trap_handler 26 27#if WORKAROUND_CVE_2022_23960 28 wa_cve_2022_23960_bhb_vector_table NEOVERSE_N1_BHB_LOOP_COUNT, neoverse_n1 29#endif /* WORKAROUND_CVE_2022_23960 */ 30 31cpu_reset_prologue neoverse_n1 32 33workaround_reset_start neoverse_n1, ERRATUM(925373), ERRATA_N1_925373 34 sysreg_lazy_start NEOVERSE_N1_CPUACTLR3_EL1 35 sysreg_lazy_set BIT(33) 36 sysreg_lazy_set BIT(34) 37 sysreg_lazy_commit NEOVERSE_N1_CPUACTLR3_EL1 38workaround_reset_end neoverse_n1, ERRATUM(925373) 39 40check_erratum_ls neoverse_n1, ERRATUM(925373), CPU_REV(0, 0) 41 42workaround_reset_start neoverse_n1, ERRATUM(936184), ERRATA_DSU_936184 43 errata_dsu_936184_wa_impl 44workaround_reset_end neoverse_n1, ERRATUM(936184) 45 46check_erratum_custom_start neoverse_n1, ERRATUM(936184) 47 branch_if_scu_not_present 2f /* label 1 is used in the macro */ 48 check_errata_dsu_936184_impl 49 2: 50 ret 51check_erratum_custom_end neoverse_n1, ERRATUM(936184) 52 53workaround_reset_start neoverse_n1, ERRATUM(1043202), ERRATA_N1_1043202 54 /* Apply instruction patching sequence */ 55 ldr x0, =0x0 56 msr CPUPSELR_EL3, x0 57 ldr x0, =0xF3BF8F2F 58 msr CPUPOR_EL3, x0 59 ldr x0, =0xFFFFFFFF 60 msr CPUPMR_EL3, x0 61 ldr x0, =0x800200071 62 msr CPUPCR_EL3, x0 63workaround_reset_end neoverse_n1, ERRATUM(1043202) 64 65check_erratum_ls neoverse_n1, ERRATUM(1043202), CPU_REV(1, 0) 66 67workaround_reset_start neoverse_n1, ERRATUM(1073348), ERRATA_N1_1073348 68 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 69workaround_reset_end neoverse_n1, ERRATUM(1073348) 70 71check_erratum_ls neoverse_n1, ERRATUM(1073348), CPU_REV(1, 0) 72 73workaround_reset_start neoverse_n1, ERRATUM(1130799), ERRATA_N1_1130799 74 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 75workaround_reset_end neoverse_n1, ERRATUM(1130799) 76 77check_erratum_ls neoverse_n1, ERRATUM(1130799), CPU_REV(2, 0) 78 79workaround_reset_start neoverse_n1, ERRATUM(1165347), ERRATA_N1_1165347 80 sysreg_lazy_start NEOVERSE_N1_CPUACTLR2_EL1 81 sysreg_lazy_set NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 82 sysreg_lazy_set NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 83 sysreg_lazy_commit NEOVERSE_N1_CPUACTLR2_EL1 84workaround_reset_end neoverse_n1, ERRATUM(1165347) 85 86check_erratum_ls neoverse_n1, ERRATUM(1165347), CPU_REV(2, 0) 87 88workaround_reset_start neoverse_n1, ERRATUM(1207823), ERRATA_N1_1207823 89 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 90workaround_reset_end neoverse_n1, ERRATUM(1207823) 91 92check_erratum_ls neoverse_n1, ERRATUM(1207823), CPU_REV(2, 0) 93 94workaround_reset_start neoverse_n1, ERRATUM(1220197), ERRATA_N1_1220197 95 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_WS_THR_L2_MASK 96workaround_reset_end neoverse_n1, ERRATUM(1220197) 97 98check_erratum_ls neoverse_n1, ERRATUM(1220197), CPU_REV(2, 0) 99 100workaround_reset_start neoverse_n1, ERRATUM(1257314), ERRATA_N1_1257314 101 sysreg_bit_set NEOVERSE_N1_CPUACTLR3_EL1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 102workaround_reset_end neoverse_n1, ERRATUM(1257314) 103 104check_erratum_ls neoverse_n1, ERRATUM(1257314), CPU_REV(3, 0) 105 106workaround_reset_start neoverse_n1, ERRATUM(1262606), ERRATA_N1_1262606 107 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 108workaround_reset_end neoverse_n1, ERRATUM(1262606) 109 110check_erratum_ls neoverse_n1, ERRATUM(1262606), CPU_REV(3, 0) 111 112workaround_reset_start neoverse_n1, ERRATUM(1262888), ERRATA_N1_1262888 113 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 114workaround_reset_end neoverse_n1, ERRATUM(1262888) 115 116check_erratum_ls neoverse_n1, ERRATUM(1262888), CPU_REV(3, 0) 117 118workaround_reset_start neoverse_n1, ERRATUM(1275112), ERRATA_N1_1275112 119 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 120workaround_reset_end neoverse_n1, ERRATUM(1275112) 121 122check_erratum_ls neoverse_n1, ERRATUM(1275112), CPU_REV(3, 0) 123 124workaround_reset_start neoverse_n1, ERRATUM(1315703), ERRATA_N1_1315703 125 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 126workaround_reset_end neoverse_n1, ERRATUM(1315703) 127 128check_erratum_ls neoverse_n1, ERRATUM(1315703), CPU_REV(3, 0) 129 130workaround_reset_start neoverse_n1, ERRATUM(1542419), ERRATA_N1_1542419, SPLIT_WA 131 /* Apply instruction patching sequence */ 132 ldr x0, =0x0 133 msr CPUPSELR_EL3, x0 134 ldr x0, =0xEE670D35 135 msr CPUPOR_EL3, x0 136 ldr x0, =0xFFFF0FFF 137 msr CPUPMR_EL3, x0 138 ldr x0, =0x08000020007D 139 msr CPUPCR_EL3, x0 140 isb 141workaround_reset_end neoverse_n1, ERRATUM(1542419) 142 143check_erratum_range neoverse_n1, ERRATUM(1542419), CPU_REV(3, 0), CPU_REV(4, 0) 144 145workaround_reset_start neoverse_n1, ERRATUM(1791580), ERRATA_N1_1791580 146 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, BIT(2) 147workaround_reset_end neoverse_n1, ERRATUM(1791580) 148 149check_erratum_ls neoverse_n1, ERRATUM(1791580), CPU_REV(4, 0) 150 151workaround_reset_start neoverse_n1, ERRATUM(1868343), ERRATA_N1_1868343 152 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 153workaround_reset_end neoverse_n1, ERRATUM(1868343) 154 155check_erratum_ls neoverse_n1, ERRATUM(1868343), CPU_REV(4, 0) 156 157workaround_reset_start neoverse_n1, ERRATUM(1946160), ERRATA_N1_1946160 158 mov x0, #3 159 msr S3_6_C15_C8_0, x0 160 ldr x0, =0x10E3900002 161 msr S3_6_C15_C8_2, x0 162 ldr x0, =0x10FFF00083 163 msr S3_6_C15_C8_3, x0 164 ldr x0, =0x2001003FF 165 msr S3_6_C15_C8_1, x0 166 mov x0, #4 167 msr S3_6_C15_C8_0, x0 168 ldr x0, =0x10E3800082 169 msr S3_6_C15_C8_2, x0 170 ldr x0, =0x10FFF00083 171 msr S3_6_C15_C8_3, x0 172 ldr x0, =0x2001003FF 173 msr S3_6_C15_C8_1, x0 174 mov x0, #5 175 msr S3_6_C15_C8_0, x0 176 ldr x0, =0x10E3800200 177 msr S3_6_C15_C8_2, x0 178 ldr x0, =0x10FFF003E0 179 msr S3_6_C15_C8_3, x0 180 ldr x0, =0x2001003FF 181 msr S3_6_C15_C8_1, x0 182 isb 183workaround_reset_end neoverse_n1, ERRATUM(1946160) 184 185check_erratum_range neoverse_n1, ERRATUM(1946160), CPU_REV(3, 0), CPU_REV(4, 1) 186 187workaround_runtime_start neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102 188 /* dsb before isb of power down sequence */ 189 dsb sy 190workaround_runtime_end neoverse_n1, ERRATUM(2743102) 191 192check_erratum_ls neoverse_n1, ERRATUM(2743102), CPU_REV(4, 1) 193 194workaround_runtime_start neoverse_n1, ERRATUM(3324349), ERRATA_N1_3324349 195 isb 196workaround_runtime_end neoverse_n1, ERRATUM(3324349) 197 198check_erratum_ls neoverse_n1, ERRATUM(3324349), CPU_REV(4, 1) 199 200workaround_reset_start neoverse_n1, ERRATUM(3888013), ERRATA_N1_3888013 201 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, BIT(22) 202workaround_reset_end neoverse_n1, ERRATUM(3888013) 203 204check_erratum_chosen neoverse_n1, ERRATUM(3888013), ERRATA_N1_3888013 205 206workaround_reset_start neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 207#if IMAGE_BL31 208 /* 209 * The Neoverse-N1 generic vectors are overridden to apply errata 210 * mitigation on exception entry from lower ELs. 211 */ 212 override_vector_table wa_cve_vbar_neoverse_n1 213#endif /* IMAGE_BL31 */ 214workaround_reset_end neoverse_n1, CVE(2022, 23960) 215 216check_erratum_chosen neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 217 218/* -------------------------------------------------- 219 * Disable speculative loads if Neoverse N1 supports 220 * SSBS. 221 * 222 * Shall clobber: x0. 223 * -------------------------------------------------- 224 */ 225func neoverse_n1_disable_speculative_loads 226 /* Check if the PE implements SSBS */ 227 mrs x0, id_aa64pfr1_el1 228 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 229 b.eq 1f 230 231 /* Disable speculative loads */ 232 msr SSBS, xzr 233 apply_erratum neoverse_n1, ERRATUM(3324349), ERRATA_N1_3324349 234 2351: 236 ret 237endfunc neoverse_n1_disable_speculative_loads 238 239cpu_reset_func_start neoverse_n1 240 bl neoverse_n1_disable_speculative_loads 241 242 /* Forces all cacheable atomic instructions to be near */ 243 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 244 isb 245 246#if ENABLE_FEAT_AMU 247 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 248 sysreg_bit_set actlr_el3, NEOVERSE_N1_ACTLR_AMEN_BIT 249 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 250 sysreg_bit_set actlr_el2, NEOVERSE_N1_ACTLR_AMEN_BIT 251 /* Enable group0 counters */ 252 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 253 msr CPUAMCNTENSET_EL0, x0 254#endif 255 256#if NEOVERSE_Nx_EXTERNAL_LLC 257 /* Some system may have External LLC, core needs to be made aware */ 258 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 259#endif 260cpu_reset_func_end neoverse_n1 261 262 /* --------------------------------------------- 263 * HW will do the cache maintenance while powering down 264 * --------------------------------------------- 265 */ 266func neoverse_n1_core_pwr_dwn 267 /* --------------------------------------------- 268 * Enable CPU power down bit in power control register 269 * --------------------------------------------- 270 */ 271 sysreg_bit_set NEOVERSE_N1_CPUPWRCTLR_EL1, NEOVERSE_N1_CORE_PWRDN_EN_MASK 272 273 apply_erratum neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102 274 275 isb 276 ret 277endfunc neoverse_n1_core_pwr_dwn 278 279/* 280 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 281 * inner-shareable invalidation to an arbitrary address followed by a DSB. 282 * 283 * x1: Exception Syndrome 284 */ 285func neoverse_n1_errata_ic_trap_handler 286 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 287 b.ne 1f 288 tlbi vae3is, xzr 289 dsb sy 290 291 # Skip the IC instruction itself 292 mrs x3, elr_el3 293 add x3, x3, #4 294 msr elr_el3, x3 295 296 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 297 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 298 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 299 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 300 301 exception_return 3021: 303 ret 304endfunc neoverse_n1_errata_ic_trap_handler 305 306 /* --------------------------------------------- 307 * This function provides neoverse_n1 specific 308 * register information for crash reporting. 309 * It needs to return with x6 pointing to 310 * a list of register names in ascii and 311 * x8 - x15 having values of registers to be 312 * reported. 313 * --------------------------------------------- 314 */ 315.section .rodata.neoverse_n1_regs, "aS" 316neoverse_n1_regs: /* The ascii list of register names to be reported */ 317 .asciz "cpuectlr_el1", "" 318 319func neoverse_n1_cpu_reg_dump 320 adr x6, neoverse_n1_regs 321 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 322 ret 323endfunc neoverse_n1_cpu_reg_dump 324 325declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 326 neoverse_n1_reset_func, \ 327 neoverse_n1_errata_ic_trap_handler, \ 328 neoverse_n1_core_pwr_dwn 329