1/* 2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9#include <cpuamu.h> 10#include <cpu_macros.S> 11#include <dsu_macros.S> 12#include <neoverse_n1.h> 13#include "wa_cve_2022_23960_bhb_vector.S" 14 15/* Hardware handled coherency */ 16#if HW_ASSISTED_COHERENCY == 0 17#error "Neoverse N1 must be compiled with HW_ASSISTED_COHERENCY enabled" 18#endif 19 20/* 64-bit only core */ 21#if CTX_INCLUDE_AARCH32_REGS == 1 22#error "Neoverse-N1 supports only AArch64. Compile with CTX_INCLUDE_AARCH32_REGS=0" 23#endif 24 25 .global neoverse_n1_errata_ic_trap_handler 26 27#if WORKAROUND_CVE_2022_23960 28 wa_cve_2022_23960_bhb_vector_table NEOVERSE_N1_BHB_LOOP_COUNT, neoverse_n1 29#endif /* WORKAROUND_CVE_2022_23960 */ 30 31cpu_reset_prologue neoverse_n1 32 33workaround_reset_start neoverse_n1, ERRATUM(936184), ERRATA_DSU_936184 34 errata_dsu_936184_wa_impl 35workaround_reset_end neoverse_n1, ERRATUM(936184) 36 37check_erratum_custom_start neoverse_n1, ERRATUM(936184) 38 branch_if_scu_not_present 2f /* label 1 is used in the macro */ 39 check_errata_dsu_936184_impl 40 2: 41 ret 42check_erratum_custom_end neoverse_n1, ERRATUM(936184) 43 44workaround_reset_start neoverse_n1, ERRATUM(1043202), ERRATA_N1_1043202 45 /* Apply instruction patching sequence */ 46 ldr x0, =0x0 47 msr CPUPSELR_EL3, x0 48 ldr x0, =0xF3BF8F2F 49 msr CPUPOR_EL3, x0 50 ldr x0, =0xFFFFFFFF 51 msr CPUPMR_EL3, x0 52 ldr x0, =0x800200071 53 msr CPUPCR_EL3, x0 54workaround_reset_end neoverse_n1, ERRATUM(1043202) 55 56check_erratum_ls neoverse_n1, ERRATUM(1043202), CPU_REV(1, 0) 57 58workaround_reset_start neoverse_n1, ERRATUM(1073348), ERRATA_N1_1073348 59 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_6 60workaround_reset_end neoverse_n1, ERRATUM(1073348) 61 62check_erratum_ls neoverse_n1, ERRATUM(1073348), CPU_REV(1, 0) 63 64workaround_reset_start neoverse_n1, ERRATUM(1130799), ERRATA_N1_1130799 65 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_59 66workaround_reset_end neoverse_n1, ERRATUM(1130799) 67 68check_erratum_ls neoverse_n1, ERRATUM(1130799), CPU_REV(2, 0) 69 70workaround_reset_start neoverse_n1, ERRATUM(1165347), ERRATA_N1_1165347 71 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_0 72 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_15 73workaround_reset_end neoverse_n1, ERRATUM(1165347) 74 75check_erratum_ls neoverse_n1, ERRATUM(1165347), CPU_REV(2, 0) 76 77workaround_reset_start neoverse_n1, ERRATUM(1207823), ERRATA_N1_1207823 78 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_11 79workaround_reset_end neoverse_n1, ERRATUM(1207823) 80 81check_erratum_ls neoverse_n1, ERRATUM(1207823), CPU_REV(2, 0) 82 83workaround_reset_start neoverse_n1, ERRATUM(1220197), ERRATA_N1_1220197 84 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_WS_THR_L2_MASK 85workaround_reset_end neoverse_n1, ERRATUM(1220197) 86 87check_erratum_ls neoverse_n1, ERRATUM(1220197), CPU_REV(2, 0) 88 89workaround_reset_start neoverse_n1, ERRATUM(1257314), ERRATA_N1_1257314 90 sysreg_bit_set NEOVERSE_N1_CPUACTLR3_EL1, NEOVERSE_N1_CPUACTLR3_EL1_BIT_10 91workaround_reset_end neoverse_n1, ERRATUM(1257314) 92 93check_erratum_ls neoverse_n1, ERRATUM(1257314), CPU_REV(3, 0) 94 95workaround_reset_start neoverse_n1, ERRATUM(1262606), ERRATA_N1_1262606 96 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 97workaround_reset_end neoverse_n1, ERRATUM(1262606) 98 99check_erratum_ls neoverse_n1, ERRATUM(1262606), CPU_REV(3, 0) 100 101workaround_reset_start neoverse_n1, ERRATUM(1262888), ERRATA_N1_1262888 102 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_MM_TLBPF_DIS_BIT 103workaround_reset_end neoverse_n1, ERRATUM(1262888) 104 105check_erratum_ls neoverse_n1, ERRATUM(1262888), CPU_REV(3, 0) 106 107workaround_reset_start neoverse_n1, ERRATUM(1275112), ERRATA_N1_1275112 108 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 109workaround_reset_end neoverse_n1, ERRATUM(1275112) 110 111check_erratum_ls neoverse_n1, ERRATUM(1275112), CPU_REV(3, 0) 112 113workaround_reset_start neoverse_n1, ERRATUM(1315703), ERRATA_N1_1315703 114 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_16 115workaround_reset_end neoverse_n1, ERRATUM(1315703) 116 117check_erratum_ls neoverse_n1, ERRATUM(1315703), CPU_REV(3, 0) 118 119workaround_reset_start neoverse_n1, ERRATUM(1542419), ERRATA_N1_1542419, SPLIT_WA 120 /* Apply instruction patching sequence */ 121 ldr x0, =0x0 122 msr CPUPSELR_EL3, x0 123 ldr x0, =0xEE670D35 124 msr CPUPOR_EL3, x0 125 ldr x0, =0xFFFF0FFF 126 msr CPUPMR_EL3, x0 127 ldr x0, =0x08000020007D 128 msr CPUPCR_EL3, x0 129 isb 130workaround_reset_end neoverse_n1, ERRATUM(1542419) 131 132check_erratum_range neoverse_n1, ERRATUM(1542419), CPU_REV(3, 0), CPU_REV(4, 0) 133 134workaround_reset_start neoverse_n1, ERRATUM(1868343), ERRATA_N1_1868343 135 sysreg_bit_set NEOVERSE_N1_CPUACTLR_EL1, NEOVERSE_N1_CPUACTLR_EL1_BIT_13 136workaround_reset_end neoverse_n1, ERRATUM(1868343) 137 138check_erratum_ls neoverse_n1, ERRATUM(1868343), CPU_REV(4, 0) 139 140workaround_reset_start neoverse_n1, ERRATUM(1946160), ERRATA_N1_1946160 141 mov x0, #3 142 msr S3_6_C15_C8_0, x0 143 ldr x0, =0x10E3900002 144 msr S3_6_C15_C8_2, x0 145 ldr x0, =0x10FFF00083 146 msr S3_6_C15_C8_3, x0 147 ldr x0, =0x2001003FF 148 msr S3_6_C15_C8_1, x0 149 mov x0, #4 150 msr S3_6_C15_C8_0, x0 151 ldr x0, =0x10E3800082 152 msr S3_6_C15_C8_2, x0 153 ldr x0, =0x10FFF00083 154 msr S3_6_C15_C8_3, x0 155 ldr x0, =0x2001003FF 156 msr S3_6_C15_C8_1, x0 157 mov x0, #5 158 msr S3_6_C15_C8_0, x0 159 ldr x0, =0x10E3800200 160 msr S3_6_C15_C8_2, x0 161 ldr x0, =0x10FFF003E0 162 msr S3_6_C15_C8_3, x0 163 ldr x0, =0x2001003FF 164 msr S3_6_C15_C8_1, x0 165 isb 166workaround_reset_end neoverse_n1, ERRATUM(1946160) 167 168check_erratum_range neoverse_n1, ERRATUM(1946160), CPU_REV(3, 0), CPU_REV(4, 1) 169 170workaround_runtime_start neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102 171 /* dsb before isb of power down sequence */ 172 dsb sy 173workaround_runtime_end neoverse_n1, ERRATUM(2743102) 174 175check_erratum_ls neoverse_n1, ERRATUM(2743102), CPU_REV(4, 1) 176 177workaround_runtime_start neoverse_n1, ERRATUM(3324349), ERRATA_N1_3324349 178 speculation_barrier 179workaround_runtime_end neoverse_n1, ERRATUM(3324349) 180 181check_erratum_ls neoverse_n1, ERRATUM(3324349), CPU_REV(4, 1) 182 183workaround_reset_start neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 184#if IMAGE_BL31 185 /* 186 * The Neoverse-N1 generic vectors are overridden to apply errata 187 * mitigation on exception entry from lower ELs. 188 */ 189 override_vector_table wa_cve_vbar_neoverse_n1 190#endif /* IMAGE_BL31 */ 191workaround_reset_end neoverse_n1, CVE(2022, 23960) 192 193check_erratum_chosen neoverse_n1, CVE(2022, 23960), WORKAROUND_CVE_2022_23960 194 195/* -------------------------------------------------- 196 * Disable speculative loads if Neoverse N1 supports 197 * SSBS. 198 * 199 * Shall clobber: x0. 200 * -------------------------------------------------- 201 */ 202func neoverse_n1_disable_speculative_loads 203 /* Check if the PE implements SSBS */ 204 mrs x0, id_aa64pfr1_el1 205 tst x0, #(ID_AA64PFR1_EL1_SSBS_MASK << ID_AA64PFR1_EL1_SSBS_SHIFT) 206 b.eq 1f 207 208 /* Disable speculative loads */ 209 msr SSBS, xzr 210 apply_erratum neoverse_n1, ERRATUM(3324349), ERRATA_N1_3324349 211 2121: 213 ret 214endfunc neoverse_n1_disable_speculative_loads 215 216cpu_reset_func_start neoverse_n1 217 bl neoverse_n1_disable_speculative_loads 218 219 /* Forces all cacheable atomic instructions to be near */ 220 sysreg_bit_set NEOVERSE_N1_CPUACTLR2_EL1, NEOVERSE_N1_CPUACTLR2_EL1_BIT_2 221 isb 222 223#if ENABLE_FEAT_AMU 224 /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ 225 sysreg_bit_set actlr_el3, NEOVERSE_N1_ACTLR_AMEN_BIT 226 /* Make sure accesses from EL0/EL1 are not trapped to EL2 */ 227 sysreg_bit_set actlr_el2, NEOVERSE_N1_ACTLR_AMEN_BIT 228 /* Enable group0 counters */ 229 mov x0, #NEOVERSE_N1_AMU_GROUP0_MASK 230 msr CPUAMCNTENSET_EL0, x0 231#endif 232 233#if NEOVERSE_Nx_EXTERNAL_LLC 234 /* Some system may have External LLC, core needs to be made aware */ 235 sysreg_bit_set NEOVERSE_N1_CPUECTLR_EL1, NEOVERSE_N1_CPUECTLR_EL1_EXTLLC_BIT 236#endif 237cpu_reset_func_end neoverse_n1 238 239 /* --------------------------------------------- 240 * HW will do the cache maintenance while powering down 241 * --------------------------------------------- 242 */ 243func neoverse_n1_core_pwr_dwn 244 /* --------------------------------------------- 245 * Enable CPU power down bit in power control register 246 * --------------------------------------------- 247 */ 248 sysreg_bit_set NEOVERSE_N1_CPUPWRCTLR_EL1, NEOVERSE_N1_CORE_PWRDN_EN_MASK 249 250 apply_erratum neoverse_n1, ERRATUM(2743102), ERRATA_N1_2743102 251 252 isb 253 ret 254endfunc neoverse_n1_core_pwr_dwn 255 256/* 257 * Handle trap of EL0 IC IVAU instructions to EL3 by executing a TLB 258 * inner-shareable invalidation to an arbitrary address followed by a DSB. 259 * 260 * x1: Exception Syndrome 261 */ 262func neoverse_n1_errata_ic_trap_handler 263 cmp x1, #NEOVERSE_N1_EC_IC_TRAP 264 b.ne 1f 265 tlbi vae3is, xzr 266 dsb sy 267 268 # Skip the IC instruction itself 269 mrs x3, elr_el3 270 add x3, x3, #4 271 msr elr_el3, x3 272 273 ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] 274 ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] 275 ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] 276 ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] 277 278 exception_return 2791: 280 ret 281endfunc neoverse_n1_errata_ic_trap_handler 282 283 /* --------------------------------------------- 284 * This function provides neoverse_n1 specific 285 * register information for crash reporting. 286 * It needs to return with x6 pointing to 287 * a list of register names in ascii and 288 * x8 - x15 having values of registers to be 289 * reported. 290 * --------------------------------------------- 291 */ 292.section .rodata.neoverse_n1_regs, "aS" 293neoverse_n1_regs: /* The ascii list of register names to be reported */ 294 .asciz "cpuectlr_el1", "" 295 296func neoverse_n1_cpu_reg_dump 297 adr x6, neoverse_n1_regs 298 mrs x8, NEOVERSE_N1_CPUECTLR_EL1 299 ret 300endfunc neoverse_n1_cpu_reg_dump 301 302declare_cpu_ops_eh neoverse_n1, NEOVERSE_N1_MIDR, \ 303 neoverse_n1_reset_func, \ 304 neoverse_n1_errata_ic_trap_handler, \ 305 neoverse_n1_core_pwr_dwn 306