1 /* 2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <cdefs.h> 9 #include <inttypes.h> 10 #include <stdbool.h> 11 #include <stdint.h> 12 13 #include <arch.h> 14 #include <arch_features.h> 15 #include <arch_helpers.h> 16 #include <common/debug.h> 17 #include <lib/el3_runtime/pubsub_events.h> 18 #include <lib/extensions/amu.h> 19 #include <lib/per_cpu/per_cpu.h> 20 #include <lib/utils_def.h> 21 #include <platform_def.h> 22 23 PER_CPU_DEFINE(amu_regs_t, amu_ctx); 24 25 static inline uint8_t read_amcgcr_el0_cg1nc(void) 26 { 27 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & 28 AMCGCR_EL0_CG1NC_MASK; 29 } 30 31 void amu_enable(cpu_context_t *ctx) 32 { 33 /* Initialize FEAT_AMUv1p1 features if present. */ 34 if (is_feat_amuv1p1_supported()) { 35 el3_state_t *state = get_el3state_ctx(ctx); 36 u_register_t reg; 37 38 /* 39 * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual 40 * offset registers at EL2 do not trap to EL3 41 */ 42 reg = read_ctx_reg(state, CTX_SCR_EL3); 43 reg |= SCR_AMVOFFEN_BIT; 44 write_ctx_reg(state, CTX_SCR_EL3, reg); 45 } 46 } 47 48 void amu_enable_per_world(per_world_context_t *per_world_ctx) 49 { 50 /* 51 * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor 52 * registers do not trap to EL3. 53 */ 54 uint64_t cptr_el3 = per_world_ctx->ctx_cptr_el3; 55 56 cptr_el3 &= ~TAM_BIT; 57 per_world_ctx->ctx_cptr_el3 = cptr_el3; 58 } 59 60 void amu_init_el3(unsigned int core_pos) 61 { 62 /* architecture is currently pinned to 4 */ 63 assert((read_amcgcr_el0() & AMCGCR_EL0_CG0NC_MASK) == CTX_AMU_GRP0_ALL); 64 65 /* Enable all architected counters by default */ 66 write_amcntenset0_el0(AMCNTENSET0_EL0_Pn_MASK); 67 68 if (!is_feat_amuv1p1_supported()) { 69 return; 70 } 71 72 if (is_feat_amu_aux_supported()) { 73 /* something went wrong if we're trying to write higher bits */ 74 assert((get_amu_aux_enables(core_pos) & ~AMCNTENSET1_EL0_Pn_MASK) == 0); 75 write_amcntenset1_el0(get_amu_aux_enables(core_pos)); 76 } 77 78 #if AMU_RESTRICT_COUNTERS 79 /* 80 * FEAT_AMUv1p1 adds a register field to restrict access to 81 * group 1 counters at all but the highest implemented EL. This 82 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time 83 * flag, when set, system register reads at lower ELs return 84 * zero. Reads from the memory mapped view are unaffected. 85 */ 86 VERBOSE("AMU group 1 counter access restricted.\n"); 87 write_amcr_el0(AMCR_CG1RZ_BIT); 88 #else 89 /* HDBG = 0 in both cases */ 90 write_amcr_el0(0); 91 #endif 92 } 93 94 void amu_init_el2_unused(void) 95 { 96 /* 97 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor 98 * registers do not trap to EL2. 99 */ 100 write_cptr_el2(read_cptr_el2() & ~CPTR_EL2_TAM_BIT); 101 102 if (is_feat_amuv1p1_supported()) { 103 /* Make sure virtual offsets are disabled */ 104 write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT); 105 } 106 } 107 108 static void amu_disable_counters(unsigned int core_pos) 109 { 110 write_amcntenclr0_el0(AMCNTENCLR0_EL0_Pn_MASK); 111 112 if (is_feat_amu_aux_supported()) { 113 write_amcntenclr1_el0(get_amu_aux_enables(core_pos)); 114 } 115 116 isb(); /* wait for the disable to take effect */ 117 } 118 119 static void *amu_context_save(const void *arg) 120 { 121 if (!is_feat_amu_supported()) { 122 return (void *)0; 123 } 124 125 unsigned int core_pos = *(unsigned int *)arg; 126 amu_regs_t *ctx = PER_CPU_CUR(amu_ctx); 127 128 /* disable counters so the save is a static snapshot for all counters */ 129 amu_disable_counters(core_pos); 130 131 write_amu_grp0_ctx_reg(ctx, 0, read_amevcntr00_el0()); 132 write_amu_grp0_ctx_reg(ctx, 1, read_amevcntr01_el0()); 133 write_amu_grp0_ctx_reg(ctx, 2, read_amevcntr02_el0()); 134 write_amu_grp0_ctx_reg(ctx, 3, read_amevcntr03_el0()); 135 136 if (is_feat_amu_aux_supported()) { 137 uint8_t num_counters = read_amcgcr_el0_cg1nc(); 138 139 switch (num_counters) { 140 case 0x10: 141 write_amu_grp1_ctx_reg(ctx, 0xf, read_amevcntr1f_el0()); 142 __fallthrough; 143 case 0x0f: 144 write_amu_grp1_ctx_reg(ctx, 0xe, read_amevcntr1e_el0()); 145 __fallthrough; 146 case 0x0e: 147 write_amu_grp1_ctx_reg(ctx, 0xd, read_amevcntr1d_el0()); 148 __fallthrough; 149 case 0x0d: 150 write_amu_grp1_ctx_reg(ctx, 0xc, read_amevcntr1c_el0()); 151 __fallthrough; 152 case 0x0c: 153 write_amu_grp1_ctx_reg(ctx, 0xb, read_amevcntr1b_el0()); 154 __fallthrough; 155 case 0x0b: 156 write_amu_grp1_ctx_reg(ctx, 0xa, read_amevcntr1a_el0()); 157 __fallthrough; 158 case 0x0a: 159 write_amu_grp1_ctx_reg(ctx, 0x9, read_amevcntr19_el0()); 160 __fallthrough; 161 case 0x09: 162 write_amu_grp1_ctx_reg(ctx, 0x8, read_amevcntr18_el0()); 163 __fallthrough; 164 case 0x08: 165 write_amu_grp1_ctx_reg(ctx, 0x7, read_amevcntr17_el0()); 166 __fallthrough; 167 case 0x07: 168 write_amu_grp1_ctx_reg(ctx, 0x6, read_amevcntr16_el0()); 169 __fallthrough; 170 case 0x06: 171 write_amu_grp1_ctx_reg(ctx, 0x5, read_amevcntr15_el0()); 172 __fallthrough; 173 case 0x05: 174 write_amu_grp1_ctx_reg(ctx, 0x4, read_amevcntr14_el0()); 175 __fallthrough; 176 case 0x04: 177 write_amu_grp1_ctx_reg(ctx, 0x3, read_amevcntr13_el0()); 178 __fallthrough; 179 case 0x03: 180 write_amu_grp1_ctx_reg(ctx, 0x2, read_amevcntr12_el0()); 181 __fallthrough; 182 case 0x02: 183 write_amu_grp1_ctx_reg(ctx, 0x1, read_amevcntr11_el0()); 184 __fallthrough; 185 case 0x01: 186 write_amu_grp1_ctx_reg(ctx, 0x0, read_amevcntr10_el0()); 187 __fallthrough; 188 case 0x00: 189 break; 190 default: 191 assert(0); /* something is wrong */ 192 } 193 } 194 195 return (void *)0; 196 } 197 198 static void *amu_context_restore(const void *arg) 199 { 200 if (!is_feat_amu_supported()) { 201 return (void *)0; 202 } 203 204 unsigned int core_pos = *(unsigned int *)arg; 205 amu_regs_t *ctx = PER_CPU_CUR(amu_ctx); 206 207 /* 208 * Counters must be disabled to write them safely. All counters start 209 * disabled on an AMU reset but AMU reset doesn't have to happen with PE 210 * reset. So don't bother disabling them if they already are. 211 */ 212 if (read_amcntenclr0_el0() != 0) { 213 amu_disable_counters(core_pos); 214 } 215 216 write_amevcntr00_el0(read_amu_grp0_ctx_reg(ctx, 0)); 217 write_amevcntr01_el0(read_amu_grp0_ctx_reg(ctx, 1)); 218 write_amevcntr02_el0(read_amu_grp0_ctx_reg(ctx, 2)); 219 write_amevcntr03_el0(read_amu_grp0_ctx_reg(ctx, 3)); 220 221 if (is_feat_amu_aux_supported()) { 222 uint8_t num_counters = read_amcgcr_el0_cg1nc(); 223 224 switch (num_counters) { 225 case 0x10: 226 write_amevcntr1f_el0(read_amu_grp1_ctx_reg(ctx, 0xf)); 227 __fallthrough; 228 case 0x0f: 229 write_amevcntr1e_el0(read_amu_grp1_ctx_reg(ctx, 0xe)); 230 __fallthrough; 231 case 0x0e: 232 write_amevcntr1d_el0(read_amu_grp1_ctx_reg(ctx, 0xd)); 233 __fallthrough; 234 case 0x0d: 235 write_amevcntr1c_el0(read_amu_grp1_ctx_reg(ctx, 0xc)); 236 __fallthrough; 237 case 0x0c: 238 write_amevcntr1b_el0(read_amu_grp1_ctx_reg(ctx, 0xb)); 239 __fallthrough; 240 case 0x0b: 241 write_amevcntr1a_el0(read_amu_grp1_ctx_reg(ctx, 0xa)); 242 __fallthrough; 243 case 0x0a: 244 write_amevcntr19_el0(read_amu_grp1_ctx_reg(ctx, 0x9)); 245 __fallthrough; 246 case 0x09: 247 write_amevcntr18_el0(read_amu_grp1_ctx_reg(ctx, 0x8)); 248 __fallthrough; 249 case 0x08: 250 write_amevcntr17_el0(read_amu_grp1_ctx_reg(ctx, 0x7)); 251 __fallthrough; 252 case 0x07: 253 write_amevcntr16_el0(read_amu_grp1_ctx_reg(ctx, 0x6)); 254 __fallthrough; 255 case 0x06: 256 write_amevcntr15_el0(read_amu_grp1_ctx_reg(ctx, 0x5)); 257 __fallthrough; 258 case 0x05: 259 write_amevcntr14_el0(read_amu_grp1_ctx_reg(ctx, 0x4)); 260 __fallthrough; 261 case 0x04: 262 write_amevcntr13_el0(read_amu_grp1_ctx_reg(ctx, 0x3)); 263 __fallthrough; 264 case 0x03: 265 write_amevcntr12_el0(read_amu_grp1_ctx_reg(ctx, 0x2)); 266 __fallthrough; 267 case 0x02: 268 write_amevcntr11_el0(read_amu_grp1_ctx_reg(ctx, 0x1)); 269 __fallthrough; 270 case 0x01: 271 write_amevcntr10_el0(read_amu_grp1_ctx_reg(ctx, 0x0)); 272 __fallthrough; 273 case 0x00: 274 break; 275 default: 276 assert(0); /* something is wrong */ 277 } 278 } 279 280 281 /* now enable them again */ 282 write_amcntenset0_el0(AMCNTENSET0_EL0_Pn_MASK); 283 if (is_feat_amu_aux_supported()) { 284 write_amcntenset1_el0(get_amu_aux_enables(core_pos)); 285 } 286 287 isb(); 288 return (void *)0; 289 } 290 291 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 292 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 293