1 /*
2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <cdefs.h>
9 #include <inttypes.h>
10 #include <stdbool.h>
11 #include <stdint.h>
12
13 #include <arch.h>
14 #include <arch_features.h>
15 #include <arch_helpers.h>
16 #include <common/debug.h>
17 #include <lib/el3_runtime/pubsub_events.h>
18 #include <lib/extensions/amu.h>
19 #include <lib/per_cpu/per_cpu.h>
20 #include <lib/utils_def.h>
21 #include <platform_def.h>
22
23 PER_CPU_DEFINE(amu_regs_t, amu_ctx);
24
read_amcgcr_el0_cg1nc(void)25 static inline uint8_t read_amcgcr_el0_cg1nc(void)
26 {
27 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) &
28 AMCGCR_EL0_CG1NC_MASK;
29 }
30
amu_enable(cpu_context_t * ctx)31 void amu_enable(cpu_context_t *ctx)
32 {
33 /* Initialize FEAT_AMUv1p1 features if present. */
34 if (is_feat_amuv1p1_supported()) {
35 el3_state_t *state = get_el3state_ctx(ctx);
36 u_register_t reg;
37
38 /*
39 * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual
40 * offset registers at EL2 do not trap to EL3
41 */
42 reg = read_ctx_reg(state, CTX_SCR_EL3);
43 reg |= SCR_AMVOFFEN_BIT;
44 write_ctx_reg(state, CTX_SCR_EL3, reg);
45 }
46 }
47
amu_enable_per_world(per_world_context_t * per_world_ctx)48 void amu_enable_per_world(per_world_context_t *per_world_ctx)
49 {
50 /*
51 * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor
52 * registers do not trap to EL3.
53 */
54 uint64_t cptr_el3 = per_world_ctx->ctx_cptr_el3;
55
56 cptr_el3 &= ~TAM_BIT;
57 per_world_ctx->ctx_cptr_el3 = cptr_el3;
58 }
59
amu_init_el3(unsigned int core_pos)60 void amu_init_el3(unsigned int core_pos)
61 {
62 /*
63 * Enable all architected counters by default. Currently, only the
64 * bottom 4 are defined and enabled. Higher bits are allocated but
65 * RAZ/WI in the latest (AMUv1p1) revision. Future revisions should
66 * consider whether the new counters need to be enabled and/or context
67 * switched.
68 */
69 write_amcntenset0_el0(AMCNTENSET0_EL0_Pn_ALWAYS_ON);
70
71 if (!is_feat_amuv1p1_supported()) {
72 return;
73 }
74
75 if (is_feat_amu_aux_supported()) {
76 /* something went wrong if we're trying to write higher bits */
77 assert((get_amu_aux_enables(core_pos) & ~AMCNTENSET1_EL0_Pn_MASK) == 0);
78 write_amcntenset1_el0(get_amu_aux_enables(core_pos));
79 }
80
81 #if AMU_RESTRICT_COUNTERS
82 /*
83 * FEAT_AMUv1p1 adds a register field to restrict access to
84 * group 1 counters at all but the highest implemented EL. This
85 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time
86 * flag, when set, system register reads at lower ELs return
87 * zero. Reads from the memory mapped view are unaffected.
88 */
89 VERBOSE("AMU group 1 counter access restricted.\n");
90 write_amcr_el0(AMCR_CG1RZ_BIT);
91 #else
92 /* HDBG = 0 in both cases */
93 write_amcr_el0(0);
94 #endif
95 }
96
amu_init_el2_unused(void)97 void amu_init_el2_unused(void)
98 {
99 /*
100 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor
101 * registers do not trap to EL2.
102 */
103 write_cptr_el2(read_cptr_el2() & ~CPTR_EL2_TAM_BIT);
104
105 if (is_feat_amuv1p1_supported()) {
106 /* Make sure virtual offsets are disabled */
107 write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT);
108 }
109 }
110
amu_disable_counters(unsigned int core_pos)111 static void amu_disable_counters(unsigned int core_pos)
112 {
113 write_amcntenclr0_el0(AMCNTENCLR0_EL0_Pn_ALL);
114
115 if (is_feat_amu_aux_supported()) {
116 write_amcntenclr1_el0(get_amu_aux_enables(core_pos));
117 }
118
119 isb(); /* wait for the disable to take effect */
120 }
121
amu_context_save(const void * arg)122 static void *amu_context_save(const void *arg)
123 {
124 if (!is_feat_amu_supported()) {
125 return (void *)0;
126 }
127
128 unsigned int core_pos = *(unsigned int *)arg;
129 amu_regs_t *ctx = PER_CPU_CUR(amu_ctx);
130
131 /* disable counters so the save is a static snapshot for all counters */
132 amu_disable_counters(core_pos);
133
134 write_amu_grp0_ctx_reg(ctx, 0, read_amevcntr00_el0());
135 write_amu_grp0_ctx_reg(ctx, 1, read_amevcntr01_el0());
136 /* update the NS copy of counters 2 and 3 since that will be picked up
137 * by ctx management when the core wakes up */
138 cm_sysregs_context_save_amu(NON_SECURE);
139
140 if (is_feat_amu_aux_supported()) {
141 uint8_t num_counters = read_amcgcr_el0_cg1nc();
142
143 switch (num_counters) {
144 case 0x10:
145 write_amu_grp1_ctx_reg(ctx, 0xf, read_amevcntr1f_el0());
146 __fallthrough;
147 case 0x0f:
148 write_amu_grp1_ctx_reg(ctx, 0xe, read_amevcntr1e_el0());
149 __fallthrough;
150 case 0x0e:
151 write_amu_grp1_ctx_reg(ctx, 0xd, read_amevcntr1d_el0());
152 __fallthrough;
153 case 0x0d:
154 write_amu_grp1_ctx_reg(ctx, 0xc, read_amevcntr1c_el0());
155 __fallthrough;
156 case 0x0c:
157 write_amu_grp1_ctx_reg(ctx, 0xb, read_amevcntr1b_el0());
158 __fallthrough;
159 case 0x0b:
160 write_amu_grp1_ctx_reg(ctx, 0xa, read_amevcntr1a_el0());
161 __fallthrough;
162 case 0x0a:
163 write_amu_grp1_ctx_reg(ctx, 0x9, read_amevcntr19_el0());
164 __fallthrough;
165 case 0x09:
166 write_amu_grp1_ctx_reg(ctx, 0x8, read_amevcntr18_el0());
167 __fallthrough;
168 case 0x08:
169 write_amu_grp1_ctx_reg(ctx, 0x7, read_amevcntr17_el0());
170 __fallthrough;
171 case 0x07:
172 write_amu_grp1_ctx_reg(ctx, 0x6, read_amevcntr16_el0());
173 __fallthrough;
174 case 0x06:
175 write_amu_grp1_ctx_reg(ctx, 0x5, read_amevcntr15_el0());
176 __fallthrough;
177 case 0x05:
178 write_amu_grp1_ctx_reg(ctx, 0x4, read_amevcntr14_el0());
179 __fallthrough;
180 case 0x04:
181 write_amu_grp1_ctx_reg(ctx, 0x3, read_amevcntr13_el0());
182 __fallthrough;
183 case 0x03:
184 write_amu_grp1_ctx_reg(ctx, 0x2, read_amevcntr12_el0());
185 __fallthrough;
186 case 0x02:
187 write_amu_grp1_ctx_reg(ctx, 0x1, read_amevcntr11_el0());
188 __fallthrough;
189 case 0x01:
190 write_amu_grp1_ctx_reg(ctx, 0x0, read_amevcntr10_el0());
191 __fallthrough;
192 case 0x00:
193 break;
194 default:
195 assert(0); /* something is wrong */
196 }
197 }
198
199 return (void *)0;
200 }
201
amu_context_restore(const void * arg)202 static void *amu_context_restore(const void *arg)
203 {
204 if (!is_feat_amu_supported()) {
205 return (void *)0;
206 }
207
208 unsigned int core_pos = *(unsigned int *)arg;
209 amu_regs_t *ctx = PER_CPU_CUR(amu_ctx);
210
211 /*
212 * Counters must be disabled to write them safely. All counters start
213 * disabled on an AMU reset but AMU reset doesn't have to happen with PE
214 * reset. So don't bother disabling them if they already are.
215 */
216 if (read_amcntenclr0_el0() != 0) {
217 amu_disable_counters(core_pos);
218 }
219
220 /* context management will pick up counters 2 and 3 later */
221 write_amevcntr00_el0(read_amu_grp0_ctx_reg(ctx, 0));
222 write_amevcntr01_el0(read_amu_grp0_ctx_reg(ctx, 1));
223
224 if (is_feat_amu_aux_supported()) {
225 uint8_t num_counters = read_amcgcr_el0_cg1nc();
226
227 switch (num_counters) {
228 case 0x10:
229 write_amevcntr1f_el0(read_amu_grp1_ctx_reg(ctx, 0xf));
230 __fallthrough;
231 case 0x0f:
232 write_amevcntr1e_el0(read_amu_grp1_ctx_reg(ctx, 0xe));
233 __fallthrough;
234 case 0x0e:
235 write_amevcntr1d_el0(read_amu_grp1_ctx_reg(ctx, 0xd));
236 __fallthrough;
237 case 0x0d:
238 write_amevcntr1c_el0(read_amu_grp1_ctx_reg(ctx, 0xc));
239 __fallthrough;
240 case 0x0c:
241 write_amevcntr1b_el0(read_amu_grp1_ctx_reg(ctx, 0xb));
242 __fallthrough;
243 case 0x0b:
244 write_amevcntr1a_el0(read_amu_grp1_ctx_reg(ctx, 0xa));
245 __fallthrough;
246 case 0x0a:
247 write_amevcntr19_el0(read_amu_grp1_ctx_reg(ctx, 0x9));
248 __fallthrough;
249 case 0x09:
250 write_amevcntr18_el0(read_amu_grp1_ctx_reg(ctx, 0x8));
251 __fallthrough;
252 case 0x08:
253 write_amevcntr17_el0(read_amu_grp1_ctx_reg(ctx, 0x7));
254 __fallthrough;
255 case 0x07:
256 write_amevcntr16_el0(read_amu_grp1_ctx_reg(ctx, 0x6));
257 __fallthrough;
258 case 0x06:
259 write_amevcntr15_el0(read_amu_grp1_ctx_reg(ctx, 0x5));
260 __fallthrough;
261 case 0x05:
262 write_amevcntr14_el0(read_amu_grp1_ctx_reg(ctx, 0x4));
263 __fallthrough;
264 case 0x04:
265 write_amevcntr13_el0(read_amu_grp1_ctx_reg(ctx, 0x3));
266 __fallthrough;
267 case 0x03:
268 write_amevcntr12_el0(read_amu_grp1_ctx_reg(ctx, 0x2));
269 __fallthrough;
270 case 0x02:
271 write_amevcntr11_el0(read_amu_grp1_ctx_reg(ctx, 0x1));
272 __fallthrough;
273 case 0x01:
274 write_amevcntr10_el0(read_amu_grp1_ctx_reg(ctx, 0x0));
275 __fallthrough;
276 case 0x00:
277 break;
278 default:
279 assert(0); /* something is wrong */
280 }
281 }
282
283
284 /* now enable them again */
285 write_amcntenset0_el0(AMCNTENSET0_EL0_Pn_ALWAYS_ON);
286 if (is_feat_amu_aux_supported()) {
287 write_amcntenset1_el0(get_amu_aux_enables(core_pos));
288 }
289
290 isb();
291 return (void *)0;
292 }
293
294 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
295 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);
296