1 /*
2 * Copyright (c) 2017-2025, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <cdefs.h>
9 #include <inttypes.h>
10 #include <stdbool.h>
11 #include <stdint.h>
12
13 #include <arch.h>
14 #include <arch_features.h>
15 #include <arch_helpers.h>
16 #include <common/debug.h>
17 #include <lib/el3_runtime/pubsub_events.h>
18 #include <lib/extensions/amu.h>
19 #include <lib/per_cpu/per_cpu.h>
20 #include <lib/utils_def.h>
21 #include <platform_def.h>
22
23 PER_CPU_DEFINE(amu_regs_t, amu_ctx);
24
read_amcgcr_el0_cg1nc(void)25 static inline uint8_t read_amcgcr_el0_cg1nc(void)
26 {
27 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) &
28 AMCGCR_EL0_CG1NC_MASK;
29 }
30
amu_enable(cpu_context_t * ctx)31 void amu_enable(cpu_context_t *ctx)
32 {
33 /* Initialize FEAT_AMUv1p1 features if present. */
34 if (is_feat_amuv1p1_supported()) {
35 el3_state_t *state = get_el3state_ctx(ctx);
36 u_register_t reg;
37
38 /*
39 * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual
40 * offset registers at EL2 do not trap to EL3
41 */
42 reg = read_ctx_reg(state, CTX_SCR_EL3);
43 reg |= SCR_AMVOFFEN_BIT;
44 write_ctx_reg(state, CTX_SCR_EL3, reg);
45 }
46 }
47
amu_enable_per_world(per_world_context_t * per_world_ctx)48 void amu_enable_per_world(per_world_context_t *per_world_ctx)
49 {
50 /*
51 * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor
52 * registers do not trap to EL3.
53 */
54 uint64_t cptr_el3 = per_world_ctx->ctx_cptr_el3;
55
56 cptr_el3 &= ~TAM_BIT;
57 per_world_ctx->ctx_cptr_el3 = cptr_el3;
58 }
59
amu_init_el3(unsigned int core_pos)60 void amu_init_el3(unsigned int core_pos)
61 {
62 /* architecture is currently pinned to 4 */
63 assert((read_amcgcr_el0() & AMCGCR_EL0_CG0NC_MASK) == CTX_AMU_GRP0_ALL);
64
65 /* Enable all architected counters by default */
66 write_amcntenset0_el0(AMCNTENSET0_EL0_Pn_MASK);
67
68 if (!is_feat_amuv1p1_supported()) {
69 return;
70 }
71
72 if (is_feat_amu_aux_supported()) {
73 /* something went wrong if we're trying to write higher bits */
74 assert((get_amu_aux_enables(core_pos) & ~AMCNTENSET1_EL0_Pn_MASK) == 0);
75 write_amcntenset1_el0(get_amu_aux_enables(core_pos));
76 }
77
78 #if AMU_RESTRICT_COUNTERS
79 /*
80 * FEAT_AMUv1p1 adds a register field to restrict access to
81 * group 1 counters at all but the highest implemented EL. This
82 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time
83 * flag, when set, system register reads at lower ELs return
84 * zero. Reads from the memory mapped view are unaffected.
85 */
86 VERBOSE("AMU group 1 counter access restricted.\n");
87 write_amcr_el0(AMCR_CG1RZ_BIT);
88 #else
89 /* HDBG = 0 in both cases */
90 write_amcr_el0(0);
91 #endif
92 }
93
amu_init_el2_unused(void)94 void amu_init_el2_unused(void)
95 {
96 /*
97 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor
98 * registers do not trap to EL2.
99 */
100 write_cptr_el2(read_cptr_el2() & ~CPTR_EL2_TAM_BIT);
101
102 if (is_feat_amuv1p1_supported()) {
103 /* Make sure virtual offsets are disabled */
104 write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT);
105 }
106 }
107
amu_context_save(const void * arg)108 static void *amu_context_save(const void *arg)
109 {
110 if (!is_feat_amu_supported()) {
111 return (void *)0;
112 }
113
114 unsigned int core_pos = *(unsigned int *)arg;
115 amu_regs_t *ctx = PER_CPU_CUR(amu_ctx);
116
117 /* disable all counters so we can write them safely later */
118 write_amcntenclr0_el0(AMCNTENCLR0_EL0_Pn_MASK);
119 if (is_feat_amu_aux_supported()) {
120 write_amcntenclr1_el0(get_amu_aux_enables(core_pos));
121 }
122
123 isb();
124
125 write_amu_grp0_ctx_reg(ctx, 0, read_amevcntr00_el0());
126 write_amu_grp0_ctx_reg(ctx, 1, read_amevcntr01_el0());
127 write_amu_grp0_ctx_reg(ctx, 2, read_amevcntr02_el0());
128 write_amu_grp0_ctx_reg(ctx, 3, read_amevcntr03_el0());
129
130 if (is_feat_amu_aux_supported()) {
131 uint8_t num_counters = read_amcgcr_el0_cg1nc();
132
133 switch (num_counters) {
134 case 0x10:
135 write_amu_grp1_ctx_reg(ctx, 0xf, read_amevcntr1f_el0());
136 __fallthrough;
137 case 0x0f:
138 write_amu_grp1_ctx_reg(ctx, 0xe, read_amevcntr1e_el0());
139 __fallthrough;
140 case 0x0e:
141 write_amu_grp1_ctx_reg(ctx, 0xd, read_amevcntr1d_el0());
142 __fallthrough;
143 case 0x0d:
144 write_amu_grp1_ctx_reg(ctx, 0xc, read_amevcntr1c_el0());
145 __fallthrough;
146 case 0x0c:
147 write_amu_grp1_ctx_reg(ctx, 0xb, read_amevcntr1b_el0());
148 __fallthrough;
149 case 0x0b:
150 write_amu_grp1_ctx_reg(ctx, 0xa, read_amevcntr1a_el0());
151 __fallthrough;
152 case 0x0a:
153 write_amu_grp1_ctx_reg(ctx, 0x9, read_amevcntr19_el0());
154 __fallthrough;
155 case 0x09:
156 write_amu_grp1_ctx_reg(ctx, 0x8, read_amevcntr18_el0());
157 __fallthrough;
158 case 0x08:
159 write_amu_grp1_ctx_reg(ctx, 0x7, read_amevcntr17_el0());
160 __fallthrough;
161 case 0x07:
162 write_amu_grp1_ctx_reg(ctx, 0x6, read_amevcntr16_el0());
163 __fallthrough;
164 case 0x06:
165 write_amu_grp1_ctx_reg(ctx, 0x5, read_amevcntr15_el0());
166 __fallthrough;
167 case 0x05:
168 write_amu_grp1_ctx_reg(ctx, 0x4, read_amevcntr14_el0());
169 __fallthrough;
170 case 0x04:
171 write_amu_grp1_ctx_reg(ctx, 0x3, read_amevcntr13_el0());
172 __fallthrough;
173 case 0x03:
174 write_amu_grp1_ctx_reg(ctx, 0x2, read_amevcntr12_el0());
175 __fallthrough;
176 case 0x02:
177 write_amu_grp1_ctx_reg(ctx, 0x1, read_amevcntr11_el0());
178 __fallthrough;
179 case 0x01:
180 write_amu_grp1_ctx_reg(ctx, 0x0, read_amevcntr10_el0());
181 __fallthrough;
182 case 0x00:
183 break;
184 default:
185 assert(0); /* something is wrong */
186 }
187 }
188
189 return (void *)0;
190 }
191
amu_context_restore(const void * arg)192 static void *amu_context_restore(const void *arg)
193 {
194 if (!is_feat_amu_supported()) {
195 return (void *)0;
196 }
197
198 unsigned int core_pos = *(unsigned int *)arg;
199 amu_regs_t *ctx = PER_CPU_CUR(amu_ctx);
200
201 write_amevcntr00_el0(read_amu_grp0_ctx_reg(ctx, 0));
202 write_amevcntr01_el0(read_amu_grp0_ctx_reg(ctx, 1));
203 write_amevcntr02_el0(read_amu_grp0_ctx_reg(ctx, 2));
204 write_amevcntr03_el0(read_amu_grp0_ctx_reg(ctx, 3));
205
206 if (is_feat_amu_aux_supported()) {
207 uint8_t num_counters = read_amcgcr_el0_cg1nc();
208
209 switch (num_counters) {
210 case 0x10:
211 write_amevcntr1f_el0(read_amu_grp1_ctx_reg(ctx, 0xf));
212 __fallthrough;
213 case 0x0f:
214 write_amevcntr1e_el0(read_amu_grp1_ctx_reg(ctx, 0xe));
215 __fallthrough;
216 case 0x0e:
217 write_amevcntr1d_el0(read_amu_grp1_ctx_reg(ctx, 0xd));
218 __fallthrough;
219 case 0x0d:
220 write_amevcntr1c_el0(read_amu_grp1_ctx_reg(ctx, 0xc));
221 __fallthrough;
222 case 0x0c:
223 write_amevcntr1b_el0(read_amu_grp1_ctx_reg(ctx, 0xb));
224 __fallthrough;
225 case 0x0b:
226 write_amevcntr1a_el0(read_amu_grp1_ctx_reg(ctx, 0xa));
227 __fallthrough;
228 case 0x0a:
229 write_amevcntr19_el0(read_amu_grp1_ctx_reg(ctx, 0x9));
230 __fallthrough;
231 case 0x09:
232 write_amevcntr18_el0(read_amu_grp1_ctx_reg(ctx, 0x8));
233 __fallthrough;
234 case 0x08:
235 write_amevcntr17_el0(read_amu_grp1_ctx_reg(ctx, 0x7));
236 __fallthrough;
237 case 0x07:
238 write_amevcntr16_el0(read_amu_grp1_ctx_reg(ctx, 0x6));
239 __fallthrough;
240 case 0x06:
241 write_amevcntr15_el0(read_amu_grp1_ctx_reg(ctx, 0x5));
242 __fallthrough;
243 case 0x05:
244 write_amevcntr14_el0(read_amu_grp1_ctx_reg(ctx, 0x4));
245 __fallthrough;
246 case 0x04:
247 write_amevcntr13_el0(read_amu_grp1_ctx_reg(ctx, 0x3));
248 __fallthrough;
249 case 0x03:
250 write_amevcntr12_el0(read_amu_grp1_ctx_reg(ctx, 0x2));
251 __fallthrough;
252 case 0x02:
253 write_amevcntr11_el0(read_amu_grp1_ctx_reg(ctx, 0x1));
254 __fallthrough;
255 case 0x01:
256 write_amevcntr10_el0(read_amu_grp1_ctx_reg(ctx, 0x0));
257 __fallthrough;
258 case 0x00:
259 break;
260 default:
261 assert(0); /* something is wrong */
262 }
263 }
264
265
266 /* now enable them again */
267 write_amcntenset0_el0(AMCNTENSET0_EL0_Pn_MASK);
268 if (is_feat_amu_aux_supported()) {
269 write_amcntenset1_el0(get_amu_aux_enables(core_pos));
270 }
271
272 isb();
273 return (void *)0;
274 }
275
276 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
277 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);
278