1 /* 2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <cdefs.h> 9 #include <stdbool.h> 10 11 #include "../amu_private.h" 12 #include <arch.h> 13 #include <arch_features.h> 14 #include <arch_helpers.h> 15 #include <common/debug.h> 16 #include <lib/el3_runtime/pubsub_events.h> 17 #include <lib/extensions/amu.h> 18 19 #include <plat/common/platform.h> 20 21 struct amu_ctx { 22 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; 23 #if ENABLE_AMU_AUXILIARY_COUNTERS 24 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; 25 #endif 26 27 uint16_t group0_enable; 28 #if ENABLE_AMU_AUXILIARY_COUNTERS 29 uint16_t group1_enable; 30 #endif 31 }; 32 33 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; 34 35 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, 36 amu_ctx_group0_enable_cannot_represent_all_group0_counters); 37 38 #if ENABLE_AMU_AUXILIARY_COUNTERS 39 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, 40 amu_ctx_group1_enable_cannot_represent_all_group1_counters); 41 #endif 42 43 static inline __unused void write_hcptr_tam(uint32_t value) 44 { 45 write_hcptr((read_hcptr() & ~TAM_BIT) | 46 ((value << TAM_SHIFT) & TAM_BIT)); 47 } 48 49 static inline __unused void write_amcr_cg1rz(uint32_t value) 50 { 51 write_amcr((read_amcr() & ~AMCR_CG1RZ_BIT) | 52 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); 53 } 54 55 static inline __unused uint32_t read_amcfgr_ncg(void) 56 { 57 return (read_amcfgr() >> AMCFGR_NCG_SHIFT) & 58 AMCFGR_NCG_MASK; 59 } 60 61 static inline __unused uint32_t read_amcgcr_cg0nc(void) 62 { 63 return (read_amcgcr() >> AMCGCR_CG0NC_SHIFT) & 64 AMCGCR_CG0NC_MASK; 65 } 66 67 static inline __unused uint32_t read_amcgcr_cg1nc(void) 68 { 69 return (read_amcgcr() >> AMCGCR_CG1NC_SHIFT) & 70 AMCGCR_CG1NC_MASK; 71 } 72 73 static inline __unused uint32_t read_amcntenset0_px(void) 74 { 75 return (read_amcntenset0() >> AMCNTENSET0_Pn_SHIFT) & 76 AMCNTENSET0_Pn_MASK; 77 } 78 79 static inline __unused uint32_t read_amcntenset1_px(void) 80 { 81 return (read_amcntenset1() >> AMCNTENSET1_Pn_SHIFT) & 82 AMCNTENSET1_Pn_MASK; 83 } 84 85 static inline __unused void write_amcntenset0_px(uint32_t px) 86 { 87 uint32_t value = read_amcntenset0(); 88 89 value &= ~AMCNTENSET0_Pn_MASK; 90 value |= (px << AMCNTENSET0_Pn_SHIFT) & 91 AMCNTENSET0_Pn_MASK; 92 93 write_amcntenset0(value); 94 } 95 96 static inline __unused void write_amcntenset1_px(uint32_t px) 97 { 98 uint32_t value = read_amcntenset1(); 99 100 value &= ~AMCNTENSET1_Pn_MASK; 101 value |= (px << AMCNTENSET1_Pn_SHIFT) & 102 AMCNTENSET1_Pn_MASK; 103 104 write_amcntenset1(value); 105 } 106 107 static inline __unused void write_amcntenclr0_px(uint32_t px) 108 { 109 uint32_t value = read_amcntenclr0(); 110 111 value &= ~AMCNTENCLR0_Pn_MASK; 112 value |= (px << AMCNTENCLR0_Pn_SHIFT) & AMCNTENCLR0_Pn_MASK; 113 114 write_amcntenclr0(value); 115 } 116 117 static inline __unused void write_amcntenclr1_px(uint32_t px) 118 { 119 uint32_t value = read_amcntenclr1(); 120 121 value &= ~AMCNTENCLR1_Pn_MASK; 122 value |= (px << AMCNTENCLR1_Pn_SHIFT) & AMCNTENCLR1_Pn_MASK; 123 124 write_amcntenclr1(value); 125 } 126 127 #if ENABLE_AMU_AUXILIARY_COUNTERS 128 static __unused bool amu_group1_supported(void) 129 { 130 return read_amcfgr_ncg() > 0U; 131 } 132 #endif 133 134 /* 135 * Enable counters. This function is meant to be invoked by the context 136 * management library before exiting from EL3. 137 */ 138 void amu_enable(bool el2_unused) 139 { 140 uint32_t amcfgr_ncg; /* Number of counter groups */ 141 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ 142 143 uint32_t amcntenset0_px = 0x0; /* Group 0 enable mask */ 144 uint32_t amcntenset1_px = 0x0; /* Group 1 enable mask */ 145 146 if (el2_unused) { 147 /* 148 * HCPTR.TAM: Set to zero so any accesses to the Activity 149 * Monitor registers do not trap to EL2. 150 */ 151 write_hcptr_tam(0U); 152 } 153 154 /* 155 * Retrieve the number of architected counters. All of these counters 156 * are enabled by default. 157 */ 158 159 amcgcr_cg0nc = read_amcgcr_cg0nc(); 160 amcntenset0_px = (UINT32_C(1) << (amcgcr_cg0nc)) - 1U; 161 162 assert(amcgcr_cg0nc <= AMU_AMCGCR_CG0NC_MAX); 163 164 /* 165 * The platform may opt to enable specific auxiliary counters. This can 166 * be done via the common FCONF getter, or via the platform-implemented 167 * function. 168 */ 169 170 #if ENABLE_AMU_AUXILIARY_COUNTERS 171 const struct amu_topology *topology; 172 173 #if ENABLE_AMU_FCONF 174 topology = FCONF_GET_PROPERTY(amu, config, topology); 175 #else 176 topology = plat_amu_topology(); 177 #endif /* ENABLE_AMU_FCONF */ 178 179 if (topology != NULL) { 180 unsigned int core_pos = plat_my_core_pos(); 181 182 amcntenset1_el0_px = topology->cores[core_pos].enable; 183 } else { 184 ERROR("AMU: failed to generate AMU topology\n"); 185 } 186 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ 187 188 /* 189 * Enable the requested counters. 190 */ 191 192 write_amcntenset0_px(amcntenset0_px); 193 194 amcfgr_ncg = read_amcfgr_ncg(); 195 if (amcfgr_ncg > 0U) { 196 write_amcntenset1_px(amcntenset1_px); 197 198 #if !ENABLE_AMU_AUXILIARY_COUNTERS 199 VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); 200 #endif 201 } 202 203 /* Bail out if FEAT_AMUv1p1 features are not present. */ 204 if (!is_feat_amuv1p1_supported()) { 205 return; 206 } 207 208 #if AMU_RESTRICT_COUNTERS 209 /* 210 * FEAT_AMUv1p1 adds a register field to restrict access to group 1 211 * counters at all but the highest implemented EL. This is controlled 212 * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system 213 * register reads at lower ELs return zero. Reads from the memory 214 * mapped view are unaffected. 215 */ 216 VERBOSE("AMU group 1 counter access restricted.\n"); 217 write_amcr_cg1rz(1U); 218 #else 219 write_amcr_cg1rz(0U); 220 #endif 221 } 222 223 /* Read the group 0 counter identified by the given `idx`. */ 224 static uint64_t amu_group0_cnt_read(unsigned int idx) 225 { 226 assert(is_feat_amu_supported()); 227 assert(idx < read_amcgcr_cg0nc()); 228 229 return amu_group0_cnt_read_internal(idx); 230 } 231 232 /* Write the group 0 counter identified by the given `idx` with `val` */ 233 static void amu_group0_cnt_write(unsigned int idx, uint64_t val) 234 { 235 assert(is_feat_amu_supported()); 236 assert(idx < read_amcgcr_cg0nc()); 237 238 amu_group0_cnt_write_internal(idx, val); 239 isb(); 240 } 241 242 #if ENABLE_AMU_AUXILIARY_COUNTERS 243 /* Read the group 1 counter identified by the given `idx` */ 244 static uint64_t amu_group1_cnt_read(unsigned int idx) 245 { 246 assert(is_feat_amu_supported()); 247 assert(amu_group1_supported()); 248 assert(idx < read_amcgcr_cg1nc()); 249 250 return amu_group1_cnt_read_internal(idx); 251 } 252 253 /* Write the group 1 counter identified by the given `idx` with `val` */ 254 static void amu_group1_cnt_write(unsigned int idx, uint64_t val) 255 { 256 assert(is_feat_amu_supported()); 257 assert(amu_group1_supported()); 258 assert(idx < read_amcgcr_cg1nc()); 259 260 amu_group1_cnt_write_internal(idx, val); 261 isb(); 262 } 263 #endif 264 265 static void *amu_context_save(const void *arg) 266 { 267 uint32_t i; 268 269 unsigned int core_pos; 270 struct amu_ctx *ctx; 271 272 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ 273 274 #if ENABLE_AMU_AUXILIARY_COUNTERS 275 uint32_t amcfgr_ncg; /* Number of counter groups */ 276 uint32_t amcgcr_cg1nc; /* Number of group 1 counters */ 277 #endif 278 279 if (!is_feat_amu_supported()) { 280 return (void *)0; 281 } 282 283 core_pos = plat_my_core_pos(); 284 ctx = &amu_ctxs_[core_pos]; 285 286 amcgcr_cg0nc = read_amcgcr_cg0nc(); 287 288 #if ENABLE_AMU_AUXILIARY_COUNTERS 289 amcfgr_ncg = read_amcfgr_ncg(); 290 amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U; 291 #endif 292 293 /* 294 * Disable all AMU counters. 295 */ 296 297 ctx->group0_enable = read_amcntenset0_px(); 298 write_amcntenclr0_px(ctx->group0_enable); 299 300 #if ENABLE_AMU_AUXILIARY_COUNTERS 301 if (amcfgr_ncg > 0U) { 302 ctx->group1_enable = read_amcntenset1_px(); 303 write_amcntenclr1_px(ctx->group1_enable); 304 } 305 #endif 306 307 /* 308 * Save the counters to the local context. 309 */ 310 311 isb(); /* Ensure counters have been stopped */ 312 313 for (i = 0U; i < amcgcr_cg0nc; i++) { 314 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 315 } 316 317 #if ENABLE_AMU_AUXILIARY_COUNTERS 318 for (i = 0U; i < amcgcr_cg1nc; i++) { 319 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 320 } 321 #endif 322 323 return (void *)0; 324 } 325 326 static void *amu_context_restore(const void *arg) 327 { 328 uint32_t i; 329 330 unsigned int core_pos; 331 struct amu_ctx *ctx; 332 333 uint32_t amcfgr_ncg; /* Number of counter groups */ 334 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ 335 336 #if ENABLE_AMU_AUXILIARY_COUNTERS 337 uint32_t amcgcr_cg1nc; /* Number of group 1 counters */ 338 #endif 339 340 if (!is_feat_amu_supported()) { 341 return (void *)0; 342 } 343 344 core_pos = plat_my_core_pos(); 345 ctx = &amu_ctxs_[core_pos]; 346 347 amcfgr_ncg = read_amcfgr_ncg(); 348 amcgcr_cg0nc = read_amcgcr_cg0nc(); 349 350 #if ENABLE_AMU_AUXILIARY_COUNTERS 351 amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U; 352 #endif 353 354 /* 355 * Sanity check that all counters were disabled when the context was 356 * previously saved. 357 */ 358 359 assert(read_amcntenset0_px() == 0U); 360 361 if (amcfgr_ncg > 0U) { 362 assert(read_amcntenset1_px() == 0U); 363 } 364 365 /* 366 * Restore the counter values from the local context. 367 */ 368 369 for (i = 0U; i < amcgcr_cg0nc; i++) { 370 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 371 } 372 373 #if ENABLE_AMU_AUXILIARY_COUNTERS 374 for (i = 0U; i < amcgcr_cg1nc; i++) { 375 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 376 } 377 #endif 378 379 /* 380 * Re-enable counters that were disabled during context save. 381 */ 382 383 write_amcntenset0_px(ctx->group0_enable); 384 385 #if ENABLE_AMU_AUXILIARY_COUNTERS 386 if (amcfgr_ncg > 0U) { 387 write_amcntenset1_px(ctx->group1_enable); 388 } 389 #endif 390 391 return (void *)0; 392 } 393 394 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 395 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 396