1 /* 2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <cdefs.h> 9 #include <stdbool.h> 10 11 #include "../amu_private.h" 12 #include <arch.h> 13 #include <arch_helpers.h> 14 #include <common/debug.h> 15 #include <lib/el3_runtime/pubsub_events.h> 16 #include <lib/extensions/amu.h> 17 18 #include <plat/common/platform.h> 19 20 struct amu_ctx { 21 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; 22 #if ENABLE_AMU_AUXILIARY_COUNTERS 23 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; 24 #endif 25 26 uint16_t group0_enable; 27 #if ENABLE_AMU_AUXILIARY_COUNTERS 28 uint16_t group1_enable; 29 #endif 30 }; 31 32 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; 33 34 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, 35 amu_ctx_group0_enable_cannot_represent_all_group0_counters); 36 37 #if ENABLE_AMU_AUXILIARY_COUNTERS 38 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, 39 amu_ctx_group1_enable_cannot_represent_all_group1_counters); 40 #endif 41 42 static inline __unused uint32_t read_id_pfr0_amu(void) 43 { 44 return (read_id_pfr0() >> ID_PFR0_AMU_SHIFT) & 45 ID_PFR0_AMU_MASK; 46 } 47 48 static inline __unused void write_hcptr_tam(uint32_t value) 49 { 50 write_hcptr((read_hcptr() & ~TAM_BIT) | 51 ((value << TAM_SHIFT) & TAM_BIT)); 52 } 53 54 static inline __unused void write_amcr_cg1rz(uint32_t value) 55 { 56 write_amcr((read_amcr() & ~AMCR_CG1RZ_BIT) | 57 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); 58 } 59 60 static inline __unused uint32_t read_amcfgr_ncg(void) 61 { 62 return (read_amcfgr() >> AMCFGR_NCG_SHIFT) & 63 AMCFGR_NCG_MASK; 64 } 65 66 static inline __unused uint32_t read_amcgcr_cg0nc(void) 67 { 68 return (read_amcgcr() >> AMCGCR_CG0NC_SHIFT) & 69 AMCGCR_CG0NC_MASK; 70 } 71 72 static inline __unused uint32_t read_amcgcr_cg1nc(void) 73 { 74 return (read_amcgcr() >> AMCGCR_CG1NC_SHIFT) & 75 AMCGCR_CG1NC_MASK; 76 } 77 78 static inline __unused uint32_t read_amcntenset0_px(void) 79 { 80 return (read_amcntenset0() >> AMCNTENSET0_Pn_SHIFT) & 81 AMCNTENSET0_Pn_MASK; 82 } 83 84 static inline __unused uint32_t read_amcntenset1_px(void) 85 { 86 return (read_amcntenset1() >> AMCNTENSET1_Pn_SHIFT) & 87 AMCNTENSET1_Pn_MASK; 88 } 89 90 static inline __unused void write_amcntenset0_px(uint32_t px) 91 { 92 uint32_t value = read_amcntenset0(); 93 94 value &= ~AMCNTENSET0_Pn_MASK; 95 value |= (px << AMCNTENSET0_Pn_SHIFT) & 96 AMCNTENSET0_Pn_MASK; 97 98 write_amcntenset0(value); 99 } 100 101 static inline __unused void write_amcntenset1_px(uint32_t px) 102 { 103 uint32_t value = read_amcntenset1(); 104 105 value &= ~AMCNTENSET1_Pn_MASK; 106 value |= (px << AMCNTENSET1_Pn_SHIFT) & 107 AMCNTENSET1_Pn_MASK; 108 109 write_amcntenset1(value); 110 } 111 112 static inline __unused void write_amcntenclr0_px(uint32_t px) 113 { 114 uint32_t value = read_amcntenclr0(); 115 116 value &= ~AMCNTENCLR0_Pn_MASK; 117 value |= (px << AMCNTENCLR0_Pn_SHIFT) & AMCNTENCLR0_Pn_MASK; 118 119 write_amcntenclr0(value); 120 } 121 122 static inline __unused void write_amcntenclr1_px(uint32_t px) 123 { 124 uint32_t value = read_amcntenclr1(); 125 126 value &= ~AMCNTENCLR1_Pn_MASK; 127 value |= (px << AMCNTENCLR1_Pn_SHIFT) & AMCNTENCLR1_Pn_MASK; 128 129 write_amcntenclr1(value); 130 } 131 132 static __unused bool amu_supported(void) 133 { 134 return read_id_pfr0_amu() >= ID_PFR0_AMU_V1; 135 } 136 137 #if ENABLE_AMU_AUXILIARY_COUNTERS 138 static __unused bool amu_group1_supported(void) 139 { 140 return read_amcfgr_ncg() > 0U; 141 } 142 #endif 143 144 /* 145 * Enable counters. This function is meant to be invoked by the context 146 * management library before exiting from EL3. 147 */ 148 void amu_enable(bool el2_unused) 149 { 150 uint32_t id_pfr0_amu; /* AMU version */ 151 152 uint32_t amcfgr_ncg; /* Number of counter groups */ 153 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ 154 155 uint32_t amcntenset0_px = 0x0; /* Group 0 enable mask */ 156 uint32_t amcntenset1_px = 0x0; /* Group 1 enable mask */ 157 158 id_pfr0_amu = read_id_pfr0_amu(); 159 if (id_pfr0_amu == ID_PFR0_AMU_NOT_SUPPORTED) { 160 /* 161 * If the AMU is unsupported, nothing needs to be done. 162 */ 163 164 return; 165 } 166 167 if (el2_unused) { 168 /* 169 * HCPTR.TAM: Set to zero so any accesses to the Activity 170 * Monitor registers do not trap to EL2. 171 */ 172 write_hcptr_tam(0U); 173 } 174 175 /* 176 * Retrieve the number of architected counters. All of these counters 177 * are enabled by default. 178 */ 179 180 amcgcr_cg0nc = read_amcgcr_cg0nc(); 181 amcntenset0_px = (UINT32_C(1) << (amcgcr_cg0nc)) - 1U; 182 183 assert(amcgcr_cg0nc <= AMU_AMCGCR_CG0NC_MAX); 184 185 /* 186 * The platform may opt to enable specific auxiliary counters. This can 187 * be done via the common FCONF getter, or via the platform-implemented 188 * function. 189 */ 190 191 #if ENABLE_AMU_AUXILIARY_COUNTERS 192 const struct amu_topology *topology; 193 194 #if ENABLE_AMU_FCONF 195 topology = FCONF_GET_PROPERTY(amu, config, topology); 196 #else 197 topology = plat_amu_topology(); 198 #endif /* ENABLE_AMU_FCONF */ 199 200 if (topology != NULL) { 201 unsigned int core_pos = plat_my_core_pos(); 202 203 amcntenset1_el0_px = topology->cores[core_pos].enable; 204 } else { 205 ERROR("AMU: failed to generate AMU topology\n"); 206 } 207 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ 208 209 /* 210 * Enable the requested counters. 211 */ 212 213 write_amcntenset0_px(amcntenset0_px); 214 215 amcfgr_ncg = read_amcfgr_ncg(); 216 if (amcfgr_ncg > 0U) { 217 write_amcntenset1_px(amcntenset1_px); 218 219 #if !ENABLE_AMU_AUXILIARY_COUNTERS 220 VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); 221 #endif 222 } 223 224 /* Initialize FEAT_AMUv1p1 features if present. */ 225 if (id_pfr0_amu < ID_PFR0_AMU_V1P1) { 226 return; 227 } 228 229 #if AMU_RESTRICT_COUNTERS 230 /* 231 * FEAT_AMUv1p1 adds a register field to restrict access to group 1 232 * counters at all but the highest implemented EL. This is controlled 233 * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system 234 * register reads at lower ELs return zero. Reads from the memory 235 * mapped view are unaffected. 236 */ 237 VERBOSE("AMU group 1 counter access restricted.\n"); 238 write_amcr_cg1rz(1U); 239 #else 240 write_amcr_cg1rz(0U); 241 #endif 242 } 243 244 /* Read the group 0 counter identified by the given `idx`. */ 245 static uint64_t amu_group0_cnt_read(unsigned int idx) 246 { 247 assert(amu_supported()); 248 assert(idx < read_amcgcr_cg0nc()); 249 250 return amu_group0_cnt_read_internal(idx); 251 } 252 253 /* Write the group 0 counter identified by the given `idx` with `val` */ 254 static void amu_group0_cnt_write(unsigned int idx, uint64_t val) 255 { 256 assert(amu_supported()); 257 assert(idx < read_amcgcr_cg0nc()); 258 259 amu_group0_cnt_write_internal(idx, val); 260 isb(); 261 } 262 263 #if ENABLE_AMU_AUXILIARY_COUNTERS 264 /* Read the group 1 counter identified by the given `idx` */ 265 static uint64_t amu_group1_cnt_read(unsigned int idx) 266 { 267 assert(amu_supported()); 268 assert(amu_group1_supported()); 269 assert(idx < read_amcgcr_cg1nc()); 270 271 return amu_group1_cnt_read_internal(idx); 272 } 273 274 /* Write the group 1 counter identified by the given `idx` with `val` */ 275 static void amu_group1_cnt_write(unsigned int idx, uint64_t val) 276 { 277 assert(amu_supported()); 278 assert(amu_group1_supported()); 279 assert(idx < read_amcgcr_cg1nc()); 280 281 amu_group1_cnt_write_internal(idx, val); 282 isb(); 283 } 284 #endif 285 286 static void *amu_context_save(const void *arg) 287 { 288 uint32_t i; 289 290 unsigned int core_pos; 291 struct amu_ctx *ctx; 292 293 uint32_t id_pfr0_amu; /* AMU version */ 294 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ 295 296 #if ENABLE_AMU_AUXILIARY_COUNTERS 297 uint32_t amcfgr_ncg; /* Number of counter groups */ 298 uint32_t amcgcr_cg1nc; /* Number of group 1 counters */ 299 #endif 300 301 id_pfr0_amu = read_id_pfr0_amu(); 302 if (id_pfr0_amu == ID_PFR0_AMU_NOT_SUPPORTED) { 303 return (void *)0; 304 } 305 306 core_pos = plat_my_core_pos(); 307 ctx = &amu_ctxs_[core_pos]; 308 309 amcgcr_cg0nc = read_amcgcr_cg0nc(); 310 311 #if ENABLE_AMU_AUXILIARY_COUNTERS 312 amcfgr_ncg = read_amcfgr_ncg(); 313 amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U; 314 #endif 315 316 /* 317 * Disable all AMU counters. 318 */ 319 320 ctx->group0_enable = read_amcntenset0_px(); 321 write_amcntenclr0_px(ctx->group0_enable); 322 323 #if ENABLE_AMU_AUXILIARY_COUNTERS 324 if (amcfgr_ncg > 0U) { 325 ctx->group1_enable = read_amcntenset1_px(); 326 write_amcntenclr1_px(ctx->group1_enable); 327 } 328 #endif 329 330 /* 331 * Save the counters to the local context. 332 */ 333 334 isb(); /* Ensure counters have been stopped */ 335 336 for (i = 0U; i < amcgcr_cg0nc; i++) { 337 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 338 } 339 340 #if ENABLE_AMU_AUXILIARY_COUNTERS 341 for (i = 0U; i < amcgcr_cg1nc; i++) { 342 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 343 } 344 #endif 345 346 return (void *)0; 347 } 348 349 static void *amu_context_restore(const void *arg) 350 { 351 uint32_t i; 352 353 unsigned int core_pos; 354 struct amu_ctx *ctx; 355 356 uint32_t id_pfr0_amu; /* AMU version */ 357 358 uint32_t amcfgr_ncg; /* Number of counter groups */ 359 uint32_t amcgcr_cg0nc; /* Number of group 0 counters */ 360 361 #if ENABLE_AMU_AUXILIARY_COUNTERS 362 uint32_t amcgcr_cg1nc; /* Number of group 1 counters */ 363 #endif 364 365 id_pfr0_amu = read_id_pfr0_amu(); 366 if (id_pfr0_amu == ID_PFR0_AMU_NOT_SUPPORTED) { 367 return (void *)0; 368 } 369 370 core_pos = plat_my_core_pos(); 371 ctx = &amu_ctxs_[core_pos]; 372 373 amcfgr_ncg = read_amcfgr_ncg(); 374 amcgcr_cg0nc = read_amcgcr_cg0nc(); 375 376 #if ENABLE_AMU_AUXILIARY_COUNTERS 377 amcgcr_cg1nc = (amcfgr_ncg > 0U) ? read_amcgcr_cg1nc() : 0U; 378 #endif 379 380 /* 381 * Sanity check that all counters were disabled when the context was 382 * previously saved. 383 */ 384 385 assert(read_amcntenset0_px() == 0U); 386 387 if (amcfgr_ncg > 0U) { 388 assert(read_amcntenset1_px() == 0U); 389 } 390 391 /* 392 * Restore the counter values from the local context. 393 */ 394 395 for (i = 0U; i < amcgcr_cg0nc; i++) { 396 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 397 } 398 399 #if ENABLE_AMU_AUXILIARY_COUNTERS 400 for (i = 0U; i < amcgcr_cg1nc; i++) { 401 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 402 } 403 #endif 404 405 /* 406 * Re-enable counters that were disabled during context save. 407 */ 408 409 write_amcntenset0_px(ctx->group0_enable); 410 411 #if ENABLE_AMU_AUXILIARY_COUNTERS 412 if (amcfgr_ncg > 0U) { 413 write_amcntenset1_px(ctx->group1_enable); 414 } 415 #endif 416 417 return (void *)0; 418 } 419 420 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 421 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 422