1 /* 2 * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <cdefs.h> 9 #include <inttypes.h> 10 #include <stdbool.h> 11 #include <stdint.h> 12 13 #include "../amu_private.h" 14 #include <arch.h> 15 #include <arch_features.h> 16 #include <arch_helpers.h> 17 #include <common/debug.h> 18 #include <lib/el3_runtime/pubsub_events.h> 19 #include <lib/extensions/amu.h> 20 21 #include <plat/common/platform.h> 22 23 #if ENABLE_AMU_FCONF 24 # include <lib/fconf/fconf.h> 25 # include <lib/fconf/fconf_amu_getter.h> 26 #endif 27 28 struct amu_ctx { 29 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; 30 #if ENABLE_AMU_AUXILIARY_COUNTERS 31 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; 32 #endif 33 34 /* Architected event counter 1 does not have an offset register */ 35 uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U]; 36 #if ENABLE_AMU_AUXILIARY_COUNTERS 37 uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS]; 38 #endif 39 40 uint16_t group0_enable; 41 #if ENABLE_AMU_AUXILIARY_COUNTERS 42 uint16_t group1_enable; 43 #endif 44 }; 45 46 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; 47 48 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, 49 amu_ctx_group0_enable_cannot_represent_all_group0_counters); 50 51 #if ENABLE_AMU_AUXILIARY_COUNTERS 52 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, 53 amu_ctx_group1_enable_cannot_represent_all_group1_counters); 54 #endif 55 56 static inline __unused uint64_t read_hcr_el2_amvoffen(void) 57 { 58 return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >> 59 HCR_AMVOFFEN_SHIFT; 60 } 61 62 static inline __unused void write_cptr_el2_tam(uint64_t value) 63 { 64 write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) | 65 ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT)); 66 } 67 68 static inline __unused void ctx_write_scr_el3_amvoffen(cpu_context_t *ctx, uint64_t amvoffen) 69 { 70 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3); 71 72 value &= ~SCR_AMVOFFEN_BIT; 73 value |= (amvoffen << SCR_AMVOFFEN_SHIFT) & SCR_AMVOFFEN_BIT; 74 75 write_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3, value); 76 } 77 78 static inline __unused void write_hcr_el2_amvoffen(uint64_t value) 79 { 80 write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) | 81 ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT)); 82 } 83 84 static inline __unused void write_amcr_el0_cg1rz(uint64_t value) 85 { 86 write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) | 87 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); 88 } 89 90 static inline __unused uint64_t read_amcfgr_el0_ncg(void) 91 { 92 return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) & 93 AMCFGR_EL0_NCG_MASK; 94 } 95 96 static inline __unused uint64_t read_amcgcr_el0_cg0nc(void) 97 { 98 return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) & 99 AMCGCR_EL0_CG0NC_MASK; 100 } 101 102 static inline __unused uint64_t read_amcg1idr_el0_voff(void) 103 { 104 return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 105 AMCG1IDR_VOFF_MASK; 106 } 107 108 static inline __unused uint64_t read_amcgcr_el0_cg1nc(void) 109 { 110 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & 111 AMCGCR_EL0_CG1NC_MASK; 112 } 113 114 static inline __unused uint64_t read_amcntenset0_el0_px(void) 115 { 116 return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) & 117 AMCNTENSET0_EL0_Pn_MASK; 118 } 119 120 static inline __unused uint64_t read_amcntenset1_el0_px(void) 121 { 122 return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) & 123 AMCNTENSET1_EL0_Pn_MASK; 124 } 125 126 static inline __unused void write_amcntenset0_el0_px(uint64_t px) 127 { 128 uint64_t value = read_amcntenset0_el0(); 129 130 value &= ~AMCNTENSET0_EL0_Pn_MASK; 131 value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK; 132 133 write_amcntenset0_el0(value); 134 } 135 136 static inline __unused void write_amcntenset1_el0_px(uint64_t px) 137 { 138 uint64_t value = read_amcntenset1_el0(); 139 140 value &= ~AMCNTENSET1_EL0_Pn_MASK; 141 value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK; 142 143 write_amcntenset1_el0(value); 144 } 145 146 static inline __unused void write_amcntenclr0_el0_px(uint64_t px) 147 { 148 uint64_t value = read_amcntenclr0_el0(); 149 150 value &= ~AMCNTENCLR0_EL0_Pn_MASK; 151 value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK; 152 153 write_amcntenclr0_el0(value); 154 } 155 156 static inline __unused void write_amcntenclr1_el0_px(uint64_t px) 157 { 158 uint64_t value = read_amcntenclr1_el0(); 159 160 value &= ~AMCNTENCLR1_EL0_Pn_MASK; 161 value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK; 162 163 write_amcntenclr1_el0(value); 164 } 165 166 #if ENABLE_AMU_AUXILIARY_COUNTERS 167 static __unused bool amu_group1_supported(void) 168 { 169 return read_amcfgr_el0_ncg() > 0U; 170 } 171 #endif 172 173 /* 174 * Enable counters. This function is meant to be invoked by the context 175 * management library before exiting from EL3. 176 */ 177 void amu_enable(cpu_context_t *ctx) 178 { 179 /* Initialize FEAT_AMUv1p1 features if present. */ 180 if (is_feat_amuv1p1_supported()) { 181 /* 182 * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual 183 * offset registers at EL2 do not trap to EL3 184 */ 185 ctx_write_scr_el3_amvoffen(ctx, 1U); 186 } 187 } 188 189 void amu_enable_per_world(per_world_context_t *per_world_ctx) 190 { 191 /* 192 * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor 193 * registers do not trap to EL3. 194 */ 195 uint64_t cptr_el3 = per_world_ctx->ctx_cptr_el3; 196 197 cptr_el3 &= ~TAM_BIT; 198 per_world_ctx->ctx_cptr_el3 = cptr_el3; 199 } 200 201 void amu_init_el3(void) 202 { 203 uint64_t group0_impl_ctr = read_amcgcr_el0_cg0nc(); 204 uint64_t group0_en_mask = (1 << (group0_impl_ctr)) - 1U; 205 uint64_t num_ctr_groups = read_amcfgr_el0_ncg(); 206 207 /* Enable all architected counters by default */ 208 write_amcntenset0_el0_px(group0_en_mask); 209 210 #if ENABLE_AMU_AUXILIARY_COUNTERS 211 if (num_ctr_groups > 0U) { 212 uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */ 213 const struct amu_topology *topology; 214 215 /* 216 * The platform may opt to enable specific auxiliary counters. 217 * This can be done via the common FCONF getter, or via the 218 * platform-implemented function. 219 */ 220 #if ENABLE_AMU_FCONF 221 topology = FCONF_GET_PROPERTY(amu, config, topology); 222 #else 223 topology = plat_amu_topology(); 224 #endif /* ENABLE_AMU_FCONF */ 225 226 if (topology != NULL) { 227 unsigned int core_pos = plat_my_core_pos(); 228 229 amcntenset1_el0_px = topology->cores[core_pos].enable; 230 } else { 231 ERROR("AMU: failed to generate AMU topology\n"); 232 } 233 234 write_amcntenset1_el0_px(amcntenset1_el0_px); 235 } 236 #else /* ENABLE_AMU_AUXILIARY_COUNTERS */ 237 if (num_ctr_groups > 0U) { 238 VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); 239 } 240 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ 241 242 if (is_feat_amuv1p1_supported()) { 243 #if AMU_RESTRICT_COUNTERS 244 /* 245 * FEAT_AMUv1p1 adds a register field to restrict access to 246 * group 1 counters at all but the highest implemented EL. This 247 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time 248 * flag, when set, system register reads at lower ELs return 249 * zero. Reads from the memory mapped view are unaffected. 250 */ 251 VERBOSE("AMU group 1 counter access restricted.\n"); 252 write_amcr_el0_cg1rz(1U); 253 #else 254 write_amcr_el0_cg1rz(0U); 255 #endif 256 } 257 } 258 259 void amu_init_el2_unused(void) 260 { 261 /* 262 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor 263 * registers do not trap to EL2. 264 */ 265 write_cptr_el2_tam(0U); 266 267 /* Initialize FEAT_AMUv1p1 features if present. */ 268 if (is_feat_amuv1p1_supported()) { 269 /* Make sure virtual offsets are disabled if EL2 not used. */ 270 write_hcr_el2_amvoffen(0U); 271 } 272 } 273 274 /* Read the group 0 counter identified by the given `idx`. */ 275 static uint64_t amu_group0_cnt_read(unsigned int idx) 276 { 277 assert(is_feat_amu_supported()); 278 assert(idx < read_amcgcr_el0_cg0nc()); 279 280 return amu_group0_cnt_read_internal(idx); 281 } 282 283 /* Write the group 0 counter identified by the given `idx` with `val` */ 284 static void amu_group0_cnt_write(unsigned int idx, uint64_t val) 285 { 286 assert(is_feat_amu_supported()); 287 assert(idx < read_amcgcr_el0_cg0nc()); 288 289 amu_group0_cnt_write_internal(idx, val); 290 isb(); 291 } 292 293 /* 294 * Unlike with auxiliary counters, we cannot detect at runtime whether an 295 * architected counter supports a virtual offset. These are instead fixed 296 * according to FEAT_AMUv1p1, but this switch will need to be updated if later 297 * revisions of FEAT_AMU add additional architected counters. 298 */ 299 static bool amu_group0_voffset_supported(uint64_t idx) 300 { 301 switch (idx) { 302 case 0U: 303 case 2U: 304 case 3U: 305 return true; 306 307 case 1U: 308 return false; 309 310 default: 311 ERROR("AMU: can't set up virtual offset for unknown " 312 "architected counter %" PRIu64 "!\n", idx); 313 314 panic(); 315 } 316 } 317 318 /* 319 * Read the group 0 offset register for a given index. Index must be 0, 2, 320 * or 3, the register for 1 does not exist. 321 * 322 * Using this function requires FEAT_AMUv1p1 support. 323 */ 324 static uint64_t amu_group0_voffset_read(unsigned int idx) 325 { 326 assert(is_feat_amuv1p1_supported()); 327 assert(idx < read_amcgcr_el0_cg0nc()); 328 assert(idx != 1U); 329 330 return amu_group0_voffset_read_internal(idx); 331 } 332 333 /* 334 * Write the group 0 offset register for a given index. Index must be 0, 2, or 335 * 3, the register for 1 does not exist. 336 * 337 * Using this function requires FEAT_AMUv1p1 support. 338 */ 339 static void amu_group0_voffset_write(unsigned int idx, uint64_t val) 340 { 341 assert(is_feat_amuv1p1_supported()); 342 assert(idx < read_amcgcr_el0_cg0nc()); 343 assert(idx != 1U); 344 345 amu_group0_voffset_write_internal(idx, val); 346 isb(); 347 } 348 349 #if ENABLE_AMU_AUXILIARY_COUNTERS 350 /* Read the group 1 counter identified by the given `idx` */ 351 static uint64_t amu_group1_cnt_read(unsigned int idx) 352 { 353 assert(is_feat_amu_supported()); 354 assert(amu_group1_supported()); 355 assert(idx < read_amcgcr_el0_cg1nc()); 356 357 return amu_group1_cnt_read_internal(idx); 358 } 359 360 /* Write the group 1 counter identified by the given `idx` with `val` */ 361 static void amu_group1_cnt_write(unsigned int idx, uint64_t val) 362 { 363 assert(is_feat_amu_supported()); 364 assert(amu_group1_supported()); 365 assert(idx < read_amcgcr_el0_cg1nc()); 366 367 amu_group1_cnt_write_internal(idx, val); 368 isb(); 369 } 370 371 /* 372 * Read the group 1 offset register for a given index. 373 * 374 * Using this function requires FEAT_AMUv1p1 support. 375 */ 376 static uint64_t amu_group1_voffset_read(unsigned int idx) 377 { 378 assert(is_feat_amuv1p1_supported()); 379 assert(amu_group1_supported()); 380 assert(idx < read_amcgcr_el0_cg1nc()); 381 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 382 383 return amu_group1_voffset_read_internal(idx); 384 } 385 386 /* 387 * Write the group 1 offset register for a given index. 388 * 389 * Using this function requires FEAT_AMUv1p1 support. 390 */ 391 static void amu_group1_voffset_write(unsigned int idx, uint64_t val) 392 { 393 assert(is_feat_amuv1p1_supported()); 394 assert(amu_group1_supported()); 395 assert(idx < read_amcgcr_el0_cg1nc()); 396 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 397 398 amu_group1_voffset_write_internal(idx, val); 399 isb(); 400 } 401 #endif 402 403 static void *amu_context_save(const void *arg) 404 { 405 uint64_t i, j; 406 407 unsigned int core_pos; 408 struct amu_ctx *ctx; 409 410 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */ 411 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 412 413 #if ENABLE_AMU_AUXILIARY_COUNTERS 414 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 415 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 416 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 417 #endif 418 419 if (!is_feat_amu_supported()) { 420 return (void *)0; 421 } 422 423 core_pos = plat_my_core_pos(); 424 ctx = &amu_ctxs_[core_pos]; 425 426 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 427 if (is_feat_amuv1p1_supported()) { 428 hcr_el2_amvoffen = read_hcr_el2_amvoffen(); 429 } 430 431 #if ENABLE_AMU_AUXILIARY_COUNTERS 432 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 433 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 434 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 435 #endif 436 437 /* 438 * Disable all AMU counters. 439 */ 440 441 ctx->group0_enable = read_amcntenset0_el0_px(); 442 write_amcntenclr0_el0_px(ctx->group0_enable); 443 444 #if ENABLE_AMU_AUXILIARY_COUNTERS 445 if (amcfgr_el0_ncg > 0U) { 446 ctx->group1_enable = read_amcntenset1_el0_px(); 447 write_amcntenclr1_el0_px(ctx->group1_enable); 448 } 449 #endif 450 451 /* 452 * Save the counters to the local context. 453 */ 454 455 isb(); /* Ensure counters have been stopped */ 456 457 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 458 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 459 } 460 461 #if ENABLE_AMU_AUXILIARY_COUNTERS 462 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 463 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 464 } 465 #endif 466 467 /* 468 * Save virtual offsets for counters that offer them. 469 */ 470 471 if (hcr_el2_amvoffen != 0U) { 472 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 473 if (!amu_group0_voffset_supported(i)) { 474 continue; /* No virtual offset */ 475 } 476 477 ctx->group0_voffsets[j++] = amu_group0_voffset_read(i); 478 } 479 480 #if ENABLE_AMU_AUXILIARY_COUNTERS 481 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 482 if ((amcg1idr_el0_voff >> i) & 1U) { 483 continue; /* No virtual offset */ 484 } 485 486 ctx->group1_voffsets[j++] = amu_group1_voffset_read(i); 487 } 488 #endif 489 } 490 491 return (void *)0; 492 } 493 494 static void *amu_context_restore(const void *arg) 495 { 496 uint64_t i, j; 497 498 unsigned int core_pos; 499 struct amu_ctx *ctx; 500 501 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */ 502 503 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 504 505 #if ENABLE_AMU_AUXILIARY_COUNTERS 506 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 507 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 508 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 509 #endif 510 511 if (!is_feat_amu_supported()) { 512 return (void *)0; 513 } 514 515 core_pos = plat_my_core_pos(); 516 ctx = &amu_ctxs_[core_pos]; 517 518 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 519 520 if (is_feat_amuv1p1_supported()) { 521 hcr_el2_amvoffen = read_hcr_el2_amvoffen(); 522 } 523 524 #if ENABLE_AMU_AUXILIARY_COUNTERS 525 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 526 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 527 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 528 #endif 529 530 /* 531 * Restore the counter values from the local context. 532 */ 533 534 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 535 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 536 } 537 538 #if ENABLE_AMU_AUXILIARY_COUNTERS 539 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 540 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 541 } 542 #endif 543 544 /* 545 * Restore virtual offsets for counters that offer them. 546 */ 547 548 if (hcr_el2_amvoffen != 0U) { 549 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 550 if (!amu_group0_voffset_supported(i)) { 551 continue; /* No virtual offset */ 552 } 553 554 amu_group0_voffset_write(i, ctx->group0_voffsets[j++]); 555 } 556 557 #if ENABLE_AMU_AUXILIARY_COUNTERS 558 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 559 if ((amcg1idr_el0_voff >> i) & 1U) { 560 continue; /* No virtual offset */ 561 } 562 563 amu_group1_voffset_write(i, ctx->group1_voffsets[j++]); 564 } 565 #endif 566 } 567 568 /* 569 * Re-enable counters that were disabled during context save. 570 */ 571 572 write_amcntenset0_el0_px(ctx->group0_enable); 573 574 #if ENABLE_AMU_AUXILIARY_COUNTERS 575 if (amcfgr_el0_ncg > 0) { 576 write_amcntenset1_el0_px(ctx->group1_enable); 577 } 578 #endif 579 580 #if ENABLE_MPMM 581 mpmm_enable(); 582 #endif 583 584 return (void *)0; 585 } 586 587 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 588 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 589