1 /* 2 * Copyright (c) 2017-2023, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <cdefs.h> 9 #include <inttypes.h> 10 #include <stdbool.h> 11 #include <stdint.h> 12 13 #include "../amu_private.h" 14 #include <arch.h> 15 #include <arch_features.h> 16 #include <arch_helpers.h> 17 #include <common/debug.h> 18 #include <lib/el3_runtime/pubsub_events.h> 19 #include <lib/extensions/amu.h> 20 21 #include <plat/common/platform.h> 22 23 #if ENABLE_AMU_FCONF 24 # include <lib/fconf/fconf.h> 25 # include <lib/fconf/fconf_amu_getter.h> 26 #endif 27 28 #if ENABLE_MPMM 29 # include <lib/mpmm/mpmm.h> 30 #endif 31 32 struct amu_ctx { 33 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; 34 #if ENABLE_AMU_AUXILIARY_COUNTERS 35 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; 36 #endif 37 38 /* Architected event counter 1 does not have an offset register */ 39 uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U]; 40 #if ENABLE_AMU_AUXILIARY_COUNTERS 41 uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS]; 42 #endif 43 44 uint16_t group0_enable; 45 #if ENABLE_AMU_AUXILIARY_COUNTERS 46 uint16_t group1_enable; 47 #endif 48 }; 49 50 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; 51 52 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, 53 amu_ctx_group0_enable_cannot_represent_all_group0_counters); 54 55 #if ENABLE_AMU_AUXILIARY_COUNTERS 56 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, 57 amu_ctx_group1_enable_cannot_represent_all_group1_counters); 58 #endif 59 60 static inline __unused uint64_t read_hcr_el2_amvoffen(void) 61 { 62 return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >> 63 HCR_AMVOFFEN_SHIFT; 64 } 65 66 static inline __unused void write_cptr_el2_tam(uint64_t value) 67 { 68 write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) | 69 ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT)); 70 } 71 72 static inline __unused void ctx_write_scr_el3_amvoffen(cpu_context_t *ctx, uint64_t amvoffen) 73 { 74 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3); 75 76 value &= ~SCR_AMVOFFEN_BIT; 77 value |= (amvoffen << SCR_AMVOFFEN_SHIFT) & SCR_AMVOFFEN_BIT; 78 79 write_ctx_reg(get_el3state_ctx(ctx), CTX_SCR_EL3, value); 80 } 81 82 static inline __unused void write_hcr_el2_amvoffen(uint64_t value) 83 { 84 write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) | 85 ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT)); 86 } 87 88 static inline __unused void write_amcr_el0_cg1rz(uint64_t value) 89 { 90 write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) | 91 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); 92 } 93 94 static inline __unused uint64_t read_amcfgr_el0_ncg(void) 95 { 96 return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) & 97 AMCFGR_EL0_NCG_MASK; 98 } 99 100 static inline __unused uint64_t read_amcgcr_el0_cg0nc(void) 101 { 102 return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) & 103 AMCGCR_EL0_CG0NC_MASK; 104 } 105 106 static inline __unused uint64_t read_amcg1idr_el0_voff(void) 107 { 108 return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 109 AMCG1IDR_VOFF_MASK; 110 } 111 112 static inline __unused uint64_t read_amcgcr_el0_cg1nc(void) 113 { 114 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & 115 AMCGCR_EL0_CG1NC_MASK; 116 } 117 118 static inline __unused uint64_t read_amcntenset0_el0_px(void) 119 { 120 return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) & 121 AMCNTENSET0_EL0_Pn_MASK; 122 } 123 124 static inline __unused uint64_t read_amcntenset1_el0_px(void) 125 { 126 return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) & 127 AMCNTENSET1_EL0_Pn_MASK; 128 } 129 130 static inline __unused void write_amcntenset0_el0_px(uint64_t px) 131 { 132 uint64_t value = read_amcntenset0_el0(); 133 134 value &= ~AMCNTENSET0_EL0_Pn_MASK; 135 value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK; 136 137 write_amcntenset0_el0(value); 138 } 139 140 static inline __unused void write_amcntenset1_el0_px(uint64_t px) 141 { 142 uint64_t value = read_amcntenset1_el0(); 143 144 value &= ~AMCNTENSET1_EL0_Pn_MASK; 145 value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK; 146 147 write_amcntenset1_el0(value); 148 } 149 150 static inline __unused void write_amcntenclr0_el0_px(uint64_t px) 151 { 152 uint64_t value = read_amcntenclr0_el0(); 153 154 value &= ~AMCNTENCLR0_EL0_Pn_MASK; 155 value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK; 156 157 write_amcntenclr0_el0(value); 158 } 159 160 static inline __unused void write_amcntenclr1_el0_px(uint64_t px) 161 { 162 uint64_t value = read_amcntenclr1_el0(); 163 164 value &= ~AMCNTENCLR1_EL0_Pn_MASK; 165 value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK; 166 167 write_amcntenclr1_el0(value); 168 } 169 170 #if ENABLE_AMU_AUXILIARY_COUNTERS 171 static __unused bool amu_group1_supported(void) 172 { 173 return read_amcfgr_el0_ncg() > 0U; 174 } 175 #endif 176 177 /* 178 * Enable counters. This function is meant to be invoked by the context 179 * management library before exiting from EL3. 180 */ 181 void amu_enable(cpu_context_t *ctx) 182 { 183 /* Initialize FEAT_AMUv1p1 features if present. */ 184 if (is_feat_amuv1p1_supported()) { 185 /* 186 * Set SCR_EL3.AMVOFFEN to one so that accesses to virtual 187 * offset registers at EL2 do not trap to EL3 188 */ 189 ctx_write_scr_el3_amvoffen(ctx, 1U); 190 } 191 } 192 193 void amu_enable_per_world(per_world_context_t *per_world_ctx) 194 { 195 /* 196 * Set CPTR_EL3.TAM to zero so that any accesses to the Activity Monitor 197 * registers do not trap to EL3. 198 */ 199 uint64_t cptr_el3 = per_world_ctx->ctx_cptr_el3; 200 201 cptr_el3 &= ~TAM_BIT; 202 per_world_ctx->ctx_cptr_el3 = cptr_el3; 203 } 204 205 void amu_init_el3(void) 206 { 207 uint64_t group0_impl_ctr = read_amcgcr_el0_cg0nc(); 208 uint64_t group0_en_mask = (1 << (group0_impl_ctr)) - 1U; 209 uint64_t num_ctr_groups = read_amcfgr_el0_ncg(); 210 211 /* Enable all architected counters by default */ 212 write_amcntenset0_el0_px(group0_en_mask); 213 214 #if ENABLE_AMU_AUXILIARY_COUNTERS 215 if (num_ctr_groups > 0U) { 216 uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */ 217 const struct amu_topology *topology; 218 219 /* 220 * The platform may opt to enable specific auxiliary counters. 221 * This can be done via the common FCONF getter, or via the 222 * platform-implemented function. 223 */ 224 #if ENABLE_AMU_FCONF 225 topology = FCONF_GET_PROPERTY(amu, config, topology); 226 #else 227 topology = plat_amu_topology(); 228 #endif /* ENABLE_AMU_FCONF */ 229 230 if (topology != NULL) { 231 unsigned int core_pos = plat_my_core_pos(); 232 233 amcntenset1_el0_px = topology->cores[core_pos].enable; 234 } else { 235 ERROR("AMU: failed to generate AMU topology\n"); 236 } 237 238 write_amcntenset1_el0_px(amcntenset1_el0_px); 239 } 240 #else /* ENABLE_AMU_AUXILIARY_COUNTERS */ 241 if (num_ctr_groups > 0U) { 242 VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); 243 } 244 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ 245 246 if (is_feat_amuv1p1_supported()) { 247 #if AMU_RESTRICT_COUNTERS 248 /* 249 * FEAT_AMUv1p1 adds a register field to restrict access to 250 * group 1 counters at all but the highest implemented EL. This 251 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time 252 * flag, when set, system register reads at lower ELs return 253 * zero. Reads from the memory mapped view are unaffected. 254 */ 255 VERBOSE("AMU group 1 counter access restricted.\n"); 256 write_amcr_el0_cg1rz(1U); 257 #else 258 write_amcr_el0_cg1rz(0U); 259 #endif 260 } 261 262 #if ENABLE_MPMM 263 mpmm_enable(); 264 #endif 265 } 266 267 void amu_init_el2_unused(void) 268 { 269 /* 270 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity Monitor 271 * registers do not trap to EL2. 272 */ 273 write_cptr_el2_tam(0U); 274 275 /* Initialize FEAT_AMUv1p1 features if present. */ 276 if (is_feat_amuv1p1_supported()) { 277 /* Make sure virtual offsets are disabled if EL2 not used. */ 278 write_hcr_el2_amvoffen(0U); 279 } 280 } 281 282 /* Read the group 0 counter identified by the given `idx`. */ 283 static uint64_t amu_group0_cnt_read(unsigned int idx) 284 { 285 assert(is_feat_amu_supported()); 286 assert(idx < read_amcgcr_el0_cg0nc()); 287 288 return amu_group0_cnt_read_internal(idx); 289 } 290 291 /* Write the group 0 counter identified by the given `idx` with `val` */ 292 static void amu_group0_cnt_write(unsigned int idx, uint64_t val) 293 { 294 assert(is_feat_amu_supported()); 295 assert(idx < read_amcgcr_el0_cg0nc()); 296 297 amu_group0_cnt_write_internal(idx, val); 298 isb(); 299 } 300 301 /* 302 * Unlike with auxiliary counters, we cannot detect at runtime whether an 303 * architected counter supports a virtual offset. These are instead fixed 304 * according to FEAT_AMUv1p1, but this switch will need to be updated if later 305 * revisions of FEAT_AMU add additional architected counters. 306 */ 307 static bool amu_group0_voffset_supported(uint64_t idx) 308 { 309 switch (idx) { 310 case 0U: 311 case 2U: 312 case 3U: 313 return true; 314 315 case 1U: 316 return false; 317 318 default: 319 ERROR("AMU: can't set up virtual offset for unknown " 320 "architected counter %" PRIu64 "!\n", idx); 321 322 panic(); 323 } 324 } 325 326 /* 327 * Read the group 0 offset register for a given index. Index must be 0, 2, 328 * or 3, the register for 1 does not exist. 329 * 330 * Using this function requires FEAT_AMUv1p1 support. 331 */ 332 static uint64_t amu_group0_voffset_read(unsigned int idx) 333 { 334 assert(is_feat_amuv1p1_supported()); 335 assert(idx < read_amcgcr_el0_cg0nc()); 336 assert(idx != 1U); 337 338 return amu_group0_voffset_read_internal(idx); 339 } 340 341 /* 342 * Write the group 0 offset register for a given index. Index must be 0, 2, or 343 * 3, the register for 1 does not exist. 344 * 345 * Using this function requires FEAT_AMUv1p1 support. 346 */ 347 static void amu_group0_voffset_write(unsigned int idx, uint64_t val) 348 { 349 assert(is_feat_amuv1p1_supported()); 350 assert(idx < read_amcgcr_el0_cg0nc()); 351 assert(idx != 1U); 352 353 amu_group0_voffset_write_internal(idx, val); 354 isb(); 355 } 356 357 #if ENABLE_AMU_AUXILIARY_COUNTERS 358 /* Read the group 1 counter identified by the given `idx` */ 359 static uint64_t amu_group1_cnt_read(unsigned int idx) 360 { 361 assert(is_feat_amu_supported()); 362 assert(amu_group1_supported()); 363 assert(idx < read_amcgcr_el0_cg1nc()); 364 365 return amu_group1_cnt_read_internal(idx); 366 } 367 368 /* Write the group 1 counter identified by the given `idx` with `val` */ 369 static void amu_group1_cnt_write(unsigned int idx, uint64_t val) 370 { 371 assert(is_feat_amu_supported()); 372 assert(amu_group1_supported()); 373 assert(idx < read_amcgcr_el0_cg1nc()); 374 375 amu_group1_cnt_write_internal(idx, val); 376 isb(); 377 } 378 379 /* 380 * Read the group 1 offset register for a given index. 381 * 382 * Using this function requires FEAT_AMUv1p1 support. 383 */ 384 static uint64_t amu_group1_voffset_read(unsigned int idx) 385 { 386 assert(is_feat_amuv1p1_supported()); 387 assert(amu_group1_supported()); 388 assert(idx < read_amcgcr_el0_cg1nc()); 389 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 390 391 return amu_group1_voffset_read_internal(idx); 392 } 393 394 /* 395 * Write the group 1 offset register for a given index. 396 * 397 * Using this function requires FEAT_AMUv1p1 support. 398 */ 399 static void amu_group1_voffset_write(unsigned int idx, uint64_t val) 400 { 401 assert(is_feat_amuv1p1_supported()); 402 assert(amu_group1_supported()); 403 assert(idx < read_amcgcr_el0_cg1nc()); 404 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 405 406 amu_group1_voffset_write_internal(idx, val); 407 isb(); 408 } 409 #endif 410 411 static void *amu_context_save(const void *arg) 412 { 413 uint64_t i, j; 414 415 unsigned int core_pos; 416 struct amu_ctx *ctx; 417 418 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */ 419 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 420 421 #if ENABLE_AMU_AUXILIARY_COUNTERS 422 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 423 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 424 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 425 #endif 426 427 if (!is_feat_amu_supported()) { 428 return (void *)0; 429 } 430 431 core_pos = plat_my_core_pos(); 432 ctx = &amu_ctxs_[core_pos]; 433 434 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 435 if (is_feat_amuv1p1_supported()) { 436 hcr_el2_amvoffen = read_hcr_el2_amvoffen(); 437 } 438 439 #if ENABLE_AMU_AUXILIARY_COUNTERS 440 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 441 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 442 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 443 #endif 444 445 /* 446 * Disable all AMU counters. 447 */ 448 449 ctx->group0_enable = read_amcntenset0_el0_px(); 450 write_amcntenclr0_el0_px(ctx->group0_enable); 451 452 #if ENABLE_AMU_AUXILIARY_COUNTERS 453 if (amcfgr_el0_ncg > 0U) { 454 ctx->group1_enable = read_amcntenset1_el0_px(); 455 write_amcntenclr1_el0_px(ctx->group1_enable); 456 } 457 #endif 458 459 /* 460 * Save the counters to the local context. 461 */ 462 463 isb(); /* Ensure counters have been stopped */ 464 465 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 466 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 467 } 468 469 #if ENABLE_AMU_AUXILIARY_COUNTERS 470 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 471 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 472 } 473 #endif 474 475 /* 476 * Save virtual offsets for counters that offer them. 477 */ 478 479 if (hcr_el2_amvoffen != 0U) { 480 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 481 if (!amu_group0_voffset_supported(i)) { 482 continue; /* No virtual offset */ 483 } 484 485 ctx->group0_voffsets[j++] = amu_group0_voffset_read(i); 486 } 487 488 #if ENABLE_AMU_AUXILIARY_COUNTERS 489 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 490 if ((amcg1idr_el0_voff >> i) & 1U) { 491 continue; /* No virtual offset */ 492 } 493 494 ctx->group1_voffsets[j++] = amu_group1_voffset_read(i); 495 } 496 #endif 497 } 498 499 return (void *)0; 500 } 501 502 static void *amu_context_restore(const void *arg) 503 { 504 uint64_t i, j; 505 506 unsigned int core_pos; 507 struct amu_ctx *ctx; 508 509 uint64_t hcr_el2_amvoffen = 0; /* AMU virtual offsets enabled */ 510 511 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 512 513 #if ENABLE_AMU_AUXILIARY_COUNTERS 514 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 515 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 516 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 517 #endif 518 519 if (!is_feat_amu_supported()) { 520 return (void *)0; 521 } 522 523 core_pos = plat_my_core_pos(); 524 ctx = &amu_ctxs_[core_pos]; 525 526 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 527 528 if (is_feat_amuv1p1_supported()) { 529 hcr_el2_amvoffen = read_hcr_el2_amvoffen(); 530 } 531 532 #if ENABLE_AMU_AUXILIARY_COUNTERS 533 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 534 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 535 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 536 #endif 537 538 /* 539 * Restore the counter values from the local context. 540 */ 541 542 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 543 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 544 } 545 546 #if ENABLE_AMU_AUXILIARY_COUNTERS 547 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 548 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 549 } 550 #endif 551 552 /* 553 * Restore virtual offsets for counters that offer them. 554 */ 555 556 if (hcr_el2_amvoffen != 0U) { 557 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 558 if (!amu_group0_voffset_supported(i)) { 559 continue; /* No virtual offset */ 560 } 561 562 amu_group0_voffset_write(i, ctx->group0_voffsets[j++]); 563 } 564 565 #if ENABLE_AMU_AUXILIARY_COUNTERS 566 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 567 if ((amcg1idr_el0_voff >> i) & 1U) { 568 continue; /* No virtual offset */ 569 } 570 571 amu_group1_voffset_write(i, ctx->group1_voffsets[j++]); 572 } 573 #endif 574 } 575 576 /* 577 * Re-enable counters that were disabled during context save. 578 */ 579 580 write_amcntenset0_el0_px(ctx->group0_enable); 581 582 #if ENABLE_AMU_AUXILIARY_COUNTERS 583 if (amcfgr_el0_ncg > 0) { 584 write_amcntenset1_el0_px(ctx->group1_enable); 585 } 586 #endif 587 588 #if ENABLE_MPMM 589 mpmm_enable(); 590 #endif 591 592 return (void *)0; 593 } 594 595 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 596 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 597