1 /* 2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <cdefs.h> 9 #include <inttypes.h> 10 #include <stdbool.h> 11 #include <stdint.h> 12 13 #include "../amu_private.h" 14 #include <arch.h> 15 #include <arch_features.h> 16 #include <arch_helpers.h> 17 #include <common/debug.h> 18 #include <lib/el3_runtime/pubsub_events.h> 19 #include <lib/extensions/amu.h> 20 21 #include <plat/common/platform.h> 22 23 #if ENABLE_AMU_FCONF 24 # include <lib/fconf/fconf.h> 25 # include <lib/fconf/fconf_amu_getter.h> 26 #endif 27 28 #if ENABLE_MPMM 29 # include <lib/mpmm/mpmm.h> 30 #endif 31 32 struct amu_ctx { 33 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; 34 #if ENABLE_AMU_AUXILIARY_COUNTERS 35 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; 36 #endif 37 38 /* Architected event counter 1 does not have an offset register */ 39 uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U]; 40 #if ENABLE_AMU_AUXILIARY_COUNTERS 41 uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS]; 42 #endif 43 44 uint16_t group0_enable; 45 #if ENABLE_AMU_AUXILIARY_COUNTERS 46 uint16_t group1_enable; 47 #endif 48 }; 49 50 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; 51 52 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, 53 amu_ctx_group0_enable_cannot_represent_all_group0_counters); 54 55 #if ENABLE_AMU_AUXILIARY_COUNTERS 56 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, 57 amu_ctx_group1_enable_cannot_represent_all_group1_counters); 58 #endif 59 60 static inline __unused uint64_t read_id_aa64pfr0_el1_amu(void) 61 { 62 return (read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & 63 ID_AA64PFR0_AMU_MASK; 64 } 65 66 static inline __unused uint64_t read_hcr_el2_amvoffen(void) 67 { 68 return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >> 69 HCR_AMVOFFEN_SHIFT; 70 } 71 72 static inline __unused void write_cptr_el2_tam(uint64_t value) 73 { 74 write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) | 75 ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT)); 76 } 77 78 static inline __unused void write_cptr_el3_tam(cpu_context_t *ctx, uint64_t tam) 79 { 80 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); 81 82 value &= ~TAM_BIT; 83 value |= (tam << TAM_SHIFT) & TAM_BIT; 84 85 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, value); 86 } 87 88 static inline __unused void write_hcr_el2_amvoffen(uint64_t value) 89 { 90 write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) | 91 ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT)); 92 } 93 94 static inline __unused void write_amcr_el0_cg1rz(uint64_t value) 95 { 96 write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) | 97 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); 98 } 99 100 static inline __unused uint64_t read_amcfgr_el0_ncg(void) 101 { 102 return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) & 103 AMCFGR_EL0_NCG_MASK; 104 } 105 106 static inline __unused uint64_t read_amcgcr_el0_cg0nc(void) 107 { 108 return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) & 109 AMCGCR_EL0_CG0NC_MASK; 110 } 111 112 static inline __unused uint64_t read_amcg1idr_el0_voff(void) 113 { 114 return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 115 AMCG1IDR_VOFF_MASK; 116 } 117 118 static inline __unused uint64_t read_amcgcr_el0_cg1nc(void) 119 { 120 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & 121 AMCGCR_EL0_CG1NC_MASK; 122 } 123 124 static inline __unused uint64_t read_amcntenset0_el0_px(void) 125 { 126 return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) & 127 AMCNTENSET0_EL0_Pn_MASK; 128 } 129 130 static inline __unused uint64_t read_amcntenset1_el0_px(void) 131 { 132 return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) & 133 AMCNTENSET1_EL0_Pn_MASK; 134 } 135 136 static inline __unused void write_amcntenset0_el0_px(uint64_t px) 137 { 138 uint64_t value = read_amcntenset0_el0(); 139 140 value &= ~AMCNTENSET0_EL0_Pn_MASK; 141 value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK; 142 143 write_amcntenset0_el0(value); 144 } 145 146 static inline __unused void write_amcntenset1_el0_px(uint64_t px) 147 { 148 uint64_t value = read_amcntenset1_el0(); 149 150 value &= ~AMCNTENSET1_EL0_Pn_MASK; 151 value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK; 152 153 write_amcntenset1_el0(value); 154 } 155 156 static inline __unused void write_amcntenclr0_el0_px(uint64_t px) 157 { 158 uint64_t value = read_amcntenclr0_el0(); 159 160 value &= ~AMCNTENCLR0_EL0_Pn_MASK; 161 value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK; 162 163 write_amcntenclr0_el0(value); 164 } 165 166 static inline __unused void write_amcntenclr1_el0_px(uint64_t px) 167 { 168 uint64_t value = read_amcntenclr1_el0(); 169 170 value &= ~AMCNTENCLR1_EL0_Pn_MASK; 171 value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK; 172 173 write_amcntenclr1_el0(value); 174 } 175 176 static __unused bool amu_supported(void) 177 { 178 return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1; 179 } 180 181 static __unused bool amu_v1p1_supported(void) 182 { 183 return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1P1; 184 } 185 186 #if ENABLE_AMU_AUXILIARY_COUNTERS 187 static __unused bool amu_group1_supported(void) 188 { 189 return read_amcfgr_el0_ncg() > 0U; 190 } 191 #endif 192 193 /* 194 * Enable counters. This function is meant to be invoked by the context 195 * management library before exiting from EL3. 196 */ 197 void amu_enable(bool el2_unused, cpu_context_t *ctx) 198 { 199 uint64_t id_aa64pfr0_el1_amu; /* AMU version */ 200 201 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 202 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 203 204 uint64_t amcntenset0_el0_px = 0x0; /* Group 0 enable mask */ 205 uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */ 206 207 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); 208 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 209 /* 210 * If the AMU is unsupported, nothing needs to be done. 211 */ 212 213 return; 214 } 215 216 if (el2_unused) { 217 /* 218 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity 219 * Monitor registers do not trap to EL2. 220 */ 221 write_cptr_el2_tam(0U); 222 } 223 224 /* 225 * Retrieve and update the CPTR_EL3 value from the context mentioned 226 * in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to 227 * the Activity Monitor registers do not trap to EL3. 228 */ 229 write_cptr_el3_tam(ctx, 0U); 230 231 /* 232 * Retrieve the number of architected counters. All of these counters 233 * are enabled by default. 234 */ 235 236 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 237 amcntenset0_el0_px = (UINT64_C(1) << (amcgcr_el0_cg0nc)) - 1U; 238 239 assert(amcgcr_el0_cg0nc <= AMU_AMCGCR_CG0NC_MAX); 240 241 /* 242 * The platform may opt to enable specific auxiliary counters. This can 243 * be done via the common FCONF getter, or via the platform-implemented 244 * function. 245 */ 246 247 #if ENABLE_AMU_AUXILIARY_COUNTERS 248 const struct amu_topology *topology; 249 250 #if ENABLE_AMU_FCONF 251 topology = FCONF_GET_PROPERTY(amu, config, topology); 252 #else 253 topology = plat_amu_topology(); 254 #endif /* ENABLE_AMU_FCONF */ 255 256 if (topology != NULL) { 257 unsigned int core_pos = plat_my_core_pos(); 258 259 amcntenset1_el0_px = topology->cores[core_pos].enable; 260 } else { 261 ERROR("AMU: failed to generate AMU topology\n"); 262 } 263 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ 264 265 /* 266 * Enable the requested counters. 267 */ 268 269 write_amcntenset0_el0_px(amcntenset0_el0_px); 270 271 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 272 if (amcfgr_el0_ncg > 0U) { 273 write_amcntenset1_el0_px(amcntenset1_el0_px); 274 275 #if !ENABLE_AMU_AUXILIARY_COUNTERS 276 VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); 277 #endif 278 } 279 280 /* Initialize FEAT_AMUv1p1 features if present. */ 281 if (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) { 282 if (el2_unused) { 283 /* 284 * Make sure virtual offsets are disabled if EL2 not 285 * used. 286 */ 287 write_hcr_el2_amvoffen(0U); 288 } 289 290 #if AMU_RESTRICT_COUNTERS 291 /* 292 * FEAT_AMUv1p1 adds a register field to restrict access to 293 * group 1 counters at all but the highest implemented EL. This 294 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time 295 * flag, when set, system register reads at lower ELs return 296 * zero. Reads from the memory mapped view are unaffected. 297 */ 298 VERBOSE("AMU group 1 counter access restricted.\n"); 299 write_amcr_el0_cg1rz(1U); 300 #else 301 write_amcr_el0_cg1rz(0U); 302 #endif 303 } 304 305 #if ENABLE_MPMM 306 mpmm_enable(); 307 #endif 308 } 309 310 /* Read the group 0 counter identified by the given `idx`. */ 311 static uint64_t amu_group0_cnt_read(unsigned int idx) 312 { 313 assert(amu_supported()); 314 assert(idx < read_amcgcr_el0_cg0nc()); 315 316 return amu_group0_cnt_read_internal(idx); 317 } 318 319 /* Write the group 0 counter identified by the given `idx` with `val` */ 320 static void amu_group0_cnt_write(unsigned int idx, uint64_t val) 321 { 322 assert(amu_supported()); 323 assert(idx < read_amcgcr_el0_cg0nc()); 324 325 amu_group0_cnt_write_internal(idx, val); 326 isb(); 327 } 328 329 /* 330 * Unlike with auxiliary counters, we cannot detect at runtime whether an 331 * architected counter supports a virtual offset. These are instead fixed 332 * according to FEAT_AMUv1p1, but this switch will need to be updated if later 333 * revisions of FEAT_AMU add additional architected counters. 334 */ 335 static bool amu_group0_voffset_supported(uint64_t idx) 336 { 337 switch (idx) { 338 case 0U: 339 case 2U: 340 case 3U: 341 return true; 342 343 case 1U: 344 return false; 345 346 default: 347 ERROR("AMU: can't set up virtual offset for unknown " 348 "architected counter %" PRIu64 "!\n", idx); 349 350 panic(); 351 } 352 } 353 354 /* 355 * Read the group 0 offset register for a given index. Index must be 0, 2, 356 * or 3, the register for 1 does not exist. 357 * 358 * Using this function requires FEAT_AMUv1p1 support. 359 */ 360 static uint64_t amu_group0_voffset_read(unsigned int idx) 361 { 362 assert(amu_v1p1_supported()); 363 assert(idx < read_amcgcr_el0_cg0nc()); 364 assert(idx != 1U); 365 366 return amu_group0_voffset_read_internal(idx); 367 } 368 369 /* 370 * Write the group 0 offset register for a given index. Index must be 0, 2, or 371 * 3, the register for 1 does not exist. 372 * 373 * Using this function requires FEAT_AMUv1p1 support. 374 */ 375 static void amu_group0_voffset_write(unsigned int idx, uint64_t val) 376 { 377 assert(amu_v1p1_supported()); 378 assert(idx < read_amcgcr_el0_cg0nc()); 379 assert(idx != 1U); 380 381 amu_group0_voffset_write_internal(idx, val); 382 isb(); 383 } 384 385 #if ENABLE_AMU_AUXILIARY_COUNTERS 386 /* Read the group 1 counter identified by the given `idx` */ 387 static uint64_t amu_group1_cnt_read(unsigned int idx) 388 { 389 assert(amu_supported()); 390 assert(amu_group1_supported()); 391 assert(idx < read_amcgcr_el0_cg1nc()); 392 393 return amu_group1_cnt_read_internal(idx); 394 } 395 396 /* Write the group 1 counter identified by the given `idx` with `val` */ 397 static void amu_group1_cnt_write(unsigned int idx, uint64_t val) 398 { 399 assert(amu_supported()); 400 assert(amu_group1_supported()); 401 assert(idx < read_amcgcr_el0_cg1nc()); 402 403 amu_group1_cnt_write_internal(idx, val); 404 isb(); 405 } 406 407 /* 408 * Read the group 1 offset register for a given index. 409 * 410 * Using this function requires FEAT_AMUv1p1 support. 411 */ 412 static uint64_t amu_group1_voffset_read(unsigned int idx) 413 { 414 assert(amu_v1p1_supported()); 415 assert(amu_group1_supported()); 416 assert(idx < read_amcgcr_el0_cg1nc()); 417 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 418 419 return amu_group1_voffset_read_internal(idx); 420 } 421 422 /* 423 * Write the group 1 offset register for a given index. 424 * 425 * Using this function requires FEAT_AMUv1p1 support. 426 */ 427 static void amu_group1_voffset_write(unsigned int idx, uint64_t val) 428 { 429 assert(amu_v1p1_supported()); 430 assert(amu_group1_supported()); 431 assert(idx < read_amcgcr_el0_cg1nc()); 432 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 433 434 amu_group1_voffset_write_internal(idx, val); 435 isb(); 436 } 437 #endif 438 439 static void *amu_context_save(const void *arg) 440 { 441 uint64_t i, j; 442 443 unsigned int core_pos; 444 struct amu_ctx *ctx; 445 446 uint64_t id_aa64pfr0_el1_amu; /* AMU version */ 447 uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ 448 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 449 450 #if ENABLE_AMU_AUXILIARY_COUNTERS 451 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 452 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 453 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 454 #endif 455 456 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); 457 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 458 return (void *)0; 459 } 460 461 core_pos = plat_my_core_pos(); 462 ctx = &amu_ctxs_[core_pos]; 463 464 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 465 hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? 466 read_hcr_el2_amvoffen() : 0U; 467 468 #if ENABLE_AMU_AUXILIARY_COUNTERS 469 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 470 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 471 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 472 #endif 473 474 /* 475 * Disable all AMU counters. 476 */ 477 478 ctx->group0_enable = read_amcntenset0_el0_px(); 479 write_amcntenclr0_el0_px(ctx->group0_enable); 480 481 #if ENABLE_AMU_AUXILIARY_COUNTERS 482 if (amcfgr_el0_ncg > 0U) { 483 ctx->group1_enable = read_amcntenset1_el0_px(); 484 write_amcntenclr1_el0_px(ctx->group1_enable); 485 } 486 #endif 487 488 /* 489 * Save the counters to the local context. 490 */ 491 492 isb(); /* Ensure counters have been stopped */ 493 494 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 495 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 496 } 497 498 #if ENABLE_AMU_AUXILIARY_COUNTERS 499 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 500 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 501 } 502 #endif 503 504 /* 505 * Save virtual offsets for counters that offer them. 506 */ 507 508 if (hcr_el2_amvoffen != 0U) { 509 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 510 if (!amu_group0_voffset_supported(i)) { 511 continue; /* No virtual offset */ 512 } 513 514 ctx->group0_voffsets[j++] = amu_group0_voffset_read(i); 515 } 516 517 #if ENABLE_AMU_AUXILIARY_COUNTERS 518 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 519 if ((amcg1idr_el0_voff >> i) & 1U) { 520 continue; /* No virtual offset */ 521 } 522 523 ctx->group1_voffsets[j++] = amu_group1_voffset_read(i); 524 } 525 #endif 526 } 527 528 return (void *)0; 529 } 530 531 static void *amu_context_restore(const void *arg) 532 { 533 uint64_t i, j; 534 535 unsigned int core_pos; 536 struct amu_ctx *ctx; 537 538 uint64_t id_aa64pfr0_el1_amu; /* AMU version */ 539 540 uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ 541 542 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 543 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 544 545 #if ENABLE_AMU_AUXILIARY_COUNTERS 546 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 547 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 548 #endif 549 550 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); 551 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 552 return (void *)0; 553 } 554 555 core_pos = plat_my_core_pos(); 556 ctx = &amu_ctxs_[core_pos]; 557 558 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 559 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 560 561 hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? 562 read_hcr_el2_amvoffen() : 0U; 563 564 #if ENABLE_AMU_AUXILIARY_COUNTERS 565 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 566 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 567 #endif 568 569 /* 570 * Sanity check that all counters were disabled when the context was 571 * previously saved. 572 */ 573 574 assert(read_amcntenset0_el0_px() == 0U); 575 576 if (amcfgr_el0_ncg > 0U) { 577 assert(read_amcntenset1_el0_px() == 0U); 578 } 579 580 /* 581 * Restore the counter values from the local context. 582 */ 583 584 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 585 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 586 } 587 588 #if ENABLE_AMU_AUXILIARY_COUNTERS 589 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 590 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 591 } 592 #endif 593 594 /* 595 * Restore virtual offsets for counters that offer them. 596 */ 597 598 if (hcr_el2_amvoffen != 0U) { 599 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 600 if (!amu_group0_voffset_supported(i)) { 601 continue; /* No virtual offset */ 602 } 603 604 amu_group0_voffset_write(i, ctx->group0_voffsets[j++]); 605 } 606 607 #if ENABLE_AMU_AUXILIARY_COUNTERS 608 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 609 if ((amcg1idr_el0_voff >> i) & 1U) { 610 continue; /* No virtual offset */ 611 } 612 613 amu_group1_voffset_write(i, ctx->group1_voffsets[j++]); 614 } 615 #endif 616 } 617 618 /* 619 * Re-enable counters that were disabled during context save. 620 */ 621 622 write_amcntenset0_el0_px(ctx->group0_enable); 623 624 #if ENABLE_AMU_AUXILIARY_COUNTERS 625 if (amcfgr_el0_ncg > 0) { 626 write_amcntenset1_el0_px(ctx->group1_enable); 627 } 628 #endif 629 630 #if ENABLE_MPMM 631 mpmm_enable(); 632 #endif 633 634 return (void *)0; 635 } 636 637 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 638 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 639