1 /* 2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <cdefs.h> 9 #include <stdbool.h> 10 11 #include "../amu_private.h" 12 #include <arch.h> 13 #include <arch_features.h> 14 #include <arch_helpers.h> 15 #include <common/debug.h> 16 #include <lib/el3_runtime/pubsub_events.h> 17 #include <lib/extensions/amu.h> 18 19 #include <plat/common/platform.h> 20 21 #if ENABLE_AMU_FCONF 22 # include <lib/fconf/fconf.h> 23 # include <lib/fconf/fconf_amu_getter.h> 24 #endif 25 26 #if ENABLE_MPMM 27 # include <lib/mpmm/mpmm.h> 28 #endif 29 30 struct amu_ctx { 31 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; 32 #if ENABLE_AMU_AUXILIARY_COUNTERS 33 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; 34 #endif 35 36 /* Architected event counter 1 does not have an offset register */ 37 uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U]; 38 #if ENABLE_AMU_AUXILIARY_COUNTERS 39 uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS]; 40 #endif 41 42 uint16_t group0_enable; 43 #if ENABLE_AMU_AUXILIARY_COUNTERS 44 uint16_t group1_enable; 45 #endif 46 }; 47 48 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; 49 50 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, 51 amu_ctx_group0_enable_cannot_represent_all_group0_counters); 52 53 #if ENABLE_AMU_AUXILIARY_COUNTERS 54 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, 55 amu_ctx_group1_enable_cannot_represent_all_group1_counters); 56 #endif 57 58 static inline __unused uint64_t read_id_aa64pfr0_el1_amu(void) 59 { 60 return (read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & 61 ID_AA64PFR0_AMU_MASK; 62 } 63 64 static inline __unused uint64_t read_hcr_el2_amvoffen(void) 65 { 66 return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >> 67 HCR_AMVOFFEN_SHIFT; 68 } 69 70 static inline __unused void write_cptr_el2_tam(uint64_t value) 71 { 72 write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) | 73 ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT)); 74 } 75 76 static inline __unused void write_cptr_el3_tam(cpu_context_t *ctx, uint64_t tam) 77 { 78 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); 79 80 value &= ~TAM_BIT; 81 value |= (tam << TAM_SHIFT) & TAM_BIT; 82 83 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, value); 84 } 85 86 static inline __unused void write_hcr_el2_amvoffen(uint64_t value) 87 { 88 write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) | 89 ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT)); 90 } 91 92 static inline __unused void write_amcr_el0_cg1rz(uint64_t value) 93 { 94 write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) | 95 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); 96 } 97 98 static inline __unused uint64_t read_amcfgr_el0_ncg(void) 99 { 100 return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) & 101 AMCFGR_EL0_NCG_MASK; 102 } 103 104 static inline __unused uint64_t read_amcgcr_el0_cg0nc(void) 105 { 106 return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) & 107 AMCGCR_EL0_CG0NC_MASK; 108 } 109 110 static inline __unused uint64_t read_amcg1idr_el0_voff(void) 111 { 112 return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 113 AMCG1IDR_VOFF_MASK; 114 } 115 116 static inline __unused uint64_t read_amcgcr_el0_cg1nc(void) 117 { 118 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & 119 AMCGCR_EL0_CG1NC_MASK; 120 } 121 122 static inline __unused uint64_t read_amcntenset0_el0_px(void) 123 { 124 return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) & 125 AMCNTENSET0_EL0_Pn_MASK; 126 } 127 128 static inline __unused uint64_t read_amcntenset1_el0_px(void) 129 { 130 return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) & 131 AMCNTENSET1_EL0_Pn_MASK; 132 } 133 134 static inline __unused void write_amcntenset0_el0_px(uint64_t px) 135 { 136 uint64_t value = read_amcntenset0_el0(); 137 138 value &= ~AMCNTENSET0_EL0_Pn_MASK; 139 value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK; 140 141 write_amcntenset0_el0(value); 142 } 143 144 static inline __unused void write_amcntenset1_el0_px(uint64_t px) 145 { 146 uint64_t value = read_amcntenset1_el0(); 147 148 value &= ~AMCNTENSET1_EL0_Pn_MASK; 149 value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK; 150 151 write_amcntenset1_el0(value); 152 } 153 154 static inline __unused void write_amcntenclr0_el0_px(uint64_t px) 155 { 156 uint64_t value = read_amcntenclr0_el0(); 157 158 value &= ~AMCNTENCLR0_EL0_Pn_MASK; 159 value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK; 160 161 write_amcntenclr0_el0(value); 162 } 163 164 static inline __unused void write_amcntenclr1_el0_px(uint64_t px) 165 { 166 uint64_t value = read_amcntenclr1_el0(); 167 168 value &= ~AMCNTENCLR1_EL0_Pn_MASK; 169 value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK; 170 171 write_amcntenclr1_el0(value); 172 } 173 174 static __unused bool amu_supported(void) 175 { 176 return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1; 177 } 178 179 static __unused bool amu_v1p1_supported(void) 180 { 181 return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1P1; 182 } 183 184 #if ENABLE_AMU_AUXILIARY_COUNTERS 185 static __unused bool amu_group1_supported(void) 186 { 187 return read_amcfgr_el0_ncg() > 0U; 188 } 189 #endif 190 191 /* 192 * Enable counters. This function is meant to be invoked by the context 193 * management library before exiting from EL3. 194 */ 195 void amu_enable(bool el2_unused, cpu_context_t *ctx) 196 { 197 uint64_t id_aa64pfr0_el1_amu; /* AMU version */ 198 199 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 200 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 201 202 uint64_t amcntenset0_el0_px = 0x0; /* Group 0 enable mask */ 203 uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */ 204 205 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); 206 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 207 /* 208 * If the AMU is unsupported, nothing needs to be done. 209 */ 210 211 return; 212 } 213 214 if (el2_unused) { 215 /* 216 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity 217 * Monitor registers do not trap to EL2. 218 */ 219 write_cptr_el2_tam(0U); 220 } 221 222 /* 223 * Retrieve and update the CPTR_EL3 value from the context mentioned 224 * in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to 225 * the Activity Monitor registers do not trap to EL3. 226 */ 227 write_cptr_el3_tam(ctx, 0U); 228 229 /* 230 * Retrieve the number of architected counters. All of these counters 231 * are enabled by default. 232 */ 233 234 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 235 amcntenset0_el0_px = (UINT64_C(1) << (amcgcr_el0_cg0nc)) - 1U; 236 237 assert(amcgcr_el0_cg0nc <= AMU_AMCGCR_CG0NC_MAX); 238 239 /* 240 * The platform may opt to enable specific auxiliary counters. This can 241 * be done via the common FCONF getter, or via the platform-implemented 242 * function. 243 */ 244 245 #if ENABLE_AMU_AUXILIARY_COUNTERS 246 const struct amu_topology *topology; 247 248 #if ENABLE_AMU_FCONF 249 topology = FCONF_GET_PROPERTY(amu, config, topology); 250 #else 251 topology = plat_amu_topology(); 252 #endif /* ENABLE_AMU_FCONF */ 253 254 if (topology != NULL) { 255 unsigned int core_pos = plat_my_core_pos(); 256 257 amcntenset1_el0_px = topology->cores[core_pos].enable; 258 } else { 259 ERROR("AMU: failed to generate AMU topology\n"); 260 } 261 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ 262 263 /* 264 * Enable the requested counters. 265 */ 266 267 write_amcntenset0_el0_px(amcntenset0_el0_px); 268 269 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 270 if (amcfgr_el0_ncg > 0U) { 271 write_amcntenset1_el0_px(amcntenset1_el0_px); 272 273 #if !ENABLE_AMU_AUXILIARY_COUNTERS 274 VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); 275 #endif 276 } 277 278 /* Initialize FEAT_AMUv1p1 features if present. */ 279 if (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) { 280 if (el2_unused) { 281 /* 282 * Make sure virtual offsets are disabled if EL2 not 283 * used. 284 */ 285 write_hcr_el2_amvoffen(0U); 286 } 287 288 #if AMU_RESTRICT_COUNTERS 289 /* 290 * FEAT_AMUv1p1 adds a register field to restrict access to 291 * group 1 counters at all but the highest implemented EL. This 292 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time 293 * flag, when set, system register reads at lower ELs return 294 * zero. Reads from the memory mapped view are unaffected. 295 */ 296 VERBOSE("AMU group 1 counter access restricted.\n"); 297 write_amcr_el0_cg1rz(1U); 298 #else 299 write_amcr_el0_cg1rz(0U); 300 #endif 301 } 302 303 #if ENABLE_MPMM 304 mpmm_enable(); 305 #endif 306 } 307 308 /* Read the group 0 counter identified by the given `idx`. */ 309 static uint64_t amu_group0_cnt_read(unsigned int idx) 310 { 311 assert(amu_supported()); 312 assert(idx < read_amcgcr_el0_cg0nc()); 313 314 return amu_group0_cnt_read_internal(idx); 315 } 316 317 /* Write the group 0 counter identified by the given `idx` with `val` */ 318 static void amu_group0_cnt_write(unsigned int idx, uint64_t val) 319 { 320 assert(amu_supported()); 321 assert(idx < read_amcgcr_el0_cg0nc()); 322 323 amu_group0_cnt_write_internal(idx, val); 324 isb(); 325 } 326 327 /* 328 * Unlike with auxiliary counters, we cannot detect at runtime whether an 329 * architected counter supports a virtual offset. These are instead fixed 330 * according to FEAT_AMUv1p1, but this switch will need to be updated if later 331 * revisions of FEAT_AMU add additional architected counters. 332 */ 333 static bool amu_group0_voffset_supported(uint64_t idx) 334 { 335 switch (idx) { 336 case 0U: 337 case 2U: 338 case 3U: 339 return true; 340 341 case 1U: 342 return false; 343 344 default: 345 ERROR("AMU: can't set up virtual offset for unknown " 346 "architected counter %llu!\n", idx); 347 348 panic(); 349 } 350 } 351 352 /* 353 * Read the group 0 offset register for a given index. Index must be 0, 2, 354 * or 3, the register for 1 does not exist. 355 * 356 * Using this function requires FEAT_AMUv1p1 support. 357 */ 358 static uint64_t amu_group0_voffset_read(unsigned int idx) 359 { 360 assert(amu_v1p1_supported()); 361 assert(idx < read_amcgcr_el0_cg0nc()); 362 assert(idx != 1U); 363 364 return amu_group0_voffset_read_internal(idx); 365 } 366 367 /* 368 * Write the group 0 offset register for a given index. Index must be 0, 2, or 369 * 3, the register for 1 does not exist. 370 * 371 * Using this function requires FEAT_AMUv1p1 support. 372 */ 373 static void amu_group0_voffset_write(unsigned int idx, uint64_t val) 374 { 375 assert(amu_v1p1_supported()); 376 assert(idx < read_amcgcr_el0_cg0nc()); 377 assert(idx != 1U); 378 379 amu_group0_voffset_write_internal(idx, val); 380 isb(); 381 } 382 383 #if ENABLE_AMU_AUXILIARY_COUNTERS 384 /* Read the group 1 counter identified by the given `idx` */ 385 static uint64_t amu_group1_cnt_read(unsigned int idx) 386 { 387 assert(amu_supported()); 388 assert(amu_group1_supported()); 389 assert(idx < read_amcgcr_el0_cg1nc()); 390 391 return amu_group1_cnt_read_internal(idx); 392 } 393 394 /* Write the group 1 counter identified by the given `idx` with `val` */ 395 static void amu_group1_cnt_write(unsigned int idx, uint64_t val) 396 { 397 assert(amu_supported()); 398 assert(amu_group1_supported()); 399 assert(idx < read_amcgcr_el0_cg1nc()); 400 401 amu_group1_cnt_write_internal(idx, val); 402 isb(); 403 } 404 405 /* 406 * Read the group 1 offset register for a given index. 407 * 408 * Using this function requires FEAT_AMUv1p1 support. 409 */ 410 static uint64_t amu_group1_voffset_read(unsigned int idx) 411 { 412 assert(amu_v1p1_supported()); 413 assert(amu_group1_supported()); 414 assert(idx < read_amcgcr_el0_cg1nc()); 415 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 416 417 return amu_group1_voffset_read_internal(idx); 418 } 419 420 /* 421 * Write the group 1 offset register for a given index. 422 * 423 * Using this function requires FEAT_AMUv1p1 support. 424 */ 425 static void amu_group1_voffset_write(unsigned int idx, uint64_t val) 426 { 427 assert(amu_v1p1_supported()); 428 assert(amu_group1_supported()); 429 assert(idx < read_amcgcr_el0_cg1nc()); 430 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); 431 432 amu_group1_voffset_write_internal(idx, val); 433 isb(); 434 } 435 #endif 436 437 static void *amu_context_save(const void *arg) 438 { 439 uint64_t i, j; 440 441 unsigned int core_pos; 442 struct amu_ctx *ctx; 443 444 uint64_t id_aa64pfr0_el1_amu; /* AMU version */ 445 uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ 446 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 447 448 #if ENABLE_AMU_AUXILIARY_COUNTERS 449 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 450 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 451 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 452 #endif 453 454 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); 455 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 456 return (void *)0; 457 } 458 459 core_pos = plat_my_core_pos(); 460 ctx = &amu_ctxs_[core_pos]; 461 462 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 463 hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? 464 read_hcr_el2_amvoffen() : 0U; 465 466 #if ENABLE_AMU_AUXILIARY_COUNTERS 467 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 468 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 469 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 470 #endif 471 472 /* 473 * Disable all AMU counters. 474 */ 475 476 ctx->group0_enable = read_amcntenset0_el0_px(); 477 write_amcntenclr0_el0_px(ctx->group0_enable); 478 479 #if ENABLE_AMU_AUXILIARY_COUNTERS 480 if (amcfgr_el0_ncg > 0U) { 481 ctx->group1_enable = read_amcntenset1_el0_px(); 482 write_amcntenclr1_el0_px(ctx->group1_enable); 483 } 484 #endif 485 486 /* 487 * Save the counters to the local context. 488 */ 489 490 isb(); /* Ensure counters have been stopped */ 491 492 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 493 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 494 } 495 496 #if ENABLE_AMU_AUXILIARY_COUNTERS 497 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 498 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 499 } 500 #endif 501 502 /* 503 * Save virtual offsets for counters that offer them. 504 */ 505 506 if (hcr_el2_amvoffen != 0U) { 507 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 508 if (!amu_group0_voffset_supported(i)) { 509 continue; /* No virtual offset */ 510 } 511 512 ctx->group0_voffsets[j++] = amu_group0_voffset_read(i); 513 } 514 515 #if ENABLE_AMU_AUXILIARY_COUNTERS 516 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 517 if ((amcg1idr_el0_voff >> i) & 1U) { 518 continue; /* No virtual offset */ 519 } 520 521 ctx->group1_voffsets[j++] = amu_group1_voffset_read(i); 522 } 523 #endif 524 } 525 526 return (void *)0; 527 } 528 529 static void *amu_context_restore(const void *arg) 530 { 531 uint64_t i, j; 532 533 unsigned int core_pos; 534 struct amu_ctx *ctx; 535 536 uint64_t id_aa64pfr0_el1_amu; /* AMU version */ 537 538 uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ 539 540 uint64_t amcfgr_el0_ncg; /* Number of counter groups */ 541 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ 542 543 #if ENABLE_AMU_AUXILIARY_COUNTERS 544 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ 545 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ 546 #endif 547 548 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); 549 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 550 return (void *)0; 551 } 552 553 core_pos = plat_my_core_pos(); 554 ctx = &amu_ctxs_[core_pos]; 555 556 amcfgr_el0_ncg = read_amcfgr_el0_ncg(); 557 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); 558 559 hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? 560 read_hcr_el2_amvoffen() : 0U; 561 562 #if ENABLE_AMU_AUXILIARY_COUNTERS 563 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; 564 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; 565 #endif 566 567 /* 568 * Sanity check that all counters were disabled when the context was 569 * previously saved. 570 */ 571 572 assert(read_amcntenset0_el0_px() == 0U); 573 574 if (amcfgr_el0_ncg > 0U) { 575 assert(read_amcntenset1_el0_px() == 0U); 576 } 577 578 /* 579 * Restore the counter values from the local context. 580 */ 581 582 for (i = 0U; i < amcgcr_el0_cg0nc; i++) { 583 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 584 } 585 586 #if ENABLE_AMU_AUXILIARY_COUNTERS 587 for (i = 0U; i < amcgcr_el0_cg1nc; i++) { 588 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 589 } 590 #endif 591 592 /* 593 * Restore virtual offsets for counters that offer them. 594 */ 595 596 if (hcr_el2_amvoffen != 0U) { 597 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { 598 if (!amu_group0_voffset_supported(i)) { 599 continue; /* No virtual offset */ 600 } 601 602 amu_group0_voffset_write(i, ctx->group0_voffsets[j++]); 603 } 604 605 #if ENABLE_AMU_AUXILIARY_COUNTERS 606 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { 607 if ((amcg1idr_el0_voff >> i) & 1U) { 608 continue; /* No virtual offset */ 609 } 610 611 amu_group1_voffset_write(i, ctx->group1_voffsets[j++]); 612 } 613 #endif 614 } 615 616 /* 617 * Re-enable counters that were disabled during context save. 618 */ 619 620 write_amcntenset0_el0_px(ctx->group0_enable); 621 622 #if ENABLE_AMU_AUXILIARY_COUNTERS 623 if (amcfgr_el0_ncg > 0) { 624 write_amcntenset1_el0_px(ctx->group1_enable); 625 } 626 #endif 627 628 #if ENABLE_MPMM 629 mpmm_enable(); 630 #endif 631 632 return (void *)0; 633 } 634 635 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 636 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 637