1 /* 2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <stdbool.h> 9 10 #include <arch.h> 11 #include <arch_features.h> 12 #include <arch_helpers.h> 13 14 #include <lib/el3_runtime/pubsub_events.h> 15 #include <lib/extensions/amu.h> 16 #include <lib/extensions/amu_private.h> 17 18 #include <plat/common/platform.h> 19 20 static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; 21 22 /* 23 * Get AMU version value from aa64pfr0. 24 * Return values 25 * ID_AA64PFR0_AMU_V1: FEAT_AMUv1 supported (introduced in ARM v8.4) 26 * ID_AA64PFR0_AMU_V1P1: FEAT_AMUv1p1 supported (introduced in ARM v8.6) 27 * ID_AA64PFR0_AMU_NOT_SUPPORTED: not supported 28 */ 29 unsigned int amu_get_version(void) 30 { 31 return (unsigned int)(read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & 32 ID_AA64PFR0_AMU_MASK; 33 } 34 35 #if AMU_GROUP1_NR_COUNTERS 36 /* Check if group 1 counters is implemented */ 37 bool amu_group1_supported(void) 38 { 39 uint64_t features = read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT; 40 41 return (features & AMCFGR_EL0_NCG_MASK) == 1U; 42 } 43 #endif 44 45 /* 46 * Enable counters. This function is meant to be invoked 47 * by the context management library before exiting from EL3. 48 */ 49 void amu_enable(bool el2_unused, cpu_context_t *ctx) 50 { 51 uint64_t v; 52 unsigned int amu_version = amu_get_version(); 53 54 if (amu_version == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 55 return; 56 } 57 58 #if AMU_GROUP1_NR_COUNTERS 59 /* Check and set presence of group 1 counters */ 60 if (!amu_group1_supported()) { 61 ERROR("AMU Counter Group 1 is not implemented\n"); 62 panic(); 63 } 64 65 /* Check number of group 1 counters */ 66 uint64_t cnt_num = (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & 67 AMCGCR_EL0_CG1NC_MASK; 68 VERBOSE("%s%llu. %s%u\n", 69 "Number of AMU Group 1 Counters ", cnt_num, 70 "Requested number ", AMU_GROUP1_NR_COUNTERS); 71 72 if (cnt_num < AMU_GROUP1_NR_COUNTERS) { 73 ERROR("%s%llu is less than %s%u\n", 74 "Number of AMU Group 1 Counters ", cnt_num, 75 "Requested number ", AMU_GROUP1_NR_COUNTERS); 76 panic(); 77 } 78 #endif 79 80 if (el2_unused) { 81 /* 82 * CPTR_EL2.TAM: Set to zero so any accesses to 83 * the Activity Monitor registers do not trap to EL2. 84 */ 85 v = read_cptr_el2(); 86 v &= ~CPTR_EL2_TAM_BIT; 87 write_cptr_el2(v); 88 } 89 90 /* 91 * Retrieve and update the CPTR_EL3 value from the context mentioned 92 * in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to 93 * the Activity Monitor registers do not trap to EL3. 94 */ 95 v = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); 96 v &= ~TAM_BIT; 97 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, v); 98 99 /* Enable group 0 counters */ 100 write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); 101 102 #if AMU_GROUP1_NR_COUNTERS 103 /* Enable group 1 counters */ 104 write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); 105 #endif 106 107 /* Initialize FEAT_AMUv1p1 features if present. */ 108 if (amu_version < ID_AA64PFR0_AMU_V1P1) { 109 return; 110 } 111 112 if (el2_unused) { 113 /* Make sure virtual offsets are disabled if EL2 not used. */ 114 write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT); 115 } 116 117 #if AMU_RESTRICT_COUNTERS 118 /* 119 * FEAT_AMUv1p1 adds a register field to restrict access to group 1 120 * counters at all but the highest implemented EL. This is controlled 121 * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system 122 * register reads at lower ELs return zero. Reads from the memory 123 * mapped view are unaffected. 124 */ 125 VERBOSE("AMU group 1 counter access restricted.\n"); 126 write_amcr_el0(read_amcr_el0() | AMCR_CG1RZ_BIT); 127 #else 128 write_amcr_el0(read_amcr_el0() & ~AMCR_CG1RZ_BIT); 129 #endif 130 } 131 132 /* Read the group 0 counter identified by the given `idx`. */ 133 uint64_t amu_group0_cnt_read(unsigned int idx) 134 { 135 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 136 assert(idx < AMU_GROUP0_NR_COUNTERS); 137 138 return amu_group0_cnt_read_internal(idx); 139 } 140 141 /* Write the group 0 counter identified by the given `idx` with `val` */ 142 void amu_group0_cnt_write(unsigned int idx, uint64_t val) 143 { 144 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 145 assert(idx < AMU_GROUP0_NR_COUNTERS); 146 147 amu_group0_cnt_write_internal(idx, val); 148 isb(); 149 } 150 151 /* 152 * Read the group 0 offset register for a given index. Index must be 0, 2, 153 * or 3, the register for 1 does not exist. 154 * 155 * Using this function requires FEAT_AMUv1p1 support. 156 */ 157 uint64_t amu_group0_voffset_read(unsigned int idx) 158 { 159 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 160 assert(idx < AMU_GROUP0_NR_COUNTERS); 161 assert(idx != 1U); 162 163 return amu_group0_voffset_read_internal(idx); 164 } 165 166 /* 167 * Write the group 0 offset register for a given index. Index must be 0, 2, or 168 * 3, the register for 1 does not exist. 169 * 170 * Using this function requires FEAT_AMUv1p1 support. 171 */ 172 void amu_group0_voffset_write(unsigned int idx, uint64_t val) 173 { 174 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 175 assert(idx < AMU_GROUP0_NR_COUNTERS); 176 assert(idx != 1U); 177 178 amu_group0_voffset_write_internal(idx, val); 179 isb(); 180 } 181 182 #if AMU_GROUP1_NR_COUNTERS 183 /* Read the group 1 counter identified by the given `idx` */ 184 uint64_t amu_group1_cnt_read(unsigned int idx) 185 { 186 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 187 assert(amu_group1_supported()); 188 assert(idx < AMU_GROUP1_NR_COUNTERS); 189 190 return amu_group1_cnt_read_internal(idx); 191 } 192 193 /* Write the group 1 counter identified by the given `idx` with `val` */ 194 void amu_group1_cnt_write(unsigned int idx, uint64_t val) 195 { 196 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 197 assert(amu_group1_supported()); 198 assert(idx < AMU_GROUP1_NR_COUNTERS); 199 200 amu_group1_cnt_write_internal(idx, val); 201 isb(); 202 } 203 204 /* 205 * Read the group 1 offset register for a given index. 206 * 207 * Using this function requires FEAT_AMUv1p1 support. 208 */ 209 uint64_t amu_group1_voffset_read(unsigned int idx) 210 { 211 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 212 assert(amu_group1_supported()); 213 assert(idx < AMU_GROUP1_NR_COUNTERS); 214 assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 215 (1ULL << idx)) != 0ULL); 216 217 return amu_group1_voffset_read_internal(idx); 218 } 219 220 /* 221 * Write the group 1 offset register for a given index. 222 * 223 * Using this function requires FEAT_AMUv1p1 support. 224 */ 225 void amu_group1_voffset_write(unsigned int idx, uint64_t val) 226 { 227 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 228 assert(amu_group1_supported()); 229 assert(idx < AMU_GROUP1_NR_COUNTERS); 230 assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 231 (1ULL << idx)) != 0ULL); 232 233 amu_group1_voffset_write_internal(idx, val); 234 isb(); 235 } 236 237 /* 238 * Program the event type register for the given `idx` with 239 * the event number `val` 240 */ 241 void amu_group1_set_evtype(unsigned int idx, unsigned int val) 242 { 243 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 244 assert(amu_group1_supported()); 245 assert(idx < AMU_GROUP1_NR_COUNTERS); 246 247 amu_group1_set_evtype_internal(idx, val); 248 isb(); 249 } 250 #endif /* AMU_GROUP1_NR_COUNTERS */ 251 252 static void *amu_context_save(const void *arg) 253 { 254 struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; 255 unsigned int i; 256 257 if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 258 return (void *)-1; 259 } 260 261 #if AMU_GROUP1_NR_COUNTERS 262 if (!amu_group1_supported()) { 263 return (void *)-1; 264 } 265 #endif 266 /* Assert that group 0/1 counter configuration is what we expect */ 267 assert(read_amcntenset0_el0() == AMU_GROUP0_COUNTERS_MASK); 268 269 #if AMU_GROUP1_NR_COUNTERS 270 assert(read_amcntenset1_el0() == AMU_GROUP1_COUNTERS_MASK); 271 #endif 272 /* 273 * Disable group 0/1 counters to avoid other observers like SCP sampling 274 * counter values from the future via the memory mapped view. 275 */ 276 write_amcntenclr0_el0(AMU_GROUP0_COUNTERS_MASK); 277 278 #if AMU_GROUP1_NR_COUNTERS 279 write_amcntenclr1_el0(AMU_GROUP1_COUNTERS_MASK); 280 #endif 281 isb(); 282 283 /* Save all group 0 counters */ 284 for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { 285 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 286 } 287 288 /* Save group 0 virtual offsets if supported and enabled. */ 289 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 290 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 291 /* Not using a loop because count is fixed and index 1 DNE. */ 292 ctx->group0_voffsets[0U] = amu_group0_voffset_read(0U); 293 ctx->group0_voffsets[1U] = amu_group0_voffset_read(2U); 294 ctx->group0_voffsets[2U] = amu_group0_voffset_read(3U); 295 } 296 297 #if AMU_GROUP1_NR_COUNTERS 298 /* Save group 1 counters */ 299 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 300 if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { 301 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 302 } 303 } 304 305 /* Save group 1 virtual offsets if supported and enabled. */ 306 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 307 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 308 u_register_t amcg1idr = read_amcg1idr_el0() >> 309 AMCG1IDR_VOFF_SHIFT; 310 amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; 311 312 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 313 if (((amcg1idr >> i) & 1ULL) != 0ULL) { 314 ctx->group1_voffsets[i] = 315 amu_group1_voffset_read(i); 316 } 317 } 318 } 319 #endif 320 return (void *)0; 321 } 322 323 static void *amu_context_restore(const void *arg) 324 { 325 struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; 326 unsigned int i; 327 328 if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 329 return (void *)-1; 330 } 331 332 #if AMU_GROUP1_NR_COUNTERS 333 if (!amu_group1_supported()) { 334 return (void *)-1; 335 } 336 #endif 337 /* Counters were disabled in `amu_context_save()` */ 338 assert(read_amcntenset0_el0() == 0U); 339 340 #if AMU_GROUP1_NR_COUNTERS 341 assert(read_amcntenset1_el0() == 0U); 342 #endif 343 344 /* Restore all group 0 counters */ 345 for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { 346 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 347 } 348 349 /* Restore group 0 virtual offsets if supported and enabled. */ 350 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 351 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 352 /* Not using a loop because count is fixed and index 1 DNE. */ 353 amu_group0_voffset_write(0U, ctx->group0_voffsets[0U]); 354 amu_group0_voffset_write(2U, ctx->group0_voffsets[1U]); 355 amu_group0_voffset_write(3U, ctx->group0_voffsets[2U]); 356 } 357 358 /* Restore group 0 counter configuration */ 359 write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); 360 361 #if AMU_GROUP1_NR_COUNTERS 362 /* Restore group 1 counters */ 363 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 364 if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { 365 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 366 } 367 } 368 369 /* Restore group 1 virtual offsets if supported and enabled. */ 370 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 371 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 372 u_register_t amcg1idr = read_amcg1idr_el0() >> 373 AMCG1IDR_VOFF_SHIFT; 374 amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; 375 376 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 377 if (((amcg1idr >> i) & 1ULL) != 0ULL) { 378 amu_group1_voffset_write(i, 379 ctx->group1_voffsets[i]); 380 } 381 } 382 } 383 384 /* Restore group 1 counter configuration */ 385 write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); 386 #endif 387 388 return (void *)0; 389 } 390 391 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 392 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 393