1 /* 2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <stdbool.h> 9 10 #include <arch.h> 11 #include <arch_features.h> 12 #include <arch_helpers.h> 13 14 #include <lib/el3_runtime/pubsub_events.h> 15 #include <lib/extensions/amu.h> 16 #include <lib/extensions/amu_private.h> 17 18 #include <plat/common/platform.h> 19 20 static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; 21 22 /* 23 * Get AMU version value from aa64pfr0. 24 * Return values 25 * ID_AA64PFR0_AMU_V1: FEAT_AMUv1 supported (introduced in ARM v8.4) 26 * ID_AA64PFR0_AMU_V1P1: FEAT_AMUv1p1 supported (introduced in ARM v8.6) 27 * ID_AA64PFR0_AMU_NOT_SUPPORTED: not supported 28 */ 29 static unsigned int amu_get_version(void) 30 { 31 return (unsigned int)(read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & 32 ID_AA64PFR0_AMU_MASK; 33 } 34 35 #if AMU_GROUP1_NR_COUNTERS 36 /* Check if group 1 counters is implemented */ 37 static bool amu_group1_supported(void) 38 { 39 uint64_t features = read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT; 40 41 return (features & AMCFGR_EL0_NCG_MASK) == 1U; 42 } 43 #endif 44 45 /* 46 * Enable counters. This function is meant to be invoked 47 * by the context management library before exiting from EL3. 48 */ 49 void amu_enable(bool el2_unused, cpu_context_t *ctx) 50 { 51 uint64_t v; 52 unsigned int amu_version = amu_get_version(); 53 54 if (amu_version == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 55 return; 56 } 57 58 #if AMU_GROUP1_NR_COUNTERS 59 /* Check and set presence of group 1 counters */ 60 if (!amu_group1_supported()) { 61 ERROR("AMU Counter Group 1 is not implemented\n"); 62 panic(); 63 } 64 65 /* Check number of group 1 counters */ 66 uint64_t cnt_num = (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & 67 AMCGCR_EL0_CG1NC_MASK; 68 VERBOSE("%s%llu. %s%u\n", 69 "Number of AMU Group 1 Counters ", cnt_num, 70 "Requested number ", AMU_GROUP1_NR_COUNTERS); 71 72 if (cnt_num < AMU_GROUP1_NR_COUNTERS) { 73 ERROR("%s%llu is less than %s%u\n", 74 "Number of AMU Group 1 Counters ", cnt_num, 75 "Requested number ", AMU_GROUP1_NR_COUNTERS); 76 panic(); 77 } 78 #endif 79 80 if (el2_unused) { 81 /* 82 * CPTR_EL2.TAM: Set to zero so any accesses to 83 * the Activity Monitor registers do not trap to EL2. 84 */ 85 v = read_cptr_el2(); 86 v &= ~CPTR_EL2_TAM_BIT; 87 write_cptr_el2(v); 88 } 89 90 /* 91 * Retrieve and update the CPTR_EL3 value from the context mentioned 92 * in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to 93 * the Activity Monitor registers do not trap to EL3. 94 */ 95 v = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); 96 v &= ~TAM_BIT; 97 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, v); 98 99 /* Enable group 0 counters */ 100 write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); 101 102 #if AMU_GROUP1_NR_COUNTERS 103 /* Enable group 1 counters */ 104 write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); 105 #endif 106 107 /* Initialize FEAT_AMUv1p1 features if present. */ 108 if (amu_version < ID_AA64PFR0_AMU_V1P1) { 109 return; 110 } 111 112 if (el2_unused) { 113 /* Make sure virtual offsets are disabled if EL2 not used. */ 114 write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT); 115 } 116 117 #if AMU_RESTRICT_COUNTERS 118 /* 119 * FEAT_AMUv1p1 adds a register field to restrict access to group 1 120 * counters at all but the highest implemented EL. This is controlled 121 * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system 122 * register reads at lower ELs return zero. Reads from the memory 123 * mapped view are unaffected. 124 */ 125 VERBOSE("AMU group 1 counter access restricted.\n"); 126 write_amcr_el0(read_amcr_el0() | AMCR_CG1RZ_BIT); 127 #else 128 write_amcr_el0(read_amcr_el0() & ~AMCR_CG1RZ_BIT); 129 #endif 130 } 131 132 /* Read the group 0 counter identified by the given `idx`. */ 133 static uint64_t amu_group0_cnt_read(unsigned int idx) 134 { 135 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 136 assert(idx < AMU_GROUP0_NR_COUNTERS); 137 138 return amu_group0_cnt_read_internal(idx); 139 } 140 141 /* Write the group 0 counter identified by the given `idx` with `val` */ 142 static void amu_group0_cnt_write(unsigned int idx, uint64_t val) 143 { 144 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 145 assert(idx < AMU_GROUP0_NR_COUNTERS); 146 147 amu_group0_cnt_write_internal(idx, val); 148 isb(); 149 } 150 151 /* 152 * Read the group 0 offset register for a given index. Index must be 0, 2, 153 * or 3, the register for 1 does not exist. 154 * 155 * Using this function requires FEAT_AMUv1p1 support. 156 */ 157 static uint64_t amu_group0_voffset_read(unsigned int idx) 158 { 159 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 160 assert(idx < AMU_GROUP0_NR_COUNTERS); 161 assert(idx != 1U); 162 163 return amu_group0_voffset_read_internal(idx); 164 } 165 166 /* 167 * Write the group 0 offset register for a given index. Index must be 0, 2, or 168 * 3, the register for 1 does not exist. 169 * 170 * Using this function requires FEAT_AMUv1p1 support. 171 */ 172 static void amu_group0_voffset_write(unsigned int idx, uint64_t val) 173 { 174 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 175 assert(idx < AMU_GROUP0_NR_COUNTERS); 176 assert(idx != 1U); 177 178 amu_group0_voffset_write_internal(idx, val); 179 isb(); 180 } 181 182 #if AMU_GROUP1_NR_COUNTERS 183 /* Read the group 1 counter identified by the given `idx` */ 184 static uint64_t amu_group1_cnt_read(unsigned int idx) 185 { 186 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 187 assert(amu_group1_supported()); 188 assert(idx < AMU_GROUP1_NR_COUNTERS); 189 190 return amu_group1_cnt_read_internal(idx); 191 } 192 193 /* Write the group 1 counter identified by the given `idx` with `val` */ 194 static void amu_group1_cnt_write(unsigned int idx, uint64_t val) 195 { 196 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); 197 assert(amu_group1_supported()); 198 assert(idx < AMU_GROUP1_NR_COUNTERS); 199 200 amu_group1_cnt_write_internal(idx, val); 201 isb(); 202 } 203 204 /* 205 * Read the group 1 offset register for a given index. 206 * 207 * Using this function requires FEAT_AMUv1p1 support. 208 */ 209 static uint64_t amu_group1_voffset_read(unsigned int idx) 210 { 211 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 212 assert(amu_group1_supported()); 213 assert(idx < AMU_GROUP1_NR_COUNTERS); 214 assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 215 (1ULL << idx)) != 0ULL); 216 217 return amu_group1_voffset_read_internal(idx); 218 } 219 220 /* 221 * Write the group 1 offset register for a given index. 222 * 223 * Using this function requires FEAT_AMUv1p1 support. 224 */ 225 static void amu_group1_voffset_write(unsigned int idx, uint64_t val) 226 { 227 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); 228 assert(amu_group1_supported()); 229 assert(idx < AMU_GROUP1_NR_COUNTERS); 230 assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & 231 (1ULL << idx)) != 0ULL); 232 233 amu_group1_voffset_write_internal(idx, val); 234 isb(); 235 } 236 #endif /* AMU_GROUP1_NR_COUNTERS */ 237 238 static void *amu_context_save(const void *arg) 239 { 240 struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; 241 unsigned int i; 242 243 if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 244 return (void *)-1; 245 } 246 247 #if AMU_GROUP1_NR_COUNTERS 248 if (!amu_group1_supported()) { 249 return (void *)-1; 250 } 251 #endif 252 /* Assert that group 0/1 counter configuration is what we expect */ 253 assert(read_amcntenset0_el0() == AMU_GROUP0_COUNTERS_MASK); 254 255 #if AMU_GROUP1_NR_COUNTERS 256 assert(read_amcntenset1_el0() == AMU_GROUP1_COUNTERS_MASK); 257 #endif 258 /* 259 * Disable group 0/1 counters to avoid other observers like SCP sampling 260 * counter values from the future via the memory mapped view. 261 */ 262 write_amcntenclr0_el0(AMU_GROUP0_COUNTERS_MASK); 263 264 #if AMU_GROUP1_NR_COUNTERS 265 write_amcntenclr1_el0(AMU_GROUP1_COUNTERS_MASK); 266 #endif 267 isb(); 268 269 /* Save all group 0 counters */ 270 for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { 271 ctx->group0_cnts[i] = amu_group0_cnt_read(i); 272 } 273 274 /* Save group 0 virtual offsets if supported and enabled. */ 275 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 276 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 277 /* Not using a loop because count is fixed and index 1 DNE. */ 278 ctx->group0_voffsets[0U] = amu_group0_voffset_read(0U); 279 ctx->group0_voffsets[1U] = amu_group0_voffset_read(2U); 280 ctx->group0_voffsets[2U] = amu_group0_voffset_read(3U); 281 } 282 283 #if AMU_GROUP1_NR_COUNTERS 284 /* Save group 1 counters */ 285 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 286 if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { 287 ctx->group1_cnts[i] = amu_group1_cnt_read(i); 288 } 289 } 290 291 /* Save group 1 virtual offsets if supported and enabled. */ 292 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 293 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 294 u_register_t amcg1idr = read_amcg1idr_el0() >> 295 AMCG1IDR_VOFF_SHIFT; 296 amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; 297 298 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 299 if (((amcg1idr >> i) & 1ULL) != 0ULL) { 300 ctx->group1_voffsets[i] = 301 amu_group1_voffset_read(i); 302 } 303 } 304 } 305 #endif 306 return (void *)0; 307 } 308 309 static void *amu_context_restore(const void *arg) 310 { 311 struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; 312 unsigned int i; 313 314 if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { 315 return (void *)-1; 316 } 317 318 #if AMU_GROUP1_NR_COUNTERS 319 if (!amu_group1_supported()) { 320 return (void *)-1; 321 } 322 #endif 323 /* Counters were disabled in `amu_context_save()` */ 324 assert(read_amcntenset0_el0() == 0U); 325 326 #if AMU_GROUP1_NR_COUNTERS 327 assert(read_amcntenset1_el0() == 0U); 328 #endif 329 330 /* Restore all group 0 counters */ 331 for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { 332 amu_group0_cnt_write(i, ctx->group0_cnts[i]); 333 } 334 335 /* Restore group 0 virtual offsets if supported and enabled. */ 336 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 337 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 338 /* Not using a loop because count is fixed and index 1 DNE. */ 339 amu_group0_voffset_write(0U, ctx->group0_voffsets[0U]); 340 amu_group0_voffset_write(2U, ctx->group0_voffsets[1U]); 341 amu_group0_voffset_write(3U, ctx->group0_voffsets[2U]); 342 } 343 344 /* Restore group 0 counter configuration */ 345 write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); 346 347 #if AMU_GROUP1_NR_COUNTERS 348 /* Restore group 1 counters */ 349 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 350 if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { 351 amu_group1_cnt_write(i, ctx->group1_cnts[i]); 352 } 353 } 354 355 /* Restore group 1 virtual offsets if supported and enabled. */ 356 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && 357 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { 358 u_register_t amcg1idr = read_amcg1idr_el0() >> 359 AMCG1IDR_VOFF_SHIFT; 360 amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; 361 362 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { 363 if (((amcg1idr >> i) & 1ULL) != 0ULL) { 364 amu_group1_voffset_write(i, 365 ctx->group1_voffsets[i]); 366 } 367 } 368 } 369 370 /* Restore group 1 counter configuration */ 371 write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); 372 #endif 373 374 return (void *)0; 375 } 376 377 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); 378 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); 379