1 /* 2 * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <string.h> 9 10 #include <arch.h> 11 #include <arch_features.h> 12 #include <arch_helpers.h> 13 #include <common/bl_common.h> 14 #include <common/debug.h> 15 #include <context.h> 16 #include <drivers/delay_timer.h> 17 #include <lib/el3_runtime/context_mgmt.h> 18 #include <lib/extensions/spe.h> 19 #include <lib/pmf/pmf.h> 20 #include <lib/runtime_instr.h> 21 #include <lib/utils.h> 22 #include <plat/common/platform.h> 23 24 #include "psci_private.h" 25 26 /* 27 * SPD power management operations, expected to be supplied by the registered 28 * SPD on successful SP initialization 29 */ 30 const spd_pm_ops_t *psci_spd_pm; 31 32 /* 33 * PSCI requested local power state map. This array is used to store the local 34 * power states requested by a CPU for power levels from level 1 to 35 * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power 36 * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a 37 * CPU are the same. 38 * 39 * During state coordination, the platform is passed an array containing the 40 * local states requested for a particular non cpu power domain by each cpu 41 * within the domain. 42 * 43 * TODO: Dense packing of the requested states will cause cache thrashing 44 * when multiple power domains write to it. If we allocate the requested 45 * states at each power level in a cache-line aligned per-domain memory, 46 * the cache thrashing can be avoided. 47 */ 48 static plat_local_state_t 49 psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT]; 50 51 unsigned int psci_plat_core_count; 52 53 /******************************************************************************* 54 * Arrays that hold the platform's power domain tree information for state 55 * management of power domains. 56 * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain 57 * which is an ancestor of a CPU power domain. 58 * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain 59 ******************************************************************************/ 60 non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS] 61 #if USE_COHERENT_MEM 62 __section(".tzfw_coherent_mem") 63 #endif 64 ; 65 66 /* Lock for PSCI state coordination */ 67 DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); 68 69 cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; 70 71 /******************************************************************************* 72 * Pointer to functions exported by the platform to complete power mgmt. ops 73 ******************************************************************************/ 74 const plat_psci_ops_t *psci_plat_pm_ops; 75 76 /****************************************************************************** 77 * Check that the maximum power level supported by the platform makes sense 78 *****************************************************************************/ 79 CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) && 80 (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL), 81 assert_platform_max_pwrlvl_check); 82 83 #if PSCI_OS_INIT_MODE 84 /******************************************************************************* 85 * The power state coordination mode used in CPU_SUSPEND. 86 * Defaults to platform-coordinated mode. 87 ******************************************************************************/ 88 suspend_mode_t psci_suspend_mode = PLAT_COORD; 89 #endif 90 91 /* 92 * The plat_local_state used by the platform is one of these types: RUN, 93 * RETENTION and OFF. The platform can define further sub-states for each type 94 * apart from RUN. This categorization is done to verify the sanity of the 95 * psci_power_state passed by the platform and to print debug information. The 96 * categorization is done on the basis of the following conditions: 97 * 98 * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN. 99 * 100 * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is 101 * STATE_TYPE_RETN. 102 * 103 * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is 104 * STATE_TYPE_OFF. 105 */ 106 typedef enum plat_local_state_type { 107 STATE_TYPE_RUN = 0, 108 STATE_TYPE_RETN, 109 STATE_TYPE_OFF 110 } plat_local_state_type_t; 111 112 /* Function used to categorize plat_local_state. */ 113 static plat_local_state_type_t find_local_state_type(plat_local_state_t state) 114 { 115 if (state != 0U) { 116 if (state > PLAT_MAX_RET_STATE) { 117 return STATE_TYPE_OFF; 118 } else { 119 return STATE_TYPE_RETN; 120 } 121 } else { 122 return STATE_TYPE_RUN; 123 } 124 } 125 126 /****************************************************************************** 127 * Check that the maximum retention level supported by the platform is less 128 * than the maximum off level. 129 *****************************************************************************/ 130 CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE, 131 assert_platform_max_off_and_retn_state_check); 132 133 /****************************************************************************** 134 * This function ensures that the power state parameter in a CPU_SUSPEND request 135 * is valid. If so, it returns the requested states for each power level. 136 *****************************************************************************/ 137 int psci_validate_power_state(unsigned int power_state, 138 psci_power_state_t *state_info) 139 { 140 /* Check SBZ bits in power state are zero */ 141 if (psci_check_power_state(power_state) != 0U) { 142 return PSCI_E_INVALID_PARAMS; 143 } 144 assert(psci_plat_pm_ops->validate_power_state != NULL); 145 146 /* Validate the power_state using platform pm_ops */ 147 return psci_plat_pm_ops->validate_power_state(power_state, state_info); 148 } 149 150 /****************************************************************************** 151 * This function retrieves the `psci_power_state_t` for system suspend from 152 * the platform. 153 *****************************************************************************/ 154 void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info) 155 { 156 /* 157 * Assert that the required pm_ops hook is implemented to ensure that 158 * the capability detected during psci_setup() is valid. 159 */ 160 assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL); 161 162 /* 163 * Query the platform for the power_state required for system suspend 164 */ 165 psci_plat_pm_ops->get_sys_suspend_power_state(state_info); 166 } 167 168 #if PSCI_OS_INIT_MODE 169 /******************************************************************************* 170 * This function verifies that all the other cores at the 'end_pwrlvl' have been 171 * idled and the current CPU is the last running CPU at the 'end_pwrlvl'. 172 * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false) 173 * otherwise. 174 ******************************************************************************/ 175 static bool psci_is_last_cpu_to_idle_at_pwrlvl(unsigned int my_idx, unsigned int end_pwrlvl) 176 { 177 unsigned int lvl; 178 unsigned int parent_idx = 0; 179 unsigned int cpu_start_idx, ncpus, cpu_idx; 180 plat_local_state_t local_state; 181 182 if (end_pwrlvl == PSCI_CPU_PWR_LVL) { 183 return true; 184 } 185 186 parent_idx = psci_cpu_pd_nodes[my_idx].parent_node; 187 for (lvl = PSCI_CPU_PWR_LVL + U(1); lvl < end_pwrlvl; lvl++) { 188 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 189 } 190 191 cpu_start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; 192 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; 193 194 for (cpu_idx = cpu_start_idx; cpu_idx < cpu_start_idx + ncpus; 195 cpu_idx++) { 196 local_state = psci_get_cpu_local_state_by_idx(cpu_idx); 197 if (cpu_idx == my_idx) { 198 assert(is_local_state_run(local_state) != 0); 199 continue; 200 } 201 202 if (is_local_state_run(local_state) != 0) { 203 return false; 204 } 205 } 206 207 return true; 208 } 209 #endif 210 211 /******************************************************************************* 212 * This function verifies that all the other cores in the system have been 213 * turned OFF and the current CPU is the last running CPU in the system. 214 * Returns true, if the current CPU is the last ON CPU or false otherwise. 215 ******************************************************************************/ 216 bool psci_is_last_on_cpu(unsigned int my_idx) 217 { 218 for (unsigned int cpu_idx = 0U; cpu_idx < psci_plat_core_count; cpu_idx++) { 219 if (cpu_idx == my_idx) { 220 assert(psci_get_aff_info_state() == AFF_STATE_ON); 221 continue; 222 } 223 224 if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) { 225 VERBOSE("core=%u other than current core=%u %s\n", 226 cpu_idx, my_idx, "running in the system"); 227 return false; 228 } 229 } 230 231 return true; 232 } 233 234 /******************************************************************************* 235 * This function verifies that all cores in the system have been turned ON. 236 * Returns true, if all CPUs are ON or false otherwise. 237 ******************************************************************************/ 238 static bool psci_are_all_cpus_on(void) 239 { 240 unsigned int cpu_idx; 241 242 for (cpu_idx = 0U; cpu_idx < psci_plat_core_count; cpu_idx++) { 243 if (psci_get_aff_info_state_by_idx(cpu_idx) == AFF_STATE_OFF) { 244 return false; 245 } 246 } 247 248 return true; 249 } 250 251 /******************************************************************************* 252 * Counts the number of CPUs in the system that are currently in the ON or 253 * ON_PENDING state. 254 * 255 * @note This function does not acquire any power domain locks. It must only be 256 * called in contexts where it is guaranteed that PSCI state transitions 257 * are not concurrently happening, or where locks are already held. 258 * 259 * @return The number of CPUs currently in AFF_STATE_ON or AFF_STATE_ON_PENDING. 260 ******************************************************************************/ 261 static unsigned int psci_num_cpus_running(void) 262 { 263 unsigned int cpu_idx; 264 unsigned int no_of_cpus = 0U; 265 aff_info_state_t aff_state; 266 267 for (cpu_idx = 0U; cpu_idx < psci_plat_core_count; cpu_idx++) { 268 aff_state = psci_get_aff_info_state_by_idx(cpu_idx); 269 if (aff_state == AFF_STATE_ON || 270 aff_state == AFF_STATE_ON_PENDING) { 271 no_of_cpus++; 272 } 273 } 274 275 return no_of_cpus; 276 } 277 278 /******************************************************************************* 279 * Routine to return the maximum power level to traverse to after a cpu has 280 * been physically powered up. It is expected to be called immediately after 281 * reset from assembler code. 282 ******************************************************************************/ 283 static unsigned int get_power_on_target_pwrlvl(void) 284 { 285 unsigned int pwrlvl; 286 287 /* 288 * Assume that this cpu was suspended and retrieve its target power 289 * level. If it wasn't, the cpu is off so this will be PLAT_MAX_PWR_LVL. 290 */ 291 pwrlvl = psci_get_suspend_pwrlvl(); 292 assert(pwrlvl < PSCI_INVALID_PWR_LVL); 293 return pwrlvl; 294 } 295 296 /****************************************************************************** 297 * Helper function to update the requested local power state array. This array 298 * does not store the requested state for the CPU power level. Hence an 299 * assertion is added to prevent us from accessing the CPU power level. 300 *****************************************************************************/ 301 static void psci_set_req_local_pwr_state(unsigned int pwrlvl, 302 unsigned int cpu_idx, 303 plat_local_state_t req_pwr_state) 304 { 305 assert(pwrlvl > PSCI_CPU_PWR_LVL); 306 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 307 (cpu_idx < psci_plat_core_count)) { 308 psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state; 309 } 310 } 311 312 /****************************************************************************** 313 * This function initializes the psci_req_local_pwr_states. 314 *****************************************************************************/ 315 void __init psci_init_req_local_pwr_states(void) 316 { 317 /* Initialize the requested state of all non CPU power domains as OFF */ 318 unsigned int pwrlvl; 319 unsigned int core; 320 321 for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) { 322 for (core = 0; core < psci_plat_core_count; core++) { 323 psci_req_local_pwr_states[pwrlvl][core] = 324 PLAT_MAX_OFF_STATE; 325 } 326 } 327 } 328 329 /****************************************************************************** 330 * Helper function to return a reference to an array containing the local power 331 * states requested by each cpu for a power domain at 'pwrlvl'. The size of the 332 * array will be the number of cpu power domains of which this power domain is 333 * an ancestor. These requested states will be used to determine a suitable 334 * target state for this power domain during psci state coordination. An 335 * assertion is added to prevent us from accessing the CPU power level. 336 *****************************************************************************/ 337 static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl, 338 unsigned int cpu_idx) 339 { 340 assert(pwrlvl > PSCI_CPU_PWR_LVL); 341 342 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 343 (cpu_idx < psci_plat_core_count)) { 344 return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx]; 345 } else 346 return NULL; 347 } 348 349 #if PSCI_OS_INIT_MODE 350 /****************************************************************************** 351 * Helper function to save a copy of the psci_req_local_pwr_states (prev) for a 352 * CPU (cpu_idx), and update psci_req_local_pwr_states with the new requested 353 * local power states (state_info). 354 *****************************************************************************/ 355 void psci_update_req_local_pwr_states(unsigned int end_pwrlvl, 356 unsigned int cpu_idx, 357 psci_power_state_t *state_info, 358 plat_local_state_t *prev) 359 { 360 unsigned int lvl; 361 #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL 362 unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; 363 #else 364 unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL; 365 #endif 366 plat_local_state_t req_state; 367 368 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) { 369 /* Save the previous requested local power state */ 370 prev[lvl - 1U] = *psci_get_req_local_pwr_states(lvl, cpu_idx); 371 372 /* Update the new requested local power state */ 373 if (lvl <= end_pwrlvl) { 374 req_state = state_info->pwr_domain_state[lvl]; 375 } else { 376 req_state = state_info->pwr_domain_state[end_pwrlvl]; 377 } 378 psci_set_req_local_pwr_state(lvl, cpu_idx, req_state); 379 } 380 } 381 382 /****************************************************************************** 383 * Helper function to restore the previously saved requested local power states 384 * (prev) for a CPU (cpu_idx) to psci_req_local_pwr_states. 385 *****************************************************************************/ 386 void psci_restore_req_local_pwr_states(unsigned int cpu_idx, 387 plat_local_state_t *prev) 388 { 389 unsigned int lvl; 390 #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL 391 unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; 392 #else 393 unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL; 394 #endif 395 396 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) { 397 /* Restore the previous requested local power state */ 398 psci_set_req_local_pwr_state(lvl, cpu_idx, prev[lvl - 1U]); 399 } 400 } 401 #endif 402 403 /* 404 * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent 405 * memory. 406 * 407 * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory, 408 * it's accessed by both cached and non-cached participants. To serve the common 409 * minimum, perform a cache flush before read and after write so that non-cached 410 * participants operate on latest data in main memory. 411 * 412 * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent 413 * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent. 414 * In both cases, no cache operations are required. 415 */ 416 417 /* 418 * Retrieve local state of non-CPU power domain node from a non-cached CPU, 419 * after any required cache maintenance operation. 420 */ 421 static plat_local_state_t get_non_cpu_pd_node_local_state( 422 unsigned int parent_idx) 423 { 424 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 425 flush_dcache_range( 426 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 427 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 428 #endif 429 return psci_non_cpu_pd_nodes[parent_idx].local_state; 430 } 431 432 /* 433 * Update local state of non-CPU power domain node from a cached CPU; perform 434 * any required cache maintenance operation afterwards. 435 */ 436 static void set_non_cpu_pd_node_local_state(unsigned int parent_idx, 437 plat_local_state_t state) 438 { 439 psci_non_cpu_pd_nodes[parent_idx].local_state = state; 440 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 441 flush_dcache_range( 442 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 443 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 444 #endif 445 } 446 447 /****************************************************************************** 448 * Helper function to return the current local power state of each power domain 449 * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This 450 * function will be called after a cpu is powered on to find the local state 451 * each power domain has emerged from. 452 *****************************************************************************/ 453 void psci_get_target_local_pwr_states(unsigned int cpu_idx, unsigned int end_pwrlvl, 454 psci_power_state_t *target_state) 455 { 456 unsigned int parent_idx, lvl; 457 plat_local_state_t *pd_state = target_state->pwr_domain_state; 458 459 pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state(); 460 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 461 462 /* Copy the local power state from node to state_info */ 463 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 464 pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx); 465 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 466 } 467 468 /* Set the the higher levels to RUN */ 469 for (; lvl <= PLAT_MAX_PWR_LVL; lvl++) { 470 target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 471 } 472 } 473 474 /****************************************************************************** 475 * Helper function to set the target local power state that each power domain 476 * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will 477 * enter. This function will be called after coordination of requested power 478 * states has been done for each power level. 479 *****************************************************************************/ 480 void psci_set_target_local_pwr_states(unsigned int cpu_idx, unsigned int end_pwrlvl, 481 const psci_power_state_t *target_state) 482 { 483 unsigned int parent_idx, lvl; 484 const plat_local_state_t *pd_state = target_state->pwr_domain_state; 485 486 psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]); 487 488 /* 489 * Need to flush as local_state might be accessed with Data Cache 490 * disabled during power on 491 */ 492 psci_flush_cpu_data(psci_svc_cpu_data.local_state); 493 494 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 495 496 /* Copy the local_state from state_info */ 497 for (lvl = 1U; lvl <= end_pwrlvl; lvl++) { 498 set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]); 499 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 500 } 501 } 502 503 /******************************************************************************* 504 * PSCI helper function to get the parent nodes corresponding to a cpu_index. 505 ******************************************************************************/ 506 void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx, 507 unsigned int end_lvl, 508 unsigned int *node_index) 509 { 510 unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node; 511 unsigned int i; 512 unsigned int *node = node_index; 513 514 for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) { 515 *node = parent_node; 516 node++; 517 parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node; 518 } 519 } 520 521 /****************************************************************************** 522 * This function is invoked post CPU power up and initialization. It sets the 523 * affinity info state, target power state and requested power state for the 524 * current CPU and all its ancestor power domains to RUN. 525 *****************************************************************************/ 526 void psci_set_pwr_domains_to_run(unsigned int cpu_idx, unsigned int end_pwrlvl) 527 { 528 unsigned int parent_idx, lvl; 529 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 530 531 /* Reset the local_state to RUN for the non cpu power domains. */ 532 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 533 set_non_cpu_pd_node_local_state(parent_idx, 534 PSCI_LOCAL_STATE_RUN); 535 psci_set_req_local_pwr_state(lvl, 536 cpu_idx, 537 PSCI_LOCAL_STATE_RUN); 538 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 539 } 540 541 /* Set the affinity info state to ON */ 542 psci_set_aff_info_state(AFF_STATE_ON); 543 544 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); 545 psci_flush_cpu_data(psci_svc_cpu_data); 546 } 547 548 /****************************************************************************** 549 * This function is used in platform-coordinated mode. 550 * 551 * This function is passed the local power states requested for each power 552 * domain (state_info) between the current CPU domain and its ancestors until 553 * the target power level (end_pwrlvl). It updates the array of requested power 554 * states with this information. 555 * 556 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it 557 * retrieves the states requested by all the cpus of which the power domain at 558 * that level is an ancestor. It passes this information to the platform to 559 * coordinate and return the target power state. If the target state for a level 560 * is RUN then subsequent levels are not considered. At the CPU level, state 561 * coordination is not required. Hence, the requested and the target states are 562 * the same. 563 * 564 * The 'state_info' is updated with the target state for each level between the 565 * CPU and the 'end_pwrlvl' and returned to the caller. 566 * 567 * This function will only be invoked with data cache enabled and while 568 * powering down a core. 569 *****************************************************************************/ 570 void psci_do_state_coordination(unsigned int cpu_idx, unsigned int end_pwrlvl, 571 psci_power_state_t *state_info) 572 { 573 unsigned int lvl, parent_idx; 574 unsigned int start_idx; 575 unsigned int ncpus; 576 plat_local_state_t target_state; 577 578 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); 579 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 580 581 /* For level 0, the requested state will be equivalent 582 to target state */ 583 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 584 585 /* First update the requested power state */ 586 psci_set_req_local_pwr_state(lvl, cpu_idx, 587 state_info->pwr_domain_state[lvl]); 588 589 /* Get the requested power states for this power level */ 590 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; 591 plat_local_state_t const *req_states = psci_get_req_local_pwr_states(lvl, 592 start_idx); 593 594 /* 595 * Let the platform coordinate amongst the requested states at 596 * this power level and return the target local power state. 597 */ 598 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; 599 target_state = plat_get_target_pwr_state(lvl, 600 req_states, 601 ncpus); 602 603 state_info->pwr_domain_state[lvl] = target_state; 604 605 /* Break early if the negotiated target power state is RUN */ 606 if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) { 607 break; 608 } 609 610 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 611 } 612 613 /* 614 * This is for cases when we break out of the above loop early because 615 * the target power state is RUN at a power level < end_pwlvl. 616 * We update the requested power state from state_info and then 617 * set the target state as RUN. 618 */ 619 for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) { 620 psci_set_req_local_pwr_state(lvl, cpu_idx, 621 state_info->pwr_domain_state[lvl]); 622 state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 623 624 } 625 } 626 627 #if PSCI_OS_INIT_MODE 628 /****************************************************************************** 629 * This function is used in OS-initiated mode. 630 * 631 * This function is passed the local power states requested for each power 632 * domain (state_info) between the current CPU domain and its ancestors until 633 * the target power level (end_pwrlvl), and ensures the requested power states 634 * are valid. It updates the array of requested power states with this 635 * information. 636 * 637 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it 638 * retrieves the states requested by all the cpus of which the power domain at 639 * that level is an ancestor. It passes this information to the platform to 640 * coordinate and return the target power state. If the requested state does 641 * not match the target state, the request is denied. 642 * 643 * The 'state_info' is not modified. 644 * 645 * This function will only be invoked with data cache enabled and while 646 * powering down a core. 647 *****************************************************************************/ 648 int psci_validate_state_coordination(unsigned int cpu_idx, unsigned int end_pwrlvl, 649 psci_power_state_t *state_info) 650 { 651 int rc = PSCI_E_SUCCESS; 652 unsigned int lvl, parent_idx; 653 unsigned int start_idx; 654 unsigned int ncpus; 655 plat_local_state_t target_state, *req_states; 656 plat_local_state_t prev[PLAT_MAX_PWR_LVL]; 657 658 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); 659 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 660 661 /* 662 * Save a copy of the previous requested local power states and update 663 * the new requested local power states. 664 */ 665 psci_update_req_local_pwr_states(end_pwrlvl, cpu_idx, state_info, prev); 666 667 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 668 /* Get the requested power states for this power level */ 669 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; 670 req_states = psci_get_req_local_pwr_states(lvl, start_idx); 671 672 /* 673 * Let the platform coordinate amongst the requested states at 674 * this power level and return the target local power state. 675 */ 676 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; 677 target_state = plat_get_target_pwr_state(lvl, 678 req_states, 679 ncpus); 680 681 /* 682 * Verify that the requested power state matches the target 683 * local power state. 684 */ 685 if (state_info->pwr_domain_state[lvl] != target_state) { 686 if (target_state == PSCI_LOCAL_STATE_RUN) { 687 rc = PSCI_E_DENIED; 688 } else { 689 rc = PSCI_E_INVALID_PARAMS; 690 } 691 goto exit; 692 } 693 694 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 695 } 696 697 /* 698 * Verify that the current core is the last running core at the 699 * specified power level. 700 */ 701 lvl = state_info->last_at_pwrlvl; 702 if (!psci_is_last_cpu_to_idle_at_pwrlvl(cpu_idx, lvl)) { 703 rc = PSCI_E_DENIED; 704 } 705 706 exit: 707 if (rc != PSCI_E_SUCCESS) { 708 /* Restore the previous requested local power states. */ 709 psci_restore_req_local_pwr_states(cpu_idx, prev); 710 return rc; 711 } 712 713 return rc; 714 } 715 #endif 716 717 /****************************************************************************** 718 * This function validates a suspend request by making sure that if a standby 719 * state is requested then no power level is turned off and the highest power 720 * level is placed in a standby/retention state. 721 * 722 * It also ensures that the state level X will enter is not shallower than the 723 * state level X + 1 will enter. 724 * 725 * This validation will be enabled only for DEBUG builds as the platform is 726 * expected to perform these validations as well. 727 *****************************************************************************/ 728 int psci_validate_suspend_req(const psci_power_state_t *state_info, 729 unsigned int is_power_down_state) 730 { 731 unsigned int max_off_lvl, target_lvl, max_retn_lvl; 732 plat_local_state_t state; 733 plat_local_state_type_t req_state_type, deepest_state_type; 734 int i; 735 736 /* Find the target suspend power level */ 737 target_lvl = psci_find_target_suspend_lvl(state_info); 738 if (target_lvl == PSCI_INVALID_PWR_LVL) 739 return PSCI_E_INVALID_PARAMS; 740 741 /* All power domain levels are in a RUN state to begin with */ 742 deepest_state_type = STATE_TYPE_RUN; 743 744 for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) { 745 state = state_info->pwr_domain_state[i]; 746 req_state_type = find_local_state_type(state); 747 748 /* 749 * While traversing from the highest power level to the lowest, 750 * the state requested for lower levels has to be the same or 751 * deeper i.e. equal to or greater than the state at the higher 752 * levels. If this condition is true, then the requested state 753 * becomes the deepest state encountered so far. 754 */ 755 if (req_state_type < deepest_state_type) 756 return PSCI_E_INVALID_PARAMS; 757 deepest_state_type = req_state_type; 758 } 759 760 /* Find the highest off power level */ 761 max_off_lvl = psci_find_max_off_lvl(state_info); 762 763 /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */ 764 max_retn_lvl = PSCI_INVALID_PWR_LVL; 765 if (target_lvl != max_off_lvl) 766 max_retn_lvl = target_lvl; 767 768 /* 769 * If this is not a request for a power down state then max off level 770 * has to be invalid and max retention level has to be a valid power 771 * level. 772 */ 773 if ((is_power_down_state == 0U) && 774 ((max_off_lvl != PSCI_INVALID_PWR_LVL) || 775 (max_retn_lvl == PSCI_INVALID_PWR_LVL))) 776 return PSCI_E_INVALID_PARAMS; 777 778 return PSCI_E_SUCCESS; 779 } 780 781 /****************************************************************************** 782 * This function finds the highest power level which will be powered down 783 * amongst all the power levels specified in the 'state_info' structure 784 *****************************************************************************/ 785 unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info) 786 { 787 int i; 788 789 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 790 if (is_local_state_off(state_info->pwr_domain_state[i]) != 0) { 791 return (unsigned int) i; 792 } 793 } 794 795 return PSCI_INVALID_PWR_LVL; 796 } 797 798 /****************************************************************************** 799 * This functions finds the level of the highest power domain which will be 800 * placed in a low power state during a suspend operation. 801 *****************************************************************************/ 802 unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info) 803 { 804 int i; 805 806 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 807 if (is_local_state_run(state_info->pwr_domain_state[i]) == 0) 808 return (unsigned int) i; 809 } 810 811 return PSCI_INVALID_PWR_LVL; 812 } 813 814 /******************************************************************************* 815 * This function is passed the highest level in the topology tree that the 816 * operation should be applied to and a list of node indexes. It picks up locks 817 * from the node index list in order of increasing power domain level in the 818 * range specified. 819 ******************************************************************************/ 820 void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, 821 const unsigned int *parent_nodes) 822 { 823 unsigned int parent_idx; 824 unsigned int level; 825 826 /* No locking required for level 0. Hence start locking from level 1 */ 827 for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) { 828 parent_idx = parent_nodes[level - 1U]; 829 psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]); 830 } 831 } 832 833 /******************************************************************************* 834 * This function is passed the highest level in the topology tree that the 835 * operation should be applied to and a list of node indexes. It releases the 836 * locks in order of decreasing power domain level in the range specified. 837 ******************************************************************************/ 838 void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, 839 const unsigned int *parent_nodes) 840 { 841 unsigned int parent_idx; 842 unsigned int level; 843 844 /* Unlock top down. No unlocking required for level 0. */ 845 for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) { 846 parent_idx = parent_nodes[level - 1U]; 847 psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]); 848 } 849 } 850 851 /******************************************************************************* 852 * This function determines the full entrypoint information for the requested 853 * PSCI entrypoint on power on/resume and returns it. 854 ******************************************************************************/ 855 #ifdef __aarch64__ 856 static int psci_get_ns_ep_info(entry_point_info_t *ep, 857 uintptr_t entrypoint, 858 u_register_t context_id) 859 { 860 u_register_t ep_attr, sctlr; 861 unsigned int daif, ee, mode; 862 u_register_t ns_scr_el3 = read_scr_el3(); 863 u_register_t ns_sctlr_el1 = read_sctlr_el1(); 864 865 sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 866 read_sctlr_el2() : ns_sctlr_el1; 867 ee = 0; 868 869 ep_attr = NON_SECURE | EP_ST_DISABLE; 870 if ((sctlr & SCTLR_EE_BIT) != 0U) { 871 ep_attr |= EP_EE_BIG; 872 ee = 1; 873 } 874 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 875 876 ep->pc = entrypoint; 877 zeromem(&ep->args, sizeof(ep->args)); 878 ep->args.arg0 = context_id; 879 880 /* 881 * Figure out whether the cpu enters the non-secure address space 882 * in aarch32 or aarch64 883 */ 884 if ((ns_scr_el3 & SCR_RW_BIT) != 0U) { 885 886 /* 887 * Check whether a Thumb entry point has been provided for an 888 * aarch64 EL 889 */ 890 if ((entrypoint & 0x1UL) != 0UL) 891 return PSCI_E_INVALID_ADDRESS; 892 893 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1; 894 895 ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX, 896 DISABLE_ALL_EXCEPTIONS); 897 } else { 898 899 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 900 MODE32_hyp : MODE32_svc; 901 902 /* 903 * TODO: Choose async. exception bits if HYP mode is not 904 * implemented according to the values of SCR.{AW, FW} bits 905 */ 906 daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT; 907 908 ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee, 909 daif); 910 } 911 912 return PSCI_E_SUCCESS; 913 } 914 #else /* !__aarch64__ */ 915 static int psci_get_ns_ep_info(entry_point_info_t *ep, 916 uintptr_t entrypoint, 917 u_register_t context_id) 918 { 919 u_register_t ep_attr; 920 unsigned int aif, ee, mode; 921 u_register_t scr = read_scr(); 922 u_register_t ns_sctlr, sctlr; 923 924 /* Switch to non secure state */ 925 write_scr(scr | SCR_NS_BIT); 926 isb(); 927 ns_sctlr = read_sctlr(); 928 929 sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr; 930 931 /* Return to original state */ 932 write_scr(scr); 933 isb(); 934 ee = 0; 935 936 ep_attr = NON_SECURE | EP_ST_DISABLE; 937 if (sctlr & SCTLR_EE_BIT) { 938 ep_attr |= EP_EE_BIG; 939 ee = 1; 940 } 941 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 942 943 ep->pc = entrypoint; 944 zeromem(&ep->args, sizeof(ep->args)); 945 ep->args.arg0 = context_id; 946 947 mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc; 948 949 /* 950 * TODO: Choose async. exception bits if HYP mode is not 951 * implemented according to the values of SCR.{AW, FW} bits 952 */ 953 aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT; 954 955 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif); 956 957 return PSCI_E_SUCCESS; 958 } 959 960 #endif /* __aarch64__ */ 961 962 /******************************************************************************* 963 * This function validates the entrypoint with the platform layer if the 964 * appropriate pm_ops hook is exported by the platform and returns the 965 * 'entry_point_info'. 966 ******************************************************************************/ 967 int psci_validate_entry_point(entry_point_info_t *ep, 968 uintptr_t entrypoint, 969 u_register_t context_id) 970 { 971 int rc; 972 973 /* Validate the entrypoint using platform psci_ops */ 974 if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) { 975 rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint); 976 if (rc != PSCI_E_SUCCESS) { 977 return PSCI_E_INVALID_ADDRESS; 978 } 979 } 980 981 /* 982 * Verify and derive the re-entry information for 983 * the non-secure world from the non-secure state from 984 * where this call originated. 985 */ 986 rc = psci_get_ns_ep_info(ep, entrypoint, context_id); 987 return rc; 988 } 989 990 /******************************************************************************* 991 * Generic handler which is called when a cpu is physically powered on. It 992 * traverses the node information and finds the highest power level powered 993 * off and performs generic, architectural, platform setup and state management 994 * to power on that power level and power levels below it. 995 * e.g. For a cpu that's been powered on, it will call the platform specific 996 * code to enable the gic cpu interface and for a cluster it will enable 997 * coherency at the interconnect level in addition to gic cpu interface. 998 ******************************************************************************/ 999 void psci_warmboot_entrypoint(void) 1000 { 1001 unsigned int end_pwrlvl; 1002 unsigned int cpu_idx = plat_my_core_pos(); 1003 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 1004 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; 1005 1006 /* Init registers that never change for the lifetime of TF-A */ 1007 cm_manage_extensions_el3(cpu_idx); 1008 1009 /* 1010 * Verify that we have been explicitly turned ON or resumed from 1011 * suspend. 1012 */ 1013 if (psci_get_aff_info_state() == AFF_STATE_OFF) { 1014 ERROR("Unexpected affinity info state.\n"); 1015 panic(); 1016 } 1017 1018 /* 1019 * Get the maximum power domain level to traverse to after this cpu 1020 * has been physically powered up. 1021 */ 1022 end_pwrlvl = get_power_on_target_pwrlvl(); 1023 1024 /* Get the parent nodes */ 1025 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); 1026 1027 /* 1028 * This function acquires the lock corresponding to each power level so 1029 * that by the time all locks are taken, the system topology is snapshot 1030 * and state management can be done safely. 1031 */ 1032 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 1033 1034 psci_get_target_local_pwr_states(cpu_idx, end_pwrlvl, &state_info); 1035 1036 #if ENABLE_PSCI_STAT 1037 plat_psci_stat_accounting_stop(&state_info); 1038 #endif 1039 1040 /* 1041 * This CPU could be resuming from suspend or it could have just been 1042 * turned on. To distinguish between these 2 cases, we examine the 1043 * affinity state of the CPU: 1044 * - If the affinity state is ON_PENDING then it has just been 1045 * turned on. 1046 * - Else it is resuming from suspend. 1047 * 1048 * Depending on the type of warm reset identified, choose the right set 1049 * of power management handler and perform the generic, architecture 1050 * and platform specific handling. 1051 */ 1052 if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING) { 1053 psci_cpu_on_finish(cpu_idx, &state_info); 1054 } else { 1055 unsigned int max_off_lvl = psci_find_max_off_lvl(&state_info); 1056 1057 assert(max_off_lvl != PSCI_INVALID_PWR_LVL); 1058 psci_cpu_suspend_to_powerdown_finish(cpu_idx, max_off_lvl, &state_info); 1059 } 1060 1061 /* 1062 * Caches and (importantly) coherency are on so we can rely on seeing 1063 * whatever the primary gave us without explicit cache maintenance 1064 */ 1065 entry_point_info_t *ep = get_cpu_data(warmboot_ep_info); 1066 cm_init_my_context(ep); 1067 1068 /* 1069 * Generic management: Now we just need to retrieve the 1070 * information that we had stashed away during the cpu_on 1071 * call to set this cpu on its way. 1072 */ 1073 cm_prepare_el3_exit_ns(); 1074 1075 /* 1076 * Set the requested and target state of this CPU and all the higher 1077 * power domains which are ancestors of this CPU to run. 1078 */ 1079 psci_set_pwr_domains_to_run(cpu_idx, end_pwrlvl); 1080 1081 #if ENABLE_PSCI_STAT 1082 psci_stats_update_pwr_up(cpu_idx, end_pwrlvl, &state_info); 1083 #endif 1084 1085 /* 1086 * This loop releases the lock corresponding to each power level 1087 * in the reverse order to which they were acquired. 1088 */ 1089 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 1090 } 1091 1092 /******************************************************************************* 1093 * This function initializes the set of hooks that PSCI invokes as part of power 1094 * management operation. The power management hooks are expected to be provided 1095 * by the SPD, after it finishes all its initialization 1096 ******************************************************************************/ 1097 void psci_register_spd_pm_hook(const spd_pm_ops_t *pm) 1098 { 1099 assert(pm != NULL); 1100 psci_spd_pm = pm; 1101 1102 if (pm->svc_migrate != NULL) 1103 psci_caps |= define_psci_cap(PSCI_MIG_AARCH64); 1104 1105 if (pm->svc_migrate_info != NULL) 1106 psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) 1107 | define_psci_cap(PSCI_MIG_INFO_TYPE); 1108 } 1109 1110 /******************************************************************************* 1111 * This function invokes the migrate info hook in the spd_pm_ops. It performs 1112 * the necessary return value validation. If the Secure Payload is UP and 1113 * migrate capable, it returns the mpidr of the CPU on which the Secure payload 1114 * is resident through the mpidr parameter. Else the value of the parameter on 1115 * return is undefined. 1116 ******************************************************************************/ 1117 int psci_spd_migrate_info(u_register_t *mpidr) 1118 { 1119 int rc; 1120 1121 if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL)) 1122 return PSCI_E_NOT_SUPPORTED; 1123 1124 rc = psci_spd_pm->svc_migrate_info(mpidr); 1125 1126 assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) || 1127 (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED)); 1128 1129 return rc; 1130 } 1131 1132 1133 /******************************************************************************* 1134 * This function prints the state of all power domains present in the 1135 * system 1136 ******************************************************************************/ 1137 void psci_print_power_domain_map(void) 1138 { 1139 #if LOG_LEVEL >= LOG_LEVEL_INFO 1140 unsigned int idx; 1141 plat_local_state_t state; 1142 plat_local_state_type_t state_type; 1143 1144 /* This array maps to the PSCI_STATE_X definitions in psci.h */ 1145 static const char * const psci_state_type_str[] = { 1146 "ON", 1147 "RETENTION", 1148 "OFF", 1149 }; 1150 1151 INFO("PSCI Power Domain Map:\n"); 1152 for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count); 1153 idx++) { 1154 state_type = find_local_state_type( 1155 psci_non_cpu_pd_nodes[idx].local_state); 1156 INFO(" Domain Node : Level %u, parent_node %u," 1157 " State %s (0x%x)\n", 1158 psci_non_cpu_pd_nodes[idx].level, 1159 psci_non_cpu_pd_nodes[idx].parent_node, 1160 psci_state_type_str[state_type], 1161 psci_non_cpu_pd_nodes[idx].local_state); 1162 } 1163 1164 for (idx = 0; idx < psci_plat_core_count; idx++) { 1165 state = psci_get_cpu_local_state_by_idx(idx); 1166 state_type = find_local_state_type(state); 1167 INFO(" CPU Node : MPID 0x%llx, parent_node %u," 1168 " State %s (0x%x)\n", 1169 (unsigned long long)psci_cpu_pd_nodes[idx].mpidr, 1170 psci_cpu_pd_nodes[idx].parent_node, 1171 psci_state_type_str[state_type], 1172 psci_get_cpu_local_state_by_idx(idx)); 1173 } 1174 #endif 1175 } 1176 1177 /****************************************************************************** 1178 * Return whether any secondaries were powered up with CPU_ON call. A CPU that 1179 * have ever been powered up would have set its MPDIR value to something other 1180 * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to 1181 * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is 1182 * meaningful only when called on the primary CPU during early boot. 1183 *****************************************************************************/ 1184 int psci_secondaries_brought_up(void) 1185 { 1186 unsigned int idx, n_valid = 0U; 1187 1188 for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) { 1189 if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR) 1190 n_valid++; 1191 } 1192 1193 assert(n_valid > 0U); 1194 1195 return (n_valid > 1U) ? 1 : 0; 1196 } 1197 1198 /******************************************************************************* 1199 * Initiate power down sequence, by calling power down operations registered for 1200 * this CPU. 1201 ******************************************************************************/ 1202 void psci_pwrdown_cpu_start(unsigned int power_level) 1203 { 1204 #if ENABLE_RUNTIME_INSTRUMENTATION 1205 1206 /* 1207 * Flush cache line so that even if CPU power down happens 1208 * the timestamp update is reflected in memory. 1209 */ 1210 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 1211 RT_INSTR_ENTER_CFLUSH, 1212 PMF_CACHE_MAINT); 1213 #endif 1214 1215 #if HW_ASSISTED_COHERENCY 1216 /* 1217 * With hardware-assisted coherency, the CPU drivers only initiate the 1218 * power down sequence, without performing cache-maintenance operations 1219 * in software. Data caches enabled both before and after this call. 1220 */ 1221 prepare_cpu_pwr_dwn(power_level); 1222 #else 1223 /* 1224 * Without hardware-assisted coherency, the CPU drivers disable data 1225 * caches, then perform cache-maintenance operations in software. 1226 * 1227 * This also calls prepare_cpu_pwr_dwn() to initiate power down 1228 * sequence, but that function will return with data caches disabled. 1229 * We must ensure that the stack memory is flushed out to memory before 1230 * we start popping from it again. 1231 */ 1232 psci_do_pwrdown_cache_maintenance(power_level); 1233 #endif 1234 1235 #if ENABLE_RUNTIME_INSTRUMENTATION 1236 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 1237 RT_INSTR_EXIT_CFLUSH, 1238 PMF_NO_CACHE_MAINT); 1239 #endif 1240 } 1241 1242 /******************************************************************************* 1243 * Finish a terminal power down sequence, ending with a wfi. In case of wakeup 1244 * will retry the sleep and panic if it persists. 1245 ******************************************************************************/ 1246 void __dead2 psci_pwrdown_cpu_end_terminal(void) 1247 { 1248 #if ERRATA_SME_POWER_DOWN 1249 /* 1250 * force SME off to not get power down rejected. Getting here is 1251 * terminal so we don't care if we lose context because of another 1252 * wakeup 1253 */ 1254 if (is_feat_sme_supported()) { 1255 write_svcr(0); 1256 isb(); 1257 } 1258 #endif /* ERRATA_SME_POWER_DOWN */ 1259 1260 /* 1261 * Execute a wfi which, in most cases, will allow the power controller 1262 * to physically power down this cpu. Under some circumstances that may 1263 * be denied. Hopefully this is transient, retrying a few times should 1264 * power down. 1265 */ 1266 for (int i = 0; i < 32; i++) 1267 psci_power_down_wfi(); 1268 1269 /* Wake up wasn't transient. System is probably in a bad state. */ 1270 ERROR("Could not power off CPU.\n"); 1271 panic(); 1272 } 1273 1274 /******************************************************************************* 1275 * Finish a non-terminal power down sequence, ending with a wfi. In case of 1276 * wakeup will unwind any CPU specific actions and return. 1277 ******************************************************************************/ 1278 1279 void psci_pwrdown_cpu_end_wakeup(unsigned int power_level) 1280 { 1281 /* 1282 * Usually, will be terminal. In some circumstances the powerdown will 1283 * be denied and we'll need to unwind 1284 */ 1285 psci_power_down_wfi(); 1286 1287 /* 1288 * Waking up does not require hardware-assisted coherency, but that is 1289 * the case for every core that can wake up. Untangling the cache 1290 * coherency code from powerdown is a non-trivial effort which isn't 1291 * needed for our purposes. 1292 */ 1293 #if !FEAT_PABANDON 1294 ERROR("Systems without FEAT_PABANDON shouldn't wake up.\n"); 1295 panic(); 1296 #else /* FEAT_PABANDON */ 1297 1298 /* 1299 * Begin unwinding. Everything can be shared with CPU_ON and co later, 1300 * except the CPU specific bit. Cores that have hardware-assisted 1301 * coherency don't have much to do so just calling the hook again is 1302 * the simplest way to achieve this 1303 */ 1304 prepare_cpu_pwr_dwn(power_level); 1305 #endif /* FEAT_PABANDON */ 1306 } 1307 1308 /******************************************************************************* 1309 * This function invokes the callback 'stop_func()' with the 'mpidr' of each 1310 * online PE. Caller can pass suitable method to stop a remote core. 1311 * 1312 * 'wait_ms' is the timeout value in milliseconds for the other cores to 1313 * transition to power down state. Passing '0' makes it non-blocking. 1314 * 1315 * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the 1316 * given timeout. 1317 ******************************************************************************/ 1318 int psci_stop_other_cores(unsigned int this_cpu_idx, unsigned int wait_ms, 1319 void (*stop_func)(u_register_t mpidr)) 1320 { 1321 /* Invoke stop_func for each core */ 1322 for (unsigned int idx = 0U; idx < psci_plat_core_count; idx++) { 1323 /* skip current CPU */ 1324 if (idx == this_cpu_idx) { 1325 continue; 1326 } 1327 1328 /* Check if the CPU is ON */ 1329 if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) { 1330 (*stop_func)(psci_cpu_pd_nodes[idx].mpidr); 1331 } 1332 } 1333 1334 /* Need to wait for other cores to shutdown */ 1335 if (wait_ms != 0U) { 1336 for (uint32_t delay_ms = wait_ms; ((delay_ms != 0U) && 1337 (!psci_is_last_on_cpu(this_cpu_idx))); delay_ms--) { 1338 mdelay(1U); 1339 } 1340 1341 if (!psci_is_last_on_cpu(this_cpu_idx)) { 1342 WARN("Failed to stop all cores!\n"); 1343 psci_print_power_domain_map(); 1344 return PSCI_E_DENIED; 1345 } 1346 } 1347 1348 return PSCI_E_SUCCESS; 1349 } 1350 1351 /******************************************************************************* 1352 * This function verifies that all the other cores in the system have been 1353 * turned OFF and the current CPU is the last running CPU in the system. 1354 * Returns true if the current CPU is the last ON CPU or false otherwise. 1355 * 1356 * This API has following differences with psci_is_last_on_cpu 1357 * 1. PSCI states are locked 1358 ******************************************************************************/ 1359 bool psci_is_last_on_cpu_safe(unsigned int this_core) 1360 { 1361 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 1362 1363 psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); 1364 1365 psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1366 1367 if (!psci_is_last_on_cpu(this_core)) { 1368 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1369 return false; 1370 } 1371 1372 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1373 1374 return true; 1375 } 1376 1377 /******************************************************************************* 1378 * This function verifies that all cores in the system have been turned ON. 1379 * Returns true, if all CPUs are ON or false otherwise. 1380 * 1381 * This API has following differences with psci_are_all_cpus_on 1382 * 1. PSCI states are locked 1383 ******************************************************************************/ 1384 bool psci_are_all_cpus_on_safe(unsigned int this_core) 1385 { 1386 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 1387 1388 psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); 1389 1390 psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1391 1392 if (!psci_are_all_cpus_on()) { 1393 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1394 return false; 1395 } 1396 1397 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1398 1399 return true; 1400 } 1401 1402 /******************************************************************************* 1403 * Safely counts the number of CPUs in the system that are currently in the ON 1404 * or ON_PENDING state. 1405 * 1406 * This function acquires and releases the necessary power domain locks to 1407 * ensure consistency of the CPU state information. 1408 * 1409 * @param this_core The index of the current core making the query. 1410 * 1411 * @return The number of CPUs currently in AFF_STATE_ON or AFF_STATE_ON_PENDING. 1412 ******************************************************************************/ 1413 unsigned int psci_num_cpus_running_on_safe(unsigned int this_core) 1414 { 1415 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 1416 unsigned int no_of_cpus; 1417 1418 psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); 1419 1420 psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1421 1422 no_of_cpus = psci_num_cpus_running(); 1423 1424 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1425 1426 return no_of_cpus; 1427 } 1428