1 /* 2 * Copyright (c) 2013-2022, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <string.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <drivers/delay_timer.h> 16 #include <lib/el3_runtime/context_mgmt.h> 17 #include <lib/utils.h> 18 #include <plat/common/platform.h> 19 20 #include "psci_private.h" 21 22 /* 23 * SPD power management operations, expected to be supplied by the registered 24 * SPD on successful SP initialization 25 */ 26 const spd_pm_ops_t *psci_spd_pm; 27 28 /* 29 * PSCI requested local power state map. This array is used to store the local 30 * power states requested by a CPU for power levels from level 1 to 31 * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power 32 * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a 33 * CPU are the same. 34 * 35 * During state coordination, the platform is passed an array containing the 36 * local states requested for a particular non cpu power domain by each cpu 37 * within the domain. 38 * 39 * TODO: Dense packing of the requested states will cause cache thrashing 40 * when multiple power domains write to it. If we allocate the requested 41 * states at each power level in a cache-line aligned per-domain memory, 42 * the cache thrashing can be avoided. 43 */ 44 static plat_local_state_t 45 psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT]; 46 47 unsigned int psci_plat_core_count; 48 49 /******************************************************************************* 50 * Arrays that hold the platform's power domain tree information for state 51 * management of power domains. 52 * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain 53 * which is an ancestor of a CPU power domain. 54 * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain 55 ******************************************************************************/ 56 non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS] 57 #if USE_COHERENT_MEM 58 __section("tzfw_coherent_mem") 59 #endif 60 ; 61 62 /* Lock for PSCI state coordination */ 63 DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); 64 65 cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; 66 67 /******************************************************************************* 68 * Pointer to functions exported by the platform to complete power mgmt. ops 69 ******************************************************************************/ 70 const plat_psci_ops_t *psci_plat_pm_ops; 71 72 /****************************************************************************** 73 * Check that the maximum power level supported by the platform makes sense 74 *****************************************************************************/ 75 CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) && 76 (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL), 77 assert_platform_max_pwrlvl_check); 78 79 /* 80 * The plat_local_state used by the platform is one of these types: RUN, 81 * RETENTION and OFF. The platform can define further sub-states for each type 82 * apart from RUN. This categorization is done to verify the sanity of the 83 * psci_power_state passed by the platform and to print debug information. The 84 * categorization is done on the basis of the following conditions: 85 * 86 * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN. 87 * 88 * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is 89 * STATE_TYPE_RETN. 90 * 91 * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is 92 * STATE_TYPE_OFF. 93 */ 94 typedef enum plat_local_state_type { 95 STATE_TYPE_RUN = 0, 96 STATE_TYPE_RETN, 97 STATE_TYPE_OFF 98 } plat_local_state_type_t; 99 100 /* Function used to categorize plat_local_state. */ 101 static plat_local_state_type_t find_local_state_type(plat_local_state_t state) 102 { 103 if (state != 0U) { 104 if (state > PLAT_MAX_RET_STATE) { 105 return STATE_TYPE_OFF; 106 } else { 107 return STATE_TYPE_RETN; 108 } 109 } else { 110 return STATE_TYPE_RUN; 111 } 112 } 113 114 /****************************************************************************** 115 * Check that the maximum retention level supported by the platform is less 116 * than the maximum off level. 117 *****************************************************************************/ 118 CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE, 119 assert_platform_max_off_and_retn_state_check); 120 121 /****************************************************************************** 122 * This function ensures that the power state parameter in a CPU_SUSPEND request 123 * is valid. If so, it returns the requested states for each power level. 124 *****************************************************************************/ 125 int psci_validate_power_state(unsigned int power_state, 126 psci_power_state_t *state_info) 127 { 128 /* Check SBZ bits in power state are zero */ 129 if (psci_check_power_state(power_state) != 0U) 130 return PSCI_E_INVALID_PARAMS; 131 132 assert(psci_plat_pm_ops->validate_power_state != NULL); 133 134 /* Validate the power_state using platform pm_ops */ 135 return psci_plat_pm_ops->validate_power_state(power_state, state_info); 136 } 137 138 /****************************************************************************** 139 * This function retrieves the `psci_power_state_t` for system suspend from 140 * the platform. 141 *****************************************************************************/ 142 void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info) 143 { 144 /* 145 * Assert that the required pm_ops hook is implemented to ensure that 146 * the capability detected during psci_setup() is valid. 147 */ 148 assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL); 149 150 /* 151 * Query the platform for the power_state required for system suspend 152 */ 153 psci_plat_pm_ops->get_sys_suspend_power_state(state_info); 154 } 155 156 /******************************************************************************* 157 * This function verifies that the all the other cores in the system have been 158 * turned OFF and the current CPU is the last running CPU in the system. 159 * Returns true, if the current CPU is the last ON CPU or false otherwise. 160 ******************************************************************************/ 161 bool psci_is_last_on_cpu(void) 162 { 163 unsigned int cpu_idx, my_idx = plat_my_core_pos(); 164 165 for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) { 166 if (cpu_idx == my_idx) { 167 assert(psci_get_aff_info_state() == AFF_STATE_ON); 168 continue; 169 } 170 171 if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) { 172 VERBOSE("core=%u other than current core=%u %s\n", 173 cpu_idx, my_idx, "running in the system"); 174 return false; 175 } 176 } 177 178 return true; 179 } 180 181 /******************************************************************************* 182 * Routine to return the maximum power level to traverse to after a cpu has 183 * been physically powered up. It is expected to be called immediately after 184 * reset from assembler code. 185 ******************************************************************************/ 186 static unsigned int get_power_on_target_pwrlvl(void) 187 { 188 unsigned int pwrlvl; 189 190 /* 191 * Assume that this cpu was suspended and retrieve its target power 192 * level. If it is invalid then it could only have been turned off 193 * earlier. PLAT_MAX_PWR_LVL will be the highest power level a 194 * cpu can be turned off to. 195 */ 196 pwrlvl = psci_get_suspend_pwrlvl(); 197 if (pwrlvl == PSCI_INVALID_PWR_LVL) 198 pwrlvl = PLAT_MAX_PWR_LVL; 199 assert(pwrlvl < PSCI_INVALID_PWR_LVL); 200 return pwrlvl; 201 } 202 203 /****************************************************************************** 204 * Helper function to update the requested local power state array. This array 205 * does not store the requested state for the CPU power level. Hence an 206 * assertion is added to prevent us from accessing the CPU power level. 207 *****************************************************************************/ 208 static void psci_set_req_local_pwr_state(unsigned int pwrlvl, 209 unsigned int cpu_idx, 210 plat_local_state_t req_pwr_state) 211 { 212 assert(pwrlvl > PSCI_CPU_PWR_LVL); 213 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 214 (cpu_idx < psci_plat_core_count)) { 215 psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state; 216 } 217 } 218 219 /****************************************************************************** 220 * This function initializes the psci_req_local_pwr_states. 221 *****************************************************************************/ 222 void __init psci_init_req_local_pwr_states(void) 223 { 224 /* Initialize the requested state of all non CPU power domains as OFF */ 225 unsigned int pwrlvl; 226 unsigned int core; 227 228 for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) { 229 for (core = 0; core < psci_plat_core_count; core++) { 230 psci_req_local_pwr_states[pwrlvl][core] = 231 PLAT_MAX_OFF_STATE; 232 } 233 } 234 } 235 236 /****************************************************************************** 237 * Helper function to return a reference to an array containing the local power 238 * states requested by each cpu for a power domain at 'pwrlvl'. The size of the 239 * array will be the number of cpu power domains of which this power domain is 240 * an ancestor. These requested states will be used to determine a suitable 241 * target state for this power domain during psci state coordination. An 242 * assertion is added to prevent us from accessing the CPU power level. 243 *****************************************************************************/ 244 static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl, 245 unsigned int cpu_idx) 246 { 247 assert(pwrlvl > PSCI_CPU_PWR_LVL); 248 249 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 250 (cpu_idx < psci_plat_core_count)) { 251 return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx]; 252 } else 253 return NULL; 254 } 255 256 /* 257 * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent 258 * memory. 259 * 260 * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory, 261 * it's accessed by both cached and non-cached participants. To serve the common 262 * minimum, perform a cache flush before read and after write so that non-cached 263 * participants operate on latest data in main memory. 264 * 265 * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent 266 * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent. 267 * In both cases, no cache operations are required. 268 */ 269 270 /* 271 * Retrieve local state of non-CPU power domain node from a non-cached CPU, 272 * after any required cache maintenance operation. 273 */ 274 static plat_local_state_t get_non_cpu_pd_node_local_state( 275 unsigned int parent_idx) 276 { 277 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 278 flush_dcache_range( 279 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 280 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 281 #endif 282 return psci_non_cpu_pd_nodes[parent_idx].local_state; 283 } 284 285 /* 286 * Update local state of non-CPU power domain node from a cached CPU; perform 287 * any required cache maintenance operation afterwards. 288 */ 289 static void set_non_cpu_pd_node_local_state(unsigned int parent_idx, 290 plat_local_state_t state) 291 { 292 psci_non_cpu_pd_nodes[parent_idx].local_state = state; 293 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 294 flush_dcache_range( 295 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 296 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 297 #endif 298 } 299 300 /****************************************************************************** 301 * Helper function to return the current local power state of each power domain 302 * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This 303 * function will be called after a cpu is powered on to find the local state 304 * each power domain has emerged from. 305 *****************************************************************************/ 306 void psci_get_target_local_pwr_states(unsigned int end_pwrlvl, 307 psci_power_state_t *target_state) 308 { 309 unsigned int parent_idx, lvl; 310 plat_local_state_t *pd_state = target_state->pwr_domain_state; 311 312 pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state(); 313 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 314 315 /* Copy the local power state from node to state_info */ 316 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 317 pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx); 318 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 319 } 320 321 /* Set the the higher levels to RUN */ 322 for (; lvl <= PLAT_MAX_PWR_LVL; lvl++) 323 target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 324 } 325 326 /****************************************************************************** 327 * Helper function to set the target local power state that each power domain 328 * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will 329 * enter. This function will be called after coordination of requested power 330 * states has been done for each power level. 331 *****************************************************************************/ 332 static void psci_set_target_local_pwr_states(unsigned int end_pwrlvl, 333 const psci_power_state_t *target_state) 334 { 335 unsigned int parent_idx, lvl; 336 const plat_local_state_t *pd_state = target_state->pwr_domain_state; 337 338 psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]); 339 340 /* 341 * Need to flush as local_state might be accessed with Data Cache 342 * disabled during power on 343 */ 344 psci_flush_cpu_data(psci_svc_cpu_data.local_state); 345 346 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 347 348 /* Copy the local_state from state_info */ 349 for (lvl = 1U; lvl <= end_pwrlvl; lvl++) { 350 set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]); 351 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 352 } 353 } 354 355 356 /******************************************************************************* 357 * PSCI helper function to get the parent nodes corresponding to a cpu_index. 358 ******************************************************************************/ 359 void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx, 360 unsigned int end_lvl, 361 unsigned int *node_index) 362 { 363 unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node; 364 unsigned int i; 365 unsigned int *node = node_index; 366 367 for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) { 368 *node = parent_node; 369 node++; 370 parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node; 371 } 372 } 373 374 /****************************************************************************** 375 * This function is invoked post CPU power up and initialization. It sets the 376 * affinity info state, target power state and requested power state for the 377 * current CPU and all its ancestor power domains to RUN. 378 *****************************************************************************/ 379 void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl) 380 { 381 unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl; 382 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 383 384 /* Reset the local_state to RUN for the non cpu power domains. */ 385 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 386 set_non_cpu_pd_node_local_state(parent_idx, 387 PSCI_LOCAL_STATE_RUN); 388 psci_set_req_local_pwr_state(lvl, 389 cpu_idx, 390 PSCI_LOCAL_STATE_RUN); 391 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 392 } 393 394 /* Set the affinity info state to ON */ 395 psci_set_aff_info_state(AFF_STATE_ON); 396 397 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); 398 psci_flush_cpu_data(psci_svc_cpu_data); 399 } 400 401 /****************************************************************************** 402 * This function is passed the local power states requested for each power 403 * domain (state_info) between the current CPU domain and its ancestors until 404 * the target power level (end_pwrlvl). It updates the array of requested power 405 * states with this information. 406 * 407 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it 408 * retrieves the states requested by all the cpus of which the power domain at 409 * that level is an ancestor. It passes this information to the platform to 410 * coordinate and return the target power state. If the target state for a level 411 * is RUN then subsequent levels are not considered. At the CPU level, state 412 * coordination is not required. Hence, the requested and the target states are 413 * the same. 414 * 415 * The 'state_info' is updated with the target state for each level between the 416 * CPU and the 'end_pwrlvl' and returned to the caller. 417 * 418 * This function will only be invoked with data cache enabled and while 419 * powering down a core. 420 *****************************************************************************/ 421 void psci_do_state_coordination(unsigned int end_pwrlvl, 422 psci_power_state_t *state_info) 423 { 424 unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); 425 unsigned int start_idx; 426 unsigned int ncpus; 427 plat_local_state_t target_state, *req_states; 428 429 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); 430 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 431 432 /* For level 0, the requested state will be equivalent 433 to target state */ 434 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 435 436 /* First update the requested power state */ 437 psci_set_req_local_pwr_state(lvl, cpu_idx, 438 state_info->pwr_domain_state[lvl]); 439 440 /* Get the requested power states for this power level */ 441 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; 442 req_states = psci_get_req_local_pwr_states(lvl, start_idx); 443 444 /* 445 * Let the platform coordinate amongst the requested states at 446 * this power level and return the target local power state. 447 */ 448 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; 449 target_state = plat_get_target_pwr_state(lvl, 450 req_states, 451 ncpus); 452 453 state_info->pwr_domain_state[lvl] = target_state; 454 455 /* Break early if the negotiated target power state is RUN */ 456 if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) 457 break; 458 459 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 460 } 461 462 /* 463 * This is for cases when we break out of the above loop early because 464 * the target power state is RUN at a power level < end_pwlvl. 465 * We update the requested power state from state_info and then 466 * set the target state as RUN. 467 */ 468 for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) { 469 psci_set_req_local_pwr_state(lvl, cpu_idx, 470 state_info->pwr_domain_state[lvl]); 471 state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 472 473 } 474 475 /* Update the target state in the power domain nodes */ 476 psci_set_target_local_pwr_states(end_pwrlvl, state_info); 477 } 478 479 /****************************************************************************** 480 * This function validates a suspend request by making sure that if a standby 481 * state is requested then no power level is turned off and the highest power 482 * level is placed in a standby/retention state. 483 * 484 * It also ensures that the state level X will enter is not shallower than the 485 * state level X + 1 will enter. 486 * 487 * This validation will be enabled only for DEBUG builds as the platform is 488 * expected to perform these validations as well. 489 *****************************************************************************/ 490 int psci_validate_suspend_req(const psci_power_state_t *state_info, 491 unsigned int is_power_down_state) 492 { 493 unsigned int max_off_lvl, target_lvl, max_retn_lvl; 494 plat_local_state_t state; 495 plat_local_state_type_t req_state_type, deepest_state_type; 496 int i; 497 498 /* Find the target suspend power level */ 499 target_lvl = psci_find_target_suspend_lvl(state_info); 500 if (target_lvl == PSCI_INVALID_PWR_LVL) 501 return PSCI_E_INVALID_PARAMS; 502 503 /* All power domain levels are in a RUN state to begin with */ 504 deepest_state_type = STATE_TYPE_RUN; 505 506 for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) { 507 state = state_info->pwr_domain_state[i]; 508 req_state_type = find_local_state_type(state); 509 510 /* 511 * While traversing from the highest power level to the lowest, 512 * the state requested for lower levels has to be the same or 513 * deeper i.e. equal to or greater than the state at the higher 514 * levels. If this condition is true, then the requested state 515 * becomes the deepest state encountered so far. 516 */ 517 if (req_state_type < deepest_state_type) 518 return PSCI_E_INVALID_PARAMS; 519 deepest_state_type = req_state_type; 520 } 521 522 /* Find the highest off power level */ 523 max_off_lvl = psci_find_max_off_lvl(state_info); 524 525 /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */ 526 max_retn_lvl = PSCI_INVALID_PWR_LVL; 527 if (target_lvl != max_off_lvl) 528 max_retn_lvl = target_lvl; 529 530 /* 531 * If this is not a request for a power down state then max off level 532 * has to be invalid and max retention level has to be a valid power 533 * level. 534 */ 535 if ((is_power_down_state == 0U) && 536 ((max_off_lvl != PSCI_INVALID_PWR_LVL) || 537 (max_retn_lvl == PSCI_INVALID_PWR_LVL))) 538 return PSCI_E_INVALID_PARAMS; 539 540 return PSCI_E_SUCCESS; 541 } 542 543 /****************************************************************************** 544 * This function finds the highest power level which will be powered down 545 * amongst all the power levels specified in the 'state_info' structure 546 *****************************************************************************/ 547 unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info) 548 { 549 int i; 550 551 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 552 if (is_local_state_off(state_info->pwr_domain_state[i]) != 0) 553 return (unsigned int) i; 554 } 555 556 return PSCI_INVALID_PWR_LVL; 557 } 558 559 /****************************************************************************** 560 * This functions finds the level of the highest power domain which will be 561 * placed in a low power state during a suspend operation. 562 *****************************************************************************/ 563 unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info) 564 { 565 int i; 566 567 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 568 if (is_local_state_run(state_info->pwr_domain_state[i]) == 0) 569 return (unsigned int) i; 570 } 571 572 return PSCI_INVALID_PWR_LVL; 573 } 574 575 /******************************************************************************* 576 * This function is passed the highest level in the topology tree that the 577 * operation should be applied to and a list of node indexes. It picks up locks 578 * from the node index list in order of increasing power domain level in the 579 * range specified. 580 ******************************************************************************/ 581 void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, 582 const unsigned int *parent_nodes) 583 { 584 unsigned int parent_idx; 585 unsigned int level; 586 587 /* No locking required for level 0. Hence start locking from level 1 */ 588 for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) { 589 parent_idx = parent_nodes[level - 1U]; 590 psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]); 591 } 592 } 593 594 /******************************************************************************* 595 * This function is passed the highest level in the topology tree that the 596 * operation should be applied to and a list of node indexes. It releases the 597 * locks in order of decreasing power domain level in the range specified. 598 ******************************************************************************/ 599 void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, 600 const unsigned int *parent_nodes) 601 { 602 unsigned int parent_idx; 603 unsigned int level; 604 605 /* Unlock top down. No unlocking required for level 0. */ 606 for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) { 607 parent_idx = parent_nodes[level - 1U]; 608 psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]); 609 } 610 } 611 612 /******************************************************************************* 613 * Simple routine to determine whether a mpidr is valid or not. 614 ******************************************************************************/ 615 int psci_validate_mpidr(u_register_t mpidr) 616 { 617 if (plat_core_pos_by_mpidr(mpidr) < 0) 618 return PSCI_E_INVALID_PARAMS; 619 620 return PSCI_E_SUCCESS; 621 } 622 623 /******************************************************************************* 624 * This function determines the full entrypoint information for the requested 625 * PSCI entrypoint on power on/resume and returns it. 626 ******************************************************************************/ 627 #ifdef __aarch64__ 628 static int psci_get_ns_ep_info(entry_point_info_t *ep, 629 uintptr_t entrypoint, 630 u_register_t context_id) 631 { 632 u_register_t ep_attr, sctlr; 633 unsigned int daif, ee, mode; 634 u_register_t ns_scr_el3 = read_scr_el3(); 635 u_register_t ns_sctlr_el1 = read_sctlr_el1(); 636 637 sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 638 read_sctlr_el2() : ns_sctlr_el1; 639 ee = 0; 640 641 ep_attr = NON_SECURE | EP_ST_DISABLE; 642 if ((sctlr & SCTLR_EE_BIT) != 0U) { 643 ep_attr |= EP_EE_BIG; 644 ee = 1; 645 } 646 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 647 648 ep->pc = entrypoint; 649 zeromem(&ep->args, sizeof(ep->args)); 650 ep->args.arg0 = context_id; 651 652 /* 653 * Figure out whether the cpu enters the non-secure address space 654 * in aarch32 or aarch64 655 */ 656 if ((ns_scr_el3 & SCR_RW_BIT) != 0U) { 657 658 /* 659 * Check whether a Thumb entry point has been provided for an 660 * aarch64 EL 661 */ 662 if ((entrypoint & 0x1UL) != 0UL) 663 return PSCI_E_INVALID_ADDRESS; 664 665 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1; 666 667 ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX, 668 DISABLE_ALL_EXCEPTIONS); 669 } else { 670 671 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 672 MODE32_hyp : MODE32_svc; 673 674 /* 675 * TODO: Choose async. exception bits if HYP mode is not 676 * implemented according to the values of SCR.{AW, FW} bits 677 */ 678 daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT; 679 680 ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee, 681 daif); 682 } 683 684 return PSCI_E_SUCCESS; 685 } 686 #else /* !__aarch64__ */ 687 static int psci_get_ns_ep_info(entry_point_info_t *ep, 688 uintptr_t entrypoint, 689 u_register_t context_id) 690 { 691 u_register_t ep_attr; 692 unsigned int aif, ee, mode; 693 u_register_t scr = read_scr(); 694 u_register_t ns_sctlr, sctlr; 695 696 /* Switch to non secure state */ 697 write_scr(scr | SCR_NS_BIT); 698 isb(); 699 ns_sctlr = read_sctlr(); 700 701 sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr; 702 703 /* Return to original state */ 704 write_scr(scr); 705 isb(); 706 ee = 0; 707 708 ep_attr = NON_SECURE | EP_ST_DISABLE; 709 if (sctlr & SCTLR_EE_BIT) { 710 ep_attr |= EP_EE_BIG; 711 ee = 1; 712 } 713 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 714 715 ep->pc = entrypoint; 716 zeromem(&ep->args, sizeof(ep->args)); 717 ep->args.arg0 = context_id; 718 719 mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc; 720 721 /* 722 * TODO: Choose async. exception bits if HYP mode is not 723 * implemented according to the values of SCR.{AW, FW} bits 724 */ 725 aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT; 726 727 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif); 728 729 return PSCI_E_SUCCESS; 730 } 731 732 #endif /* __aarch64__ */ 733 734 /******************************************************************************* 735 * This function validates the entrypoint with the platform layer if the 736 * appropriate pm_ops hook is exported by the platform and returns the 737 * 'entry_point_info'. 738 ******************************************************************************/ 739 int psci_validate_entry_point(entry_point_info_t *ep, 740 uintptr_t entrypoint, 741 u_register_t context_id) 742 { 743 int rc; 744 745 /* Validate the entrypoint using platform psci_ops */ 746 if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) { 747 rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint); 748 if (rc != PSCI_E_SUCCESS) 749 return PSCI_E_INVALID_ADDRESS; 750 } 751 752 /* 753 * Verify and derive the re-entry information for 754 * the non-secure world from the non-secure state from 755 * where this call originated. 756 */ 757 rc = psci_get_ns_ep_info(ep, entrypoint, context_id); 758 return rc; 759 } 760 761 /******************************************************************************* 762 * Generic handler which is called when a cpu is physically powered on. It 763 * traverses the node information and finds the highest power level powered 764 * off and performs generic, architectural, platform setup and state management 765 * to power on that power level and power levels below it. 766 * e.g. For a cpu that's been powered on, it will call the platform specific 767 * code to enable the gic cpu interface and for a cluster it will enable 768 * coherency at the interconnect level in addition to gic cpu interface. 769 ******************************************************************************/ 770 void psci_warmboot_entrypoint(void) 771 { 772 unsigned int end_pwrlvl; 773 unsigned int cpu_idx = plat_my_core_pos(); 774 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 775 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; 776 777 /* 778 * Verify that we have been explicitly turned ON or resumed from 779 * suspend. 780 */ 781 if (psci_get_aff_info_state() == AFF_STATE_OFF) { 782 ERROR("Unexpected affinity info state.\n"); 783 panic(); 784 } 785 786 /* 787 * Get the maximum power domain level to traverse to after this cpu 788 * has been physically powered up. 789 */ 790 end_pwrlvl = get_power_on_target_pwrlvl(); 791 792 /* Get the parent nodes */ 793 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); 794 795 /* 796 * This function acquires the lock corresponding to each power level so 797 * that by the time all locks are taken, the system topology is snapshot 798 * and state management can be done safely. 799 */ 800 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 801 802 psci_get_target_local_pwr_states(end_pwrlvl, &state_info); 803 804 #if ENABLE_PSCI_STAT 805 plat_psci_stat_accounting_stop(&state_info); 806 #endif 807 808 /* 809 * This CPU could be resuming from suspend or it could have just been 810 * turned on. To distinguish between these 2 cases, we examine the 811 * affinity state of the CPU: 812 * - If the affinity state is ON_PENDING then it has just been 813 * turned on. 814 * - Else it is resuming from suspend. 815 * 816 * Depending on the type of warm reset identified, choose the right set 817 * of power management handler and perform the generic, architecture 818 * and platform specific handling. 819 */ 820 if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING) 821 psci_cpu_on_finish(cpu_idx, &state_info); 822 else 823 psci_cpu_suspend_finish(cpu_idx, &state_info); 824 825 /* 826 * Set the requested and target state of this CPU and all the higher 827 * power domains which are ancestors of this CPU to run. 828 */ 829 psci_set_pwr_domains_to_run(end_pwrlvl); 830 831 #if ENABLE_PSCI_STAT 832 /* 833 * Update PSCI stats. 834 * Caches are off when writing stats data on the power down path. 835 * Since caches are now enabled, it's necessary to do cache 836 * maintenance before reading that same data. 837 */ 838 psci_stats_update_pwr_up(end_pwrlvl, &state_info); 839 #endif 840 841 /* 842 * This loop releases the lock corresponding to each power level 843 * in the reverse order to which they were acquired. 844 */ 845 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 846 } 847 848 /******************************************************************************* 849 * This function initializes the set of hooks that PSCI invokes as part of power 850 * management operation. The power management hooks are expected to be provided 851 * by the SPD, after it finishes all its initialization 852 ******************************************************************************/ 853 void psci_register_spd_pm_hook(const spd_pm_ops_t *pm) 854 { 855 assert(pm != NULL); 856 psci_spd_pm = pm; 857 858 if (pm->svc_migrate != NULL) 859 psci_caps |= define_psci_cap(PSCI_MIG_AARCH64); 860 861 if (pm->svc_migrate_info != NULL) 862 psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) 863 | define_psci_cap(PSCI_MIG_INFO_TYPE); 864 } 865 866 /******************************************************************************* 867 * This function invokes the migrate info hook in the spd_pm_ops. It performs 868 * the necessary return value validation. If the Secure Payload is UP and 869 * migrate capable, it returns the mpidr of the CPU on which the Secure payload 870 * is resident through the mpidr parameter. Else the value of the parameter on 871 * return is undefined. 872 ******************************************************************************/ 873 int psci_spd_migrate_info(u_register_t *mpidr) 874 { 875 int rc; 876 877 if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL)) 878 return PSCI_E_NOT_SUPPORTED; 879 880 rc = psci_spd_pm->svc_migrate_info(mpidr); 881 882 assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) || 883 (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED)); 884 885 return rc; 886 } 887 888 889 /******************************************************************************* 890 * This function prints the state of all power domains present in the 891 * system 892 ******************************************************************************/ 893 void psci_print_power_domain_map(void) 894 { 895 #if LOG_LEVEL >= LOG_LEVEL_INFO 896 unsigned int idx; 897 plat_local_state_t state; 898 plat_local_state_type_t state_type; 899 900 /* This array maps to the PSCI_STATE_X definitions in psci.h */ 901 static const char * const psci_state_type_str[] = { 902 "ON", 903 "RETENTION", 904 "OFF", 905 }; 906 907 INFO("PSCI Power Domain Map:\n"); 908 for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count); 909 idx++) { 910 state_type = find_local_state_type( 911 psci_non_cpu_pd_nodes[idx].local_state); 912 INFO(" Domain Node : Level %u, parent_node %u," 913 " State %s (0x%x)\n", 914 psci_non_cpu_pd_nodes[idx].level, 915 psci_non_cpu_pd_nodes[idx].parent_node, 916 psci_state_type_str[state_type], 917 psci_non_cpu_pd_nodes[idx].local_state); 918 } 919 920 for (idx = 0; idx < psci_plat_core_count; idx++) { 921 state = psci_get_cpu_local_state_by_idx(idx); 922 state_type = find_local_state_type(state); 923 INFO(" CPU Node : MPID 0x%llx, parent_node %u," 924 " State %s (0x%x)\n", 925 (unsigned long long)psci_cpu_pd_nodes[idx].mpidr, 926 psci_cpu_pd_nodes[idx].parent_node, 927 psci_state_type_str[state_type], 928 psci_get_cpu_local_state_by_idx(idx)); 929 } 930 #endif 931 } 932 933 /****************************************************************************** 934 * Return whether any secondaries were powered up with CPU_ON call. A CPU that 935 * have ever been powered up would have set its MPDIR value to something other 936 * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to 937 * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is 938 * meaningful only when called on the primary CPU during early boot. 939 *****************************************************************************/ 940 int psci_secondaries_brought_up(void) 941 { 942 unsigned int idx, n_valid = 0U; 943 944 for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) { 945 if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR) 946 n_valid++; 947 } 948 949 assert(n_valid > 0U); 950 951 return (n_valid > 1U) ? 1 : 0; 952 } 953 954 /******************************************************************************* 955 * Initiate power down sequence, by calling power down operations registered for 956 * this CPU. 957 ******************************************************************************/ 958 void psci_pwrdown_cpu(unsigned int power_level) 959 { 960 #if HW_ASSISTED_COHERENCY 961 /* 962 * With hardware-assisted coherency, the CPU drivers only initiate the 963 * power down sequence, without performing cache-maintenance operations 964 * in software. Data caches enabled both before and after this call. 965 */ 966 prepare_cpu_pwr_dwn(power_level); 967 #else 968 /* 969 * Without hardware-assisted coherency, the CPU drivers disable data 970 * caches, then perform cache-maintenance operations in software. 971 * 972 * This also calls prepare_cpu_pwr_dwn() to initiate power down 973 * sequence, but that function will return with data caches disabled. 974 * We must ensure that the stack memory is flushed out to memory before 975 * we start popping from it again. 976 */ 977 psci_do_pwrdown_cache_maintenance(power_level); 978 #endif 979 } 980 981 /******************************************************************************* 982 * This function invokes the callback 'stop_func()' with the 'mpidr' of each 983 * online PE. Caller can pass suitable method to stop a remote core. 984 * 985 * 'wait_ms' is the timeout value in milliseconds for the other cores to 986 * transition to power down state. Passing '0' makes it non-blocking. 987 * 988 * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the 989 * given timeout. 990 ******************************************************************************/ 991 int psci_stop_other_cores(unsigned int wait_ms, 992 void (*stop_func)(u_register_t mpidr)) 993 { 994 unsigned int idx, this_cpu_idx; 995 996 this_cpu_idx = plat_my_core_pos(); 997 998 /* Invoke stop_func for each core */ 999 for (idx = 0U; idx < psci_plat_core_count; idx++) { 1000 /* skip current CPU */ 1001 if (idx == this_cpu_idx) { 1002 continue; 1003 } 1004 1005 /* Check if the CPU is ON */ 1006 if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) { 1007 (*stop_func)(psci_cpu_pd_nodes[idx].mpidr); 1008 } 1009 } 1010 1011 /* Need to wait for other cores to shutdown */ 1012 if (wait_ms != 0U) { 1013 while ((wait_ms-- != 0U) && (!psci_is_last_on_cpu())) { 1014 mdelay(1U); 1015 } 1016 1017 if (!psci_is_last_on_cpu()) { 1018 WARN("Failed to stop all cores!\n"); 1019 psci_print_power_domain_map(); 1020 return PSCI_E_DENIED; 1021 } 1022 } 1023 1024 return PSCI_E_SUCCESS; 1025 } 1026 1027 /******************************************************************************* 1028 * This function verifies that all the other cores in the system have been 1029 * turned OFF and the current CPU is the last running CPU in the system. 1030 * Returns true if the current CPU is the last ON CPU or false otherwise. 1031 * 1032 * This API has following differences with psci_is_last_on_cpu 1033 * 1. PSCI states are locked 1034 ******************************************************************************/ 1035 bool psci_is_last_on_cpu_safe(void) 1036 { 1037 unsigned int this_core = plat_my_core_pos(); 1038 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 1039 1040 psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); 1041 1042 psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1043 1044 if (!psci_is_last_on_cpu()) { 1045 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1046 return false; 1047 } 1048 1049 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1050 1051 return true; 1052 } 1053