1 /* 2 * Copyright (c) 2013-2022, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <string.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <drivers/delay_timer.h> 16 #include <lib/el3_runtime/context_mgmt.h> 17 #include <lib/utils.h> 18 #include <plat/common/platform.h> 19 20 #include "psci_private.h" 21 22 /* 23 * SPD power management operations, expected to be supplied by the registered 24 * SPD on successful SP initialization 25 */ 26 const spd_pm_ops_t *psci_spd_pm; 27 28 /* 29 * PSCI requested local power state map. This array is used to store the local 30 * power states requested by a CPU for power levels from level 1 to 31 * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power 32 * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a 33 * CPU are the same. 34 * 35 * During state coordination, the platform is passed an array containing the 36 * local states requested for a particular non cpu power domain by each cpu 37 * within the domain. 38 * 39 * TODO: Dense packing of the requested states will cause cache thrashing 40 * when multiple power domains write to it. If we allocate the requested 41 * states at each power level in a cache-line aligned per-domain memory, 42 * the cache thrashing can be avoided. 43 */ 44 static plat_local_state_t 45 psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT]; 46 47 unsigned int psci_plat_core_count; 48 49 /******************************************************************************* 50 * Arrays that hold the platform's power domain tree information for state 51 * management of power domains. 52 * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain 53 * which is an ancestor of a CPU power domain. 54 * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain 55 ******************************************************************************/ 56 non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS] 57 #if USE_COHERENT_MEM 58 __section("tzfw_coherent_mem") 59 #endif 60 ; 61 62 /* Lock for PSCI state coordination */ 63 DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); 64 65 cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; 66 67 /******************************************************************************* 68 * Pointer to functions exported by the platform to complete power mgmt. ops 69 ******************************************************************************/ 70 const plat_psci_ops_t *psci_plat_pm_ops; 71 72 /****************************************************************************** 73 * Check that the maximum power level supported by the platform makes sense 74 *****************************************************************************/ 75 CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) && 76 (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL), 77 assert_platform_max_pwrlvl_check); 78 79 /* 80 * The plat_local_state used by the platform is one of these types: RUN, 81 * RETENTION and OFF. The platform can define further sub-states for each type 82 * apart from RUN. This categorization is done to verify the sanity of the 83 * psci_power_state passed by the platform and to print debug information. The 84 * categorization is done on the basis of the following conditions: 85 * 86 * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN. 87 * 88 * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is 89 * STATE_TYPE_RETN. 90 * 91 * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is 92 * STATE_TYPE_OFF. 93 */ 94 typedef enum plat_local_state_type { 95 STATE_TYPE_RUN = 0, 96 STATE_TYPE_RETN, 97 STATE_TYPE_OFF 98 } plat_local_state_type_t; 99 100 /* Function used to categorize plat_local_state. */ 101 static plat_local_state_type_t find_local_state_type(plat_local_state_t state) 102 { 103 if (state != 0U) { 104 if (state > PLAT_MAX_RET_STATE) { 105 return STATE_TYPE_OFF; 106 } else { 107 return STATE_TYPE_RETN; 108 } 109 } else { 110 return STATE_TYPE_RUN; 111 } 112 } 113 114 /****************************************************************************** 115 * Check that the maximum retention level supported by the platform is less 116 * than the maximum off level. 117 *****************************************************************************/ 118 CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE, 119 assert_platform_max_off_and_retn_state_check); 120 121 /****************************************************************************** 122 * This function ensures that the power state parameter in a CPU_SUSPEND request 123 * is valid. If so, it returns the requested states for each power level. 124 *****************************************************************************/ 125 int psci_validate_power_state(unsigned int power_state, 126 psci_power_state_t *state_info) 127 { 128 /* Check SBZ bits in power state are zero */ 129 if (psci_check_power_state(power_state) != 0U) 130 return PSCI_E_INVALID_PARAMS; 131 132 assert(psci_plat_pm_ops->validate_power_state != NULL); 133 134 /* Validate the power_state using platform pm_ops */ 135 return psci_plat_pm_ops->validate_power_state(power_state, state_info); 136 } 137 138 /****************************************************************************** 139 * This function retrieves the `psci_power_state_t` for system suspend from 140 * the platform. 141 *****************************************************************************/ 142 void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info) 143 { 144 /* 145 * Assert that the required pm_ops hook is implemented to ensure that 146 * the capability detected during psci_setup() is valid. 147 */ 148 assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL); 149 150 /* 151 * Query the platform for the power_state required for system suspend 152 */ 153 psci_plat_pm_ops->get_sys_suspend_power_state(state_info); 154 } 155 156 /******************************************************************************* 157 * This function verifies that the all the other cores in the system have been 158 * turned OFF and the current CPU is the last running CPU in the system. 159 * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false) 160 * otherwise. 161 ******************************************************************************/ 162 unsigned int psci_is_last_on_cpu(void) 163 { 164 unsigned int cpu_idx, my_idx = plat_my_core_pos(); 165 166 for (cpu_idx = 0; cpu_idx < psci_plat_core_count; 167 cpu_idx++) { 168 if (cpu_idx == my_idx) { 169 assert(psci_get_aff_info_state() == AFF_STATE_ON); 170 continue; 171 } 172 173 if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) 174 return 0; 175 } 176 177 return 1; 178 } 179 180 /******************************************************************************* 181 * Routine to return the maximum power level to traverse to after a cpu has 182 * been physically powered up. It is expected to be called immediately after 183 * reset from assembler code. 184 ******************************************************************************/ 185 static unsigned int get_power_on_target_pwrlvl(void) 186 { 187 unsigned int pwrlvl; 188 189 /* 190 * Assume that this cpu was suspended and retrieve its target power 191 * level. If it is invalid then it could only have been turned off 192 * earlier. PLAT_MAX_PWR_LVL will be the highest power level a 193 * cpu can be turned off to. 194 */ 195 pwrlvl = psci_get_suspend_pwrlvl(); 196 if (pwrlvl == PSCI_INVALID_PWR_LVL) 197 pwrlvl = PLAT_MAX_PWR_LVL; 198 assert(pwrlvl < PSCI_INVALID_PWR_LVL); 199 return pwrlvl; 200 } 201 202 /****************************************************************************** 203 * Helper function to update the requested local power state array. This array 204 * does not store the requested state for the CPU power level. Hence an 205 * assertion is added to prevent us from accessing the CPU power level. 206 *****************************************************************************/ 207 static void psci_set_req_local_pwr_state(unsigned int pwrlvl, 208 unsigned int cpu_idx, 209 plat_local_state_t req_pwr_state) 210 { 211 assert(pwrlvl > PSCI_CPU_PWR_LVL); 212 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 213 (cpu_idx < psci_plat_core_count)) { 214 psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state; 215 } 216 } 217 218 /****************************************************************************** 219 * This function initializes the psci_req_local_pwr_states. 220 *****************************************************************************/ 221 void __init psci_init_req_local_pwr_states(void) 222 { 223 /* Initialize the requested state of all non CPU power domains as OFF */ 224 unsigned int pwrlvl; 225 unsigned int core; 226 227 for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) { 228 for (core = 0; core < psci_plat_core_count; core++) { 229 psci_req_local_pwr_states[pwrlvl][core] = 230 PLAT_MAX_OFF_STATE; 231 } 232 } 233 } 234 235 /****************************************************************************** 236 * Helper function to return a reference to an array containing the local power 237 * states requested by each cpu for a power domain at 'pwrlvl'. The size of the 238 * array will be the number of cpu power domains of which this power domain is 239 * an ancestor. These requested states will be used to determine a suitable 240 * target state for this power domain during psci state coordination. An 241 * assertion is added to prevent us from accessing the CPU power level. 242 *****************************************************************************/ 243 static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl, 244 unsigned int cpu_idx) 245 { 246 assert(pwrlvl > PSCI_CPU_PWR_LVL); 247 248 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 249 (cpu_idx < psci_plat_core_count)) { 250 return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx]; 251 } else 252 return NULL; 253 } 254 255 /* 256 * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent 257 * memory. 258 * 259 * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory, 260 * it's accessed by both cached and non-cached participants. To serve the common 261 * minimum, perform a cache flush before read and after write so that non-cached 262 * participants operate on latest data in main memory. 263 * 264 * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent 265 * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent. 266 * In both cases, no cache operations are required. 267 */ 268 269 /* 270 * Retrieve local state of non-CPU power domain node from a non-cached CPU, 271 * after any required cache maintenance operation. 272 */ 273 static plat_local_state_t get_non_cpu_pd_node_local_state( 274 unsigned int parent_idx) 275 { 276 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 277 flush_dcache_range( 278 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 279 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 280 #endif 281 return psci_non_cpu_pd_nodes[parent_idx].local_state; 282 } 283 284 /* 285 * Update local state of non-CPU power domain node from a cached CPU; perform 286 * any required cache maintenance operation afterwards. 287 */ 288 static void set_non_cpu_pd_node_local_state(unsigned int parent_idx, 289 plat_local_state_t state) 290 { 291 psci_non_cpu_pd_nodes[parent_idx].local_state = state; 292 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 293 flush_dcache_range( 294 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 295 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 296 #endif 297 } 298 299 /****************************************************************************** 300 * Helper function to return the current local power state of each power domain 301 * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This 302 * function will be called after a cpu is powered on to find the local state 303 * each power domain has emerged from. 304 *****************************************************************************/ 305 void psci_get_target_local_pwr_states(unsigned int end_pwrlvl, 306 psci_power_state_t *target_state) 307 { 308 unsigned int parent_idx, lvl; 309 plat_local_state_t *pd_state = target_state->pwr_domain_state; 310 311 pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state(); 312 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 313 314 /* Copy the local power state from node to state_info */ 315 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 316 pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx); 317 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 318 } 319 320 /* Set the the higher levels to RUN */ 321 for (; lvl <= PLAT_MAX_PWR_LVL; lvl++) 322 target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 323 } 324 325 /****************************************************************************** 326 * Helper function to set the target local power state that each power domain 327 * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will 328 * enter. This function will be called after coordination of requested power 329 * states has been done for each power level. 330 *****************************************************************************/ 331 static void psci_set_target_local_pwr_states(unsigned int end_pwrlvl, 332 const psci_power_state_t *target_state) 333 { 334 unsigned int parent_idx, lvl; 335 const plat_local_state_t *pd_state = target_state->pwr_domain_state; 336 337 psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]); 338 339 /* 340 * Need to flush as local_state might be accessed with Data Cache 341 * disabled during power on 342 */ 343 psci_flush_cpu_data(psci_svc_cpu_data.local_state); 344 345 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 346 347 /* Copy the local_state from state_info */ 348 for (lvl = 1U; lvl <= end_pwrlvl; lvl++) { 349 set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]); 350 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 351 } 352 } 353 354 355 /******************************************************************************* 356 * PSCI helper function to get the parent nodes corresponding to a cpu_index. 357 ******************************************************************************/ 358 void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx, 359 unsigned int end_lvl, 360 unsigned int *node_index) 361 { 362 unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node; 363 unsigned int i; 364 unsigned int *node = node_index; 365 366 for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) { 367 *node = parent_node; 368 node++; 369 parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node; 370 } 371 } 372 373 /****************************************************************************** 374 * This function is invoked post CPU power up and initialization. It sets the 375 * affinity info state, target power state and requested power state for the 376 * current CPU and all its ancestor power domains to RUN. 377 *****************************************************************************/ 378 void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl) 379 { 380 unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl; 381 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 382 383 /* Reset the local_state to RUN for the non cpu power domains. */ 384 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 385 set_non_cpu_pd_node_local_state(parent_idx, 386 PSCI_LOCAL_STATE_RUN); 387 psci_set_req_local_pwr_state(lvl, 388 cpu_idx, 389 PSCI_LOCAL_STATE_RUN); 390 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 391 } 392 393 /* Set the affinity info state to ON */ 394 psci_set_aff_info_state(AFF_STATE_ON); 395 396 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); 397 psci_flush_cpu_data(psci_svc_cpu_data); 398 } 399 400 /****************************************************************************** 401 * This function is passed the local power states requested for each power 402 * domain (state_info) between the current CPU domain and its ancestors until 403 * the target power level (end_pwrlvl). It updates the array of requested power 404 * states with this information. 405 * 406 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it 407 * retrieves the states requested by all the cpus of which the power domain at 408 * that level is an ancestor. It passes this information to the platform to 409 * coordinate and return the target power state. If the target state for a level 410 * is RUN then subsequent levels are not considered. At the CPU level, state 411 * coordination is not required. Hence, the requested and the target states are 412 * the same. 413 * 414 * The 'state_info' is updated with the target state for each level between the 415 * CPU and the 'end_pwrlvl' and returned to the caller. 416 * 417 * This function will only be invoked with data cache enabled and while 418 * powering down a core. 419 *****************************************************************************/ 420 void psci_do_state_coordination(unsigned int end_pwrlvl, 421 psci_power_state_t *state_info) 422 { 423 unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); 424 unsigned int start_idx; 425 unsigned int ncpus; 426 plat_local_state_t target_state, *req_states; 427 428 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); 429 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 430 431 /* For level 0, the requested state will be equivalent 432 to target state */ 433 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 434 435 /* First update the requested power state */ 436 psci_set_req_local_pwr_state(lvl, cpu_idx, 437 state_info->pwr_domain_state[lvl]); 438 439 /* Get the requested power states for this power level */ 440 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; 441 req_states = psci_get_req_local_pwr_states(lvl, start_idx); 442 443 /* 444 * Let the platform coordinate amongst the requested states at 445 * this power level and return the target local power state. 446 */ 447 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; 448 target_state = plat_get_target_pwr_state(lvl, 449 req_states, 450 ncpus); 451 452 state_info->pwr_domain_state[lvl] = target_state; 453 454 /* Break early if the negotiated target power state is RUN */ 455 if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) 456 break; 457 458 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 459 } 460 461 /* 462 * This is for cases when we break out of the above loop early because 463 * the target power state is RUN at a power level < end_pwlvl. 464 * We update the requested power state from state_info and then 465 * set the target state as RUN. 466 */ 467 for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) { 468 psci_set_req_local_pwr_state(lvl, cpu_idx, 469 state_info->pwr_domain_state[lvl]); 470 state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 471 472 } 473 474 /* Update the target state in the power domain nodes */ 475 psci_set_target_local_pwr_states(end_pwrlvl, state_info); 476 } 477 478 /****************************************************************************** 479 * This function validates a suspend request by making sure that if a standby 480 * state is requested then no power level is turned off and the highest power 481 * level is placed in a standby/retention state. 482 * 483 * It also ensures that the state level X will enter is not shallower than the 484 * state level X + 1 will enter. 485 * 486 * This validation will be enabled only for DEBUG builds as the platform is 487 * expected to perform these validations as well. 488 *****************************************************************************/ 489 int psci_validate_suspend_req(const psci_power_state_t *state_info, 490 unsigned int is_power_down_state) 491 { 492 unsigned int max_off_lvl, target_lvl, max_retn_lvl; 493 plat_local_state_t state; 494 plat_local_state_type_t req_state_type, deepest_state_type; 495 int i; 496 497 /* Find the target suspend power level */ 498 target_lvl = psci_find_target_suspend_lvl(state_info); 499 if (target_lvl == PSCI_INVALID_PWR_LVL) 500 return PSCI_E_INVALID_PARAMS; 501 502 /* All power domain levels are in a RUN state to begin with */ 503 deepest_state_type = STATE_TYPE_RUN; 504 505 for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) { 506 state = state_info->pwr_domain_state[i]; 507 req_state_type = find_local_state_type(state); 508 509 /* 510 * While traversing from the highest power level to the lowest, 511 * the state requested for lower levels has to be the same or 512 * deeper i.e. equal to or greater than the state at the higher 513 * levels. If this condition is true, then the requested state 514 * becomes the deepest state encountered so far. 515 */ 516 if (req_state_type < deepest_state_type) 517 return PSCI_E_INVALID_PARAMS; 518 deepest_state_type = req_state_type; 519 } 520 521 /* Find the highest off power level */ 522 max_off_lvl = psci_find_max_off_lvl(state_info); 523 524 /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */ 525 max_retn_lvl = PSCI_INVALID_PWR_LVL; 526 if (target_lvl != max_off_lvl) 527 max_retn_lvl = target_lvl; 528 529 /* 530 * If this is not a request for a power down state then max off level 531 * has to be invalid and max retention level has to be a valid power 532 * level. 533 */ 534 if ((is_power_down_state == 0U) && 535 ((max_off_lvl != PSCI_INVALID_PWR_LVL) || 536 (max_retn_lvl == PSCI_INVALID_PWR_LVL))) 537 return PSCI_E_INVALID_PARAMS; 538 539 return PSCI_E_SUCCESS; 540 } 541 542 /****************************************************************************** 543 * This function finds the highest power level which will be powered down 544 * amongst all the power levels specified in the 'state_info' structure 545 *****************************************************************************/ 546 unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info) 547 { 548 int i; 549 550 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 551 if (is_local_state_off(state_info->pwr_domain_state[i]) != 0) 552 return (unsigned int) i; 553 } 554 555 return PSCI_INVALID_PWR_LVL; 556 } 557 558 /****************************************************************************** 559 * This functions finds the level of the highest power domain which will be 560 * placed in a low power state during a suspend operation. 561 *****************************************************************************/ 562 unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info) 563 { 564 int i; 565 566 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 567 if (is_local_state_run(state_info->pwr_domain_state[i]) == 0) 568 return (unsigned int) i; 569 } 570 571 return PSCI_INVALID_PWR_LVL; 572 } 573 574 /******************************************************************************* 575 * This function is passed the highest level in the topology tree that the 576 * operation should be applied to and a list of node indexes. It picks up locks 577 * from the node index list in order of increasing power domain level in the 578 * range specified. 579 ******************************************************************************/ 580 void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, 581 const unsigned int *parent_nodes) 582 { 583 unsigned int parent_idx; 584 unsigned int level; 585 586 /* No locking required for level 0. Hence start locking from level 1 */ 587 for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) { 588 parent_idx = parent_nodes[level - 1U]; 589 psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]); 590 } 591 } 592 593 /******************************************************************************* 594 * This function is passed the highest level in the topology tree that the 595 * operation should be applied to and a list of node indexes. It releases the 596 * locks in order of decreasing power domain level in the range specified. 597 ******************************************************************************/ 598 void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, 599 const unsigned int *parent_nodes) 600 { 601 unsigned int parent_idx; 602 unsigned int level; 603 604 /* Unlock top down. No unlocking required for level 0. */ 605 for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) { 606 parent_idx = parent_nodes[level - 1U]; 607 psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]); 608 } 609 } 610 611 /******************************************************************************* 612 * Simple routine to determine whether a mpidr is valid or not. 613 ******************************************************************************/ 614 int psci_validate_mpidr(u_register_t mpidr) 615 { 616 if (plat_core_pos_by_mpidr(mpidr) < 0) 617 return PSCI_E_INVALID_PARAMS; 618 619 return PSCI_E_SUCCESS; 620 } 621 622 /******************************************************************************* 623 * This function determines the full entrypoint information for the requested 624 * PSCI entrypoint on power on/resume and returns it. 625 ******************************************************************************/ 626 #ifdef __aarch64__ 627 static int psci_get_ns_ep_info(entry_point_info_t *ep, 628 uintptr_t entrypoint, 629 u_register_t context_id) 630 { 631 u_register_t ep_attr, sctlr; 632 unsigned int daif, ee, mode; 633 u_register_t ns_scr_el3 = read_scr_el3(); 634 u_register_t ns_sctlr_el1 = read_sctlr_el1(); 635 636 sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 637 read_sctlr_el2() : ns_sctlr_el1; 638 ee = 0; 639 640 ep_attr = NON_SECURE | EP_ST_DISABLE; 641 if ((sctlr & SCTLR_EE_BIT) != 0U) { 642 ep_attr |= EP_EE_BIG; 643 ee = 1; 644 } 645 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 646 647 ep->pc = entrypoint; 648 zeromem(&ep->args, sizeof(ep->args)); 649 ep->args.arg0 = context_id; 650 651 /* 652 * Figure out whether the cpu enters the non-secure address space 653 * in aarch32 or aarch64 654 */ 655 if ((ns_scr_el3 & SCR_RW_BIT) != 0U) { 656 657 /* 658 * Check whether a Thumb entry point has been provided for an 659 * aarch64 EL 660 */ 661 if ((entrypoint & 0x1UL) != 0UL) 662 return PSCI_E_INVALID_ADDRESS; 663 664 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1; 665 666 ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX, 667 DISABLE_ALL_EXCEPTIONS); 668 } else { 669 670 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 671 MODE32_hyp : MODE32_svc; 672 673 /* 674 * TODO: Choose async. exception bits if HYP mode is not 675 * implemented according to the values of SCR.{AW, FW} bits 676 */ 677 daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT; 678 679 ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee, 680 daif); 681 } 682 683 return PSCI_E_SUCCESS; 684 } 685 #else /* !__aarch64__ */ 686 static int psci_get_ns_ep_info(entry_point_info_t *ep, 687 uintptr_t entrypoint, 688 u_register_t context_id) 689 { 690 u_register_t ep_attr; 691 unsigned int aif, ee, mode; 692 u_register_t scr = read_scr(); 693 u_register_t ns_sctlr, sctlr; 694 695 /* Switch to non secure state */ 696 write_scr(scr | SCR_NS_BIT); 697 isb(); 698 ns_sctlr = read_sctlr(); 699 700 sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr; 701 702 /* Return to original state */ 703 write_scr(scr); 704 isb(); 705 ee = 0; 706 707 ep_attr = NON_SECURE | EP_ST_DISABLE; 708 if (sctlr & SCTLR_EE_BIT) { 709 ep_attr |= EP_EE_BIG; 710 ee = 1; 711 } 712 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 713 714 ep->pc = entrypoint; 715 zeromem(&ep->args, sizeof(ep->args)); 716 ep->args.arg0 = context_id; 717 718 mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc; 719 720 /* 721 * TODO: Choose async. exception bits if HYP mode is not 722 * implemented according to the values of SCR.{AW, FW} bits 723 */ 724 aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT; 725 726 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif); 727 728 return PSCI_E_SUCCESS; 729 } 730 731 #endif /* __aarch64__ */ 732 733 /******************************************************************************* 734 * This function validates the entrypoint with the platform layer if the 735 * appropriate pm_ops hook is exported by the platform and returns the 736 * 'entry_point_info'. 737 ******************************************************************************/ 738 int psci_validate_entry_point(entry_point_info_t *ep, 739 uintptr_t entrypoint, 740 u_register_t context_id) 741 { 742 int rc; 743 744 /* Validate the entrypoint using platform psci_ops */ 745 if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) { 746 rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint); 747 if (rc != PSCI_E_SUCCESS) 748 return PSCI_E_INVALID_ADDRESS; 749 } 750 751 /* 752 * Verify and derive the re-entry information for 753 * the non-secure world from the non-secure state from 754 * where this call originated. 755 */ 756 rc = psci_get_ns_ep_info(ep, entrypoint, context_id); 757 return rc; 758 } 759 760 /******************************************************************************* 761 * Generic handler which is called when a cpu is physically powered on. It 762 * traverses the node information and finds the highest power level powered 763 * off and performs generic, architectural, platform setup and state management 764 * to power on that power level and power levels below it. 765 * e.g. For a cpu that's been powered on, it will call the platform specific 766 * code to enable the gic cpu interface and for a cluster it will enable 767 * coherency at the interconnect level in addition to gic cpu interface. 768 ******************************************************************************/ 769 void psci_warmboot_entrypoint(void) 770 { 771 unsigned int end_pwrlvl; 772 unsigned int cpu_idx = plat_my_core_pos(); 773 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 774 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; 775 776 /* 777 * Verify that we have been explicitly turned ON or resumed from 778 * suspend. 779 */ 780 if (psci_get_aff_info_state() == AFF_STATE_OFF) { 781 ERROR("Unexpected affinity info state.\n"); 782 panic(); 783 } 784 785 /* 786 * Get the maximum power domain level to traverse to after this cpu 787 * has been physically powered up. 788 */ 789 end_pwrlvl = get_power_on_target_pwrlvl(); 790 791 /* Get the parent nodes */ 792 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); 793 794 /* 795 * This function acquires the lock corresponding to each power level so 796 * that by the time all locks are taken, the system topology is snapshot 797 * and state management can be done safely. 798 */ 799 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 800 801 psci_get_target_local_pwr_states(end_pwrlvl, &state_info); 802 803 #if ENABLE_PSCI_STAT 804 plat_psci_stat_accounting_stop(&state_info); 805 #endif 806 807 /* 808 * This CPU could be resuming from suspend or it could have just been 809 * turned on. To distinguish between these 2 cases, we examine the 810 * affinity state of the CPU: 811 * - If the affinity state is ON_PENDING then it has just been 812 * turned on. 813 * - Else it is resuming from suspend. 814 * 815 * Depending on the type of warm reset identified, choose the right set 816 * of power management handler and perform the generic, architecture 817 * and platform specific handling. 818 */ 819 if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING) 820 psci_cpu_on_finish(cpu_idx, &state_info); 821 else 822 psci_cpu_suspend_finish(cpu_idx, &state_info); 823 824 /* 825 * Set the requested and target state of this CPU and all the higher 826 * power domains which are ancestors of this CPU to run. 827 */ 828 psci_set_pwr_domains_to_run(end_pwrlvl); 829 830 #if ENABLE_PSCI_STAT 831 /* 832 * Update PSCI stats. 833 * Caches are off when writing stats data on the power down path. 834 * Since caches are now enabled, it's necessary to do cache 835 * maintenance before reading that same data. 836 */ 837 psci_stats_update_pwr_up(end_pwrlvl, &state_info); 838 #endif 839 840 /* 841 * This loop releases the lock corresponding to each power level 842 * in the reverse order to which they were acquired. 843 */ 844 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 845 } 846 847 /******************************************************************************* 848 * This function initializes the set of hooks that PSCI invokes as part of power 849 * management operation. The power management hooks are expected to be provided 850 * by the SPD, after it finishes all its initialization 851 ******************************************************************************/ 852 void psci_register_spd_pm_hook(const spd_pm_ops_t *pm) 853 { 854 assert(pm != NULL); 855 psci_spd_pm = pm; 856 857 if (pm->svc_migrate != NULL) 858 psci_caps |= define_psci_cap(PSCI_MIG_AARCH64); 859 860 if (pm->svc_migrate_info != NULL) 861 psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) 862 | define_psci_cap(PSCI_MIG_INFO_TYPE); 863 } 864 865 /******************************************************************************* 866 * This function invokes the migrate info hook in the spd_pm_ops. It performs 867 * the necessary return value validation. If the Secure Payload is UP and 868 * migrate capable, it returns the mpidr of the CPU on which the Secure payload 869 * is resident through the mpidr parameter. Else the value of the parameter on 870 * return is undefined. 871 ******************************************************************************/ 872 int psci_spd_migrate_info(u_register_t *mpidr) 873 { 874 int rc; 875 876 if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL)) 877 return PSCI_E_NOT_SUPPORTED; 878 879 rc = psci_spd_pm->svc_migrate_info(mpidr); 880 881 assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) || 882 (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED)); 883 884 return rc; 885 } 886 887 888 /******************************************************************************* 889 * This function prints the state of all power domains present in the 890 * system 891 ******************************************************************************/ 892 void psci_print_power_domain_map(void) 893 { 894 #if LOG_LEVEL >= LOG_LEVEL_INFO 895 unsigned int idx; 896 plat_local_state_t state; 897 plat_local_state_type_t state_type; 898 899 /* This array maps to the PSCI_STATE_X definitions in psci.h */ 900 static const char * const psci_state_type_str[] = { 901 "ON", 902 "RETENTION", 903 "OFF", 904 }; 905 906 INFO("PSCI Power Domain Map:\n"); 907 for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count); 908 idx++) { 909 state_type = find_local_state_type( 910 psci_non_cpu_pd_nodes[idx].local_state); 911 INFO(" Domain Node : Level %u, parent_node %u," 912 " State %s (0x%x)\n", 913 psci_non_cpu_pd_nodes[idx].level, 914 psci_non_cpu_pd_nodes[idx].parent_node, 915 psci_state_type_str[state_type], 916 psci_non_cpu_pd_nodes[idx].local_state); 917 } 918 919 for (idx = 0; idx < psci_plat_core_count; idx++) { 920 state = psci_get_cpu_local_state_by_idx(idx); 921 state_type = find_local_state_type(state); 922 INFO(" CPU Node : MPID 0x%llx, parent_node %u," 923 " State %s (0x%x)\n", 924 (unsigned long long)psci_cpu_pd_nodes[idx].mpidr, 925 psci_cpu_pd_nodes[idx].parent_node, 926 psci_state_type_str[state_type], 927 psci_get_cpu_local_state_by_idx(idx)); 928 } 929 #endif 930 } 931 932 /****************************************************************************** 933 * Return whether any secondaries were powered up with CPU_ON call. A CPU that 934 * have ever been powered up would have set its MPDIR value to something other 935 * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to 936 * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is 937 * meaningful only when called on the primary CPU during early boot. 938 *****************************************************************************/ 939 int psci_secondaries_brought_up(void) 940 { 941 unsigned int idx, n_valid = 0U; 942 943 for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) { 944 if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR) 945 n_valid++; 946 } 947 948 assert(n_valid > 0U); 949 950 return (n_valid > 1U) ? 1 : 0; 951 } 952 953 /******************************************************************************* 954 * Initiate power down sequence, by calling power down operations registered for 955 * this CPU. 956 ******************************************************************************/ 957 void psci_do_pwrdown_sequence(unsigned int power_level) 958 { 959 #if HW_ASSISTED_COHERENCY 960 /* 961 * With hardware-assisted coherency, the CPU drivers only initiate the 962 * power down sequence, without performing cache-maintenance operations 963 * in software. Data caches enabled both before and after this call. 964 */ 965 prepare_cpu_pwr_dwn(power_level); 966 #else 967 /* 968 * Without hardware-assisted coherency, the CPU drivers disable data 969 * caches, then perform cache-maintenance operations in software. 970 * 971 * This also calls prepare_cpu_pwr_dwn() to initiate power down 972 * sequence, but that function will return with data caches disabled. 973 * We must ensure that the stack memory is flushed out to memory before 974 * we start popping from it again. 975 */ 976 psci_do_pwrdown_cache_maintenance(power_level); 977 #endif 978 } 979 980 /******************************************************************************* 981 * This function invokes the callback 'stop_func()' with the 'mpidr' of each 982 * online PE. Caller can pass suitable method to stop a remote core. 983 * 984 * 'wait_ms' is the timeout value in milliseconds for the other cores to 985 * transition to power down state. Passing '0' makes it non-blocking. 986 * 987 * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the 988 * given timeout. 989 ******************************************************************************/ 990 int psci_stop_other_cores(unsigned int wait_ms, 991 void (*stop_func)(u_register_t mpidr)) 992 { 993 unsigned int idx, this_cpu_idx; 994 995 this_cpu_idx = plat_my_core_pos(); 996 997 /* Invoke stop_func for each core */ 998 for (idx = 0U; idx < psci_plat_core_count; idx++) { 999 /* skip current CPU */ 1000 if (idx == this_cpu_idx) { 1001 continue; 1002 } 1003 1004 /* Check if the CPU is ON */ 1005 if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) { 1006 (*stop_func)(psci_cpu_pd_nodes[idx].mpidr); 1007 } 1008 } 1009 1010 /* Need to wait for other cores to shutdown */ 1011 if (wait_ms != 0U) { 1012 while ((wait_ms-- != 0U) && (psci_is_last_on_cpu() != 0U)) { 1013 mdelay(1U); 1014 } 1015 1016 if (psci_is_last_on_cpu() != 0U) { 1017 WARN("Failed to stop all cores!\n"); 1018 psci_print_power_domain_map(); 1019 return PSCI_E_DENIED; 1020 } 1021 } 1022 1023 return PSCI_E_SUCCESS; 1024 } 1025 1026 /******************************************************************************* 1027 * This function verifies that all the other cores in the system have been 1028 * turned OFF and the current CPU is the last running CPU in the system. 1029 * Returns true if the current CPU is the last ON CPU or false otherwise. 1030 * 1031 * This API has following differences with psci_is_last_on_cpu 1032 * 1. PSCI states are locked 1033 * 2. It caters for "forest" topology instead of just "tree" 1034 * TODO : Revisit both API's and unify them 1035 ******************************************************************************/ 1036 bool psci_is_last_on_cpu_safe(void) 1037 { 1038 unsigned int this_core = plat_my_core_pos(); 1039 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 1040 unsigned int i = 0; 1041 1042 /* 1043 * Traverse the forest of PSCI nodes, nodes with no parents 1044 * (invalid-nodes) are the root nodes. 1045 */ 1046 while ((psci_non_cpu_pd_nodes[i].parent_node == 1047 PSCI_PARENT_NODE_INVALID) && 1048 (i < PSCI_NUM_NON_CPU_PWR_DOMAINS)) { 1049 psci_get_parent_pwr_domain_nodes( 1050 psci_non_cpu_pd_nodes[i].cpu_start_idx, 1051 PLAT_MAX_PWR_LVL, parent_nodes); 1052 1053 psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1054 1055 for (unsigned int core = 0U; 1056 core < psci_non_cpu_pd_nodes[i].ncpus; core++) { 1057 if (core == this_core) { 1058 continue; 1059 } 1060 1061 if (psci_get_aff_info_state_by_idx(core) != 1062 AFF_STATE_OFF) { 1063 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, 1064 parent_nodes); 1065 VERBOSE("core=%u other than boot core=%u %s\n", 1066 core, this_core, "running in the system"); 1067 1068 return false; 1069 } 1070 } 1071 1072 psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); 1073 i++; 1074 } 1075 1076 return true; 1077 } 1078