1 /* 2 * Copyright (c) 2013-2020, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <string.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <drivers/delay_timer.h> 16 #include <lib/el3_runtime/context_mgmt.h> 17 #include <lib/utils.h> 18 #include <plat/common/platform.h> 19 20 #include "psci_private.h" 21 22 /* 23 * SPD power management operations, expected to be supplied by the registered 24 * SPD on successful SP initialization 25 */ 26 const spd_pm_ops_t *psci_spd_pm; 27 28 /* 29 * PSCI requested local power state map. This array is used to store the local 30 * power states requested by a CPU for power levels from level 1 to 31 * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power 32 * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a 33 * CPU are the same. 34 * 35 * During state coordination, the platform is passed an array containing the 36 * local states requested for a particular non cpu power domain by each cpu 37 * within the domain. 38 * 39 * TODO: Dense packing of the requested states will cause cache thrashing 40 * when multiple power domains write to it. If we allocate the requested 41 * states at each power level in a cache-line aligned per-domain memory, 42 * the cache thrashing can be avoided. 43 */ 44 static plat_local_state_t 45 psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT]; 46 47 unsigned int psci_plat_core_count; 48 49 /******************************************************************************* 50 * Arrays that hold the platform's power domain tree information for state 51 * management of power domains. 52 * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain 53 * which is an ancestor of a CPU power domain. 54 * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain 55 ******************************************************************************/ 56 non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS] 57 #if USE_COHERENT_MEM 58 __section("tzfw_coherent_mem") 59 #endif 60 ; 61 62 /* Lock for PSCI state coordination */ 63 DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); 64 65 cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; 66 67 /******************************************************************************* 68 * Pointer to functions exported by the platform to complete power mgmt. ops 69 ******************************************************************************/ 70 const plat_psci_ops_t *psci_plat_pm_ops; 71 72 /****************************************************************************** 73 * Check that the maximum power level supported by the platform makes sense 74 *****************************************************************************/ 75 CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) && 76 (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL), 77 assert_platform_max_pwrlvl_check); 78 79 /* 80 * The plat_local_state used by the platform is one of these types: RUN, 81 * RETENTION and OFF. The platform can define further sub-states for each type 82 * apart from RUN. This categorization is done to verify the sanity of the 83 * psci_power_state passed by the platform and to print debug information. The 84 * categorization is done on the basis of the following conditions: 85 * 86 * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN. 87 * 88 * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is 89 * STATE_TYPE_RETN. 90 * 91 * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is 92 * STATE_TYPE_OFF. 93 */ 94 typedef enum plat_local_state_type { 95 STATE_TYPE_RUN = 0, 96 STATE_TYPE_RETN, 97 STATE_TYPE_OFF 98 } plat_local_state_type_t; 99 100 /* Function used to categorize plat_local_state. */ 101 static plat_local_state_type_t find_local_state_type(plat_local_state_t state) 102 { 103 if (state != 0U) { 104 if (state > PLAT_MAX_RET_STATE) { 105 return STATE_TYPE_OFF; 106 } else { 107 return STATE_TYPE_RETN; 108 } 109 } else { 110 return STATE_TYPE_RUN; 111 } 112 } 113 114 /****************************************************************************** 115 * Check that the maximum retention level supported by the platform is less 116 * than the maximum off level. 117 *****************************************************************************/ 118 CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE, 119 assert_platform_max_off_and_retn_state_check); 120 121 /****************************************************************************** 122 * This function ensures that the power state parameter in a CPU_SUSPEND request 123 * is valid. If so, it returns the requested states for each power level. 124 *****************************************************************************/ 125 int psci_validate_power_state(unsigned int power_state, 126 psci_power_state_t *state_info) 127 { 128 /* Check SBZ bits in power state are zero */ 129 if (psci_check_power_state(power_state) != 0U) 130 return PSCI_E_INVALID_PARAMS; 131 132 assert(psci_plat_pm_ops->validate_power_state != NULL); 133 134 /* Validate the power_state using platform pm_ops */ 135 return psci_plat_pm_ops->validate_power_state(power_state, state_info); 136 } 137 138 /****************************************************************************** 139 * This function retrieves the `psci_power_state_t` for system suspend from 140 * the platform. 141 *****************************************************************************/ 142 void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info) 143 { 144 /* 145 * Assert that the required pm_ops hook is implemented to ensure that 146 * the capability detected during psci_setup() is valid. 147 */ 148 assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL); 149 150 /* 151 * Query the platform for the power_state required for system suspend 152 */ 153 psci_plat_pm_ops->get_sys_suspend_power_state(state_info); 154 } 155 156 /******************************************************************************* 157 * This function verifies that the all the other cores in the system have been 158 * turned OFF and the current CPU is the last running CPU in the system. 159 * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false) 160 * otherwise. 161 ******************************************************************************/ 162 unsigned int psci_is_last_on_cpu(void) 163 { 164 unsigned int cpu_idx, my_idx = plat_my_core_pos(); 165 166 for (cpu_idx = 0; cpu_idx < psci_plat_core_count; 167 cpu_idx++) { 168 if (cpu_idx == my_idx) { 169 assert(psci_get_aff_info_state() == AFF_STATE_ON); 170 continue; 171 } 172 173 if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) 174 return 0; 175 } 176 177 return 1; 178 } 179 180 /******************************************************************************* 181 * Routine to return the maximum power level to traverse to after a cpu has 182 * been physically powered up. It is expected to be called immediately after 183 * reset from assembler code. 184 ******************************************************************************/ 185 static unsigned int get_power_on_target_pwrlvl(void) 186 { 187 unsigned int pwrlvl; 188 189 /* 190 * Assume that this cpu was suspended and retrieve its target power 191 * level. If it is invalid then it could only have been turned off 192 * earlier. PLAT_MAX_PWR_LVL will be the highest power level a 193 * cpu can be turned off to. 194 */ 195 pwrlvl = psci_get_suspend_pwrlvl(); 196 if (pwrlvl == PSCI_INVALID_PWR_LVL) 197 pwrlvl = PLAT_MAX_PWR_LVL; 198 assert(pwrlvl < PSCI_INVALID_PWR_LVL); 199 return pwrlvl; 200 } 201 202 /****************************************************************************** 203 * Helper function to update the requested local power state array. This array 204 * does not store the requested state for the CPU power level. Hence an 205 * assertion is added to prevent us from accessing the CPU power level. 206 *****************************************************************************/ 207 static void psci_set_req_local_pwr_state(unsigned int pwrlvl, 208 unsigned int cpu_idx, 209 plat_local_state_t req_pwr_state) 210 { 211 assert(pwrlvl > PSCI_CPU_PWR_LVL); 212 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 213 (cpu_idx < psci_plat_core_count)) { 214 psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state; 215 } 216 } 217 218 /****************************************************************************** 219 * This function initializes the psci_req_local_pwr_states. 220 *****************************************************************************/ 221 void __init psci_init_req_local_pwr_states(void) 222 { 223 /* Initialize the requested state of all non CPU power domains as OFF */ 224 unsigned int pwrlvl; 225 unsigned int core; 226 227 for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) { 228 for (core = 0; core < psci_plat_core_count; core++) { 229 psci_req_local_pwr_states[pwrlvl][core] = 230 PLAT_MAX_OFF_STATE; 231 } 232 } 233 } 234 235 /****************************************************************************** 236 * Helper function to return a reference to an array containing the local power 237 * states requested by each cpu for a power domain at 'pwrlvl'. The size of the 238 * array will be the number of cpu power domains of which this power domain is 239 * an ancestor. These requested states will be used to determine a suitable 240 * target state for this power domain during psci state coordination. An 241 * assertion is added to prevent us from accessing the CPU power level. 242 *****************************************************************************/ 243 static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl, 244 unsigned int cpu_idx) 245 { 246 assert(pwrlvl > PSCI_CPU_PWR_LVL); 247 248 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 249 (cpu_idx < psci_plat_core_count)) { 250 return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx]; 251 } else 252 return NULL; 253 } 254 255 /* 256 * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent 257 * memory. 258 * 259 * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory, 260 * it's accessed by both cached and non-cached participants. To serve the common 261 * minimum, perform a cache flush before read and after write so that non-cached 262 * participants operate on latest data in main memory. 263 * 264 * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent 265 * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent. 266 * In both cases, no cache operations are required. 267 */ 268 269 /* 270 * Retrieve local state of non-CPU power domain node from a non-cached CPU, 271 * after any required cache maintenance operation. 272 */ 273 static plat_local_state_t get_non_cpu_pd_node_local_state( 274 unsigned int parent_idx) 275 { 276 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 277 flush_dcache_range( 278 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 279 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 280 #endif 281 return psci_non_cpu_pd_nodes[parent_idx].local_state; 282 } 283 284 /* 285 * Update local state of non-CPU power domain node from a cached CPU; perform 286 * any required cache maintenance operation afterwards. 287 */ 288 static void set_non_cpu_pd_node_local_state(unsigned int parent_idx, 289 plat_local_state_t state) 290 { 291 psci_non_cpu_pd_nodes[parent_idx].local_state = state; 292 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 293 flush_dcache_range( 294 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 295 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 296 #endif 297 } 298 299 /****************************************************************************** 300 * Helper function to return the current local power state of each power domain 301 * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This 302 * function will be called after a cpu is powered on to find the local state 303 * each power domain has emerged from. 304 *****************************************************************************/ 305 void psci_get_target_local_pwr_states(unsigned int end_pwrlvl, 306 psci_power_state_t *target_state) 307 { 308 unsigned int parent_idx, lvl; 309 plat_local_state_t *pd_state = target_state->pwr_domain_state; 310 311 pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state(); 312 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 313 314 /* Copy the local power state from node to state_info */ 315 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 316 pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx); 317 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 318 } 319 320 /* Set the the higher levels to RUN */ 321 for (; lvl <= PLAT_MAX_PWR_LVL; lvl++) 322 target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 323 } 324 325 /****************************************************************************** 326 * Helper function to set the target local power state that each power domain 327 * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will 328 * enter. This function will be called after coordination of requested power 329 * states has been done for each power level. 330 *****************************************************************************/ 331 static void psci_set_target_local_pwr_states(unsigned int end_pwrlvl, 332 const psci_power_state_t *target_state) 333 { 334 unsigned int parent_idx, lvl; 335 const plat_local_state_t *pd_state = target_state->pwr_domain_state; 336 337 psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]); 338 339 /* 340 * Need to flush as local_state might be accessed with Data Cache 341 * disabled during power on 342 */ 343 psci_flush_cpu_data(psci_svc_cpu_data.local_state); 344 345 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 346 347 /* Copy the local_state from state_info */ 348 for (lvl = 1U; lvl <= end_pwrlvl; lvl++) { 349 set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]); 350 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 351 } 352 } 353 354 355 /******************************************************************************* 356 * PSCI helper function to get the parent nodes corresponding to a cpu_index. 357 ******************************************************************************/ 358 void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx, 359 unsigned int end_lvl, 360 unsigned int *node_index) 361 { 362 unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node; 363 unsigned int i; 364 unsigned int *node = node_index; 365 366 for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) { 367 *node = parent_node; 368 node++; 369 parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node; 370 } 371 } 372 373 /****************************************************************************** 374 * This function is invoked post CPU power up and initialization. It sets the 375 * affinity info state, target power state and requested power state for the 376 * current CPU and all its ancestor power domains to RUN. 377 *****************************************************************************/ 378 void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl) 379 { 380 unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl; 381 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 382 383 /* Reset the local_state to RUN for the non cpu power domains. */ 384 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 385 set_non_cpu_pd_node_local_state(parent_idx, 386 PSCI_LOCAL_STATE_RUN); 387 psci_set_req_local_pwr_state(lvl, 388 cpu_idx, 389 PSCI_LOCAL_STATE_RUN); 390 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 391 } 392 393 /* Set the affinity info state to ON */ 394 psci_set_aff_info_state(AFF_STATE_ON); 395 396 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); 397 psci_flush_cpu_data(psci_svc_cpu_data); 398 } 399 400 /****************************************************************************** 401 * This function is passed the local power states requested for each power 402 * domain (state_info) between the current CPU domain and its ancestors until 403 * the target power level (end_pwrlvl). It updates the array of requested power 404 * states with this information. 405 * 406 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it 407 * retrieves the states requested by all the cpus of which the power domain at 408 * that level is an ancestor. It passes this information to the platform to 409 * coordinate and return the target power state. If the target state for a level 410 * is RUN then subsequent levels are not considered. At the CPU level, state 411 * coordination is not required. Hence, the requested and the target states are 412 * the same. 413 * 414 * The 'state_info' is updated with the target state for each level between the 415 * CPU and the 'end_pwrlvl' and returned to the caller. 416 * 417 * This function will only be invoked with data cache enabled and while 418 * powering down a core. 419 *****************************************************************************/ 420 void psci_do_state_coordination(unsigned int end_pwrlvl, 421 psci_power_state_t *state_info) 422 { 423 unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); 424 unsigned int start_idx; 425 unsigned int ncpus; 426 plat_local_state_t target_state, *req_states; 427 428 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); 429 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 430 431 /* For level 0, the requested state will be equivalent 432 to target state */ 433 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 434 435 /* First update the requested power state */ 436 psci_set_req_local_pwr_state(lvl, cpu_idx, 437 state_info->pwr_domain_state[lvl]); 438 439 /* Get the requested power states for this power level */ 440 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; 441 req_states = psci_get_req_local_pwr_states(lvl, start_idx); 442 443 /* 444 * Let the platform coordinate amongst the requested states at 445 * this power level and return the target local power state. 446 */ 447 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; 448 target_state = plat_get_target_pwr_state(lvl, 449 req_states, 450 ncpus); 451 452 state_info->pwr_domain_state[lvl] = target_state; 453 454 /* Break early if the negotiated target power state is RUN */ 455 if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) 456 break; 457 458 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 459 } 460 461 /* 462 * This is for cases when we break out of the above loop early because 463 * the target power state is RUN at a power level < end_pwlvl. 464 * We update the requested power state from state_info and then 465 * set the target state as RUN. 466 */ 467 for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) { 468 psci_set_req_local_pwr_state(lvl, cpu_idx, 469 state_info->pwr_domain_state[lvl]); 470 state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 471 472 } 473 474 /* Update the target state in the power domain nodes */ 475 psci_set_target_local_pwr_states(end_pwrlvl, state_info); 476 } 477 478 /****************************************************************************** 479 * This function validates a suspend request by making sure that if a standby 480 * state is requested then no power level is turned off and the highest power 481 * level is placed in a standby/retention state. 482 * 483 * It also ensures that the state level X will enter is not shallower than the 484 * state level X + 1 will enter. 485 * 486 * This validation will be enabled only for DEBUG builds as the platform is 487 * expected to perform these validations as well. 488 *****************************************************************************/ 489 int psci_validate_suspend_req(const psci_power_state_t *state_info, 490 unsigned int is_power_down_state) 491 { 492 unsigned int max_off_lvl, target_lvl, max_retn_lvl; 493 plat_local_state_t state; 494 plat_local_state_type_t req_state_type, deepest_state_type; 495 int i; 496 497 /* Find the target suspend power level */ 498 target_lvl = psci_find_target_suspend_lvl(state_info); 499 if (target_lvl == PSCI_INVALID_PWR_LVL) 500 return PSCI_E_INVALID_PARAMS; 501 502 /* All power domain levels are in a RUN state to begin with */ 503 deepest_state_type = STATE_TYPE_RUN; 504 505 for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) { 506 state = state_info->pwr_domain_state[i]; 507 req_state_type = find_local_state_type(state); 508 509 /* 510 * While traversing from the highest power level to the lowest, 511 * the state requested for lower levels has to be the same or 512 * deeper i.e. equal to or greater than the state at the higher 513 * levels. If this condition is true, then the requested state 514 * becomes the deepest state encountered so far. 515 */ 516 if (req_state_type < deepest_state_type) 517 return PSCI_E_INVALID_PARAMS; 518 deepest_state_type = req_state_type; 519 } 520 521 /* Find the highest off power level */ 522 max_off_lvl = psci_find_max_off_lvl(state_info); 523 524 /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */ 525 max_retn_lvl = PSCI_INVALID_PWR_LVL; 526 if (target_lvl != max_off_lvl) 527 max_retn_lvl = target_lvl; 528 529 /* 530 * If this is not a request for a power down state then max off level 531 * has to be invalid and max retention level has to be a valid power 532 * level. 533 */ 534 if ((is_power_down_state == 0U) && 535 ((max_off_lvl != PSCI_INVALID_PWR_LVL) || 536 (max_retn_lvl == PSCI_INVALID_PWR_LVL))) 537 return PSCI_E_INVALID_PARAMS; 538 539 return PSCI_E_SUCCESS; 540 } 541 542 /****************************************************************************** 543 * This function finds the highest power level which will be powered down 544 * amongst all the power levels specified in the 'state_info' structure 545 *****************************************************************************/ 546 unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info) 547 { 548 int i; 549 550 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 551 if (is_local_state_off(state_info->pwr_domain_state[i]) != 0) 552 return (unsigned int) i; 553 } 554 555 return PSCI_INVALID_PWR_LVL; 556 } 557 558 /****************************************************************************** 559 * This functions finds the level of the highest power domain which will be 560 * placed in a low power state during a suspend operation. 561 *****************************************************************************/ 562 unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info) 563 { 564 int i; 565 566 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 567 if (is_local_state_run(state_info->pwr_domain_state[i]) == 0) 568 return (unsigned int) i; 569 } 570 571 return PSCI_INVALID_PWR_LVL; 572 } 573 574 /******************************************************************************* 575 * This function is passed the highest level in the topology tree that the 576 * operation should be applied to and a list of node indexes. It picks up locks 577 * from the node index list in order of increasing power domain level in the 578 * range specified. 579 ******************************************************************************/ 580 void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, 581 const unsigned int *parent_nodes) 582 { 583 unsigned int parent_idx; 584 unsigned int level; 585 586 /* No locking required for level 0. Hence start locking from level 1 */ 587 for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) { 588 parent_idx = parent_nodes[level - 1U]; 589 psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]); 590 } 591 } 592 593 /******************************************************************************* 594 * This function is passed the highest level in the topology tree that the 595 * operation should be applied to and a list of node indexes. It releases the 596 * locks in order of decreasing power domain level in the range specified. 597 ******************************************************************************/ 598 void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, 599 const unsigned int *parent_nodes) 600 { 601 unsigned int parent_idx; 602 unsigned int level; 603 604 /* Unlock top down. No unlocking required for level 0. */ 605 for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) { 606 parent_idx = parent_nodes[level - 1U]; 607 psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]); 608 } 609 } 610 611 /******************************************************************************* 612 * Simple routine to determine whether a mpidr is valid or not. 613 ******************************************************************************/ 614 int psci_validate_mpidr(u_register_t mpidr) 615 { 616 if (plat_core_pos_by_mpidr(mpidr) < 0) 617 return PSCI_E_INVALID_PARAMS; 618 619 return PSCI_E_SUCCESS; 620 } 621 622 /******************************************************************************* 623 * This function determines the full entrypoint information for the requested 624 * PSCI entrypoint on power on/resume and returns it. 625 ******************************************************************************/ 626 #ifdef __aarch64__ 627 static int psci_get_ns_ep_info(entry_point_info_t *ep, 628 uintptr_t entrypoint, 629 u_register_t context_id) 630 { 631 u_register_t ep_attr, sctlr; 632 unsigned int daif, ee, mode; 633 u_register_t ns_scr_el3 = read_scr_el3(); 634 u_register_t ns_sctlr_el1 = read_sctlr_el1(); 635 636 sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 637 read_sctlr_el2() : ns_sctlr_el1; 638 ee = 0; 639 640 ep_attr = NON_SECURE | EP_ST_DISABLE; 641 if ((sctlr & SCTLR_EE_BIT) != 0U) { 642 ep_attr |= EP_EE_BIG; 643 ee = 1; 644 } 645 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 646 647 ep->pc = entrypoint; 648 zeromem(&ep->args, sizeof(ep->args)); 649 ep->args.arg0 = context_id; 650 651 /* 652 * Figure out whether the cpu enters the non-secure address space 653 * in aarch32 or aarch64 654 */ 655 if ((ns_scr_el3 & SCR_RW_BIT) != 0U) { 656 657 /* 658 * Check whether a Thumb entry point has been provided for an 659 * aarch64 EL 660 */ 661 if ((entrypoint & 0x1UL) != 0UL) 662 return PSCI_E_INVALID_ADDRESS; 663 664 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1; 665 666 ep->spsr = SPSR_64(mode, MODE_SP_ELX, DISABLE_ALL_EXCEPTIONS); 667 } else { 668 669 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 670 MODE32_hyp : MODE32_svc; 671 672 /* 673 * TODO: Choose async. exception bits if HYP mode is not 674 * implemented according to the values of SCR.{AW, FW} bits 675 */ 676 daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT; 677 678 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, daif); 679 } 680 681 return PSCI_E_SUCCESS; 682 } 683 #else /* !__aarch64__ */ 684 static int psci_get_ns_ep_info(entry_point_info_t *ep, 685 uintptr_t entrypoint, 686 u_register_t context_id) 687 { 688 u_register_t ep_attr; 689 unsigned int aif, ee, mode; 690 u_register_t scr = read_scr(); 691 u_register_t ns_sctlr, sctlr; 692 693 /* Switch to non secure state */ 694 write_scr(scr | SCR_NS_BIT); 695 isb(); 696 ns_sctlr = read_sctlr(); 697 698 sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr; 699 700 /* Return to original state */ 701 write_scr(scr); 702 isb(); 703 ee = 0; 704 705 ep_attr = NON_SECURE | EP_ST_DISABLE; 706 if (sctlr & SCTLR_EE_BIT) { 707 ep_attr |= EP_EE_BIG; 708 ee = 1; 709 } 710 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 711 712 ep->pc = entrypoint; 713 zeromem(&ep->args, sizeof(ep->args)); 714 ep->args.arg0 = context_id; 715 716 mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc; 717 718 /* 719 * TODO: Choose async. exception bits if HYP mode is not 720 * implemented according to the values of SCR.{AW, FW} bits 721 */ 722 aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT; 723 724 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif); 725 726 return PSCI_E_SUCCESS; 727 } 728 729 #endif /* __aarch64__ */ 730 731 /******************************************************************************* 732 * This function validates the entrypoint with the platform layer if the 733 * appropriate pm_ops hook is exported by the platform and returns the 734 * 'entry_point_info'. 735 ******************************************************************************/ 736 int psci_validate_entry_point(entry_point_info_t *ep, 737 uintptr_t entrypoint, 738 u_register_t context_id) 739 { 740 int rc; 741 742 /* Validate the entrypoint using platform psci_ops */ 743 if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) { 744 rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint); 745 if (rc != PSCI_E_SUCCESS) 746 return PSCI_E_INVALID_ADDRESS; 747 } 748 749 /* 750 * Verify and derive the re-entry information for 751 * the non-secure world from the non-secure state from 752 * where this call originated. 753 */ 754 rc = psci_get_ns_ep_info(ep, entrypoint, context_id); 755 return rc; 756 } 757 758 /******************************************************************************* 759 * Generic handler which is called when a cpu is physically powered on. It 760 * traverses the node information and finds the highest power level powered 761 * off and performs generic, architectural, platform setup and state management 762 * to power on that power level and power levels below it. 763 * e.g. For a cpu that's been powered on, it will call the platform specific 764 * code to enable the gic cpu interface and for a cluster it will enable 765 * coherency at the interconnect level in addition to gic cpu interface. 766 ******************************************************************************/ 767 void psci_warmboot_entrypoint(void) 768 { 769 unsigned int end_pwrlvl; 770 unsigned int cpu_idx = plat_my_core_pos(); 771 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 772 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; 773 774 /* 775 * Verify that we have been explicitly turned ON or resumed from 776 * suspend. 777 */ 778 if (psci_get_aff_info_state() == AFF_STATE_OFF) { 779 ERROR("Unexpected affinity info state.\n"); 780 panic(); 781 } 782 783 /* 784 * Get the maximum power domain level to traverse to after this cpu 785 * has been physically powered up. 786 */ 787 end_pwrlvl = get_power_on_target_pwrlvl(); 788 789 /* Get the parent nodes */ 790 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); 791 792 /* 793 * This function acquires the lock corresponding to each power level so 794 * that by the time all locks are taken, the system topology is snapshot 795 * and state management can be done safely. 796 */ 797 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 798 799 psci_get_target_local_pwr_states(end_pwrlvl, &state_info); 800 801 #if ENABLE_PSCI_STAT 802 plat_psci_stat_accounting_stop(&state_info); 803 #endif 804 805 /* 806 * This CPU could be resuming from suspend or it could have just been 807 * turned on. To distinguish between these 2 cases, we examine the 808 * affinity state of the CPU: 809 * - If the affinity state is ON_PENDING then it has just been 810 * turned on. 811 * - Else it is resuming from suspend. 812 * 813 * Depending on the type of warm reset identified, choose the right set 814 * of power management handler and perform the generic, architecture 815 * and platform specific handling. 816 */ 817 if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING) 818 psci_cpu_on_finish(cpu_idx, &state_info); 819 else 820 psci_cpu_suspend_finish(cpu_idx, &state_info); 821 822 /* 823 * Set the requested and target state of this CPU and all the higher 824 * power domains which are ancestors of this CPU to run. 825 */ 826 psci_set_pwr_domains_to_run(end_pwrlvl); 827 828 #if ENABLE_PSCI_STAT 829 /* 830 * Update PSCI stats. 831 * Caches are off when writing stats data on the power down path. 832 * Since caches are now enabled, it's necessary to do cache 833 * maintenance before reading that same data. 834 */ 835 psci_stats_update_pwr_up(end_pwrlvl, &state_info); 836 #endif 837 838 /* 839 * This loop releases the lock corresponding to each power level 840 * in the reverse order to which they were acquired. 841 */ 842 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 843 } 844 845 /******************************************************************************* 846 * This function initializes the set of hooks that PSCI invokes as part of power 847 * management operation. The power management hooks are expected to be provided 848 * by the SPD, after it finishes all its initialization 849 ******************************************************************************/ 850 void psci_register_spd_pm_hook(const spd_pm_ops_t *pm) 851 { 852 assert(pm != NULL); 853 psci_spd_pm = pm; 854 855 if (pm->svc_migrate != NULL) 856 psci_caps |= define_psci_cap(PSCI_MIG_AARCH64); 857 858 if (pm->svc_migrate_info != NULL) 859 psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) 860 | define_psci_cap(PSCI_MIG_INFO_TYPE); 861 } 862 863 /******************************************************************************* 864 * This function invokes the migrate info hook in the spd_pm_ops. It performs 865 * the necessary return value validation. If the Secure Payload is UP and 866 * migrate capable, it returns the mpidr of the CPU on which the Secure payload 867 * is resident through the mpidr parameter. Else the value of the parameter on 868 * return is undefined. 869 ******************************************************************************/ 870 int psci_spd_migrate_info(u_register_t *mpidr) 871 { 872 int rc; 873 874 if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL)) 875 return PSCI_E_NOT_SUPPORTED; 876 877 rc = psci_spd_pm->svc_migrate_info(mpidr); 878 879 assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) || 880 (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED)); 881 882 return rc; 883 } 884 885 886 /******************************************************************************* 887 * This function prints the state of all power domains present in the 888 * system 889 ******************************************************************************/ 890 void psci_print_power_domain_map(void) 891 { 892 #if LOG_LEVEL >= LOG_LEVEL_INFO 893 unsigned int idx; 894 plat_local_state_t state; 895 plat_local_state_type_t state_type; 896 897 /* This array maps to the PSCI_STATE_X definitions in psci.h */ 898 static const char * const psci_state_type_str[] = { 899 "ON", 900 "RETENTION", 901 "OFF", 902 }; 903 904 INFO("PSCI Power Domain Map:\n"); 905 for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count); 906 idx++) { 907 state_type = find_local_state_type( 908 psci_non_cpu_pd_nodes[idx].local_state); 909 INFO(" Domain Node : Level %u, parent_node %d," 910 " State %s (0x%x)\n", 911 psci_non_cpu_pd_nodes[idx].level, 912 psci_non_cpu_pd_nodes[idx].parent_node, 913 psci_state_type_str[state_type], 914 psci_non_cpu_pd_nodes[idx].local_state); 915 } 916 917 for (idx = 0; idx < psci_plat_core_count; idx++) { 918 state = psci_get_cpu_local_state_by_idx(idx); 919 state_type = find_local_state_type(state); 920 INFO(" CPU Node : MPID 0x%llx, parent_node %d," 921 " State %s (0x%x)\n", 922 (unsigned long long)psci_cpu_pd_nodes[idx].mpidr, 923 psci_cpu_pd_nodes[idx].parent_node, 924 psci_state_type_str[state_type], 925 psci_get_cpu_local_state_by_idx(idx)); 926 } 927 #endif 928 } 929 930 /****************************************************************************** 931 * Return whether any secondaries were powered up with CPU_ON call. A CPU that 932 * have ever been powered up would have set its MPDIR value to something other 933 * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to 934 * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is 935 * meaningful only when called on the primary CPU during early boot. 936 *****************************************************************************/ 937 int psci_secondaries_brought_up(void) 938 { 939 unsigned int idx, n_valid = 0U; 940 941 for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) { 942 if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR) 943 n_valid++; 944 } 945 946 assert(n_valid > 0U); 947 948 return (n_valid > 1U) ? 1 : 0; 949 } 950 951 /******************************************************************************* 952 * Initiate power down sequence, by calling power down operations registered for 953 * this CPU. 954 ******************************************************************************/ 955 void psci_do_pwrdown_sequence(unsigned int power_level) 956 { 957 #if HW_ASSISTED_COHERENCY 958 /* 959 * With hardware-assisted coherency, the CPU drivers only initiate the 960 * power down sequence, without performing cache-maintenance operations 961 * in software. Data caches enabled both before and after this call. 962 */ 963 prepare_cpu_pwr_dwn(power_level); 964 #else 965 /* 966 * Without hardware-assisted coherency, the CPU drivers disable data 967 * caches, then perform cache-maintenance operations in software. 968 * 969 * This also calls prepare_cpu_pwr_dwn() to initiate power down 970 * sequence, but that function will return with data caches disabled. 971 * We must ensure that the stack memory is flushed out to memory before 972 * we start popping from it again. 973 */ 974 psci_do_pwrdown_cache_maintenance(power_level); 975 #endif 976 } 977 978 /******************************************************************************* 979 * This function invokes the callback 'stop_func()' with the 'mpidr' of each 980 * online PE. Caller can pass suitable method to stop a remote core. 981 * 982 * 'wait_ms' is the timeout value in milliseconds for the other cores to 983 * transition to power down state. Passing '0' makes it non-blocking. 984 * 985 * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the 986 * given timeout. 987 ******************************************************************************/ 988 int psci_stop_other_cores(unsigned int wait_ms, 989 void (*stop_func)(u_register_t mpidr)) 990 { 991 unsigned int idx, this_cpu_idx; 992 993 this_cpu_idx = plat_my_core_pos(); 994 995 /* Invoke stop_func for each core */ 996 for (idx = 0U; idx < psci_plat_core_count; idx++) { 997 /* skip current CPU */ 998 if (idx == this_cpu_idx) { 999 continue; 1000 } 1001 1002 /* Check if the CPU is ON */ 1003 if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) { 1004 (*stop_func)(psci_cpu_pd_nodes[idx].mpidr); 1005 } 1006 } 1007 1008 /* Need to wait for other cores to shutdown */ 1009 if (wait_ms != 0U) { 1010 while ((wait_ms-- != 0U) && (psci_is_last_on_cpu() != 0U)) { 1011 mdelay(1U); 1012 } 1013 1014 if (psci_is_last_on_cpu() != 0U) { 1015 WARN("Failed to stop all cores!\n"); 1016 psci_print_power_domain_map(); 1017 return PSCI_E_DENIED; 1018 } 1019 } 1020 1021 return PSCI_E_SUCCESS; 1022 } 1023