1 /* 2 * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <string.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <lib/el3_runtime/context_mgmt.h> 16 #include <lib/utils.h> 17 #include <plat/common/platform.h> 18 19 #include "psci_private.h" 20 21 /* 22 * SPD power management operations, expected to be supplied by the registered 23 * SPD on successful SP initialization 24 */ 25 const spd_pm_ops_t *psci_spd_pm; 26 27 /* 28 * PSCI requested local power state map. This array is used to store the local 29 * power states requested by a CPU for power levels from level 1 to 30 * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power 31 * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a 32 * CPU are the same. 33 * 34 * During state coordination, the platform is passed an array containing the 35 * local states requested for a particular non cpu power domain by each cpu 36 * within the domain. 37 * 38 * TODO: Dense packing of the requested states will cause cache thrashing 39 * when multiple power domains write to it. If we allocate the requested 40 * states at each power level in a cache-line aligned per-domain memory, 41 * the cache thrashing can be avoided. 42 */ 43 static plat_local_state_t 44 psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT]; 45 46 47 /******************************************************************************* 48 * Arrays that hold the platform's power domain tree information for state 49 * management of power domains. 50 * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain 51 * which is an ancestor of a CPU power domain. 52 * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain 53 ******************************************************************************/ 54 non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS] 55 #if USE_COHERENT_MEM 56 __section("tzfw_coherent_mem") 57 #endif 58 ; 59 60 /* Lock for PSCI state coordination */ 61 DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); 62 63 cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; 64 65 /******************************************************************************* 66 * Pointer to functions exported by the platform to complete power mgmt. ops 67 ******************************************************************************/ 68 const plat_psci_ops_t *psci_plat_pm_ops; 69 70 /****************************************************************************** 71 * Check that the maximum power level supported by the platform makes sense 72 *****************************************************************************/ 73 CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) && 74 (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL), 75 assert_platform_max_pwrlvl_check); 76 77 /* 78 * The plat_local_state used by the platform is one of these types: RUN, 79 * RETENTION and OFF. The platform can define further sub-states for each type 80 * apart from RUN. This categorization is done to verify the sanity of the 81 * psci_power_state passed by the platform and to print debug information. The 82 * categorization is done on the basis of the following conditions: 83 * 84 * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN. 85 * 86 * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is 87 * STATE_TYPE_RETN. 88 * 89 * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is 90 * STATE_TYPE_OFF. 91 */ 92 typedef enum plat_local_state_type { 93 STATE_TYPE_RUN = 0, 94 STATE_TYPE_RETN, 95 STATE_TYPE_OFF 96 } plat_local_state_type_t; 97 98 /* Function used to categorize plat_local_state. */ 99 static plat_local_state_type_t find_local_state_type(plat_local_state_t state) 100 { 101 if (state != 0U) { 102 if (state > PLAT_MAX_RET_STATE) { 103 return STATE_TYPE_OFF; 104 } else { 105 return STATE_TYPE_RETN; 106 } 107 } else { 108 return STATE_TYPE_RUN; 109 } 110 } 111 112 /****************************************************************************** 113 * Check that the maximum retention level supported by the platform is less 114 * than the maximum off level. 115 *****************************************************************************/ 116 CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE, 117 assert_platform_max_off_and_retn_state_check); 118 119 /****************************************************************************** 120 * This function ensures that the power state parameter in a CPU_SUSPEND request 121 * is valid. If so, it returns the requested states for each power level. 122 *****************************************************************************/ 123 int psci_validate_power_state(unsigned int power_state, 124 psci_power_state_t *state_info) 125 { 126 /* Check SBZ bits in power state are zero */ 127 if (psci_check_power_state(power_state) != 0U) 128 return PSCI_E_INVALID_PARAMS; 129 130 assert(psci_plat_pm_ops->validate_power_state != NULL); 131 132 /* Validate the power_state using platform pm_ops */ 133 return psci_plat_pm_ops->validate_power_state(power_state, state_info); 134 } 135 136 /****************************************************************************** 137 * This function retrieves the `psci_power_state_t` for system suspend from 138 * the platform. 139 *****************************************************************************/ 140 void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info) 141 { 142 /* 143 * Assert that the required pm_ops hook is implemented to ensure that 144 * the capability detected during psci_setup() is valid. 145 */ 146 assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL); 147 148 /* 149 * Query the platform for the power_state required for system suspend 150 */ 151 psci_plat_pm_ops->get_sys_suspend_power_state(state_info); 152 } 153 154 /******************************************************************************* 155 * This function verifies that the all the other cores in the system have been 156 * turned OFF and the current CPU is the last running CPU in the system. 157 * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false) 158 * otherwise. 159 ******************************************************************************/ 160 unsigned int psci_is_last_on_cpu(void) 161 { 162 int cpu_idx, my_idx = (int) plat_my_core_pos(); 163 164 for (cpu_idx = 0; cpu_idx < PLATFORM_CORE_COUNT; cpu_idx++) { 165 if (cpu_idx == my_idx) { 166 assert(psci_get_aff_info_state() == AFF_STATE_ON); 167 continue; 168 } 169 170 if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) 171 return 0; 172 } 173 174 return 1; 175 } 176 177 /******************************************************************************* 178 * Routine to return the maximum power level to traverse to after a cpu has 179 * been physically powered up. It is expected to be called immediately after 180 * reset from assembler code. 181 ******************************************************************************/ 182 static unsigned int get_power_on_target_pwrlvl(void) 183 { 184 unsigned int pwrlvl; 185 186 /* 187 * Assume that this cpu was suspended and retrieve its target power 188 * level. If it is invalid then it could only have been turned off 189 * earlier. PLAT_MAX_PWR_LVL will be the highest power level a 190 * cpu can be turned off to. 191 */ 192 pwrlvl = psci_get_suspend_pwrlvl(); 193 if (pwrlvl == PSCI_INVALID_PWR_LVL) 194 pwrlvl = PLAT_MAX_PWR_LVL; 195 return pwrlvl; 196 } 197 198 /****************************************************************************** 199 * Helper function to update the requested local power state array. This array 200 * does not store the requested state for the CPU power level. Hence an 201 * assertion is added to prevent us from accessing the CPU power level. 202 *****************************************************************************/ 203 static void psci_set_req_local_pwr_state(unsigned int pwrlvl, 204 unsigned int cpu_idx, 205 plat_local_state_t req_pwr_state) 206 { 207 assert(pwrlvl > PSCI_CPU_PWR_LVL); 208 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 209 (cpu_idx < PLATFORM_CORE_COUNT)) { 210 psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state; 211 } 212 } 213 214 /****************************************************************************** 215 * This function initializes the psci_req_local_pwr_states. 216 *****************************************************************************/ 217 void __init psci_init_req_local_pwr_states(void) 218 { 219 /* Initialize the requested state of all non CPU power domains as OFF */ 220 unsigned int pwrlvl; 221 int core; 222 223 for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) { 224 for (core = 0; core < PLATFORM_CORE_COUNT; core++) { 225 psci_req_local_pwr_states[pwrlvl][core] = 226 PLAT_MAX_OFF_STATE; 227 } 228 } 229 } 230 231 /****************************************************************************** 232 * Helper function to return a reference to an array containing the local power 233 * states requested by each cpu for a power domain at 'pwrlvl'. The size of the 234 * array will be the number of cpu power domains of which this power domain is 235 * an ancestor. These requested states will be used to determine a suitable 236 * target state for this power domain during psci state coordination. An 237 * assertion is added to prevent us from accessing the CPU power level. 238 *****************************************************************************/ 239 static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl, 240 int cpu_idx) 241 { 242 assert(pwrlvl > PSCI_CPU_PWR_LVL); 243 244 if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && 245 (cpu_idx < PLATFORM_CORE_COUNT)) { 246 return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx]; 247 } else 248 return NULL; 249 } 250 251 /* 252 * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent 253 * memory. 254 * 255 * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory, 256 * it's accessed by both cached and non-cached participants. To serve the common 257 * minimum, perform a cache flush before read and after write so that non-cached 258 * participants operate on latest data in main memory. 259 * 260 * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent 261 * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent. 262 * In both cases, no cache operations are required. 263 */ 264 265 /* 266 * Retrieve local state of non-CPU power domain node from a non-cached CPU, 267 * after any required cache maintenance operation. 268 */ 269 static plat_local_state_t get_non_cpu_pd_node_local_state( 270 unsigned int parent_idx) 271 { 272 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 273 flush_dcache_range( 274 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 275 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 276 #endif 277 return psci_non_cpu_pd_nodes[parent_idx].local_state; 278 } 279 280 /* 281 * Update local state of non-CPU power domain node from a cached CPU; perform 282 * any required cache maintenance operation afterwards. 283 */ 284 static void set_non_cpu_pd_node_local_state(unsigned int parent_idx, 285 plat_local_state_t state) 286 { 287 psci_non_cpu_pd_nodes[parent_idx].local_state = state; 288 #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 289 flush_dcache_range( 290 (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], 291 sizeof(psci_non_cpu_pd_nodes[parent_idx])); 292 #endif 293 } 294 295 /****************************************************************************** 296 * Helper function to return the current local power state of each power domain 297 * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This 298 * function will be called after a cpu is powered on to find the local state 299 * each power domain has emerged from. 300 *****************************************************************************/ 301 void psci_get_target_local_pwr_states(unsigned int end_pwrlvl, 302 psci_power_state_t *target_state) 303 { 304 unsigned int parent_idx, lvl; 305 plat_local_state_t *pd_state = target_state->pwr_domain_state; 306 307 pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state(); 308 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 309 310 /* Copy the local power state from node to state_info */ 311 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 312 pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx); 313 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 314 } 315 316 /* Set the the higher levels to RUN */ 317 for (; lvl <= PLAT_MAX_PWR_LVL; lvl++) 318 target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 319 } 320 321 /****************************************************************************** 322 * Helper function to set the target local power state that each power domain 323 * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will 324 * enter. This function will be called after coordination of requested power 325 * states has been done for each power level. 326 *****************************************************************************/ 327 static void psci_set_target_local_pwr_states(unsigned int end_pwrlvl, 328 const psci_power_state_t *target_state) 329 { 330 unsigned int parent_idx, lvl; 331 const plat_local_state_t *pd_state = target_state->pwr_domain_state; 332 333 psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]); 334 335 /* 336 * Need to flush as local_state might be accessed with Data Cache 337 * disabled during power on 338 */ 339 psci_flush_cpu_data(psci_svc_cpu_data.local_state); 340 341 parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; 342 343 /* Copy the local_state from state_info */ 344 for (lvl = 1U; lvl <= end_pwrlvl; lvl++) { 345 set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]); 346 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 347 } 348 } 349 350 351 /******************************************************************************* 352 * PSCI helper function to get the parent nodes corresponding to a cpu_index. 353 ******************************************************************************/ 354 void psci_get_parent_pwr_domain_nodes(int cpu_idx, 355 unsigned int end_lvl, 356 unsigned int *node_index) 357 { 358 unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node; 359 unsigned int i; 360 unsigned int *node = node_index; 361 362 for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) { 363 *node = parent_node; 364 node++; 365 parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node; 366 } 367 } 368 369 /****************************************************************************** 370 * This function is invoked post CPU power up and initialization. It sets the 371 * affinity info state, target power state and requested power state for the 372 * current CPU and all its ancestor power domains to RUN. 373 *****************************************************************************/ 374 void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl) 375 { 376 unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl; 377 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 378 379 /* Reset the local_state to RUN for the non cpu power domains. */ 380 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 381 set_non_cpu_pd_node_local_state(parent_idx, 382 PSCI_LOCAL_STATE_RUN); 383 psci_set_req_local_pwr_state(lvl, 384 cpu_idx, 385 PSCI_LOCAL_STATE_RUN); 386 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 387 } 388 389 /* Set the affinity info state to ON */ 390 psci_set_aff_info_state(AFF_STATE_ON); 391 392 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); 393 psci_flush_cpu_data(psci_svc_cpu_data); 394 } 395 396 /****************************************************************************** 397 * This function is passed the local power states requested for each power 398 * domain (state_info) between the current CPU domain and its ancestors until 399 * the target power level (end_pwrlvl). It updates the array of requested power 400 * states with this information. 401 * 402 * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it 403 * retrieves the states requested by all the cpus of which the power domain at 404 * that level is an ancestor. It passes this information to the platform to 405 * coordinate and return the target power state. If the target state for a level 406 * is RUN then subsequent levels are not considered. At the CPU level, state 407 * coordination is not required. Hence, the requested and the target states are 408 * the same. 409 * 410 * The 'state_info' is updated with the target state for each level between the 411 * CPU and the 'end_pwrlvl' and returned to the caller. 412 * 413 * This function will only be invoked with data cache enabled and while 414 * powering down a core. 415 *****************************************************************************/ 416 void psci_do_state_coordination(unsigned int end_pwrlvl, 417 psci_power_state_t *state_info) 418 { 419 unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); 420 int start_idx; 421 unsigned int ncpus; 422 plat_local_state_t target_state, *req_states; 423 424 assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); 425 parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; 426 427 /* For level 0, the requested state will be equivalent 428 to target state */ 429 for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { 430 431 /* First update the requested power state */ 432 psci_set_req_local_pwr_state(lvl, cpu_idx, 433 state_info->pwr_domain_state[lvl]); 434 435 /* Get the requested power states for this power level */ 436 start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; 437 req_states = psci_get_req_local_pwr_states(lvl, start_idx); 438 439 /* 440 * Let the platform coordinate amongst the requested states at 441 * this power level and return the target local power state. 442 */ 443 ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; 444 target_state = plat_get_target_pwr_state(lvl, 445 req_states, 446 ncpus); 447 448 state_info->pwr_domain_state[lvl] = target_state; 449 450 /* Break early if the negotiated target power state is RUN */ 451 if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) 452 break; 453 454 parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; 455 } 456 457 /* 458 * This is for cases when we break out of the above loop early because 459 * the target power state is RUN at a power level < end_pwlvl. 460 * We update the requested power state from state_info and then 461 * set the target state as RUN. 462 */ 463 for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) { 464 psci_set_req_local_pwr_state(lvl, cpu_idx, 465 state_info->pwr_domain_state[lvl]); 466 state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; 467 468 } 469 470 /* Update the target state in the power domain nodes */ 471 psci_set_target_local_pwr_states(end_pwrlvl, state_info); 472 } 473 474 /****************************************************************************** 475 * This function validates a suspend request by making sure that if a standby 476 * state is requested then no power level is turned off and the highest power 477 * level is placed in a standby/retention state. 478 * 479 * It also ensures that the state level X will enter is not shallower than the 480 * state level X + 1 will enter. 481 * 482 * This validation will be enabled only for DEBUG builds as the platform is 483 * expected to perform these validations as well. 484 *****************************************************************************/ 485 int psci_validate_suspend_req(const psci_power_state_t *state_info, 486 unsigned int is_power_down_state) 487 { 488 unsigned int max_off_lvl, target_lvl, max_retn_lvl; 489 plat_local_state_t state; 490 plat_local_state_type_t req_state_type, deepest_state_type; 491 int i; 492 493 /* Find the target suspend power level */ 494 target_lvl = psci_find_target_suspend_lvl(state_info); 495 if (target_lvl == PSCI_INVALID_PWR_LVL) 496 return PSCI_E_INVALID_PARAMS; 497 498 /* All power domain levels are in a RUN state to begin with */ 499 deepest_state_type = STATE_TYPE_RUN; 500 501 for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) { 502 state = state_info->pwr_domain_state[i]; 503 req_state_type = find_local_state_type(state); 504 505 /* 506 * While traversing from the highest power level to the lowest, 507 * the state requested for lower levels has to be the same or 508 * deeper i.e. equal to or greater than the state at the higher 509 * levels. If this condition is true, then the requested state 510 * becomes the deepest state encountered so far. 511 */ 512 if (req_state_type < deepest_state_type) 513 return PSCI_E_INVALID_PARAMS; 514 deepest_state_type = req_state_type; 515 } 516 517 /* Find the highest off power level */ 518 max_off_lvl = psci_find_max_off_lvl(state_info); 519 520 /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */ 521 max_retn_lvl = PSCI_INVALID_PWR_LVL; 522 if (target_lvl != max_off_lvl) 523 max_retn_lvl = target_lvl; 524 525 /* 526 * If this is not a request for a power down state then max off level 527 * has to be invalid and max retention level has to be a valid power 528 * level. 529 */ 530 if ((is_power_down_state == 0U) && 531 ((max_off_lvl != PSCI_INVALID_PWR_LVL) || 532 (max_retn_lvl == PSCI_INVALID_PWR_LVL))) 533 return PSCI_E_INVALID_PARAMS; 534 535 return PSCI_E_SUCCESS; 536 } 537 538 /****************************************************************************** 539 * This function finds the highest power level which will be powered down 540 * amongst all the power levels specified in the 'state_info' structure 541 *****************************************************************************/ 542 unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info) 543 { 544 int i; 545 546 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 547 if (is_local_state_off(state_info->pwr_domain_state[i]) != 0) 548 return (unsigned int) i; 549 } 550 551 return PSCI_INVALID_PWR_LVL; 552 } 553 554 /****************************************************************************** 555 * This functions finds the level of the highest power domain which will be 556 * placed in a low power state during a suspend operation. 557 *****************************************************************************/ 558 unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info) 559 { 560 int i; 561 562 for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { 563 if (is_local_state_run(state_info->pwr_domain_state[i]) == 0) 564 return (unsigned int) i; 565 } 566 567 return PSCI_INVALID_PWR_LVL; 568 } 569 570 /******************************************************************************* 571 * This function is passed the highest level in the topology tree that the 572 * operation should be applied to and a list of node indexes. It picks up locks 573 * from the node index list in order of increasing power domain level in the 574 * range specified. 575 ******************************************************************************/ 576 void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, 577 const unsigned int *parent_nodes) 578 { 579 unsigned int parent_idx; 580 unsigned int level; 581 582 /* No locking required for level 0. Hence start locking from level 1 */ 583 for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) { 584 parent_idx = parent_nodes[level - 1U]; 585 psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]); 586 } 587 } 588 589 /******************************************************************************* 590 * This function is passed the highest level in the topology tree that the 591 * operation should be applied to and a list of node indexes. It releases the 592 * locks in order of decreasing power domain level in the range specified. 593 ******************************************************************************/ 594 void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, 595 const unsigned int *parent_nodes) 596 { 597 unsigned int parent_idx; 598 unsigned int level; 599 600 /* Unlock top down. No unlocking required for level 0. */ 601 for (level = end_pwrlvl; level >= PSCI_CPU_PWR_LVL + 1U; level--) { 602 parent_idx = parent_nodes[level - 1U]; 603 psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]); 604 } 605 } 606 607 /******************************************************************************* 608 * Simple routine to determine whether a mpidr is valid or not. 609 ******************************************************************************/ 610 int psci_validate_mpidr(u_register_t mpidr) 611 { 612 if (plat_core_pos_by_mpidr(mpidr) < 0) 613 return PSCI_E_INVALID_PARAMS; 614 615 return PSCI_E_SUCCESS; 616 } 617 618 /******************************************************************************* 619 * This function determines the full entrypoint information for the requested 620 * PSCI entrypoint on power on/resume and returns it. 621 ******************************************************************************/ 622 #ifdef __aarch64__ 623 static int psci_get_ns_ep_info(entry_point_info_t *ep, 624 uintptr_t entrypoint, 625 u_register_t context_id) 626 { 627 u_register_t ep_attr, sctlr; 628 unsigned int daif, ee, mode; 629 u_register_t ns_scr_el3 = read_scr_el3(); 630 u_register_t ns_sctlr_el1 = read_sctlr_el1(); 631 632 sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 633 read_sctlr_el2() : ns_sctlr_el1; 634 ee = 0; 635 636 ep_attr = NON_SECURE | EP_ST_DISABLE; 637 if ((sctlr & SCTLR_EE_BIT) != 0U) { 638 ep_attr |= EP_EE_BIG; 639 ee = 1; 640 } 641 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 642 643 ep->pc = entrypoint; 644 zeromem(&ep->args, sizeof(ep->args)); 645 ep->args.arg0 = context_id; 646 647 /* 648 * Figure out whether the cpu enters the non-secure address space 649 * in aarch32 or aarch64 650 */ 651 if ((ns_scr_el3 & SCR_RW_BIT) != 0U) { 652 653 /* 654 * Check whether a Thumb entry point has been provided for an 655 * aarch64 EL 656 */ 657 if ((entrypoint & 0x1UL) != 0UL) 658 return PSCI_E_INVALID_ADDRESS; 659 660 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1; 661 662 ep->spsr = SPSR_64(mode, MODE_SP_ELX, DISABLE_ALL_EXCEPTIONS); 663 } else { 664 665 mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? 666 MODE32_hyp : MODE32_svc; 667 668 /* 669 * TODO: Choose async. exception bits if HYP mode is not 670 * implemented according to the values of SCR.{AW, FW} bits 671 */ 672 daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT; 673 674 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, daif); 675 } 676 677 return PSCI_E_SUCCESS; 678 } 679 #else /* !__aarch64__ */ 680 static int psci_get_ns_ep_info(entry_point_info_t *ep, 681 uintptr_t entrypoint, 682 u_register_t context_id) 683 { 684 u_register_t ep_attr; 685 unsigned int aif, ee, mode; 686 u_register_t scr = read_scr(); 687 u_register_t ns_sctlr, sctlr; 688 689 /* Switch to non secure state */ 690 write_scr(scr | SCR_NS_BIT); 691 isb(); 692 ns_sctlr = read_sctlr(); 693 694 sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr; 695 696 /* Return to original state */ 697 write_scr(scr); 698 isb(); 699 ee = 0; 700 701 ep_attr = NON_SECURE | EP_ST_DISABLE; 702 if (sctlr & SCTLR_EE_BIT) { 703 ep_attr |= EP_EE_BIG; 704 ee = 1; 705 } 706 SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); 707 708 ep->pc = entrypoint; 709 zeromem(&ep->args, sizeof(ep->args)); 710 ep->args.arg0 = context_id; 711 712 mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc; 713 714 /* 715 * TODO: Choose async. exception bits if HYP mode is not 716 * implemented according to the values of SCR.{AW, FW} bits 717 */ 718 aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT; 719 720 ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif); 721 722 return PSCI_E_SUCCESS; 723 } 724 725 #endif /* __aarch64__ */ 726 727 /******************************************************************************* 728 * This function validates the entrypoint with the platform layer if the 729 * appropriate pm_ops hook is exported by the platform and returns the 730 * 'entry_point_info'. 731 ******************************************************************************/ 732 int psci_validate_entry_point(entry_point_info_t *ep, 733 uintptr_t entrypoint, 734 u_register_t context_id) 735 { 736 int rc; 737 738 /* Validate the entrypoint using platform psci_ops */ 739 if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) { 740 rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint); 741 if (rc != PSCI_E_SUCCESS) 742 return PSCI_E_INVALID_ADDRESS; 743 } 744 745 /* 746 * Verify and derive the re-entry information for 747 * the non-secure world from the non-secure state from 748 * where this call originated. 749 */ 750 rc = psci_get_ns_ep_info(ep, entrypoint, context_id); 751 return rc; 752 } 753 754 /******************************************************************************* 755 * Generic handler which is called when a cpu is physically powered on. It 756 * traverses the node information and finds the highest power level powered 757 * off and performs generic, architectural, platform setup and state management 758 * to power on that power level and power levels below it. 759 * e.g. For a cpu that's been powered on, it will call the platform specific 760 * code to enable the gic cpu interface and for a cluster it will enable 761 * coherency at the interconnect level in addition to gic cpu interface. 762 ******************************************************************************/ 763 void psci_warmboot_entrypoint(void) 764 { 765 unsigned int end_pwrlvl; 766 int cpu_idx = (int) plat_my_core_pos(); 767 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 768 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; 769 770 /* 771 * Verify that we have been explicitly turned ON or resumed from 772 * suspend. 773 */ 774 if (psci_get_aff_info_state() == AFF_STATE_OFF) { 775 ERROR("Unexpected affinity info state"); 776 panic(); 777 } 778 779 /* 780 * Get the maximum power domain level to traverse to after this cpu 781 * has been physically powered up. 782 */ 783 end_pwrlvl = get_power_on_target_pwrlvl(); 784 785 /* Get the parent nodes */ 786 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); 787 788 /* 789 * This function acquires the lock corresponding to each power level so 790 * that by the time all locks are taken, the system topology is snapshot 791 * and state management can be done safely. 792 */ 793 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 794 795 psci_get_target_local_pwr_states(end_pwrlvl, &state_info); 796 797 #if ENABLE_PSCI_STAT 798 plat_psci_stat_accounting_stop(&state_info); 799 #endif 800 801 /* 802 * This CPU could be resuming from suspend or it could have just been 803 * turned on. To distinguish between these 2 cases, we examine the 804 * affinity state of the CPU: 805 * - If the affinity state is ON_PENDING then it has just been 806 * turned on. 807 * - Else it is resuming from suspend. 808 * 809 * Depending on the type of warm reset identified, choose the right set 810 * of power management handler and perform the generic, architecture 811 * and platform specific handling. 812 */ 813 if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING) 814 psci_cpu_on_finish(cpu_idx, &state_info); 815 else 816 psci_cpu_suspend_finish(cpu_idx, &state_info); 817 818 /* 819 * Set the requested and target state of this CPU and all the higher 820 * power domains which are ancestors of this CPU to run. 821 */ 822 psci_set_pwr_domains_to_run(end_pwrlvl); 823 824 #if ENABLE_PSCI_STAT 825 /* 826 * Update PSCI stats. 827 * Caches are off when writing stats data on the power down path. 828 * Since caches are now enabled, it's necessary to do cache 829 * maintenance before reading that same data. 830 */ 831 psci_stats_update_pwr_up(end_pwrlvl, &state_info); 832 #endif 833 834 /* 835 * This loop releases the lock corresponding to each power level 836 * in the reverse order to which they were acquired. 837 */ 838 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 839 } 840 841 /******************************************************************************* 842 * This function initializes the set of hooks that PSCI invokes as part of power 843 * management operation. The power management hooks are expected to be provided 844 * by the SPD, after it finishes all its initialization 845 ******************************************************************************/ 846 void psci_register_spd_pm_hook(const spd_pm_ops_t *pm) 847 { 848 assert(pm != NULL); 849 psci_spd_pm = pm; 850 851 if (pm->svc_migrate != NULL) 852 psci_caps |= define_psci_cap(PSCI_MIG_AARCH64); 853 854 if (pm->svc_migrate_info != NULL) 855 psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) 856 | define_psci_cap(PSCI_MIG_INFO_TYPE); 857 } 858 859 /******************************************************************************* 860 * This function invokes the migrate info hook in the spd_pm_ops. It performs 861 * the necessary return value validation. If the Secure Payload is UP and 862 * migrate capable, it returns the mpidr of the CPU on which the Secure payload 863 * is resident through the mpidr parameter. Else the value of the parameter on 864 * return is undefined. 865 ******************************************************************************/ 866 int psci_spd_migrate_info(u_register_t *mpidr) 867 { 868 int rc; 869 870 if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL)) 871 return PSCI_E_NOT_SUPPORTED; 872 873 rc = psci_spd_pm->svc_migrate_info(mpidr); 874 875 assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) || 876 (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED)); 877 878 return rc; 879 } 880 881 882 /******************************************************************************* 883 * This function prints the state of all power domains present in the 884 * system 885 ******************************************************************************/ 886 void psci_print_power_domain_map(void) 887 { 888 #if LOG_LEVEL >= LOG_LEVEL_INFO 889 int idx; 890 plat_local_state_t state; 891 plat_local_state_type_t state_type; 892 893 /* This array maps to the PSCI_STATE_X definitions in psci.h */ 894 static const char * const psci_state_type_str[] = { 895 "ON", 896 "RETENTION", 897 "OFF", 898 }; 899 900 INFO("PSCI Power Domain Map:\n"); 901 for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - PLATFORM_CORE_COUNT); 902 idx++) { 903 state_type = find_local_state_type( 904 psci_non_cpu_pd_nodes[idx].local_state); 905 INFO(" Domain Node : Level %u, parent_node %d," 906 " State %s (0x%x)\n", 907 psci_non_cpu_pd_nodes[idx].level, 908 psci_non_cpu_pd_nodes[idx].parent_node, 909 psci_state_type_str[state_type], 910 psci_non_cpu_pd_nodes[idx].local_state); 911 } 912 913 for (idx = 0; idx < PLATFORM_CORE_COUNT; idx++) { 914 state = psci_get_cpu_local_state_by_idx(idx); 915 state_type = find_local_state_type(state); 916 INFO(" CPU Node : MPID 0x%llx, parent_node %d," 917 " State %s (0x%x)\n", 918 (unsigned long long)psci_cpu_pd_nodes[idx].mpidr, 919 psci_cpu_pd_nodes[idx].parent_node, 920 psci_state_type_str[state_type], 921 psci_get_cpu_local_state_by_idx(idx)); 922 } 923 #endif 924 } 925 926 /****************************************************************************** 927 * Return whether any secondaries were powered up with CPU_ON call. A CPU that 928 * have ever been powered up would have set its MPDIR value to something other 929 * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to 930 * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is 931 * meaningful only when called on the primary CPU during early boot. 932 *****************************************************************************/ 933 int psci_secondaries_brought_up(void) 934 { 935 unsigned int idx, n_valid = 0U; 936 937 for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) { 938 if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR) 939 n_valid++; 940 } 941 942 assert(n_valid > 0U); 943 944 return (n_valid > 1U) ? 1 : 0; 945 } 946 947 /******************************************************************************* 948 * Initiate power down sequence, by calling power down operations registered for 949 * this CPU. 950 ******************************************************************************/ 951 void psci_do_pwrdown_sequence(unsigned int power_level) 952 { 953 #if HW_ASSISTED_COHERENCY 954 /* 955 * With hardware-assisted coherency, the CPU drivers only initiate the 956 * power down sequence, without performing cache-maintenance operations 957 * in software. Data caches enabled both before and after this call. 958 */ 959 prepare_cpu_pwr_dwn(power_level); 960 #else 961 /* 962 * Without hardware-assisted coherency, the CPU drivers disable data 963 * caches, then perform cache-maintenance operations in software. 964 * 965 * This also calls prepare_cpu_pwr_dwn() to initiate power down 966 * sequence, but that function will return with data caches disabled. 967 * We must ensure that the stack memory is flushed out to memory before 968 * we start popping from it again. 969 */ 970 psci_do_pwrdown_cache_maintenance(power_level); 971 #endif 972 } 973