1 /*
2 * Copyright (c) 2013-2026, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <string.h>
9
10 #include <arch.h>
11 #include <arch_features.h>
12 #include <arch_helpers.h>
13 #include <common/debug.h>
14 #include <lib/pmf/pmf.h>
15 #include <lib/runtime_instr.h>
16 #include <lib/smccc.h>
17 #include <plat/common/platform.h>
18 #include <services/arm_arch_svc.h>
19
20 #include "psci_private.h"
21
22 /*******************************************************************************
23 * PSCI frontend api for servicing SMCs. Described in the PSCI spec.
24 ******************************************************************************/
psci_cpu_on(u_register_t target_cpu,uintptr_t entrypoint,u_register_t context_id)25 int psci_cpu_on(u_register_t target_cpu,
26 uintptr_t entrypoint,
27 u_register_t context_id)
28
29 {
30 int rc;
31 entry_point_info_t *ep = NULL;
32 unsigned int target_idx = (unsigned int)plat_core_pos_by_mpidr(target_cpu);
33
34 /* Validate the target CPU */
35 if (!is_valid_mpidr(target_cpu)) {
36 return PSCI_E_INVALID_PARAMS;
37 }
38
39 ep = get_cpu_data_by_index(target_idx, warmboot_ep_info);
40 /* Validate the lower EL entry point and put it in the entry_point_info */
41 rc = psci_validate_entry_point(ep, entrypoint, context_id);
42 if (rc != PSCI_E_SUCCESS) {
43 return rc;
44 }
45
46 /*
47 * To turn this cpu on, specify which power
48 * levels need to be turned on
49 */
50 return psci_cpu_on_start(target_cpu);
51 }
52
psci_version(void)53 unsigned int psci_version(void)
54 {
55 return PSCI_MAJOR_VER | PSCI_MINOR_VER;
56 }
57
psci_cpu_suspend(unsigned int power_state,uintptr_t entrypoint,u_register_t context_id)58 int psci_cpu_suspend(unsigned int power_state,
59 uintptr_t entrypoint,
60 u_register_t context_id)
61 {
62 int rc;
63 unsigned int target_pwrlvl, is_power_down_state;
64 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} };
65 plat_local_state_t cpu_pd_state;
66 unsigned int cpu_idx = plat_my_core_pos();
67
68 /* Validate the power_state parameter */
69 rc = psci_validate_power_state(power_state, &state_info);
70 if (rc != PSCI_E_SUCCESS) {
71 assert(rc == PSCI_E_INVALID_PARAMS);
72 return rc;
73 }
74
75 /*
76 * Get the value of the state type bit from the power state parameter.
77 */
78 is_power_down_state = psci_get_pstate_type(power_state);
79
80 /* Sanity check the requested suspend levels */
81 assert(psci_validate_suspend_req(&state_info, is_power_down_state)
82 == PSCI_E_SUCCESS);
83
84 target_pwrlvl = psci_find_target_suspend_lvl(&state_info);
85 if (target_pwrlvl == PSCI_INVALID_PWR_LVL) {
86 ERROR("Invalid target power level for suspend operation\n");
87 panic();
88 }
89
90 /* Fast path for local CPU standby, won't interact with higher power levels. */
91 if (is_cpu_standby_req(is_power_down_state, target_pwrlvl)) {
92 if (psci_plat_pm_ops->cpu_standby == NULL) {
93 return PSCI_E_INVALID_PARAMS;
94 }
95
96 /*
97 * Set the state of the CPU power domain to the platform
98 * specific retention state and enter the standby state.
99 */
100 cpu_pd_state = state_info.pwr_domain_state[PSCI_CPU_PWR_LVL];
101 psci_set_cpu_local_state(cpu_pd_state);
102
103 #if ENABLE_PSCI_STAT
104 plat_psci_stat_accounting_start(&state_info);
105 #endif
106
107 #if ENABLE_RUNTIME_INSTRUMENTATION
108 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
109 RT_INSTR_ENTER_HW_LOW_PWR,
110 PMF_NO_CACHE_MAINT);
111 #endif
112
113 psci_plat_pm_ops->cpu_standby(cpu_pd_state);
114
115 /* Upon exit from standby, set the state back to RUN. */
116 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN);
117
118 #if ENABLE_RUNTIME_INSTRUMENTATION
119 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
120 RT_INSTR_EXIT_HW_LOW_PWR,
121 PMF_NO_CACHE_MAINT);
122 #endif
123
124 #if ENABLE_PSCI_STAT
125 plat_psci_stat_accounting_stop(&state_info);
126
127 /* Update PSCI stats */
128 psci_stats_update_pwr_up(cpu_idx, PSCI_CPU_PWR_LVL, &state_info);
129 #endif
130
131 return PSCI_E_SUCCESS;
132 }
133
134 /*
135 * If a power down state has been requested, we need to verify entry
136 * point and program entry information.
137 */
138 if (is_power_down_state != 0U) {
139 entry_point_info_t *ep = get_cpu_data_by_index(cpu_idx, warmboot_ep_info);
140
141 rc = psci_validate_entry_point(ep, entrypoint, context_id);
142 if (rc != PSCI_E_SUCCESS) {
143 return rc;
144 }
145 }
146
147 /*
148 * Do what is needed to enter the power down state. Upon success,
149 * enter the final wfi which will power down this CPU. This function
150 * might return if the power down was abandoned for any reason, e.g.
151 * arrival of an interrupt
152 */
153 rc = psci_cpu_suspend_start(cpu_idx,
154 target_pwrlvl,
155 &state_info,
156 is_power_down_state);
157
158 return rc;
159 }
160
161
psci_system_suspend(uintptr_t entrypoint,u_register_t context_id)162 int psci_system_suspend(uintptr_t entrypoint, u_register_t context_id)
163 {
164 int rc;
165 psci_power_state_t state_info;
166 unsigned int cpu_idx = plat_my_core_pos();
167 entry_point_info_t *ep = get_cpu_data_by_index(cpu_idx, warmboot_ep_info);
168
169 /* Check if the current CPU is the last ON CPU in the system */
170 if (!psci_is_last_on_cpu(cpu_idx)) {
171 return PSCI_E_DENIED;
172 }
173
174 /* Validate the entry point and get the entry_point_info */
175 rc = psci_validate_entry_point(ep, entrypoint, context_id);
176 if (rc != PSCI_E_SUCCESS) {
177 return rc;
178 }
179
180 /* Query the psci_power_state for system suspend */
181 psci_query_sys_suspend_pwrstate(&state_info);
182
183 /*
184 * Check if platform allows suspend to Highest power level
185 * (System level)
186 */
187 if (psci_find_target_suspend_lvl(&state_info) < PLAT_MAX_PWR_LVL) {
188 return PSCI_E_DENIED;
189 }
190 /* Ensure that the psci_power_state makes sense */
191 assert(psci_validate_suspend_req(&state_info, PSTATE_TYPE_POWERDOWN)
192 == PSCI_E_SUCCESS);
193 assert(is_local_state_off(
194 state_info.pwr_domain_state[PLAT_MAX_PWR_LVL]) != 0);
195
196 /*
197 * Do what is needed to enter the system suspend state. This function
198 * might return if the power down was abandoned for any reason, e.g.
199 * arrival of an interrupt
200 */
201 rc = psci_cpu_suspend_start(cpu_idx,
202 PLAT_MAX_PWR_LVL,
203 &state_info,
204 PSTATE_TYPE_POWERDOWN);
205
206 return rc;
207 }
208
psci_cpu_off(void)209 int psci_cpu_off(void)
210 {
211 int rc;
212 unsigned int target_pwrlvl = PLAT_MAX_PWR_LVL;
213
214 /*
215 * Do what is needed to power off this CPU and possible higher power
216 * levels if it able to do so. Upon success, enter the final wfi
217 * which will power down this CPU.
218 */
219 rc = psci_do_cpu_off(target_pwrlvl);
220
221 /*
222 * The only error cpu_off can return is E_DENIED. So check if that's
223 * indeed the case.
224 */
225 assert(rc == PSCI_E_DENIED);
226
227 return rc;
228 }
229
psci_affinity_info(u_register_t target_affinity,unsigned int lowest_affinity_level)230 int psci_affinity_info(u_register_t target_affinity,
231 unsigned int lowest_affinity_level)
232 {
233 unsigned int target_idx;
234
235 /* Validate the target affinity */
236 if (!is_valid_mpidr(target_affinity)) {
237 return PSCI_E_INVALID_PARAMS;
238 }
239
240 /* We dont support level higher than PSCI_CPU_PWR_LVL */
241 if (lowest_affinity_level > PSCI_CPU_PWR_LVL) {
242 return PSCI_E_INVALID_PARAMS;
243 }
244 /* Calculate the cpu index of the target */
245 target_idx = (unsigned int) plat_core_pos_by_mpidr(target_affinity);
246
247 /*
248 * Generic management:
249 * Perform cache maintanence ahead of reading the target CPU state to
250 * ensure that the data is not stale.
251 * There is a theoretical edge case where the cache may contain stale
252 * data for the target CPU data - this can occur under the following
253 * conditions:
254 * - the target CPU is in another cluster from the current
255 * - the target CPU was the last CPU to shutdown on its cluster
256 * - the cluster was removed from coherency as part of the CPU shutdown
257 *
258 * In this case the cache maintenace that was performed as part of the
259 * target CPUs shutdown was not seen by the current CPU's cluster. And
260 * so the cache may contain stale data for the target CPU.
261 */
262 flush_cpu_data_by_index(target_idx, psci_svc_cpu_data);
263
264 return (int)psci_get_aff_info_state_by_idx(target_idx);
265 }
266
psci_migrate(u_register_t target_cpu)267 int psci_migrate(u_register_t target_cpu)
268 {
269 int rc;
270 u_register_t resident_cpu_mpidr = 0;
271
272 /* Validate the target cpu */
273 if (!is_valid_mpidr(target_cpu)) {
274 return PSCI_E_INVALID_PARAMS;
275 }
276
277 rc = psci_spd_migrate_info(&resident_cpu_mpidr);
278 if (rc != PSCI_TOS_UP_MIG_CAP) {
279 return (rc == PSCI_TOS_NOT_UP_MIG_CAP) ?
280 PSCI_E_DENIED : PSCI_E_NOT_SUPPORTED;
281 }
282
283 /*
284 * Migrate should only be invoked on the CPU where
285 * the Secure OS is resident.
286 */
287 if (resident_cpu_mpidr != read_mpidr_el1()) {
288 return PSCI_E_NOT_PRESENT;
289 }
290
291 /* Check the validity of the specified target cpu */
292 if (!is_valid_mpidr(target_cpu)) {
293 return PSCI_E_INVALID_PARAMS;
294 }
295
296 assert((psci_spd_pm != NULL) && (psci_spd_pm->svc_migrate != NULL));
297
298 rc = psci_spd_pm->svc_migrate(read_mpidr_el1(), target_cpu);
299 assert((rc == PSCI_E_SUCCESS) || (rc == PSCI_E_INTERN_FAIL));
300
301 return rc;
302 }
303
psci_migrate_info_type(void)304 int psci_migrate_info_type(void)
305 {
306 u_register_t resident_cpu_mpidr = 0;
307
308 return psci_spd_migrate_info(&resident_cpu_mpidr);
309 }
310
psci_migrate_info_up_cpu(void)311 u_register_t psci_migrate_info_up_cpu(void)
312 {
313 u_register_t resident_cpu_mpidr = 0;
314 int rc;
315
316 /*
317 * Return value of this depends upon what
318 * psci_spd_migrate_info() returns.
319 */
320 rc = psci_spd_migrate_info(&resident_cpu_mpidr);
321 if ((rc != PSCI_TOS_NOT_UP_MIG_CAP) && (rc != PSCI_TOS_UP_MIG_CAP)) {
322 return (u_register_t)(register_t) PSCI_E_INVALID_PARAMS;
323 }
324
325 return resident_cpu_mpidr;
326 }
327
psci_node_hw_state(u_register_t target_cpu,unsigned int power_level)328 int psci_node_hw_state(u_register_t target_cpu,
329 unsigned int power_level)
330 {
331 int rc;
332
333 /* Validate target_cpu */
334 if (!is_valid_mpidr(target_cpu)) {
335 return PSCI_E_INVALID_PARAMS;
336 }
337
338 /* Validate power_level against PLAT_MAX_PWR_LVL */
339 if (power_level > PLAT_MAX_PWR_LVL) {
340 return PSCI_E_INVALID_PARAMS;
341 }
342
343 /*
344 * Dispatch this call to platform to query power controller, and pass on
345 * to the caller what it returns
346 */
347 assert(psci_plat_pm_ops->get_node_hw_state != NULL);
348 rc = psci_plat_pm_ops->get_node_hw_state(target_cpu, power_level);
349 assert(((rc >= HW_ON) && (rc <= HW_STANDBY))
350 || (rc == PSCI_E_NOT_SUPPORTED)
351 || (rc == PSCI_E_INVALID_PARAMS));
352 return rc;
353 }
354
psci_features(unsigned int psci_fid)355 int psci_features(unsigned int psci_fid)
356 {
357 unsigned int local_caps = psci_caps;
358
359 if (psci_fid == SMCCC_VERSION) {
360 return PSCI_E_SUCCESS;
361 }
362 /* Check if it is a 64 bit function */
363 if (((psci_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_64) {
364 local_caps &= PSCI_CAP_64BIT_MASK;
365 }
366 /* Check for invalid fid */
367 if (!(is_std_svc_call(psci_fid) && is_valid_fast_smc(psci_fid)
368 && is_psci_fid(psci_fid))) {
369 return PSCI_E_NOT_SUPPORTED;
370 }
371
372 /* Check if the psci fid is supported or not */
373 if ((local_caps & define_psci_cap(psci_fid)) == 0U) {
374 return PSCI_E_NOT_SUPPORTED;
375 }
376 /* Format the feature flags */
377 if ((psci_fid == PSCI_CPU_SUSPEND_AARCH32) ||
378 (psci_fid == PSCI_CPU_SUSPEND_AARCH64)) {
379 unsigned int ret = ((FF_PSTATE << FF_PSTATE_SHIFT) |
380 (FF_SUPPORTS_OS_INIT_MODE << FF_MODE_SUPPORT_SHIFT));
381 return (int)ret;
382 }
383
384 /* Return 0 for all other fid's */
385 return PSCI_E_SUCCESS;
386 }
387
388 #if PSCI_OS_INIT_MODE
psci_set_suspend_mode(unsigned int mode)389 int psci_set_suspend_mode(unsigned int mode)
390 {
391 suspend_mode_t new_mode;
392 unsigned int this_core = plat_my_core_pos();
393
394 if ((mode != (unsigned int)PLAT_COORD) &&
395 (mode != (unsigned int)OS_INIT)) {
396 return PSCI_E_INVALID_PARAMS;
397 }
398
399 new_mode = (suspend_mode_t)mode;
400
401 if (psci_suspend_mode == new_mode) {
402 return PSCI_E_SUCCESS;
403 }
404
405 if (new_mode == PLAT_COORD) {
406 /* Check if the current CPU is the last ON CPU in the system */
407 if (!psci_is_last_on_cpu_safe(this_core)) {
408 return PSCI_E_DENIED;
409 }
410 }
411
412 if (new_mode == OS_INIT) {
413 /*
414 * Check if all CPUs in the system are ON or if the current
415 * CPU is the last ON CPU in the system.
416 */
417 if (!(psci_are_all_cpus_on_safe(this_core) ||
418 psci_is_last_on_cpu_safe(this_core))) {
419 return PSCI_E_DENIED;
420 }
421 }
422
423 psci_suspend_mode = new_mode;
424 psci_flush_dcache_range((uintptr_t)&psci_suspend_mode,
425 sizeof(psci_suspend_mode));
426
427 return PSCI_E_SUCCESS;
428 }
429 #endif
430
431 /*******************************************************************************
432 * PSCI top level handler for servicing SMCs.
433 ******************************************************************************/
psci_smc_handler(uint32_t smc_fid,u_register_t x1,u_register_t x2,u_register_t x3,u_register_t x4,void * cookie,void * handle,u_register_t flags)434 u_register_t psci_smc_handler(uint32_t smc_fid,
435 u_register_t x1,
436 u_register_t x2,
437 u_register_t x3,
438 u_register_t x4,
439 void *cookie,
440 void *handle,
441 u_register_t flags)
442 {
443 (void)x4;
444 (void)cookie;
445 (void)handle;
446 u_register_t ret;
447
448 if (!is_caller_non_secure(flags)) {
449 return (u_register_t)SMC_UNK;
450 }
451
452 /* Check the fid against the capabilities */
453 if ((psci_caps & define_psci_cap(smc_fid)) == 0U) {
454 return (u_register_t)SMC_UNK;
455 }
456
457 if (((smc_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_32) {
458 /* 32-bit PSCI function, clear top parameter bits */
459
460 uint32_t r1 = (uint32_t)x1;
461 uint32_t r2 = (uint32_t)x2;
462 uint32_t r3 = (uint32_t)x3;
463
464 switch (smc_fid) {
465 case PSCI_VERSION:
466 ret = (u_register_t)psci_version();
467 break;
468
469 case PSCI_CPU_OFF:
470 ret = (u_register_t)psci_cpu_off();
471 break;
472
473 case PSCI_CPU_SUSPEND_AARCH32:
474 ret = (u_register_t)psci_cpu_suspend(r1, r2, r3);
475 break;
476
477 case PSCI_CPU_ON_AARCH32:
478 ret = (u_register_t)psci_cpu_on(r1, r2, r3);
479 break;
480
481 case PSCI_AFFINITY_INFO_AARCH32:
482 ret = (u_register_t)psci_affinity_info(r1, r2);
483 break;
484
485 case PSCI_MIG_AARCH32:
486 ret = (u_register_t)psci_migrate(r1);
487 break;
488
489 case PSCI_MIG_INFO_TYPE:
490 ret = (u_register_t)psci_migrate_info_type();
491 break;
492
493 case PSCI_MIG_INFO_UP_CPU_AARCH32:
494 ret = psci_migrate_info_up_cpu();
495 break;
496
497 case PSCI_NODE_HW_STATE_AARCH32:
498 ret = (u_register_t)psci_node_hw_state(r1, r2);
499 break;
500
501 case PSCI_SYSTEM_SUSPEND_AARCH32:
502 ret = (u_register_t)psci_system_suspend(r1, r2);
503 break;
504
505 case PSCI_SYSTEM_OFF:
506 psci_system_off();
507 /* We should never return from psci_system_off() */
508 break;
509
510 case PSCI_SYSTEM_RESET:
511 psci_system_reset();
512 /* We should never return from psci_system_reset() */
513 break;
514
515 case PSCI_FEATURES:
516 ret = (u_register_t)psci_features(r1);
517 break;
518
519 #if PSCI_OS_INIT_MODE
520 case PSCI_SET_SUSPEND_MODE:
521 ret = (u_register_t)psci_set_suspend_mode(r1);
522 break;
523 #endif
524
525 #if ENABLE_PSCI_STAT
526 case PSCI_STAT_RESIDENCY_AARCH32:
527 ret = psci_stat_residency(r1, r2);
528 break;
529
530 case PSCI_STAT_COUNT_AARCH32:
531 ret = psci_stat_count(r1, r2);
532 break;
533 #endif
534 case PSCI_MEM_PROTECT:
535 ret = psci_mem_protect(r1);
536 break;
537
538 case PSCI_MEM_CHK_RANGE_AARCH32:
539 ret = psci_mem_chk_range(r1, r2);
540 break;
541
542 case PSCI_SYSTEM_RESET2_AARCH32:
543 /* We should never return from psci_system_reset2() */
544 ret = psci_system_reset2(r1, r2);
545 break;
546
547 default:
548 WARN("Unimplemented PSCI Call: 0x%x\n", smc_fid);
549 ret = (u_register_t)SMC_UNK;
550 break;
551 }
552 } else {
553 /* 64-bit PSCI function */
554
555 switch (smc_fid) {
556 case PSCI_CPU_SUSPEND_AARCH64:
557 ret = (u_register_t)
558 psci_cpu_suspend((unsigned int)x1, x2, x3);
559 break;
560
561 case PSCI_CPU_ON_AARCH64:
562 ret = (u_register_t)psci_cpu_on(x1, x2, x3);
563 break;
564
565 case PSCI_AFFINITY_INFO_AARCH64:
566 ret = (u_register_t)
567 psci_affinity_info(x1, (unsigned int)x2);
568 break;
569
570 case PSCI_MIG_AARCH64:
571 ret = (u_register_t)psci_migrate(x1);
572 break;
573
574 case PSCI_MIG_INFO_UP_CPU_AARCH64:
575 ret = psci_migrate_info_up_cpu();
576 break;
577
578 case PSCI_NODE_HW_STATE_AARCH64:
579 ret = (u_register_t)psci_node_hw_state(
580 x1, (unsigned int) x2);
581 break;
582
583 case PSCI_SYSTEM_SUSPEND_AARCH64:
584 ret = (u_register_t)psci_system_suspend(x1, x2);
585 break;
586
587 #if ENABLE_PSCI_STAT
588 case PSCI_STAT_RESIDENCY_AARCH64:
589 ret = psci_stat_residency(x1, (unsigned int) x2);
590 break;
591
592 case PSCI_STAT_COUNT_AARCH64:
593 ret = psci_stat_count(x1, (unsigned int) x2);
594 break;
595 #endif
596
597 case PSCI_MEM_CHK_RANGE_AARCH64:
598 ret = psci_mem_chk_range(x1, x2);
599 break;
600
601 case PSCI_SYSTEM_RESET2_AARCH64:
602 /* We should never return from psci_system_reset2() */
603 ret = psci_system_reset2((uint32_t) x1, x2);
604 break;
605
606 default:
607 WARN("Unimplemented PSCI Call: 0x%x\n", smc_fid);
608 ret = (u_register_t)SMC_UNK;
609 break;
610 }
611 }
612
613 return ret;
614 }
615