1 /*
2 * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <string.h>
9
10 #include <arch.h>
11 #include <arch_features.h>
12 #include <arch_helpers.h>
13 #include <common/debug.h>
14 #include <lib/pmf/pmf.h>
15 #include <lib/runtime_instr.h>
16 #include <lib/smccc.h>
17 #include <plat/common/platform.h>
18 #include <services/arm_arch_svc.h>
19
20 #include "psci_private.h"
21
22 /*******************************************************************************
23 * PSCI frontend api for servicing SMCs. Described in the PSCI spec.
24 ******************************************************************************/
psci_cpu_on(u_register_t target_cpu,uintptr_t entrypoint,u_register_t context_id)25 int psci_cpu_on(u_register_t target_cpu,
26 uintptr_t entrypoint,
27 u_register_t context_id)
28
29 {
30 int rc;
31 entry_point_info_t *ep = NULL;
32 unsigned int target_idx = (unsigned int)plat_core_pos_by_mpidr(target_cpu);
33
34 /* Validate the target CPU */
35 if (!is_valid_mpidr(target_cpu)) {
36 return PSCI_E_INVALID_PARAMS;
37 }
38
39 ep = get_cpu_data_by_index(target_idx, warmboot_ep_info);
40 /* Validate the lower EL entry point and put it in the entry_point_info */
41 rc = psci_validate_entry_point(ep, entrypoint, context_id);
42 if (rc != PSCI_E_SUCCESS) {
43 return rc;
44 }
45
46 /*
47 * To turn this cpu on, specify which power
48 * levels need to be turned on
49 */
50 return psci_cpu_on_start(target_cpu, ep);
51 }
52
psci_version(void)53 unsigned int psci_version(void)
54 {
55 return PSCI_MAJOR_VER | PSCI_MINOR_VER;
56 }
57
psci_cpu_suspend(unsigned int power_state,uintptr_t entrypoint,u_register_t context_id)58 int psci_cpu_suspend(unsigned int power_state,
59 uintptr_t entrypoint,
60 u_register_t context_id)
61 {
62 int rc;
63 unsigned int target_pwrlvl, is_power_down_state;
64 psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} };
65 plat_local_state_t cpu_pd_state;
66 unsigned int cpu_idx = plat_my_core_pos();
67
68 #if ERRATA_SME_POWER_DOWN
69 /*
70 * If SME isn't off, attempting a real power down will only end up being
71 * rejected. If we got called with SME on, fall back to a normal
72 * suspend. We can't force SME off as in the event the power down is
73 * rejected for another reason (eg GIC) we'd lose the SME context.
74 */
75 if (is_feat_sme_supported() && read_svcr() != 0) {
76 power_state &= ~(PSTATE_TYPE_MASK << PSTATE_TYPE_SHIFT);
77 power_state &= ~(PSTATE_PWR_LVL_MASK << PSTATE_PWR_LVL_SHIFT);
78 }
79 #endif /* ERRATA_SME_POWER_DOWN */
80
81 /* Validate the power_state parameter */
82 rc = psci_validate_power_state(power_state, &state_info);
83 if (rc != PSCI_E_SUCCESS) {
84 assert(rc == PSCI_E_INVALID_PARAMS);
85 return rc;
86 }
87
88 /*
89 * Get the value of the state type bit from the power state parameter.
90 */
91 is_power_down_state = psci_get_pstate_type(power_state);
92
93 /* Sanity check the requested suspend levels */
94 assert(psci_validate_suspend_req(&state_info, is_power_down_state)
95 == PSCI_E_SUCCESS);
96
97 target_pwrlvl = psci_find_target_suspend_lvl(&state_info);
98 if (target_pwrlvl == PSCI_INVALID_PWR_LVL) {
99 ERROR("Invalid target power level for suspend operation\n");
100 panic();
101 }
102
103 /* Fast path for local CPU standby, won't interact with higher power levels. */
104 if (is_cpu_standby_req(is_power_down_state, target_pwrlvl)) {
105 if (psci_plat_pm_ops->cpu_standby == NULL) {
106 return PSCI_E_INVALID_PARAMS;
107 }
108
109 /*
110 * Set the state of the CPU power domain to the platform
111 * specific retention state and enter the standby state.
112 */
113 cpu_pd_state = state_info.pwr_domain_state[PSCI_CPU_PWR_LVL];
114 psci_set_cpu_local_state(cpu_pd_state);
115
116 #if ENABLE_PSCI_STAT
117 plat_psci_stat_accounting_start(&state_info);
118 #endif
119
120 #if ENABLE_RUNTIME_INSTRUMENTATION
121 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
122 RT_INSTR_ENTER_HW_LOW_PWR,
123 PMF_NO_CACHE_MAINT);
124 #endif
125
126 psci_plat_pm_ops->cpu_standby(cpu_pd_state);
127
128 /* Upon exit from standby, set the state back to RUN. */
129 psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN);
130
131 #if ENABLE_RUNTIME_INSTRUMENTATION
132 PMF_CAPTURE_TIMESTAMP(rt_instr_svc,
133 RT_INSTR_EXIT_HW_LOW_PWR,
134 PMF_NO_CACHE_MAINT);
135 #endif
136
137 #if ENABLE_PSCI_STAT
138 plat_psci_stat_accounting_stop(&state_info);
139
140 /* Update PSCI stats */
141 psci_stats_update_pwr_up(cpu_idx, PSCI_CPU_PWR_LVL, &state_info);
142 #endif
143
144 return PSCI_E_SUCCESS;
145 }
146
147 /*
148 * If a power down state has been requested, we need to verify entry
149 * point and program entry information.
150 */
151 if (is_power_down_state != 0U) {
152 entry_point_info_t *ep = get_cpu_data_by_index(cpu_idx, warmboot_ep_info);
153
154 rc = psci_validate_entry_point(ep, entrypoint, context_id);
155 if (rc != PSCI_E_SUCCESS) {
156 return rc;
157 }
158 }
159
160 /*
161 * Do what is needed to enter the power down state. Upon success,
162 * enter the final wfi which will power down this CPU. This function
163 * might return if the power down was abandoned for any reason, e.g.
164 * arrival of an interrupt
165 */
166 rc = psci_cpu_suspend_start(cpu_idx,
167 target_pwrlvl,
168 &state_info,
169 is_power_down_state);
170
171 return rc;
172 }
173
174
psci_system_suspend(uintptr_t entrypoint,u_register_t context_id)175 int psci_system_suspend(uintptr_t entrypoint, u_register_t context_id)
176 {
177 int rc;
178 psci_power_state_t state_info;
179 unsigned int cpu_idx = plat_my_core_pos();
180 entry_point_info_t *ep = get_cpu_data_by_index(cpu_idx, warmboot_ep_info);
181
182 /* Check if the current CPU is the last ON CPU in the system */
183 if (!psci_is_last_on_cpu(cpu_idx)) {
184 return PSCI_E_DENIED;
185 }
186
187 /* Validate the entry point and get the entry_point_info */
188 rc = psci_validate_entry_point(ep, entrypoint, context_id);
189 if (rc != PSCI_E_SUCCESS) {
190 return rc;
191 }
192
193 /* Query the psci_power_state for system suspend */
194 psci_query_sys_suspend_pwrstate(&state_info);
195
196 /*
197 * Check if platform allows suspend to Highest power level
198 * (System level)
199 */
200 if (psci_find_target_suspend_lvl(&state_info) < PLAT_MAX_PWR_LVL) {
201 return PSCI_E_DENIED;
202 }
203 /* Ensure that the psci_power_state makes sense */
204 assert(psci_validate_suspend_req(&state_info, PSTATE_TYPE_POWERDOWN)
205 == PSCI_E_SUCCESS);
206 assert(is_local_state_off(
207 state_info.pwr_domain_state[PLAT_MAX_PWR_LVL]) != 0);
208
209 /*
210 * Do what is needed to enter the system suspend state. This function
211 * might return if the power down was abandoned for any reason, e.g.
212 * arrival of an interrupt
213 */
214 rc = psci_cpu_suspend_start(cpu_idx,
215 PLAT_MAX_PWR_LVL,
216 &state_info,
217 PSTATE_TYPE_POWERDOWN);
218
219 return rc;
220 }
221
psci_cpu_off(void)222 int psci_cpu_off(void)
223 {
224 int rc;
225 unsigned int target_pwrlvl = PLAT_MAX_PWR_LVL;
226
227 /*
228 * Do what is needed to power off this CPU and possible higher power
229 * levels if it able to do so. Upon success, enter the final wfi
230 * which will power down this CPU.
231 */
232 rc = psci_do_cpu_off(target_pwrlvl);
233
234 /*
235 * The only error cpu_off can return is E_DENIED. So check if that's
236 * indeed the case.
237 */
238 assert(rc == PSCI_E_DENIED);
239
240 return rc;
241 }
242
psci_affinity_info(u_register_t target_affinity,unsigned int lowest_affinity_level)243 int psci_affinity_info(u_register_t target_affinity,
244 unsigned int lowest_affinity_level)
245 {
246 unsigned int target_idx;
247
248 /* Validate the target affinity */
249 if (!is_valid_mpidr(target_affinity)) {
250 return PSCI_E_INVALID_PARAMS;
251 }
252
253 /* We dont support level higher than PSCI_CPU_PWR_LVL */
254 if (lowest_affinity_level > PSCI_CPU_PWR_LVL) {
255 return PSCI_E_INVALID_PARAMS;
256 }
257 /* Calculate the cpu index of the target */
258 target_idx = (unsigned int) plat_core_pos_by_mpidr(target_affinity);
259
260 /*
261 * Generic management:
262 * Perform cache maintanence ahead of reading the target CPU state to
263 * ensure that the data is not stale.
264 * There is a theoretical edge case where the cache may contain stale
265 * data for the target CPU data - this can occur under the following
266 * conditions:
267 * - the target CPU is in another cluster from the current
268 * - the target CPU was the last CPU to shutdown on its cluster
269 * - the cluster was removed from coherency as part of the CPU shutdown
270 *
271 * In this case the cache maintenace that was performed as part of the
272 * target CPUs shutdown was not seen by the current CPU's cluster. And
273 * so the cache may contain stale data for the target CPU.
274 */
275 flush_cpu_data_by_index(target_idx,
276 psci_svc_cpu_data.aff_info_state);
277
278 return (int)psci_get_aff_info_state_by_idx(target_idx);
279 }
280
psci_migrate(u_register_t target_cpu)281 int psci_migrate(u_register_t target_cpu)
282 {
283 int rc;
284 u_register_t resident_cpu_mpidr = 0;
285
286 /* Validate the target cpu */
287 if (!is_valid_mpidr(target_cpu)) {
288 return PSCI_E_INVALID_PARAMS;
289 }
290
291 rc = psci_spd_migrate_info(&resident_cpu_mpidr);
292 if (rc != PSCI_TOS_UP_MIG_CAP) {
293 return (rc == PSCI_TOS_NOT_UP_MIG_CAP) ?
294 PSCI_E_DENIED : PSCI_E_NOT_SUPPORTED;
295 }
296
297 /*
298 * Migrate should only be invoked on the CPU where
299 * the Secure OS is resident.
300 */
301 if (resident_cpu_mpidr != read_mpidr_el1()) {
302 return PSCI_E_NOT_PRESENT;
303 }
304
305 /* Check the validity of the specified target cpu */
306 if (!is_valid_mpidr(target_cpu)) {
307 return PSCI_E_INVALID_PARAMS;
308 }
309
310 assert((psci_spd_pm != NULL) && (psci_spd_pm->svc_migrate != NULL));
311
312 rc = psci_spd_pm->svc_migrate(read_mpidr_el1(), target_cpu);
313 assert((rc == PSCI_E_SUCCESS) || (rc == PSCI_E_INTERN_FAIL));
314
315 return rc;
316 }
317
psci_migrate_info_type(void)318 int psci_migrate_info_type(void)
319 {
320 u_register_t resident_cpu_mpidr = 0;
321
322 return psci_spd_migrate_info(&resident_cpu_mpidr);
323 }
324
psci_migrate_info_up_cpu(void)325 u_register_t psci_migrate_info_up_cpu(void)
326 {
327 u_register_t resident_cpu_mpidr = 0;
328 int rc;
329
330 /*
331 * Return value of this depends upon what
332 * psci_spd_migrate_info() returns.
333 */
334 rc = psci_spd_migrate_info(&resident_cpu_mpidr);
335 if ((rc != PSCI_TOS_NOT_UP_MIG_CAP) && (rc != PSCI_TOS_UP_MIG_CAP)) {
336 return (u_register_t)(register_t) PSCI_E_INVALID_PARAMS;
337 }
338
339 return resident_cpu_mpidr;
340 }
341
psci_node_hw_state(u_register_t target_cpu,unsigned int power_level)342 int psci_node_hw_state(u_register_t target_cpu,
343 unsigned int power_level)
344 {
345 int rc;
346
347 /* Validate target_cpu */
348 if (!is_valid_mpidr(target_cpu)) {
349 return PSCI_E_INVALID_PARAMS;
350 }
351
352 /* Validate power_level against PLAT_MAX_PWR_LVL */
353 if (power_level > PLAT_MAX_PWR_LVL) {
354 return PSCI_E_INVALID_PARAMS;
355 }
356
357 /*
358 * Dispatch this call to platform to query power controller, and pass on
359 * to the caller what it returns
360 */
361 assert(psci_plat_pm_ops->get_node_hw_state != NULL);
362 rc = psci_plat_pm_ops->get_node_hw_state(target_cpu, power_level);
363 assert(((rc >= HW_ON) && (rc <= HW_STANDBY))
364 || (rc == PSCI_E_NOT_SUPPORTED)
365 || (rc == PSCI_E_INVALID_PARAMS));
366 return rc;
367 }
368
psci_features(unsigned int psci_fid)369 int psci_features(unsigned int psci_fid)
370 {
371 unsigned int local_caps = psci_caps;
372
373 if (psci_fid == SMCCC_VERSION) {
374 return PSCI_E_SUCCESS;
375 }
376 /* Check if it is a 64 bit function */
377 if (((psci_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_64) {
378 local_caps &= PSCI_CAP_64BIT_MASK;
379 }
380 /* Check for invalid fid */
381 if (!(is_std_svc_call(psci_fid) && is_valid_fast_smc(psci_fid)
382 && is_psci_fid(psci_fid))) {
383 return PSCI_E_NOT_SUPPORTED;
384 }
385
386 /* Check if the psci fid is supported or not */
387 if ((local_caps & define_psci_cap(psci_fid)) == 0U) {
388 return PSCI_E_NOT_SUPPORTED;
389 }
390 /* Format the feature flags */
391 if ((psci_fid == PSCI_CPU_SUSPEND_AARCH32) ||
392 (psci_fid == PSCI_CPU_SUSPEND_AARCH64)) {
393 unsigned int ret = ((FF_PSTATE << FF_PSTATE_SHIFT) |
394 (FF_SUPPORTS_OS_INIT_MODE << FF_MODE_SUPPORT_SHIFT));
395 return (int)ret;
396 }
397
398 /* Return 0 for all other fid's */
399 return PSCI_E_SUCCESS;
400 }
401
402 #if PSCI_OS_INIT_MODE
psci_set_suspend_mode(unsigned int mode)403 int psci_set_suspend_mode(unsigned int mode)
404 {
405 if (psci_suspend_mode == mode) {
406 return PSCI_E_SUCCESS;
407 }
408
409 unsigned int this_core = plat_my_core_pos();
410
411 if (mode == PLAT_COORD) {
412 /* Check if the current CPU is the last ON CPU in the system */
413 if (!psci_is_last_on_cpu_safe(this_core)) {
414 return PSCI_E_DENIED;
415 }
416 }
417
418 if (mode == OS_INIT) {
419 /*
420 * Check if all CPUs in the system are ON or if the current
421 * CPU is the last ON CPU in the system.
422 */
423 if (!(psci_are_all_cpus_on_safe(this_core) ||
424 psci_is_last_on_cpu_safe(this_core))) {
425 return PSCI_E_DENIED;
426 }
427 }
428
429 psci_suspend_mode = mode;
430 psci_flush_dcache_range((uintptr_t)&psci_suspend_mode,
431 sizeof(psci_suspend_mode));
432
433 return PSCI_E_SUCCESS;
434 }
435 #endif
436
437 /*******************************************************************************
438 * PSCI top level handler for servicing SMCs.
439 ******************************************************************************/
psci_smc_handler(uint32_t smc_fid,u_register_t x1,u_register_t x2,u_register_t x3,u_register_t x4,void * cookie,void * handle,u_register_t flags)440 u_register_t psci_smc_handler(uint32_t smc_fid,
441 u_register_t x1,
442 u_register_t x2,
443 u_register_t x3,
444 u_register_t x4,
445 void *cookie,
446 void *handle,
447 u_register_t flags)
448 {
449 (void)x4;
450 (void)cookie;
451 (void)handle;
452 u_register_t ret;
453
454 if (!is_caller_non_secure(flags)) {
455 return (u_register_t)SMC_UNK;
456 }
457
458 /* Check the fid against the capabilities */
459 if ((psci_caps & define_psci_cap(smc_fid)) == 0U) {
460 return (u_register_t)SMC_UNK;
461 }
462
463 if (((smc_fid >> FUNCID_CC_SHIFT) & FUNCID_CC_MASK) == SMC_32) {
464 /* 32-bit PSCI function, clear top parameter bits */
465
466 uint32_t r1 = (uint32_t)x1;
467 uint32_t r2 = (uint32_t)x2;
468 uint32_t r3 = (uint32_t)x3;
469
470 switch (smc_fid) {
471 case PSCI_VERSION:
472 ret = (u_register_t)psci_version();
473 break;
474
475 case PSCI_CPU_OFF:
476 ret = (u_register_t)psci_cpu_off();
477 break;
478
479 case PSCI_CPU_SUSPEND_AARCH32:
480 ret = (u_register_t)psci_cpu_suspend(r1, r2, r3);
481 break;
482
483 case PSCI_CPU_ON_AARCH32:
484 ret = (u_register_t)psci_cpu_on(r1, r2, r3);
485 break;
486
487 case PSCI_AFFINITY_INFO_AARCH32:
488 ret = (u_register_t)psci_affinity_info(r1, r2);
489 break;
490
491 case PSCI_MIG_AARCH32:
492 ret = (u_register_t)psci_migrate(r1);
493 break;
494
495 case PSCI_MIG_INFO_TYPE:
496 ret = (u_register_t)psci_migrate_info_type();
497 break;
498
499 case PSCI_MIG_INFO_UP_CPU_AARCH32:
500 ret = psci_migrate_info_up_cpu();
501 break;
502
503 case PSCI_NODE_HW_STATE_AARCH32:
504 ret = (u_register_t)psci_node_hw_state(r1, r2);
505 break;
506
507 case PSCI_SYSTEM_SUSPEND_AARCH32:
508 ret = (u_register_t)psci_system_suspend(r1, r2);
509 break;
510
511 case PSCI_SYSTEM_OFF:
512 psci_system_off();
513 /* We should never return from psci_system_off() */
514 break;
515
516 case PSCI_SYSTEM_RESET:
517 psci_system_reset();
518 /* We should never return from psci_system_reset() */
519 break;
520
521 case PSCI_FEATURES:
522 ret = (u_register_t)psci_features(r1);
523 break;
524
525 #if PSCI_OS_INIT_MODE
526 case PSCI_SET_SUSPEND_MODE:
527 ret = (u_register_t)psci_set_suspend_mode(r1);
528 break;
529 #endif
530
531 #if ENABLE_PSCI_STAT
532 case PSCI_STAT_RESIDENCY_AARCH32:
533 ret = psci_stat_residency(r1, r2);
534 break;
535
536 case PSCI_STAT_COUNT_AARCH32:
537 ret = psci_stat_count(r1, r2);
538 break;
539 #endif
540 case PSCI_MEM_PROTECT:
541 ret = psci_mem_protect(r1);
542 break;
543
544 case PSCI_MEM_CHK_RANGE_AARCH32:
545 ret = psci_mem_chk_range(r1, r2);
546 break;
547
548 case PSCI_SYSTEM_RESET2_AARCH32:
549 /* We should never return from psci_system_reset2() */
550 ret = psci_system_reset2(r1, r2);
551 break;
552
553 default:
554 WARN("Unimplemented PSCI Call: 0x%x\n", smc_fid);
555 ret = (u_register_t)SMC_UNK;
556 break;
557 }
558 } else {
559 /* 64-bit PSCI function */
560
561 switch (smc_fid) {
562 case PSCI_CPU_SUSPEND_AARCH64:
563 ret = (u_register_t)
564 psci_cpu_suspend((unsigned int)x1, x2, x3);
565 break;
566
567 case PSCI_CPU_ON_AARCH64:
568 ret = (u_register_t)psci_cpu_on(x1, x2, x3);
569 break;
570
571 case PSCI_AFFINITY_INFO_AARCH64:
572 ret = (u_register_t)
573 psci_affinity_info(x1, (unsigned int)x2);
574 break;
575
576 case PSCI_MIG_AARCH64:
577 ret = (u_register_t)psci_migrate(x1);
578 break;
579
580 case PSCI_MIG_INFO_UP_CPU_AARCH64:
581 ret = psci_migrate_info_up_cpu();
582 break;
583
584 case PSCI_NODE_HW_STATE_AARCH64:
585 ret = (u_register_t)psci_node_hw_state(
586 x1, (unsigned int) x2);
587 break;
588
589 case PSCI_SYSTEM_SUSPEND_AARCH64:
590 ret = (u_register_t)psci_system_suspend(x1, x2);
591 break;
592
593 #if ENABLE_PSCI_STAT
594 case PSCI_STAT_RESIDENCY_AARCH64:
595 ret = psci_stat_residency(x1, (unsigned int) x2);
596 break;
597
598 case PSCI_STAT_COUNT_AARCH64:
599 ret = psci_stat_count(x1, (unsigned int) x2);
600 break;
601 #endif
602
603 case PSCI_MEM_CHK_RANGE_AARCH64:
604 ret = psci_mem_chk_range(x1, x2);
605 break;
606
607 case PSCI_SYSTEM_RESET2_AARCH64:
608 /* We should never return from psci_system_reset2() */
609 ret = psci_system_reset2((uint32_t) x1, x2);
610 break;
611
612 default:
613 WARN("Unimplemented PSCI Call: 0x%x\n", smc_fid);
614 ret = (u_register_t)SMC_UNK;
615 break;
616 }
617 }
618
619 return ret;
620 }
621