1 /* 2 * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <stddef.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <lib/el3_runtime/context_mgmt.h> 16 #include <lib/el3_runtime/cpu_data.h> 17 #include <lib/el3_runtime/pubsub_events.h> 18 #include <lib/pmf/pmf.h> 19 #include <lib/runtime_instr.h> 20 #include <plat/common/platform.h> 21 22 #include "psci_private.h" 23 24 /******************************************************************************* 25 * This function does generic and platform specific operations after a wake-up 26 * from standby/retention states at multiple power levels. 27 ******************************************************************************/ 28 static void psci_cpu_suspend_to_standby_finish(unsigned int end_pwrlvl, 29 psci_power_state_t *state_info) 30 { 31 /* 32 * Plat. management: Allow the platform to do operations 33 * on waking up from retention. 34 */ 35 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); 36 37 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 38 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 39 } 40 41 /******************************************************************************* 42 * This function does generic and platform specific suspend to power down 43 * operations. 44 ******************************************************************************/ 45 static void psci_suspend_to_pwrdown_start(unsigned int end_pwrlvl, 46 unsigned int max_off_lvl, 47 const entry_point_info_t *ep, 48 const psci_power_state_t *state_info) 49 { 50 PUBLISH_EVENT(psci_suspend_pwrdown_start); 51 52 #if PSCI_OS_INIT_MODE 53 #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL 54 end_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; 55 #else 56 end_pwrlvl = PLAT_MAX_PWR_LVL; 57 #endif 58 #endif 59 60 /* Save PSCI target power level for the suspend finisher handler */ 61 psci_set_suspend_pwrlvl(end_pwrlvl); 62 63 /* 64 * Flush the target power level as it might be accessed on power up with 65 * Data cache disabled. 66 */ 67 psci_flush_cpu_data(psci_svc_cpu_data.target_pwrlvl); 68 69 /* 70 * Call the cpu suspend handler registered by the Secure Payload 71 * Dispatcher to let it do any book-keeping. If the handler encounters an 72 * error, it's expected to assert within 73 */ 74 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend != NULL)) 75 psci_spd_pm->svc_suspend(max_off_lvl); 76 77 #if !HW_ASSISTED_COHERENCY 78 /* 79 * Plat. management: Allow the platform to perform any early 80 * actions required to power down the CPU. This might be useful for 81 * HW_ASSISTED_COHERENCY = 0 platforms that can safely perform these 82 * actions with data caches enabled. 83 */ 84 if (psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early != NULL) 85 psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early(state_info); 86 #endif 87 88 /* 89 * Store the re-entry information for the non-secure world. 90 */ 91 cm_init_my_context(ep); 92 93 /* 94 * Arch. management. Initiate power down sequence. 95 */ 96 psci_pwrdown_cpu_start(max_off_lvl); 97 } 98 99 /******************************************************************************* 100 * Top level handler which is called when a cpu wants to suspend its execution. 101 * It is assumed that along with suspending the cpu power domain, power domains 102 * at higher levels until the target power level will be suspended as well. It 103 * coordinates with the platform to negotiate the target state for each of 104 * the power domain level till the target power domain level. It then performs 105 * generic, architectural, platform setup and state management required to 106 * suspend that power domain level and power domain levels below it. 107 * e.g. For a cpu that's to be suspended, it could mean programming the 108 * power controller whereas for a cluster that's to be suspended, it will call 109 * the platform specific code which will disable coherency at the interconnect 110 * level if the cpu is the last in the cluster and also the program the power 111 * controller. 112 * 113 * All the required parameter checks are performed at the beginning and after 114 * the state transition has been done, no further error is expected and it is 115 * not possible to undo any of the actions taken beyond that point. 116 ******************************************************************************/ 117 int psci_cpu_suspend_start(unsigned int idx, 118 const entry_point_info_t *ep, 119 unsigned int end_pwrlvl, 120 psci_power_state_t *state_info, 121 unsigned int is_power_down_state) 122 { 123 int rc = PSCI_E_SUCCESS; 124 bool skip_wfi = false; 125 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 126 unsigned int max_off_lvl = 0; 127 #if FEAT_PABANDON 128 cpu_context_t *ctx = cm_get_context(NON_SECURE); 129 cpu_context_t old_ctx; 130 #endif 131 132 /* 133 * This function must only be called on platforms where the 134 * CPU_SUSPEND platform hooks have been implemented. 135 */ 136 assert((psci_plat_pm_ops->pwr_domain_suspend != NULL) && 137 (psci_plat_pm_ops->pwr_domain_suspend_finish != NULL)); 138 139 /* Get the parent nodes */ 140 psci_get_parent_pwr_domain_nodes(idx, end_pwrlvl, parent_nodes); 141 142 /* 143 * This function acquires the lock corresponding to each power 144 * level so that by the time all locks are taken, the system topology 145 * is snapshot and state management can be done safely. 146 */ 147 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 148 149 /* 150 * We check if there are any pending interrupts after the delay 151 * introduced by lock contention to increase the chances of early 152 * detection that a wake-up interrupt has fired. 153 */ 154 if (read_isr_el1() != 0U) { 155 skip_wfi = true; 156 goto exit; 157 } 158 159 #if PSCI_OS_INIT_MODE 160 if (psci_suspend_mode == OS_INIT) { 161 /* 162 * This function validates the requested state info for 163 * OS-initiated mode. 164 */ 165 rc = psci_validate_state_coordination(idx, end_pwrlvl, state_info); 166 if (rc != PSCI_E_SUCCESS) { 167 skip_wfi = true; 168 goto exit; 169 } 170 } else { 171 #endif 172 /* 173 * This function is passed the requested state info and 174 * it returns the negotiated state info for each power level upto 175 * the end level specified. 176 */ 177 psci_do_state_coordination(idx, end_pwrlvl, state_info); 178 #if PSCI_OS_INIT_MODE 179 } 180 #endif 181 182 #if PSCI_OS_INIT_MODE 183 if (psci_plat_pm_ops->pwr_domain_validate_suspend != NULL) { 184 rc = psci_plat_pm_ops->pwr_domain_validate_suspend(state_info); 185 if (rc != PSCI_E_SUCCESS) { 186 skip_wfi = true; 187 goto exit; 188 } 189 } 190 #endif 191 192 /* Update the target state in the power domain nodes */ 193 psci_set_target_local_pwr_states(idx, end_pwrlvl, state_info); 194 195 #if ENABLE_PSCI_STAT 196 /* Update the last cpu for each level till end_pwrlvl */ 197 psci_stats_update_pwr_down(idx, end_pwrlvl, state_info); 198 #endif 199 200 if (is_power_down_state != 0U) { 201 /* 202 * WHen CTX_INCLUDE_EL2_REGS is usnet, we're probably runnig 203 * with some SPD that assumes the core is going off so it 204 * doesn't bother saving NS's context. Do that here until we 205 * figure out a way to make this coherent. 206 */ 207 #if FEAT_PABANDON 208 #if !CTX_INCLUDE_EL2_REGS 209 cm_el1_sysregs_context_save(NON_SECURE); 210 #endif 211 /* 212 * when the core wakes it expects its context to already be in 213 * place so we must overwrite it before powerdown. But if 214 * powerdown never happens we want the old context. Save it in 215 * case we wake up. EL2/El1 will not be touched by PSCI so don't 216 * copy */ 217 memcpy(&ctx->gpregs_ctx, &old_ctx.gpregs_ctx, sizeof(gp_regs_t)); 218 memcpy(&ctx->el3state_ctx, &old_ctx.el3state_ctx, sizeof(el3_state_t)); 219 #if DYNAMIC_WORKAROUND_CVE_2018_3639 220 memcpy(&ctx->cve_2018_3639_ctx, &old_ctx.cve_2018_3639_ctx, sizeof(cve_2018_3639_t)); 221 #endif 222 #if ERRATA_SPECULATIVE_AT 223 memcpy(&ctx->errata_speculative_at_ctx, &old_ctx.errata_speculative_at_ctx, sizeof(errata_speculative_at_t)); 224 #endif 225 #if CTX_INCLUDE_PAUTH_REGS 226 memcpy(&ctx->pauth_ctx, &old_ctx.pauth_ctx, sizeof(pauth_t)); 227 #endif 228 #endif 229 max_off_lvl = psci_find_max_off_lvl(state_info); 230 psci_suspend_to_pwrdown_start(end_pwrlvl, max_off_lvl, ep, state_info); 231 } 232 233 /* 234 * Plat. management: Allow the platform to perform the 235 * necessary actions to turn off this cpu e.g. set the 236 * platform defined mailbox with the psci entrypoint, 237 * program the power controller etc. 238 */ 239 240 psci_plat_pm_ops->pwr_domain_suspend(state_info); 241 242 #if ENABLE_PSCI_STAT 243 plat_psci_stat_accounting_start(state_info); 244 #endif 245 246 exit: 247 /* 248 * Release the locks corresponding to each power level in the 249 * reverse order to which they were acquired. 250 */ 251 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 252 253 if (skip_wfi) { 254 return rc; 255 } 256 257 #if ENABLE_RUNTIME_INSTRUMENTATION 258 /* 259 * Update the timestamp with cache off. We assume this 260 * timestamp can only be read from the current CPU and the 261 * timestamp cache line will be flushed before return to 262 * normal world on wakeup. 263 */ 264 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 265 RT_INSTR_ENTER_HW_LOW_PWR, 266 PMF_NO_CACHE_MAINT); 267 #endif 268 269 if (is_power_down_state != 0U) { 270 if (psci_plat_pm_ops->pwr_domain_pwr_down_wfi != NULL) { 271 /* This function may not return */ 272 psci_plat_pm_ops->pwr_domain_pwr_down_wfi(state_info); 273 } 274 275 psci_pwrdown_cpu_end_wakeup(max_off_lvl); 276 } else { 277 /* 278 * We will reach here if only retention/standby states have been 279 * requested at multiple power levels. This means that the cpu 280 * context will be preserved. 281 */ 282 wfi(); 283 } 284 285 #if ENABLE_RUNTIME_INSTRUMENTATION 286 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 287 RT_INSTR_EXIT_HW_LOW_PWR, 288 PMF_NO_CACHE_MAINT); 289 #endif 290 291 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 292 /* 293 * Find out which retention states this CPU has exited from until the 294 * 'end_pwrlvl'. The exit retention state could be deeper than the entry 295 * state as a result of state coordination amongst other CPUs post wfi. 296 */ 297 psci_get_target_local_pwr_states(idx, end_pwrlvl, state_info); 298 299 #if ENABLE_PSCI_STAT 300 plat_psci_stat_accounting_stop(state_info); 301 psci_stats_update_pwr_up(idx, end_pwrlvl, state_info); 302 #endif 303 304 /* 305 * Waking up means we've retained all context. Call the finishers to put 306 * the system back to a usable state. 307 */ 308 if (is_power_down_state != 0U) { 309 #if FEAT_PABANDON 310 psci_cpu_suspend_to_powerdown_finish(idx, max_off_lvl, state_info); 311 312 /* we overwrote context ourselves, put it back */ 313 memcpy(&ctx->gpregs_ctx, &old_ctx.gpregs_ctx, sizeof(gp_regs_t)); 314 memcpy(&ctx->el3state_ctx, &old_ctx.el3state_ctx, sizeof(el3_state_t)); 315 #if DYNAMIC_WORKAROUND_CVE_2018_3639 316 memcpy(&ctx->cve_2018_3639_ctx, &old_ctx.cve_2018_3639_ctx, sizeof(cve_2018_3639_t)); 317 #endif 318 #if ERRATA_SPECULATIVE_AT 319 memcpy(&ctx->errata_speculative_at_ctx, &old_ctx.errata_speculative_at_ctx, sizeof(errata_speculative_at_t)); 320 #endif 321 #if CTX_INCLUDE_PAUTH_REGS 322 memcpy(&ctx->pauth_ctx, &old_ctx.pauth_ctx, sizeof(pauth_t)); 323 #endif 324 #if !CTX_INCLUDE_EL2_REGS 325 cm_el1_sysregs_context_restore(NON_SECURE); 326 #endif 327 #endif 328 } else { 329 psci_cpu_suspend_to_standby_finish(end_pwrlvl, state_info); 330 } 331 332 /* 333 * Set the requested and target state of this CPU and all the higher 334 * power domain levels for this CPU to run. 335 */ 336 psci_set_pwr_domains_to_run(idx, end_pwrlvl); 337 338 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 339 340 return rc; 341 } 342 343 /******************************************************************************* 344 * The following functions finish an earlier suspend request. They 345 * are called by the common finisher routine in psci_common.c. The `state_info` 346 * is the psci_power_state from which this CPU has woken up from. 347 ******************************************************************************/ 348 void psci_cpu_suspend_to_powerdown_finish(unsigned int cpu_idx, unsigned int max_off_lvl, const psci_power_state_t *state_info) 349 { 350 unsigned int counter_freq; 351 352 /* Ensure we have been woken up from a suspended state */ 353 assert((psci_get_aff_info_state() == AFF_STATE_ON) && 354 (is_local_state_off( 355 state_info->pwr_domain_state[PSCI_CPU_PWR_LVL]) != 0)); 356 357 /* 358 * Plat. management: Perform the platform specific actions 359 * before we change the state of the cpu e.g. enabling the 360 * gic or zeroing the mailbox register. If anything goes 361 * wrong then assert as there is no way to recover from this 362 * situation. 363 */ 364 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); 365 366 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 367 /* Arch. management: Enable the data cache, stack memory maintenance. */ 368 psci_do_pwrup_cache_maintenance(); 369 #endif 370 371 /* Re-init the cntfrq_el0 register */ 372 counter_freq = plat_get_syscnt_freq2(); 373 write_cntfrq_el0(counter_freq); 374 375 #if ENABLE_PAUTH 376 /* Store APIAKey_EL1 key */ 377 set_cpu_data(apiakey[0], read_apiakeylo_el1()); 378 set_cpu_data(apiakey[1], read_apiakeyhi_el1()); 379 #endif /* ENABLE_PAUTH */ 380 381 /* 382 * Call the cpu suspend finish handler registered by the Secure Payload 383 * Dispatcher to let it do any bookeeping. If the handler encounters an 384 * error, it's expected to assert within 385 */ 386 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend_finish != NULL)) { 387 psci_spd_pm->svc_suspend_finish(max_off_lvl); 388 } 389 390 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 391 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 392 393 PUBLISH_EVENT(psci_suspend_pwrdown_finish); 394 } 395