1 /* 2 * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <stddef.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <drivers/arm/gic.h> 16 #include <lib/el3_runtime/context_mgmt.h> 17 #include <lib/el3_runtime/cpu_data.h> 18 #include <lib/el3_runtime/pubsub_events.h> 19 #include <lib/pmf/pmf.h> 20 #include <lib/runtime_instr.h> 21 #include <plat/common/platform.h> 22 23 #include "psci_private.h" 24 25 /******************************************************************************* 26 * This function does generic and platform specific operations after a wake-up 27 * from standby/retention states at multiple power levels. 28 ******************************************************************************/ 29 static void psci_cpu_suspend_to_standby_finish(unsigned int end_pwrlvl, 30 psci_power_state_t *state_info) 31 { 32 /* 33 * Plat. management: Allow the platform to do operations 34 * on waking up from retention. 35 */ 36 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); 37 38 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 39 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 40 } 41 42 /******************************************************************************* 43 * This function does generic and platform specific suspend to power down 44 * operations. 45 ******************************************************************************/ 46 static void psci_suspend_to_pwrdown_start(unsigned int idx, 47 unsigned int end_pwrlvl, 48 unsigned int max_off_lvl, 49 const psci_power_state_t *state_info) 50 { 51 PUBLISH_EVENT_ARG(psci_suspend_pwrdown_start, &idx); 52 53 #if PSCI_OS_INIT_MODE 54 #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL 55 end_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; 56 #else 57 end_pwrlvl = PLAT_MAX_PWR_LVL; 58 #endif 59 #endif 60 61 /* Save PSCI target power level for the suspend finisher handler */ 62 psci_set_suspend_pwrlvl(end_pwrlvl); 63 64 /* 65 * Flush the target power level as it might be accessed on power up with 66 * Data cache disabled. 67 */ 68 psci_flush_cpu_data(psci_svc_cpu_data.target_pwrlvl); 69 70 /* 71 * Call the cpu suspend handler registered by the Secure Payload 72 * Dispatcher to let it do any book-keeping. If the handler encounters an 73 * error, it's expected to assert within 74 */ 75 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend != NULL)) 76 psci_spd_pm->svc_suspend(max_off_lvl); 77 78 #if !HW_ASSISTED_COHERENCY 79 /* 80 * Plat. management: Allow the platform to perform any early 81 * actions required to power down the CPU. This might be useful for 82 * HW_ASSISTED_COHERENCY = 0 platforms that can safely perform these 83 * actions with data caches enabled. 84 */ 85 if (psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early != NULL) 86 psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early(state_info); 87 #endif 88 /* 89 * Arch. management. Initiate power down sequence. 90 */ 91 psci_pwrdown_cpu_start(max_off_lvl); 92 } 93 94 /******************************************************************************* 95 * Top level handler which is called when a cpu wants to suspend its execution. 96 * It is assumed that along with suspending the cpu power domain, power domains 97 * at higher levels until the target power level will be suspended as well. It 98 * coordinates with the platform to negotiate the target state for each of 99 * the power domain level till the target power domain level. It then performs 100 * generic, architectural, platform setup and state management required to 101 * suspend that power domain level and power domain levels below it. 102 * e.g. For a cpu that's to be suspended, it could mean programming the 103 * power controller whereas for a cluster that's to be suspended, it will call 104 * the platform specific code which will disable coherency at the interconnect 105 * level if the cpu is the last in the cluster and also the program the power 106 * controller. 107 * 108 * All the required parameter checks are performed at the beginning and after 109 * the state transition has been done, no further error is expected and it is 110 * not possible to undo any of the actions taken beyond that point. 111 ******************************************************************************/ 112 int psci_cpu_suspend_start(unsigned int idx, 113 unsigned int end_pwrlvl, 114 psci_power_state_t *state_info, 115 unsigned int is_power_down_state) 116 { 117 int rc = PSCI_E_SUCCESS; 118 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 119 unsigned int max_off_lvl = 0; 120 121 /* 122 * This function must only be called on platforms where the 123 * CPU_SUSPEND platform hooks have been implemented. 124 */ 125 assert((psci_plat_pm_ops->pwr_domain_suspend != NULL) && 126 (psci_plat_pm_ops->pwr_domain_suspend_finish != NULL)); 127 128 /* Get the parent nodes */ 129 psci_get_parent_pwr_domain_nodes(idx, end_pwrlvl, parent_nodes); 130 131 /* 132 * This function acquires the lock corresponding to each power 133 * level so that by the time all locks are taken, the system topology 134 * is snapshot and state management can be done safely. 135 */ 136 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 137 138 /* 139 * We check if there are any pending interrupts after the delay 140 * introduced by lock contention to increase the chances of early 141 * detection that a wake-up interrupt has fired. 142 */ 143 if (read_isr_el1() != 0U) { 144 goto suspend_exit; 145 } 146 147 #if PSCI_OS_INIT_MODE 148 if (psci_suspend_mode == OS_INIT) { 149 /* 150 * This function validates the requested state info for 151 * OS-initiated mode. 152 */ 153 rc = psci_validate_state_coordination(idx, end_pwrlvl, state_info); 154 if (rc != PSCI_E_SUCCESS) { 155 goto suspend_exit; 156 } 157 } else { 158 #endif 159 /* 160 * This function is passed the requested state info and 161 * it returns the negotiated state info for each power level upto 162 * the end level specified. 163 */ 164 psci_do_state_coordination(idx, end_pwrlvl, state_info); 165 #if PSCI_OS_INIT_MODE 166 } 167 #endif 168 169 #if PSCI_OS_INIT_MODE 170 if (psci_plat_pm_ops->pwr_domain_validate_suspend != NULL) { 171 rc = psci_plat_pm_ops->pwr_domain_validate_suspend(state_info); 172 if (rc != PSCI_E_SUCCESS) { 173 goto suspend_exit; 174 } 175 } 176 #endif 177 178 /* Update the target state in the power domain nodes */ 179 psci_set_target_local_pwr_states(idx, end_pwrlvl, state_info); 180 181 #if ENABLE_PSCI_STAT 182 /* Update the last cpu for each level till end_pwrlvl */ 183 psci_stats_update_pwr_down(idx, end_pwrlvl, state_info); 184 #endif 185 186 if (is_power_down_state != 0U) { 187 /* 188 * WHen CTX_INCLUDE_EL2_REGS is usnet, we're probably runnig 189 * with some SPD that assumes the core is going off so it 190 * doesn't bother saving NS's context. Do that here until we 191 * figure out a way to make this coherent. 192 */ 193 #if FEAT_PABANDON 194 #if !CTX_INCLUDE_EL2_REGS 195 cm_el1_sysregs_context_save(NON_SECURE); 196 #endif 197 #endif 198 max_off_lvl = psci_find_max_off_lvl(state_info); 199 psci_suspend_to_pwrdown_start(idx, end_pwrlvl, end_pwrlvl, state_info); 200 } 201 202 #if USE_GIC_DRIVER 203 /* turn the GIC off before we hand off to the platform */ 204 gic_cpuif_disable(idx); 205 #endif /* USE_GIC_DRIVER */ 206 207 /* 208 * Plat. management: Allow the platform to perform the 209 * necessary actions to turn off this cpu e.g. set the 210 * platform defined mailbox with the psci entrypoint, 211 * program the power controller etc. 212 */ 213 psci_plat_pm_ops->pwr_domain_suspend(state_info); 214 215 #if ENABLE_PSCI_STAT 216 plat_psci_stat_accounting_start(state_info); 217 #endif 218 219 /* 220 * Release the locks corresponding to each power level in the 221 * reverse order to which they were acquired. 222 */ 223 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 224 225 #if ENABLE_RUNTIME_INSTRUMENTATION 226 /* 227 * Update the timestamp with cache off. We assume this 228 * timestamp can only be read from the current CPU and the 229 * timestamp cache line will be flushed before return to 230 * normal world on wakeup. 231 */ 232 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 233 RT_INSTR_ENTER_HW_LOW_PWR, 234 PMF_NO_CACHE_MAINT); 235 #endif 236 237 if (is_power_down_state != 0U) { 238 if (psci_plat_pm_ops->pwr_domain_pwr_down != NULL) { 239 /* This function may not return */ 240 psci_plat_pm_ops->pwr_domain_pwr_down(state_info); 241 } 242 243 psci_pwrdown_cpu_end_wakeup(max_off_lvl); 244 } else { 245 /* 246 * We will reach here if only retention/standby states have been 247 * requested at multiple power levels. This means that the cpu 248 * context will be preserved. 249 */ 250 wfi(); 251 } 252 253 #if ENABLE_RUNTIME_INSTRUMENTATION 254 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 255 RT_INSTR_EXIT_HW_LOW_PWR, 256 PMF_NO_CACHE_MAINT); 257 #endif 258 259 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 260 /* 261 * Find out which retention states this CPU has exited from until the 262 * 'end_pwrlvl'. The exit retention state could be deeper than the entry 263 * state as a result of state coordination amongst other CPUs post wfi. 264 */ 265 psci_get_target_local_pwr_states(idx, end_pwrlvl, state_info); 266 267 #if ENABLE_PSCI_STAT 268 plat_psci_stat_accounting_stop(state_info); 269 psci_stats_update_pwr_up(idx, end_pwrlvl, state_info); 270 #endif 271 272 /* 273 * Waking up means we've retained all context. Call the finishers to put 274 * the system back to a usable state. 275 */ 276 if (is_power_down_state != 0U) { 277 #if FEAT_PABANDON 278 psci_cpu_suspend_to_powerdown_finish(idx, max_off_lvl, state_info); 279 280 #if !CTX_INCLUDE_EL2_REGS 281 cm_el1_sysregs_context_restore(NON_SECURE); 282 #endif 283 #endif 284 } else { 285 psci_cpu_suspend_to_standby_finish(end_pwrlvl, state_info); 286 } 287 288 #if USE_GIC_DRIVER 289 /* Turn GIC on after platform has had a chance to do state management */ 290 gic_cpuif_enable(idx); 291 #endif /* USE_GIC_DRIVER */ 292 293 /* 294 * Set the requested and target state of this CPU and all the higher 295 * power domain levels for this CPU to run. 296 */ 297 psci_set_pwr_domains_to_run(idx, end_pwrlvl); 298 299 suspend_exit: 300 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 301 302 return rc; 303 } 304 305 /******************************************************************************* 306 * The following functions finish an earlier suspend request. They 307 * are called by the common finisher routine in psci_common.c. The `state_info` 308 * is the psci_power_state from which this CPU has woken up from. 309 ******************************************************************************/ 310 void psci_cpu_suspend_to_powerdown_finish(unsigned int cpu_idx, unsigned int max_off_lvl, const psci_power_state_t *state_info) 311 { 312 unsigned int counter_freq; 313 314 /* Ensure we have been woken up from a suspended state */ 315 assert((psci_get_aff_info_state() == AFF_STATE_ON) && 316 (is_local_state_off( 317 state_info->pwr_domain_state[PSCI_CPU_PWR_LVL]) != 0)); 318 319 /* 320 * Plat. management: Perform the platform specific actions 321 * before we change the state of the cpu e.g. enabling the 322 * gic or zeroing the mailbox register. If anything goes 323 * wrong then assert as there is no way to recover from this 324 * situation. 325 */ 326 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); 327 328 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 329 /* Arch. management: Enable the data cache, stack memory maintenance. */ 330 psci_do_pwrup_cache_maintenance(); 331 #endif 332 333 #if USE_GIC_DRIVER 334 /* GIC on after platform has had its say and MMU is on */ 335 gic_cpuif_enable(cpu_idx); 336 #endif /* USE_GIC_DRIVER */ 337 338 /* Re-init the cntfrq_el0 register */ 339 counter_freq = plat_get_syscnt_freq2(); 340 write_cntfrq_el0(counter_freq); 341 342 /* 343 * Call the cpu suspend finish handler registered by the Secure Payload 344 * Dispatcher to let it do any bookeeping. If the handler encounters an 345 * error, it's expected to assert within 346 */ 347 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend_finish != NULL)) { 348 psci_spd_pm->svc_suspend_finish(max_off_lvl); 349 } 350 351 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 352 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 353 354 PUBLISH_EVENT_ARG(psci_suspend_pwrdown_finish, &cpu_idx); 355 } 356