1 /* 2 * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <stddef.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <lib/el3_runtime/context_mgmt.h> 16 #include <lib/el3_runtime/cpu_data.h> 17 #include <lib/el3_runtime/pubsub_events.h> 18 #include <lib/pmf/pmf.h> 19 #include <lib/runtime_instr.h> 20 #include <plat/common/platform.h> 21 22 #include "psci_private.h" 23 24 /******************************************************************************* 25 * This function does generic and platform specific operations after a wake-up 26 * from standby/retention states at multiple power levels. 27 ******************************************************************************/ 28 static void psci_cpu_suspend_to_standby_finish(unsigned int cpu_idx, 29 unsigned int end_pwrlvl, 30 psci_power_state_t *state_info) 31 { 32 /* 33 * Plat. management: Allow the platform to do operations 34 * on waking up from retention. 35 */ 36 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); 37 38 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 39 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 40 } 41 42 /******************************************************************************* 43 * This function does generic and platform specific suspend to power down 44 * operations. 45 ******************************************************************************/ 46 static void psci_suspend_to_pwrdown_start(unsigned int end_pwrlvl, 47 const entry_point_info_t *ep, 48 const psci_power_state_t *state_info) 49 { 50 unsigned int max_off_lvl = psci_find_max_off_lvl(state_info); 51 52 PUBLISH_EVENT(psci_suspend_pwrdown_start); 53 54 #if PSCI_OS_INIT_MODE 55 #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL 56 end_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; 57 #else 58 end_pwrlvl = PLAT_MAX_PWR_LVL; 59 #endif 60 #endif 61 62 /* Save PSCI target power level for the suspend finisher handler */ 63 psci_set_suspend_pwrlvl(end_pwrlvl); 64 65 /* 66 * Flush the target power level as it might be accessed on power up with 67 * Data cache disabled. 68 */ 69 psci_flush_cpu_data(psci_svc_cpu_data.target_pwrlvl); 70 71 /* 72 * Call the cpu suspend handler registered by the Secure Payload 73 * Dispatcher to let it do any book-keeping. If the handler encounters an 74 * error, it's expected to assert within 75 */ 76 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend != NULL)) 77 psci_spd_pm->svc_suspend(max_off_lvl); 78 79 #if !HW_ASSISTED_COHERENCY 80 /* 81 * Plat. management: Allow the platform to perform any early 82 * actions required to power down the CPU. This might be useful for 83 * HW_ASSISTED_COHERENCY = 0 platforms that can safely perform these 84 * actions with data caches enabled. 85 */ 86 if (psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early != NULL) 87 psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early(state_info); 88 #endif 89 90 /* 91 * Store the re-entry information for the non-secure world. 92 */ 93 cm_init_my_context(ep); 94 95 /* 96 * Arch. management. Initiate power down sequence. 97 * TODO : Introduce a mechanism to query the cache level to flush 98 * and the cpu-ops power down to perform from the platform. 99 */ 100 psci_pwrdown_cpu(max_off_lvl); 101 } 102 103 /******************************************************************************* 104 * Top level handler which is called when a cpu wants to suspend its execution. 105 * It is assumed that along with suspending the cpu power domain, power domains 106 * at higher levels until the target power level will be suspended as well. It 107 * coordinates with the platform to negotiate the target state for each of 108 * the power domain level till the target power domain level. It then performs 109 * generic, architectural, platform setup and state management required to 110 * suspend that power domain level and power domain levels below it. 111 * e.g. For a cpu that's to be suspended, it could mean programming the 112 * power controller whereas for a cluster that's to be suspended, it will call 113 * the platform specific code which will disable coherency at the interconnect 114 * level if the cpu is the last in the cluster and also the program the power 115 * controller. 116 * 117 * All the required parameter checks are performed at the beginning and after 118 * the state transition has been done, no further error is expected and it is 119 * not possible to undo any of the actions taken beyond that point. 120 ******************************************************************************/ 121 int psci_cpu_suspend_start(const entry_point_info_t *ep, 122 unsigned int end_pwrlvl, 123 psci_power_state_t *state_info, 124 unsigned int is_power_down_state) 125 { 126 int rc = PSCI_E_SUCCESS; 127 bool skip_wfi = false; 128 unsigned int idx = plat_my_core_pos(); 129 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 130 131 /* 132 * This function must only be called on platforms where the 133 * CPU_SUSPEND platform hooks have been implemented. 134 */ 135 assert((psci_plat_pm_ops->pwr_domain_suspend != NULL) && 136 (psci_plat_pm_ops->pwr_domain_suspend_finish != NULL)); 137 138 /* Get the parent nodes */ 139 psci_get_parent_pwr_domain_nodes(idx, end_pwrlvl, parent_nodes); 140 141 /* 142 * This function acquires the lock corresponding to each power 143 * level so that by the time all locks are taken, the system topology 144 * is snapshot and state management can be done safely. 145 */ 146 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 147 148 /* 149 * We check if there are any pending interrupts after the delay 150 * introduced by lock contention to increase the chances of early 151 * detection that a wake-up interrupt has fired. 152 */ 153 if (read_isr_el1() != 0U) { 154 skip_wfi = true; 155 goto exit; 156 } 157 158 #if PSCI_OS_INIT_MODE 159 if (psci_suspend_mode == OS_INIT) { 160 /* 161 * This function validates the requested state info for 162 * OS-initiated mode. 163 */ 164 rc = psci_validate_state_coordination(end_pwrlvl, state_info); 165 if (rc != PSCI_E_SUCCESS) { 166 skip_wfi = true; 167 goto exit; 168 } 169 } else { 170 #endif 171 /* 172 * This function is passed the requested state info and 173 * it returns the negotiated state info for each power level upto 174 * the end level specified. 175 */ 176 psci_do_state_coordination(end_pwrlvl, state_info); 177 #if PSCI_OS_INIT_MODE 178 } 179 #endif 180 181 #if PSCI_OS_INIT_MODE 182 if (psci_plat_pm_ops->pwr_domain_validate_suspend != NULL) { 183 rc = psci_plat_pm_ops->pwr_domain_validate_suspend(state_info); 184 if (rc != PSCI_E_SUCCESS) { 185 skip_wfi = true; 186 goto exit; 187 } 188 } 189 #endif 190 191 /* Update the target state in the power domain nodes */ 192 psci_set_target_local_pwr_states(end_pwrlvl, state_info); 193 194 #if ENABLE_PSCI_STAT 195 /* Update the last cpu for each level till end_pwrlvl */ 196 psci_stats_update_pwr_down(end_pwrlvl, state_info); 197 #endif 198 199 if (is_power_down_state != 0U) 200 psci_suspend_to_pwrdown_start(end_pwrlvl, ep, state_info); 201 202 /* 203 * Plat. management: Allow the platform to perform the 204 * necessary actions to turn off this cpu e.g. set the 205 * platform defined mailbox with the psci entrypoint, 206 * program the power controller etc. 207 */ 208 209 psci_plat_pm_ops->pwr_domain_suspend(state_info); 210 211 #if ENABLE_PSCI_STAT 212 plat_psci_stat_accounting_start(state_info); 213 #endif 214 215 exit: 216 /* 217 * Release the locks corresponding to each power level in the 218 * reverse order to which they were acquired. 219 */ 220 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 221 222 if (skip_wfi) { 223 return rc; 224 } 225 226 if (is_power_down_state != 0U) { 227 #if ENABLE_RUNTIME_INSTRUMENTATION 228 229 /* 230 * Update the timestamp with cache off. We assume this 231 * timestamp can only be read from the current CPU and the 232 * timestamp cache line will be flushed before return to 233 * normal world on wakeup. 234 */ 235 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 236 RT_INSTR_ENTER_HW_LOW_PWR, 237 PMF_NO_CACHE_MAINT); 238 #endif 239 240 /* The function calls below must not return */ 241 if (psci_plat_pm_ops->pwr_domain_pwr_down_wfi != NULL) 242 psci_plat_pm_ops->pwr_domain_pwr_down_wfi(state_info); 243 else 244 psci_power_down_wfi(); 245 } 246 247 #if ENABLE_RUNTIME_INSTRUMENTATION 248 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 249 RT_INSTR_ENTER_HW_LOW_PWR, 250 PMF_NO_CACHE_MAINT); 251 #endif 252 253 /* 254 * We will reach here if only retention/standby states have been 255 * requested at multiple power levels. This means that the cpu 256 * context will be preserved. 257 */ 258 wfi(); 259 260 #if ENABLE_RUNTIME_INSTRUMENTATION 261 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 262 RT_INSTR_EXIT_HW_LOW_PWR, 263 PMF_NO_CACHE_MAINT); 264 #endif 265 266 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 267 /* 268 * Find out which retention states this CPU has exited from until the 269 * 'end_pwrlvl'. The exit retention state could be deeper than the entry 270 * state as a result of state coordination amongst other CPUs post wfi. 271 */ 272 psci_get_target_local_pwr_states(end_pwrlvl, state_info); 273 274 #if ENABLE_PSCI_STAT 275 plat_psci_stat_accounting_stop(state_info); 276 psci_stats_update_pwr_up(end_pwrlvl, state_info); 277 #endif 278 279 /* 280 * After we wake up from context retaining suspend, call the 281 * context retaining suspend finisher. 282 */ 283 psci_cpu_suspend_to_standby_finish(idx, end_pwrlvl, state_info); 284 285 /* 286 * Set the requested and target state of this CPU and all the higher 287 * power domain levels for this CPU to run. 288 */ 289 psci_set_pwr_domains_to_run(end_pwrlvl); 290 291 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 292 293 return rc; 294 } 295 296 /******************************************************************************* 297 * The following functions finish an earlier suspend request. They 298 * are called by the common finisher routine in psci_common.c. The `state_info` 299 * is the psci_power_state from which this CPU has woken up from. 300 ******************************************************************************/ 301 void psci_cpu_suspend_to_powerdown_finish(unsigned int cpu_idx, const psci_power_state_t *state_info) 302 { 303 unsigned int counter_freq; 304 unsigned int max_off_lvl; 305 306 /* Ensure we have been woken up from a suspended state */ 307 assert((psci_get_aff_info_state() == AFF_STATE_ON) && 308 (is_local_state_off( 309 state_info->pwr_domain_state[PSCI_CPU_PWR_LVL]) != 0)); 310 311 /* 312 * Plat. management: Perform the platform specific actions 313 * before we change the state of the cpu e.g. enabling the 314 * gic or zeroing the mailbox register. If anything goes 315 * wrong then assert as there is no way to recover from this 316 * situation. 317 */ 318 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); 319 320 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 321 /* Arch. management: Enable the data cache, stack memory maintenance. */ 322 psci_do_pwrup_cache_maintenance(); 323 #endif 324 325 /* Re-init the cntfrq_el0 register */ 326 counter_freq = plat_get_syscnt_freq2(); 327 write_cntfrq_el0(counter_freq); 328 329 #if ENABLE_PAUTH 330 /* Store APIAKey_EL1 key */ 331 set_cpu_data(apiakey[0], read_apiakeylo_el1()); 332 set_cpu_data(apiakey[1], read_apiakeyhi_el1()); 333 #endif /* ENABLE_PAUTH */ 334 335 /* 336 * Call the cpu suspend finish handler registered by the Secure Payload 337 * Dispatcher to let it do any bookeeping. If the handler encounters an 338 * error, it's expected to assert within 339 */ 340 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend_finish != NULL)) { 341 max_off_lvl = psci_find_max_off_lvl(state_info); 342 assert(max_off_lvl != PSCI_INVALID_PWR_LVL); 343 psci_spd_pm->svc_suspend_finish(max_off_lvl); 344 } 345 346 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 347 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 348 349 PUBLISH_EVENT(psci_suspend_pwrdown_finish); 350 } 351