1 /* 2 * Copyright (c) 2013-2024, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <assert.h> 8 #include <stddef.h> 9 10 #include <arch.h> 11 #include <arch_helpers.h> 12 #include <common/bl_common.h> 13 #include <common/debug.h> 14 #include <context.h> 15 #include <lib/el3_runtime/context_mgmt.h> 16 #include <lib/el3_runtime/cpu_data.h> 17 #include <lib/el3_runtime/pubsub_events.h> 18 #include <lib/pmf/pmf.h> 19 #include <lib/runtime_instr.h> 20 #include <plat/common/platform.h> 21 22 #include "psci_private.h" 23 24 /******************************************************************************* 25 * This function does generic and platform specific operations after a wake-up 26 * from standby/retention states at multiple power levels. 27 ******************************************************************************/ 28 static void psci_suspend_to_standby_finisher(unsigned int cpu_idx, 29 unsigned int end_pwrlvl) 30 { 31 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 32 psci_power_state_t state_info; 33 34 /* Get the parent nodes */ 35 psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); 36 37 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 38 39 /* 40 * Find out which retention states this CPU has exited from until the 41 * 'end_pwrlvl'. The exit retention state could be deeper than the entry 42 * state as a result of state coordination amongst other CPUs post wfi. 43 */ 44 psci_get_target_local_pwr_states(end_pwrlvl, &state_info); 45 46 #if ENABLE_PSCI_STAT 47 plat_psci_stat_accounting_stop(&state_info); 48 psci_stats_update_pwr_up(end_pwrlvl, &state_info); 49 #endif 50 51 /* 52 * Plat. management: Allow the platform to do operations 53 * on waking up from retention. 54 */ 55 psci_plat_pm_ops->pwr_domain_suspend_finish(&state_info); 56 57 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 58 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 59 60 /* 61 * Set the requested and target state of this CPU and all the higher 62 * power domain levels for this CPU to run. 63 */ 64 psci_set_pwr_domains_to_run(end_pwrlvl); 65 66 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 67 } 68 69 /******************************************************************************* 70 * This function does generic and platform specific suspend to power down 71 * operations. 72 ******************************************************************************/ 73 static void psci_suspend_to_pwrdown_start(unsigned int end_pwrlvl, 74 const entry_point_info_t *ep, 75 const psci_power_state_t *state_info) 76 { 77 unsigned int max_off_lvl = psci_find_max_off_lvl(state_info); 78 79 PUBLISH_EVENT(psci_suspend_pwrdown_start); 80 81 #if PSCI_OS_INIT_MODE 82 #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL 83 end_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; 84 #else 85 end_pwrlvl = PLAT_MAX_PWR_LVL; 86 #endif 87 #endif 88 89 /* Save PSCI target power level for the suspend finisher handler */ 90 psci_set_suspend_pwrlvl(end_pwrlvl); 91 92 /* 93 * Flush the target power level as it might be accessed on power up with 94 * Data cache disabled. 95 */ 96 psci_flush_cpu_data(psci_svc_cpu_data.target_pwrlvl); 97 98 /* 99 * Call the cpu suspend handler registered by the Secure Payload 100 * Dispatcher to let it do any book-keeping. If the handler encounters an 101 * error, it's expected to assert within 102 */ 103 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend != NULL)) 104 psci_spd_pm->svc_suspend(max_off_lvl); 105 106 #if !HW_ASSISTED_COHERENCY 107 /* 108 * Plat. management: Allow the platform to perform any early 109 * actions required to power down the CPU. This might be useful for 110 * HW_ASSISTED_COHERENCY = 0 platforms that can safely perform these 111 * actions with data caches enabled. 112 */ 113 if (psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early != NULL) 114 psci_plat_pm_ops->pwr_domain_suspend_pwrdown_early(state_info); 115 #endif 116 117 /* 118 * Store the re-entry information for the non-secure world. 119 */ 120 cm_init_my_context(ep); 121 122 #if ENABLE_RUNTIME_INSTRUMENTATION 123 124 /* 125 * Flush cache line so that even if CPU power down happens 126 * the timestamp update is reflected in memory. 127 */ 128 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 129 RT_INSTR_ENTER_CFLUSH, 130 PMF_CACHE_MAINT); 131 #endif 132 133 /* 134 * Arch. management. Initiate power down sequence. 135 * TODO : Introduce a mechanism to query the cache level to flush 136 * and the cpu-ops power down to perform from the platform. 137 */ 138 psci_pwrdown_cpu(max_off_lvl); 139 140 #if ENABLE_RUNTIME_INSTRUMENTATION 141 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 142 RT_INSTR_EXIT_CFLUSH, 143 PMF_NO_CACHE_MAINT); 144 #endif 145 } 146 147 /******************************************************************************* 148 * Top level handler which is called when a cpu wants to suspend its execution. 149 * It is assumed that along with suspending the cpu power domain, power domains 150 * at higher levels until the target power level will be suspended as well. It 151 * coordinates with the platform to negotiate the target state for each of 152 * the power domain level till the target power domain level. It then performs 153 * generic, architectural, platform setup and state management required to 154 * suspend that power domain level and power domain levels below it. 155 * e.g. For a cpu that's to be suspended, it could mean programming the 156 * power controller whereas for a cluster that's to be suspended, it will call 157 * the platform specific code which will disable coherency at the interconnect 158 * level if the cpu is the last in the cluster and also the program the power 159 * controller. 160 * 161 * All the required parameter checks are performed at the beginning and after 162 * the state transition has been done, no further error is expected and it is 163 * not possible to undo any of the actions taken beyond that point. 164 ******************************************************************************/ 165 int psci_cpu_suspend_start(const entry_point_info_t *ep, 166 unsigned int end_pwrlvl, 167 psci_power_state_t *state_info, 168 unsigned int is_power_down_state) 169 { 170 int rc = PSCI_E_SUCCESS; 171 bool skip_wfi = false; 172 unsigned int idx = plat_my_core_pos(); 173 unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; 174 175 /* 176 * This function must only be called on platforms where the 177 * CPU_SUSPEND platform hooks have been implemented. 178 */ 179 assert((psci_plat_pm_ops->pwr_domain_suspend != NULL) && 180 (psci_plat_pm_ops->pwr_domain_suspend_finish != NULL)); 181 182 /* Get the parent nodes */ 183 psci_get_parent_pwr_domain_nodes(idx, end_pwrlvl, parent_nodes); 184 185 /* 186 * This function acquires the lock corresponding to each power 187 * level so that by the time all locks are taken, the system topology 188 * is snapshot and state management can be done safely. 189 */ 190 psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); 191 192 /* 193 * We check if there are any pending interrupts after the delay 194 * introduced by lock contention to increase the chances of early 195 * detection that a wake-up interrupt has fired. 196 */ 197 if (read_isr_el1() != 0U) { 198 skip_wfi = true; 199 goto exit; 200 } 201 202 #if PSCI_OS_INIT_MODE 203 if (psci_suspend_mode == OS_INIT) { 204 /* 205 * This function validates the requested state info for 206 * OS-initiated mode. 207 */ 208 rc = psci_validate_state_coordination(end_pwrlvl, state_info); 209 if (rc != PSCI_E_SUCCESS) { 210 skip_wfi = true; 211 goto exit; 212 } 213 } else { 214 #endif 215 /* 216 * This function is passed the requested state info and 217 * it returns the negotiated state info for each power level upto 218 * the end level specified. 219 */ 220 psci_do_state_coordination(end_pwrlvl, state_info); 221 #if PSCI_OS_INIT_MODE 222 } 223 #endif 224 225 #if PSCI_OS_INIT_MODE 226 if (psci_plat_pm_ops->pwr_domain_validate_suspend != NULL) { 227 rc = psci_plat_pm_ops->pwr_domain_validate_suspend(state_info); 228 if (rc != PSCI_E_SUCCESS) { 229 skip_wfi = true; 230 goto exit; 231 } 232 } 233 #endif 234 235 /* Update the target state in the power domain nodes */ 236 psci_set_target_local_pwr_states(end_pwrlvl, state_info); 237 238 #if ENABLE_PSCI_STAT 239 /* Update the last cpu for each level till end_pwrlvl */ 240 psci_stats_update_pwr_down(end_pwrlvl, state_info); 241 #endif 242 243 if (is_power_down_state != 0U) 244 psci_suspend_to_pwrdown_start(end_pwrlvl, ep, state_info); 245 246 /* 247 * Plat. management: Allow the platform to perform the 248 * necessary actions to turn off this cpu e.g. set the 249 * platform defined mailbox with the psci entrypoint, 250 * program the power controller etc. 251 */ 252 253 psci_plat_pm_ops->pwr_domain_suspend(state_info); 254 255 #if ENABLE_PSCI_STAT 256 plat_psci_stat_accounting_start(state_info); 257 #endif 258 259 exit: 260 /* 261 * Release the locks corresponding to each power level in the 262 * reverse order to which they were acquired. 263 */ 264 psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); 265 266 if (skip_wfi) { 267 return rc; 268 } 269 270 if (is_power_down_state != 0U) { 271 #if ENABLE_RUNTIME_INSTRUMENTATION 272 273 /* 274 * Update the timestamp with cache off. We assume this 275 * timestamp can only be read from the current CPU and the 276 * timestamp cache line will be flushed before return to 277 * normal world on wakeup. 278 */ 279 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 280 RT_INSTR_ENTER_HW_LOW_PWR, 281 PMF_NO_CACHE_MAINT); 282 #endif 283 284 /* The function calls below must not return */ 285 if (psci_plat_pm_ops->pwr_domain_pwr_down_wfi != NULL) 286 psci_plat_pm_ops->pwr_domain_pwr_down_wfi(state_info); 287 else 288 psci_power_down_wfi(); 289 } 290 291 #if ENABLE_RUNTIME_INSTRUMENTATION 292 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 293 RT_INSTR_ENTER_HW_LOW_PWR, 294 PMF_NO_CACHE_MAINT); 295 #endif 296 297 /* 298 * We will reach here if only retention/standby states have been 299 * requested at multiple power levels. This means that the cpu 300 * context will be preserved. 301 */ 302 wfi(); 303 304 #if ENABLE_RUNTIME_INSTRUMENTATION 305 PMF_CAPTURE_TIMESTAMP(rt_instr_svc, 306 RT_INSTR_EXIT_HW_LOW_PWR, 307 PMF_NO_CACHE_MAINT); 308 #endif 309 310 /* 311 * After we wake up from context retaining suspend, call the 312 * context retaining suspend finisher. 313 */ 314 psci_suspend_to_standby_finisher(idx, end_pwrlvl); 315 316 return rc; 317 } 318 319 /******************************************************************************* 320 * The following functions finish an earlier suspend request. They 321 * are called by the common finisher routine in psci_common.c. The `state_info` 322 * is the psci_power_state from which this CPU has woken up from. 323 ******************************************************************************/ 324 void psci_cpu_suspend_finish(unsigned int cpu_idx, const psci_power_state_t *state_info) 325 { 326 unsigned int counter_freq; 327 unsigned int max_off_lvl; 328 329 /* Ensure we have been woken up from a suspended state */ 330 assert((psci_get_aff_info_state() == AFF_STATE_ON) && 331 (is_local_state_off( 332 state_info->pwr_domain_state[PSCI_CPU_PWR_LVL]) != 0)); 333 334 /* 335 * Plat. management: Perform the platform specific actions 336 * before we change the state of the cpu e.g. enabling the 337 * gic or zeroing the mailbox register. If anything goes 338 * wrong then assert as there is no way to recover from this 339 * situation. 340 */ 341 psci_plat_pm_ops->pwr_domain_suspend_finish(state_info); 342 343 #if !(HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) 344 /* Arch. management: Enable the data cache, stack memory maintenance. */ 345 psci_do_pwrup_cache_maintenance(); 346 #endif 347 348 /* Re-init the cntfrq_el0 register */ 349 counter_freq = plat_get_syscnt_freq2(); 350 write_cntfrq_el0(counter_freq); 351 352 #if ENABLE_PAUTH 353 /* Store APIAKey_EL1 key */ 354 set_cpu_data(apiakey[0], read_apiakeylo_el1()); 355 set_cpu_data(apiakey[1], read_apiakeyhi_el1()); 356 #endif /* ENABLE_PAUTH */ 357 358 /* 359 * Call the cpu suspend finish handler registered by the Secure Payload 360 * Dispatcher to let it do any bookeeping. If the handler encounters an 361 * error, it's expected to assert within 362 */ 363 if ((psci_spd_pm != NULL) && (psci_spd_pm->svc_suspend_finish != NULL)) { 364 max_off_lvl = psci_find_max_off_lvl(state_info); 365 assert(max_off_lvl != PSCI_INVALID_PWR_LVL); 366 psci_spd_pm->svc_suspend_finish(max_off_lvl); 367 } 368 369 /* This loses its meaning when not suspending, reset so it's correct for OFF */ 370 psci_set_suspend_pwrlvl(PLAT_MAX_PWR_LVL); 371 372 PUBLISH_EVENT(psci_suspend_pwrdown_finish); 373 } 374