2  * cpuidle.c - core cpuidle infrastructure
4  * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
53 	return off || !initialized || !drv || !dev || !dev->enabled;  in cpuidle_not_available()
57  * cpuidle_play_dead - cpu off-lining
68 		return -ENODEV;  in cpuidle_play_dead()
70 	/* Find lowest-power state that supports long-term idle */  in cpuidle_play_dead()
71 	for (i = drv->state_count - 1; i >= 0; i--)  in cpuidle_play_dead()
72 		if (drv->states[i].enter_dead)  in cpuidle_play_dead()
73 			return drv->states[i].enter_dead(dev, i);  in cpuidle_play_dead()
75 	return -ENODEV;  in cpuidle_play_dead()
87 	for (i = 1; i < drv->state_count; i++) {  in find_deepest_state()
88 		struct cpuidle_state *s = &drv->states[i];  in find_deepest_state()
90 		if (dev->states_usage[i].disable ||  in find_deepest_state()
91 		    s->exit_latency_ns <= latency_req ||  in find_deepest_state()
92 		    s->exit_latency_ns > max_latency_ns ||  in find_deepest_state()
93 		    (s->flags & forbidden_flags) ||  in find_deepest_state()
94 		    (s2idle && !s->enter_s2idle))  in find_deepest_state()
97 		latency_req = s->exit_latency_ns;  in find_deepest_state()
104  * cpuidle_use_deepest_state - Set/unset governor override mode.
105  * @latency_limit_ns: Idle state exit latency limit (or no override if 0).
107  * If @latency_limit_ns is nonzero, set the current CPU to use the deepest idle
108  * state with exit latency within @latency_limit_ns (override governors going
118 		dev->forced_idle_latency_limit_ns = latency_limit_ns;  in cpuidle_use_deepest_state()
123  * cpuidle_find_deepest_state - Find the deepest available idle state.
126  * @latency_limit_ns: Idle state exit latency limit
128  * Return: the index of the deepest available idle state.
142 	struct cpuidle_state *target_state = &drv->states[index];  in enter_s2idle_proper()
148 	 * The state used here cannot be a "coupled" one, because the "coupled"  in enter_s2idle_proper()
153 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in enter_s2idle_proper()
155 	target_state->enter_s2idle(dev, drv, index);  in enter_s2idle_proper()
158 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in enter_s2idle_proper()
165 	dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);  in enter_s2idle_proper()
166 	dev->states_usage[index].s2idle_usage++;  in enter_s2idle_proper()
170  * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle.
174  * If there are states with the ->enter_s2idle callback, find the deepest of
182 	 * Find the deepest state with ->enter_s2idle present, which guarantees  in cpuidle_enter_s2idle()
196  * cpuidle_enter_state - enter the state and update stats
199  * @index: index into the states table in @drv of the state to enter
218 	target_state = &drv->states[index];  in cpuidle_enter_state()
219 	broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);  in cpuidle_enter_state()
227 		index = find_deepest_state(drv, dev, target_state->exit_latency_ns,  in cpuidle_enter_state()
231 			return -EBUSY;  in cpuidle_enter_state()
233 		target_state = &drv->states[index];  in cpuidle_enter_state()
237 	if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED)  in cpuidle_enter_state()
238 		leave_mm(dev->cpu);  in cpuidle_enter_state()
240 	/* Take note of the planned idle state. */  in cpuidle_enter_state()
243 	trace_cpu_idle(index, dev->cpu);  in cpuidle_enter_state()
247 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in cpuidle_enter_state()
249 	entered_state = target_state->enter(dev, drv, index);  in cpuidle_enter_state()
250 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in cpuidle_enter_state()
256 	trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);  in cpuidle_enter_state()
259 	/* The cpu is no longer idle or about to enter idle. */  in cpuidle_enter_state()
273 		s64 diff, delay = drv->states[entered_state].exit_latency_ns;  in cpuidle_enter_state()
283 		dev->last_residency_ns = diff;  in cpuidle_enter_state()
284 		dev->states_usage[entered_state].time_ns += diff;  in cpuidle_enter_state()
285 		dev->states_usage[entered_state].usage++;  in cpuidle_enter_state()
287 		if (diff < drv->states[entered_state].target_residency_ns) {  in cpuidle_enter_state()
288 			for (i = entered_state - 1; i >= 0; i--) {  in cpuidle_enter_state()
289 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
293 				dev->states_usage[entered_state].above++;  in cpuidle_enter_state()
297 			for (i = entered_state + 1; i < drv->state_count; i++) {  in cpuidle_enter_state()
298 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
302 				 * Update if a deeper state would have been a  in cpuidle_enter_state()
303 				 * better match for the observed idle duration.  in cpuidle_enter_state()
305 				if (diff - delay >= drv->states[i].target_residency_ns)  in cpuidle_enter_state()
306 					dev->states_usage[entered_state].below++;  in cpuidle_enter_state()
312 		dev->last_residency_ns = 0;  in cpuidle_enter_state()
313 		dev->states_usage[index].rejected++;  in cpuidle_enter_state()
320  * cpuidle_select - ask the cpuidle framework to choose an idle state
326  * Returns the index of the idle state.  The return value must not be negative.
330  * entering the returned state.
335 	return cpuidle_curr_governor->select(drv, dev, stop_tick);  in cpuidle_select()
339  * cpuidle_enter - enter into the specified idle state
343  * @index: the index in the idle state table
345  * Returns the index in the idle state, < 0 in case of error.
357 	 * ->select() callback have decided, whether to stop the tick or not.  in cpuidle_enter()
359 	WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());  in cpuidle_enter()
366 	WRITE_ONCE(dev->next_hrtimer, 0);  in cpuidle_enter()
371  * cpuidle_reflect - tell the underlying governor what was the state
375  * @index: the index in the idle state table
380 	if (cpuidle_curr_governor->reflect && index >= 0)  in cpuidle_reflect()
381 		cpuidle_curr_governor->reflect(dev, index);  in cpuidle_reflect()
385  * cpuidle_poll_time - return amount of time to poll for,
386  * governors can override dev->poll_limit_ns if necessary
398 	if (dev->poll_limit_ns)  in cpuidle_poll_time()
399 		return dev->poll_limit_ns;  in cpuidle_poll_time()
402 	for (i = 1; i < drv->state_count; i++) {  in cpuidle_poll_time()
403 		if (dev->states_usage[i].disable)  in cpuidle_poll_time()
406 		limit_ns = drv->states[i].target_residency_ns;  in cpuidle_poll_time()
410 	dev->poll_limit_ns = limit_ns;  in cpuidle_poll_time()
412 	return dev->poll_limit_ns;  in cpuidle_poll_time()
416  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
421 		/* Make sure all changes finished before we switch to new idle */  in cpuidle_install_idle_handler()
428  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
439 	 * are done looking at pointed idle states.  in cpuidle_uninstall_idle_handler()
445  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
456  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
483  * cpuidle_enable_device - enables idle PM for a CPU
495 		return -EINVAL;  in cpuidle_enable_device()
497 	if (dev->enabled)  in cpuidle_enable_device()
501 		return -EIO;  in cpuidle_enable_device()
506 		return -EIO;  in cpuidle_enable_device()
508 	if (!dev->registered)  in cpuidle_enable_device()
509 		return -EINVAL;  in cpuidle_enable_device()
515 	if (cpuidle_curr_governor->enable) {  in cpuidle_enable_device()
516 		ret = cpuidle_curr_governor->enable(drv, dev);  in cpuidle_enable_device()
523 	dev->enabled = 1;  in cpuidle_enable_device()
537  * cpuidle_disable_device - disables idle PM for a CPU
547 	if (!dev || !dev->enabled)  in cpuidle_disable_device()
553 	dev->enabled = 0;  in cpuidle_disable_device()
555 	if (cpuidle_curr_governor->disable)  in cpuidle_disable_device()
556 		cpuidle_curr_governor->disable(drv, dev);  in cpuidle_disable_device()
559 	enabled_devices--;  in cpuidle_disable_device()
568 	list_del(&dev->device_list);  in __cpuidle_unregister_device()
569 	per_cpu(cpuidle_devices, dev->cpu) = NULL;  in __cpuidle_unregister_device()
570 	module_put(drv->owner);  in __cpuidle_unregister_device()
572 	dev->registered = 0;  in __cpuidle_unregister_device()
577 	memset(dev->states_usage, 0, sizeof(dev->states_usage));  in __cpuidle_device_init()
578 	dev->last_residency_ns = 0;  in __cpuidle_device_init()
579 	dev->next_hrtimer = 0;  in __cpuidle_device_init()
583  * __cpuidle_register_device - internal register function called before register
594 	if (!try_module_get(drv->owner))  in __cpuidle_register_device()
595 		return -EINVAL;  in __cpuidle_register_device()
597 	for (i = 0; i < drv->state_count; i++) {  in __cpuidle_register_device()
598 		if (drv->states[i].flags & CPUIDLE_FLAG_UNUSABLE)  in __cpuidle_register_device()
599 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;  in __cpuidle_register_device()
601 		if (drv->states[i].flags & CPUIDLE_FLAG_OFF)  in __cpuidle_register_device()
602 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER;  in __cpuidle_register_device()
605 	per_cpu(cpuidle_devices, dev->cpu) = dev;  in __cpuidle_register_device()
606 	list_add(&dev->device_list, &cpuidle_detected_devices);  in __cpuidle_register_device()
612 		dev->registered = 1;  in __cpuidle_register_device()
618  * cpuidle_register_device - registers a CPU's idle PM feature
623 	int ret = -EBUSY;  in cpuidle_register_device()
626 		return -EINVAL;  in cpuidle_register_device()
630 	if (dev->registered)  in cpuidle_register_device()
664  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
669 	if (!dev || dev->registered == 0)  in cpuidle_unregister_device()
699 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_unregister()
731 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_register()
733 		device->cpu = cpu;  in cpuidle_register()
737 		 * On multiplatform for ARM, the coupled idle states could be  in cpuidle_register()
742 			device->coupled_cpus = *coupled_cpus;  in cpuidle_register()
759  * cpuidle_init - core initializer
764 		return -ENODEV;  in cpuidle_init()