173 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in store_vblank()  local
177 	vblank->last = last;  in store_vblank()
179 	write_seqlock(&vblank->seqlock);  in store_vblank()
180 	vblank->time = t_vblank;  in store_vblank()
181 	atomic64_add(vblank_count_inc, &vblank->count);  in store_vblank()
182 	write_sequnlock(&vblank->seqlock);  in store_vblank()
187 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_max_vblank_count()  local
189 	return vblank->max_vblank_count ?: dev->max_vblank_count;  in drm_max_vblank_count()
278 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_update_vblank_count()  local
283 	int framedur_ns = vblank->framedur_ns;  in drm_update_vblank_count()
305 		diff = (cur_vblank - vblank->last) & max_vblank_count;  in drm_update_vblank_count()
307 		u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));  in drm_update_vblank_count()
338 	if (diff > 1 && (vblank->inmodeset & 0x2)) {  in drm_update_vblank_count()
347 		    pipe, (unsigned long long)atomic64_read(&vblank->count),  in drm_update_vblank_count()
348 		    diff, cur_vblank, vblank->last);  in drm_update_vblank_count()
351 		drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);  in drm_update_vblank_count()
369 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_count()  local
375 	count = atomic64_read(&vblank->count);  in drm_vblank_count()
404 	u64 vblank;  in drm_crtc_accurate_vblank_count()  local
414 	vblank = drm_vblank_count(dev, pipe);  in drm_crtc_accurate_vblank_count()
418 	return vblank;  in drm_crtc_accurate_vblank_count()
445 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_disable_and_save()  local
462 	if (!vblank->enabled)  in drm_vblank_disable_and_save()
473 	vblank->enabled = false;  in drm_vblank_disable_and_save()
481 	struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);  in vblank_disable_fn()  local
482 	struct drm_device *dev = vblank->dev;  in vblank_disable_fn()
483 	unsigned int pipe = vblank->pipe;  in vblank_disable_fn()
487 	if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {  in vblank_disable_fn()
496 	struct drm_vblank_crtc *vblank = ptr;  in drm_vblank_init_release()  local
498 	drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&  in drm_vblank_init_release()
501 	drm_vblank_destroy_worker(vblank);  in drm_vblank_init_release()
502 	del_timer_sync(&vblank->disable_timer);  in drm_vblank_init_release()
525 	dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);  in drm_vblank_init()
526 	if (!dev->vblank)  in drm_vblank_init()
532 		struct drm_vblank_crtc *vblank = &dev->vblank[i];  in drm_vblank_init()  local
534 		vblank->dev = dev;  in drm_vblank_init()
535 		vblank->pipe = i;  in drm_vblank_init()
536 		init_waitqueue_head(&vblank->queue);  in drm_vblank_init()
537 		timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);  in drm_vblank_init()
538 		seqlock_init(&vblank->seqlock);  in drm_vblank_init()
541 					       vblank);  in drm_vblank_init()
545 		ret = drm_vblank_worker_init(vblank);  in drm_vblank_init()
586 	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;  in drm_crtc_vblank_waitqueue()
607 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_calc_timestamping_constants()  local
639 	vblank->linedur_ns  = linedur_ns;  in drm_calc_timestamping_constants()
640 	vblank->framedur_ns = framedur_ns;  in drm_calc_timestamping_constants()
641 	vblank->hwmode = *mode;  in drm_calc_timestamping_constants()
692 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()  local
712 		mode = &vblank->hwmode;  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
932 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_count_and_time()  local
942 		seq = read_seqbegin(&vblank->seqlock);  in drm_vblank_count_and_time()
943 		vblank_count = atomic64_read(&vblank->count);  in drm_vblank_count_and_time()
944 		*vblanktime = vblank->time;  in drm_vblank_count_and_time()
945 	} while (read_seqretry(&vblank->seqlock, seq));  in drm_vblank_count_and_time()
1115 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_enable()  local
1122 	if (!vblank->enabled) {  in drm_vblank_enable()
1134 			atomic_dec(&vblank->refcount);  in drm_vblank_enable()
1142 			WRITE_ONCE(vblank->enabled, true);  in drm_vblank_enable()
1153 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_get()  local
1165 	if (atomic_add_return(1, &vblank->refcount) == 1) {  in drm_vblank_get()
1168 		if (!vblank->enabled) {  in drm_vblank_get()
1169 			atomic_dec(&vblank->refcount);  in drm_vblank_get()
1196 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_vblank_put()  local
1201 	if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))  in drm_vblank_put()
1205 	if (atomic_dec_and_test(&vblank->refcount)) {  in drm_vblank_put()
1209 			vblank_disable_fn(&vblank->disable_timer);  in drm_vblank_put()
1211 			mod_timer(&vblank->disable_timer,  in drm_vblank_put()
1242 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_wait_one_vblank()  local
1256 	ret = wait_event_timeout(vblank->queue,  in drm_wait_one_vblank()
1295 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_off()  local
1311 		    pipe, vblank->enabled, vblank->inmodeset);  in drm_crtc_vblank_off()
1315 	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)  in drm_crtc_vblank_off()
1318 	wake_up(&vblank->queue);  in drm_crtc_vblank_off()
1324 	if (!vblank->inmodeset) {  in drm_crtc_vblank_off()
1325 		atomic_inc(&vblank->refcount);  in drm_crtc_vblank_off()
1326 		vblank->inmodeset = 1;  in drm_crtc_vblank_off()
1345 	drm_vblank_cancel_pending_works(vblank);  in drm_crtc_vblank_off()
1351 	vblank->hwmode.crtc_clock = 0;  in drm_crtc_vblank_off()
1354 	drm_vblank_flush_worker(vblank);  in drm_crtc_vblank_off()
1374 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_reset()  local
1381 	if (!vblank->inmodeset) {  in drm_crtc_vblank_reset()
1382 		atomic_inc(&vblank->refcount);  in drm_crtc_vblank_reset()
1383 		vblank->inmodeset = 1;  in drm_crtc_vblank_reset()
1388 	drm_WARN_ON(dev, !list_empty(&vblank->pending_work));  in drm_crtc_vblank_reset()
1414 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_set_max_vblank_count()  local
1417 	drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));  in drm_crtc_set_max_vblank_count()
1419 	vblank->max_vblank_count = max_vblank_count;  in drm_crtc_set_max_vblank_count()
1437 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_crtc_vblank_on()  local
1444 		    pipe, vblank->enabled, vblank->inmodeset);  in drm_crtc_vblank_on()
1447 	if (vblank->inmodeset) {  in drm_crtc_vblank_on()
1448 		atomic_dec(&vblank->refcount);  in drm_crtc_vblank_on()
1449 		vblank->inmodeset = 0;  in drm_crtc_vblank_on()
1458 	if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)  in drm_crtc_vblank_on()
1480 	struct drm_vblank_crtc *vblank;  in drm_vblank_restore()  local
1492 	vblank = &dev->vblank[pipe];  in drm_vblank_restore()
1494 		      drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,  in drm_vblank_restore()
1496 	framedur_ns = vblank->framedur_ns;  in drm_vblank_restore()
1503 	diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));  in drm_vblank_restore()
1510 		    diff, diff_ns, framedur_ns, cur_vblank - vblank->last);  in drm_vblank_restore()
1534 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_legacy_vblank_pre_modeset()  local
1550 	if (!vblank->inmodeset) {  in drm_legacy_vblank_pre_modeset()
1551 		vblank->inmodeset = 0x1;  in drm_legacy_vblank_pre_modeset()
1553 			vblank->inmodeset |= 0x2;  in drm_legacy_vblank_pre_modeset()
1560 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_legacy_vblank_post_modeset()  local
1569 	if (vblank->inmodeset) {  in drm_legacy_vblank_post_modeset()
1574 		if (vblank->inmodeset & 0x2)  in drm_legacy_vblank_post_modeset()
1577 		vblank->inmodeset = 0;  in drm_legacy_vblank_post_modeset()
1618 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_queue_vblank_event()  local
1650 	if (!READ_ONCE(vblank->enabled)) {  in drm_queue_vblank_event()
1739 	struct drm_vblank_crtc *vblank;  in drm_wait_vblank_ioctl()  local
1788 	vblank = &dev->vblank[pipe];  in drm_wait_vblank_ioctl()
1795 	    READ_ONCE(vblank->enabled)) {  in drm_wait_vblank_ioctl()
1842 		wait = wait_event_interruptible_timeout(vblank->queue,  in drm_wait_vblank_ioctl()
1844 				      !READ_ONCE(vblank->enabled),  in drm_wait_vblank_ioctl()
1921 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];  in drm_handle_vblank()  local
1940 	if (!vblank->enabled) {  in drm_handle_vblank()
1950 	wake_up(&vblank->queue);  in drm_handle_vblank()
1959 		       !atomic_read(&vblank->refcount));  in drm_handle_vblank()
1962 	drm_handle_vblank_works(vblank);  in drm_handle_vblank()
1967 		vblank_disable_fn(&vblank->disable_timer);  in drm_handle_vblank()
2011 	struct drm_vblank_crtc *vblank;  in drm_crtc_get_sequence_ioctl()  local
2030 	vblank = &dev->vblank[pipe];  in drm_crtc_get_sequence_ioctl()
2031 	vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);  in drm_crtc_get_sequence_ioctl()
2067 	struct drm_vblank_crtc *vblank;  in drm_crtc_queue_sequence_ioctl()  local
2095 	vblank = &dev->vblank[pipe];  in drm_crtc_queue_sequence_ioctl()
2131 	if (!READ_ONCE(vblank->enabled)) {  in drm_crtc_queue_sequence_ioctl()