136 		      struct radeon_fence **fence,  in radeon_fence_emit()  argument
142 	*fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);  in radeon_fence_emit()
143 	if ((*fence) == NULL) {  in radeon_fence_emit()
146 	(*fence)->rdev = rdev;  in radeon_fence_emit()
147 	(*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring];  in radeon_fence_emit()
148 	(*fence)->ring = ring;  in radeon_fence_emit()
149 	(*fence)->is_vm_update = false;  in radeon_fence_emit()
150 	dma_fence_init(&(*fence)->base, &radeon_fence_ops,  in radeon_fence_emit()
154 	radeon_fence_ring_emit(rdev, ring, *fence);  in radeon_fence_emit()
155 	trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq);  in radeon_fence_emit()
169 	struct radeon_fence *fence;  in radeon_fence_check_signaled()  local
172 	fence = container_of(wait, struct radeon_fence, fence_wake);  in radeon_fence_check_signaled()
178 	seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq);  in radeon_fence_check_signaled()
179 	if (seq >= fence->seq) {  in radeon_fence_check_signaled()
180 		int ret = dma_fence_signal_locked(&fence->base);  in radeon_fence_check_signaled()
183 			DMA_FENCE_TRACE(&fence->base, "signaled from irq context\n");  in radeon_fence_check_signaled()
185 			DMA_FENCE_TRACE(&fence->base, "was already signaled\n");  in radeon_fence_check_signaled()
187 		radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring);  in radeon_fence_check_signaled()
188 		__remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake);  in radeon_fence_check_signaled()
189 		dma_fence_put(&fence->base);  in radeon_fence_check_signaled()
191 		DMA_FENCE_TRACE(&fence->base, "pending\n");  in radeon_fence_check_signaled()
364 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_is_signaled()  local
365 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_is_signaled()
366 	unsigned ring = fence->ring;  in radeon_fence_is_signaled()
367 	u64 seq = fence->seq;  in radeon_fence_is_signaled()
394 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_enable_signaling()  local
395 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_enable_signaling()
397 	if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq)  in radeon_fence_enable_signaling()
401 		radeon_irq_kms_sw_irq_get(rdev, fence->ring);  in radeon_fence_enable_signaling()
403 		if (radeon_fence_activity(rdev, fence->ring))  in radeon_fence_enable_signaling()
407 		if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) {  in radeon_fence_enable_signaling()
408 			radeon_irq_kms_sw_irq_put(rdev, fence->ring);  in radeon_fence_enable_signaling()
416 		if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring))  in radeon_fence_enable_signaling()
417 			rdev->fence_drv[fence->ring].delayed_irq = true;  in radeon_fence_enable_signaling()
418 		radeon_fence_schedule_check(rdev, fence->ring);  in radeon_fence_enable_signaling()
421 	fence->fence_wake.flags = 0;  in radeon_fence_enable_signaling()
422 	fence->fence_wake.private = NULL;  in radeon_fence_enable_signaling()
423 	fence->fence_wake.func = radeon_fence_check_signaled;  in radeon_fence_enable_signaling()
424 	__add_wait_queue(&rdev->fence_queue, &fence->fence_wake);  in radeon_fence_enable_signaling()
427 	DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring);  in radeon_fence_enable_signaling()
439 bool radeon_fence_signaled(struct radeon_fence *fence)  in radeon_fence_signaled()  argument
441 	if (!fence)  in radeon_fence_signaled()
444 	if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) {  in radeon_fence_signaled()
447 		ret = dma_fence_signal(&fence->base);  in radeon_fence_signaled()
449 			DMA_FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n");  in radeon_fence_signaled()
550 long radeon_fence_wait_timeout(struct radeon_fence *fence, bool intr, long timeout)  in radeon_fence_wait_timeout()  argument
562 	if (WARN_ON_ONCE(!to_radeon_fence(&fence->base)))  in radeon_fence_wait_timeout()
563 		return dma_fence_wait(&fence->base, intr);  in radeon_fence_wait_timeout()
565 	seq[fence->ring] = fence->seq;  in radeon_fence_wait_timeout()
566 	r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout);  in radeon_fence_wait_timeout()
571 	r_sig = dma_fence_signal(&fence->base);  in radeon_fence_wait_timeout()
573 		DMA_FENCE_TRACE(&fence->base, "signaled from fence_wait\n");  in radeon_fence_wait_timeout()
588 int radeon_fence_wait(struct radeon_fence *fence, bool intr)  in radeon_fence_wait()  argument
590 	long r = radeon_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);  in radeon_fence_wait()
706 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)  in radeon_fence_ref()  argument
708 	dma_fence_get(&fence->base);  in radeon_fence_ref()
709 	return fence;  in radeon_fence_ref()
719 void radeon_fence_unref(struct radeon_fence **fence)  in radeon_fence_unref()  argument
721 	struct radeon_fence *tmp = *fence;  in radeon_fence_unref()
723 	*fence = NULL;  in radeon_fence_unref()
767 bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring)  in radeon_fence_need_sync()  argument
771 	if (!fence) {  in radeon_fence_need_sync()
775 	if (fence->ring == dst_ring) {  in radeon_fence_need_sync()
780 	fdrv = &fence->rdev->fence_drv[dst_ring];  in radeon_fence_need_sync()
781 	if (fence->seq <= fdrv->sync_seq[fence->ring]) {  in radeon_fence_need_sync()
797 void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring)  in radeon_fence_note_sync()  argument
802 	if (!fence) {  in radeon_fence_note_sync()
806 	if (fence->ring == dst_ring) {  in radeon_fence_note_sync()
811 	src = &fence->rdev->fence_drv[fence->ring];  in radeon_fence_note_sync()
812 	dst = &fence->rdev->fence_drv[dst_ring];  in radeon_fence_note_sync()
1039 static const char *radeon_fence_get_driver_name(struct dma_fence *fence)  in radeon_fence_get_driver_name()  argument
1046 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_get_timeline_name()  local
1047 	switch (fence->ring) {  in radeon_fence_get_timeline_name()
1060 static inline bool radeon_test_signaled(struct radeon_fence *fence)  in radeon_test_signaled()  argument
1062 	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags);  in radeon_test_signaled()
1071 radeon_fence_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in radeon_fence_wait_cb()  argument
1082 	struct radeon_fence *fence = to_radeon_fence(f);  in radeon_fence_default_wait()  local
1083 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_default_wait()
1101 		if (radeon_test_signaled(fence))  in radeon_fence_default_wait()