53 #define RQ_TRACE(rq, fmt, ...) do {					\  argument
54 	const struct i915_request *rq__ = (rq);				\
311 void i915_request_set_error_once(struct i915_request *rq, int error);
312 void __i915_request_skip(struct i915_request *rq);
315 void __i915_request_queue(struct i915_request *rq,
318 bool i915_request_retire(struct i915_request *rq);
319 void i915_request_retire_upto(struct i915_request *rq);
331 i915_request_get(struct i915_request *rq)  in i915_request_get()  argument
333 	return to_request(dma_fence_get(&rq->fence));  in i915_request_get()
337 i915_request_get_rcu(struct i915_request *rq)  in i915_request_get_rcu()  argument
339 	return to_request(dma_fence_get_rcu(&rq->fence));  in i915_request_get_rcu()
343 i915_request_put(struct i915_request *rq)  in i915_request_put()  argument
345 	dma_fence_put(&rq->fence);  in i915_request_put()
351 int i915_request_await_dma_fence(struct i915_request *rq,
353 int i915_request_await_execution(struct i915_request *rq,
355 				 void (*hook)(struct i915_request *rq,
358 void i915_request_add(struct i915_request *rq);
366 long i915_request_wait(struct i915_request *rq,
374 static inline bool i915_request_signaled(const struct i915_request *rq)  in i915_request_signaled()  argument
377 	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags);  in i915_request_signaled()
380 static inline bool i915_request_is_active(const struct i915_request *rq)  in i915_request_is_active()  argument
382 	return test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);  in i915_request_is_active()
385 static inline bool i915_request_in_priority_queue(const struct i915_request *rq)  in i915_request_in_priority_queue()  argument
387 	return test_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);  in i915_request_in_priority_queue()
391 i915_request_has_initial_breadcrumb(const struct i915_request *rq)  in i915_request_has_initial_breadcrumb()  argument
393 	return test_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags);  in i915_request_has_initial_breadcrumb()
404 static inline u32 __hwsp_seqno(const struct i915_request *rq)  in __hwsp_seqno()  argument
406 	const u32 *hwsp = READ_ONCE(rq->hwsp_seqno);  in __hwsp_seqno()
424 static inline u32 hwsp_seqno(const struct i915_request *rq)  in hwsp_seqno()  argument
429 	seqno = __hwsp_seqno(rq);  in hwsp_seqno()
435 static inline bool __i915_request_has_started(const struct i915_request *rq)  in __i915_request_has_started()  argument
437 	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno - 1);  in __i915_request_has_started()
466 static inline bool i915_request_started(const struct i915_request *rq)  in i915_request_started()  argument
470 	if (i915_request_signaled(rq))  in i915_request_started()
475 	if (likely(!i915_request_signaled(rq)))  in i915_request_started()
477 		result = __i915_request_has_started(rq);  in i915_request_started()
491 static inline bool i915_request_is_running(const struct i915_request *rq)  in i915_request_is_running()  argument
495 	if (!i915_request_is_active(rq))  in i915_request_is_running()
499 	result = __i915_request_has_started(rq) && i915_request_is_active(rq);  in i915_request_is_running()
521 static inline bool i915_request_is_ready(const struct i915_request *rq)  in i915_request_is_ready()  argument
523 	return !list_empty(&rq->sched.link);  in i915_request_is_ready()
526 static inline bool __i915_request_is_complete(const struct i915_request *rq)  in __i915_request_is_complete()  argument
528 	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);  in __i915_request_is_complete()
531 static inline bool i915_request_completed(const struct i915_request *rq)  in i915_request_completed()  argument
535 	if (i915_request_signaled(rq))  in i915_request_completed()
540 	if (likely(!i915_request_signaled(rq)))  in i915_request_completed()
541 		result = __i915_request_is_complete(rq);  in i915_request_completed()
547 static inline void i915_request_mark_complete(struct i915_request *rq)  in i915_request_mark_complete()  argument
549 	WRITE_ONCE(rq->hwsp_seqno, /* decouple from HWSP */  in i915_request_mark_complete()
550 		   (u32 *)&rq->fence.seqno);  in i915_request_mark_complete()
553 static inline bool i915_request_has_waitboost(const struct i915_request *rq)  in i915_request_has_waitboost()  argument
555 	return test_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags);  in i915_request_has_waitboost()
558 static inline bool i915_request_has_nopreempt(const struct i915_request *rq)  in i915_request_has_nopreempt()  argument
561 	return unlikely(test_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags));  in i915_request_has_nopreempt()
564 static inline bool i915_request_has_sentinel(const struct i915_request *rq)  in i915_request_has_sentinel()  argument
566 	return unlikely(test_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags));  in i915_request_has_sentinel()
569 static inline bool i915_request_on_hold(const struct i915_request *rq)  in i915_request_on_hold()  argument
571 	return unlikely(test_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags));  in i915_request_on_hold()
574 static inline void i915_request_set_hold(struct i915_request *rq)  in i915_request_set_hold()  argument
576 	set_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);  in i915_request_set_hold()
579 static inline void i915_request_clear_hold(struct i915_request *rq)  in i915_request_clear_hold()  argument
581 	clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);  in i915_request_clear_hold()
585 i915_request_timeline(const struct i915_request *rq)  in i915_request_timeline()  argument
588 	return rcu_dereference_protected(rq->timeline,  in i915_request_timeline()
589 					 lockdep_is_held(&rcu_access_pointer(rq->timeline)->mutex));  in i915_request_timeline()
593 i915_request_gem_context(const struct i915_request *rq)  in i915_request_gem_context()  argument
596 	return rcu_dereference_protected(rq->context->gem_context, true);  in i915_request_gem_context()
600 i915_request_active_timeline(const struct i915_request *rq)  in i915_request_active_timeline()  argument
607 	return rcu_dereference_protected(rq->timeline,  in i915_request_active_timeline()
608 					 lockdep_is_held(&rq->engine->active.lock));  in i915_request_active_timeline()