480 static inline int t4_rqes_posted(struct t4_wq *wq)  in t4_rqes_posted()  argument
482 	return wq->rq.in_use;  in t4_rqes_posted()
485 static inline int t4_rq_empty(struct t4_wq *wq)  in t4_rq_empty()  argument
487 	return wq->rq.in_use == 0;  in t4_rq_empty()
490 static inline int t4_rq_full(struct t4_wq *wq)  in t4_rq_full()  argument
492 	return wq->rq.in_use == (wq->rq.size - 1);  in t4_rq_full()
495 static inline u32 t4_rq_avail(struct t4_wq *wq)  in t4_rq_avail()  argument
497 	return wq->rq.size - 1 - wq->rq.in_use;  in t4_rq_avail()
500 static inline void t4_rq_produce(struct t4_wq *wq, u8 len16)  in t4_rq_produce()  argument
502 	wq->rq.in_use++;  in t4_rq_produce()
503 	if (++wq->rq.pidx == wq->rq.size)  in t4_rq_produce()
504 		wq->rq.pidx = 0;  in t4_rq_produce()
505 	wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_rq_produce()
506 	if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS)  in t4_rq_produce()
507 		wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_produce()
510 static inline void t4_rq_consume(struct t4_wq *wq)  in t4_rq_consume()  argument
512 	wq->rq.in_use--;  in t4_rq_consume()
513 	if (++wq->rq.cidx == wq->rq.size)  in t4_rq_consume()
514 		wq->rq.cidx = 0;  in t4_rq_consume()
517 static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq)  in t4_rq_host_wq_pidx()  argument
519 	return wq->rq.queue[wq->rq.size].status.host_wq_pidx;  in t4_rq_host_wq_pidx()
522 static inline u16 t4_rq_wq_size(struct t4_wq *wq)  in t4_rq_wq_size()  argument
524 		return wq->rq.size * T4_RQ_NUM_SLOTS;  in t4_rq_wq_size()
532 static inline int t4_sq_empty(struct t4_wq *wq)  in t4_sq_empty()  argument
534 	return wq->sq.in_use == 0;  in t4_sq_empty()
537 static inline int t4_sq_full(struct t4_wq *wq)  in t4_sq_full()  argument
539 	return wq->sq.in_use == (wq->sq.size - 1);  in t4_sq_full()
542 static inline u32 t4_sq_avail(struct t4_wq *wq)  in t4_sq_avail()  argument
544 	return wq->sq.size - 1 - wq->sq.in_use;  in t4_sq_avail()
547 static inline void t4_sq_produce(struct t4_wq *wq, u8 len16)  in t4_sq_produce()  argument
549 	wq->sq.in_use++;  in t4_sq_produce()
550 	if (++wq->sq.pidx == wq->sq.size)  in t4_sq_produce()
551 		wq->sq.pidx = 0;  in t4_sq_produce()
552 	wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);  in t4_sq_produce()
553 	if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS)  in t4_sq_produce()
554 		wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_produce()
557 static inline void t4_sq_consume(struct t4_wq *wq)  in t4_sq_consume()  argument
559 	if (wq->sq.cidx == wq->sq.flush_cidx)  in t4_sq_consume()
560 		wq->sq.flush_cidx = -1;  in t4_sq_consume()
561 	wq->sq.in_use--;  in t4_sq_consume()
562 	if (++wq->sq.cidx == wq->sq.size)  in t4_sq_consume()
563 		wq->sq.cidx = 0;  in t4_sq_consume()
566 static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq)  in t4_sq_host_wq_pidx()  argument
568 	return wq->sq.queue[wq->sq.size].status.host_wq_pidx;  in t4_sq_host_wq_pidx()
571 static inline u16 t4_sq_wq_size(struct t4_wq *wq)  in t4_sq_wq_size()  argument
573 		return wq->sq.size * T4_SQ_NUM_SLOTS;  in t4_sq_wq_size()
611 static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe)  in t4_ring_sq_db()  argument
616 	if (wq->sq.bar2_va) {  in t4_ring_sq_db()
617 		if (inc == 1 && wq->sq.bar2_qid == 0 && wqe) {  in t4_ring_sq_db()
618 			pr_debug("WC wq->sq.pidx = %d\n", wq->sq.pidx);  in t4_ring_sq_db()
620 				 (wq->sq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_sq_db()
623 			pr_debug("DB wq->sq.pidx = %d\n", wq->sq.pidx);  in t4_ring_sq_db()
624 			writel(PIDX_T5_V(inc) | QID_V(wq->sq.bar2_qid),  in t4_ring_sq_db()
625 			       wq->sq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_sq_db()
632 	writel(QID_V(wq->sq.qid) | PIDX_V(inc), wq->db);  in t4_ring_sq_db()
635 static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc,  in t4_ring_rq_db()  argument
641 	if (wq->rq.bar2_va) {  in t4_ring_rq_db()
642 		if (inc == 1 && wq->rq.bar2_qid == 0 && wqe) {  in t4_ring_rq_db()
643 			pr_debug("WC wq->rq.pidx = %d\n", wq->rq.pidx);  in t4_ring_rq_db()
645 				 (wq->rq.bar2_va + SGE_UDB_WCDOORBELL),  in t4_ring_rq_db()
648 			pr_debug("DB wq->rq.pidx = %d\n", wq->rq.pidx);  in t4_ring_rq_db()
649 			writel(PIDX_T5_V(inc) | QID_V(wq->rq.bar2_qid),  in t4_ring_rq_db()
650 			       wq->rq.bar2_va + SGE_UDB_KDOORBELL);  in t4_ring_rq_db()
657 	writel(QID_V(wq->rq.qid) | PIDX_V(inc), wq->db);  in t4_ring_rq_db()
660 static inline int t4_wq_in_error(struct t4_wq *wq)  in t4_wq_in_error()  argument
662 	return *wq->qp_errp;  in t4_wq_in_error()
665 static inline void t4_set_wq_in_error(struct t4_wq *wq, u32 srqidx)  in t4_set_wq_in_error()  argument
668 		*wq->srqidxp = srqidx;  in t4_set_wq_in_error()
669 	*wq->qp_errp = 1;  in t4_set_wq_in_error()
672 static inline void t4_disable_wq_db(struct t4_wq *wq)  in t4_disable_wq_db()  argument
674 	wq->rq.queue[wq->rq.size].status.db_off = 1;  in t4_disable_wq_db()
677 static inline void t4_enable_wq_db(struct t4_wq *wq)  in t4_enable_wq_db()  argument
679 	wq->rq.queue[wq->rq.size].status.db_off = 0;  in t4_enable_wq_db()
682 static inline int t4_wq_db_enabled(struct t4_wq *wq)  in t4_wq_db_enabled()  argument
684 	return !wq->rq.queue[wq->rq.size].status.db_off;  in t4_wq_db_enabled()