242 	struct c4iw_cq *chp = to_c4iw_cq(qhp->ibqp.send_cq);  in c4iw_flush_sq()  local
243 	struct t4_cq *cq = &chp->cq;  in c4iw_flush_sq()
335 void c4iw_flush_hw_cq(struct c4iw_cq *chp, struct c4iw_qp *flush_qhp)  in c4iw_flush_hw_cq()  argument
342 	pr_debug("cqid 0x%x\n", chp->cq.cqid);  in c4iw_flush_hw_cq()
343 	ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);  in c4iw_flush_hw_cq()
351 		qhp = get_qhp(chp->rhp, CQE_QPID(hw_cqe));  in c4iw_flush_hw_cq()
407 			flush_completed_wrs(&qhp->wq, &chp->cq);  in c4iw_flush_hw_cq()
409 			swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx];  in c4iw_flush_hw_cq()
412 			t4_swcq_produce(&chp->cq);  in c4iw_flush_hw_cq()
415 		t4_hwcq_consume(&chp->cq);  in c4iw_flush_hw_cq()
416 		ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);  in c4iw_flush_hw_cq()
754 static int __c4iw_poll_cq_one(struct c4iw_cq *chp, struct c4iw_qp *qhp,  in __c4iw_poll_cq_one()  argument
764 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit,  in __c4iw_poll_cq_one()
921 static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)  in c4iw_poll_cq_one()  argument
928 	ret = t4_next_cqe(&chp->cq, &rd_cqe);  in c4iw_poll_cq_one()
933 	qhp = get_qhp(chp->rhp, CQE_QPID(rd_cqe));  in c4iw_poll_cq_one()
939 		ret = __c4iw_poll_cq_one(chp, qhp, wc, srq);  in c4iw_poll_cq_one()
944 		ret = __c4iw_poll_cq_one(chp, NULL, wc, NULL);  in c4iw_poll_cq_one()
951 	struct c4iw_cq *chp;  in c4iw_poll_cq()  local
956 	chp = to_c4iw_cq(ibcq);  in c4iw_poll_cq()
958 	spin_lock_irqsave(&chp->lock, flags);  in c4iw_poll_cq()
961 			err = c4iw_poll_cq_one(chp, wc + npolled);  in c4iw_poll_cq()
966 	spin_unlock_irqrestore(&chp->lock, flags);  in c4iw_poll_cq()
972 	struct c4iw_cq *chp;  in c4iw_destroy_cq()  local
976 	chp = to_c4iw_cq(ib_cq);  in c4iw_destroy_cq()
978 	xa_erase_irq(&chp->rhp->cqs, chp->cq.cqid);  in c4iw_destroy_cq()
979 	atomic_dec(&chp->refcnt);  in c4iw_destroy_cq()
980 	wait_event(chp->wait, !atomic_read(&chp->refcnt));  in c4iw_destroy_cq()
984 	destroy_cq(&chp->rhp->rdev, &chp->cq,  in c4iw_destroy_cq()
985 		   ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx,  in c4iw_destroy_cq()
986 		   chp->destroy_skb, chp->wr_waitp);  in c4iw_destroy_cq()
987 	c4iw_put_wr_wait(chp->wr_waitp);  in c4iw_destroy_cq()
998 	struct c4iw_cq *chp = to_c4iw_cq(ibcq);  in c4iw_create_cq()  local
1022 	chp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);  in c4iw_create_cq()
1023 	if (!chp->wr_waitp) {  in c4iw_create_cq()
1027 	c4iw_init_wr_wait(chp->wr_waitp);  in c4iw_create_cq()
1030 	chp->destroy_skb = alloc_skb(wr_len, GFP_KERNEL);  in c4iw_create_cq()
1031 	if (!chp->destroy_skb) {  in c4iw_create_cq()
1060 			(sizeof(*chp->cq.queue) / 2) : sizeof(*chp->cq.queue));  in c4iw_create_cq()
1068 	chp->cq.size = hwentries;  in c4iw_create_cq()
1069 	chp->cq.memsize = memsize;  in c4iw_create_cq()
1070 	chp->cq.vector = vector;  in c4iw_create_cq()
1072 	ret = create_cq(&rhp->rdev, &chp->cq,  in c4iw_create_cq()
1074 			chp->wr_waitp);  in c4iw_create_cq()
1078 	chp->rhp = rhp;  in c4iw_create_cq()
1079 	chp->cq.size--;				/* status page */  in c4iw_create_cq()
1080 	chp->ibcq.cqe = entries - 2;  in c4iw_create_cq()
1081 	spin_lock_init(&chp->lock);  in c4iw_create_cq()
1082 	spin_lock_init(&chp->comp_handler_lock);  in c4iw_create_cq()
1083 	atomic_set(&chp->refcnt, 1);  in c4iw_create_cq()
1084 	init_waitqueue_head(&chp->wait);  in c4iw_create_cq()
1085 	ret = xa_insert_irq(&rhp->cqs, chp->cq.cqid, chp, GFP_KERNEL);  in c4iw_create_cq()
1100 		uresp.cqid = chp->cq.cqid;  in c4iw_create_cq()
1101 		uresp.size = chp->cq.size;  in c4iw_create_cq()
1102 		uresp.memsize = chp->cq.memsize;  in c4iw_create_cq()
1122 		mm->addr = virt_to_phys(chp->cq.queue);  in c4iw_create_cq()
1123 		mm->len = chp->cq.memsize;  in c4iw_create_cq()
1127 		mm2->addr = chp->cq.bar2_pa;  in c4iw_create_cq()
1133 		 chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize,  in c4iw_create_cq()
1134 		 &chp->cq.dma_addr);  in c4iw_create_cq()
1141 	xa_erase_irq(&rhp->cqs, chp->cq.cqid);  in c4iw_create_cq()
1143 	destroy_cq(&chp->rhp->rdev, &chp->cq,  in c4iw_create_cq()
1145 		   chp->destroy_skb, chp->wr_waitp);  in c4iw_create_cq()
1147 	kfree_skb(chp->destroy_skb);  in c4iw_create_cq()
1149 	c4iw_put_wr_wait(chp->wr_waitp);  in c4iw_create_cq()
1156 	struct c4iw_cq *chp;  in c4iw_arm_cq()  local
1160 	chp = to_c4iw_cq(ibcq);  in c4iw_arm_cq()
1161 	spin_lock_irqsave(&chp->lock, flag);  in c4iw_arm_cq()
1162 	t4_arm_cq(&chp->cq,  in c4iw_arm_cq()
1165 		ret = t4_cq_notempty(&chp->cq);  in c4iw_arm_cq()
1166 	spin_unlock_irqrestore(&chp->lock, flag);  in c4iw_arm_cq()