116 	u32 space, prod, cons;  in queue_has_space()  local
118 	prod = Q_IDX(q, q->prod);  in queue_has_space()
121 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))  in queue_has_space()
122 		space = (1 << q->max_n_shift) - (prod - cons);  in queue_has_space()
124 		space = cons - prod;  in queue_has_space()
131 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_full()
132 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);  in queue_full()
137 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_empty()
138 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);  in queue_empty()
141 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)  in queue_consumed()  argument
143 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&  in queue_consumed()
144 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||  in queue_consumed()
145 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&  in queue_consumed()
146 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));  in queue_consumed()
167 	u32 prod;  in queue_sync_prod_in()  local
175 	prod = readl(q->prod_reg);  in queue_sync_prod_in()
177 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))  in queue_sync_prod_in()
180 	q->llq.prod = prod;  in queue_sync_prod_in()
186 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;  in queue_inc_prod_n()  local
187 	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);  in queue_inc_prod_n()
339 					 u32 prod)  in arm_smmu_cmdq_build_sync_cmd()  argument
351 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *  in arm_smmu_cmdq_build_sync_cmd()
521 		.prod		= sprod,  in __arm_smmu_cmdq_poll_set_valid_map()
527 	while (llq.prod != eprod) {  in __arm_smmu_cmdq_poll_set_valid_map()
532 		swidx = BIT_WORD(Q_IDX(&llq, llq.prod));  in __arm_smmu_cmdq_poll_set_valid_map()
533 		sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;  in __arm_smmu_cmdq_poll_set_valid_map()
553 			valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;  in __arm_smmu_cmdq_poll_set_valid_map()
557 		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);  in __arm_smmu_cmdq_poll_set_valid_map()
617 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));  in __arm_smmu_cmdq_poll_until_msi()
627 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);  in __arm_smmu_cmdq_poll_until_msi()
640 	u32 prod = llq->prod;  in __arm_smmu_cmdq_poll_until_consumed()  local
646 		if (queue_consumed(llq, prod))  in __arm_smmu_cmdq_poll_until_consumed()
695 					u32 prod, int n)  in arm_smmu_cmdq_write_entries()  argument
700 		.prod		= prod,  in arm_smmu_cmdq_write_entries()
706 		prod = queue_inc_prod_n(&llq, i);  in arm_smmu_cmdq_write_entries()
707 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_write_entries()
731 	u32 prod;  in arm_smmu_cmdq_issue_cmdlist()  local
754 		head.prod = queue_inc_prod_n(&llq, n + sync) |  in arm_smmu_cmdq_issue_cmdlist()
763 	owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);  in arm_smmu_cmdq_issue_cmdlist()
764 	head.prod &= ~CMDQ_PROD_OWNED_FLAG;  in arm_smmu_cmdq_issue_cmdlist()
765 	llq.prod &= ~CMDQ_PROD_OWNED_FLAG;  in arm_smmu_cmdq_issue_cmdlist()
771 	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);  in arm_smmu_cmdq_issue_cmdlist()
773 		prod = queue_inc_prod_n(&llq, n);  in arm_smmu_cmdq_issue_cmdlist()
774 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);  in arm_smmu_cmdq_issue_cmdlist()
775 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_issue_cmdlist()
788 	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);  in arm_smmu_cmdq_issue_cmdlist()
793 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);  in arm_smmu_cmdq_issue_cmdlist()
796 		prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,  in arm_smmu_cmdq_issue_cmdlist()
797 						   &cmdq->q.llq.atomic.prod);  in arm_smmu_cmdq_issue_cmdlist()
798 		prod &= ~CMDQ_PROD_OWNED_FLAG;  in arm_smmu_cmdq_issue_cmdlist()
805 		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
811 		writel_relaxed(prod, cmdq->q.prod_reg);  in arm_smmu_cmdq_issue_cmdlist()
818 		atomic_set_release(&cmdq->owner_prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
823 		llq.prod = queue_inc_prod_n(&llq, n);  in arm_smmu_cmdq_issue_cmdlist()
828 					    llq.prod,  in arm_smmu_cmdq_issue_cmdlist()
1383 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in arm_smmu_evtq_thread()
1442 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in arm_smmu_priq_thread()
2628 	q->llq.prod = q->llq.cons = 0;  in arm_smmu_init_one_queue()
3052 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);  in arm_smmu_device_reset()
3080 	writel_relaxed(smmu->evtq.q.llq.prod,  in arm_smmu_device_reset()
3097 		writel_relaxed(smmu->priq.q.llq.prod,  in arm_smmu_device_reset()