156 	writel_relaxed(q->llq.cons, q->cons_reg);  in queue_sync_cons_out()
177 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))  in queue_sync_prod_in()
180 	q->llq.prod = prod;  in queue_sync_prod_in()
235 	if (queue_empty(&q->llq))  in queue_remove_raw()
238 	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);  in queue_remove_raw()
239 	queue_inc_cons(&q->llq);  in queue_remove_raw()
351 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *  in arm_smmu_cmdq_build_sync_cmd()
519 	struct arm_smmu_ll_queue llq = {  in __arm_smmu_cmdq_poll_set_valid_map()  local
520 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in __arm_smmu_cmdq_poll_set_valid_map()
524 	ewidx = BIT_WORD(Q_IDX(&llq, eprod));  in __arm_smmu_cmdq_poll_set_valid_map()
525 	ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;  in __arm_smmu_cmdq_poll_set_valid_map()
527 	while (llq.prod != eprod) {  in __arm_smmu_cmdq_poll_set_valid_map()
532 		swidx = BIT_WORD(Q_IDX(&llq, llq.prod));  in __arm_smmu_cmdq_poll_set_valid_map()
533 		sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;  in __arm_smmu_cmdq_poll_set_valid_map()
553 			valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;  in __arm_smmu_cmdq_poll_set_valid_map()
557 		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);  in __arm_smmu_cmdq_poll_set_valid_map()
577 					     struct arm_smmu_ll_queue *llq)  in arm_smmu_cmdq_poll_until_not_full()  argument
589 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_poll_until_not_full()
591 		llq->val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
597 		llq->val = READ_ONCE(smmu->cmdq.q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
598 		if (!queue_full(llq))  in arm_smmu_cmdq_poll_until_not_full()
612 					  struct arm_smmu_ll_queue *llq)  in __arm_smmu_cmdq_poll_until_msi()  argument
617 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));  in __arm_smmu_cmdq_poll_until_msi()
627 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);  in __arm_smmu_cmdq_poll_until_msi()
636 					       struct arm_smmu_ll_queue *llq)  in __arm_smmu_cmdq_poll_until_consumed()  argument
640 	u32 prod = llq->prod;  in __arm_smmu_cmdq_poll_until_consumed()
644 	llq->val = READ_ONCE(smmu->cmdq.q.llq.val);  in __arm_smmu_cmdq_poll_until_consumed()
646 		if (queue_consumed(llq, prod))  in __arm_smmu_cmdq_poll_until_consumed()
679 		llq->cons = readl(cmdq->q.cons_reg);  in __arm_smmu_cmdq_poll_until_consumed()
686 					 struct arm_smmu_ll_queue *llq)  in arm_smmu_cmdq_poll_until_sync()  argument
689 		return __arm_smmu_cmdq_poll_until_msi(smmu, llq);  in arm_smmu_cmdq_poll_until_sync()
691 	return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);  in arm_smmu_cmdq_poll_until_sync()
698 	struct arm_smmu_ll_queue llq = {  in arm_smmu_cmdq_write_entries()  local
699 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in arm_smmu_cmdq_write_entries()
706 		prod = queue_inc_prod_n(&llq, i);  in arm_smmu_cmdq_write_entries()
735 	struct arm_smmu_ll_queue llq = {  in arm_smmu_cmdq_issue_cmdlist()  local
736 		.max_n_shift = cmdq->q.llq.max_n_shift,  in arm_smmu_cmdq_issue_cmdlist()
737 	}, head = llq;  in arm_smmu_cmdq_issue_cmdlist()
742 	llq.val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_issue_cmdlist()
746 		while (!queue_has_space(&llq, n + sync)) {  in arm_smmu_cmdq_issue_cmdlist()
748 			if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))  in arm_smmu_cmdq_issue_cmdlist()
753 		head.cons = llq.cons;  in arm_smmu_cmdq_issue_cmdlist()
754 		head.prod = queue_inc_prod_n(&llq, n + sync) |  in arm_smmu_cmdq_issue_cmdlist()
757 		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);  in arm_smmu_cmdq_issue_cmdlist()
758 		if (old == llq.val)  in arm_smmu_cmdq_issue_cmdlist()
761 		llq.val = old;  in arm_smmu_cmdq_issue_cmdlist()
763 	owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);  in arm_smmu_cmdq_issue_cmdlist()
765 	llq.prod &= ~CMDQ_PROD_OWNED_FLAG;  in arm_smmu_cmdq_issue_cmdlist()
771 	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);  in arm_smmu_cmdq_issue_cmdlist()
773 		prod = queue_inc_prod_n(&llq, n);  in arm_smmu_cmdq_issue_cmdlist()
788 	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);  in arm_smmu_cmdq_issue_cmdlist()
793 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);  in arm_smmu_cmdq_issue_cmdlist()
797 						   &cmdq->q.llq.atomic.prod);  in arm_smmu_cmdq_issue_cmdlist()
805 		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
823 		llq.prod = queue_inc_prod_n(&llq, n);  in arm_smmu_cmdq_issue_cmdlist()
824 		ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);  in arm_smmu_cmdq_issue_cmdlist()
828 					    llq.prod,  in arm_smmu_cmdq_issue_cmdlist()
838 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);  in arm_smmu_cmdq_issue_cmdlist()
1359 	struct arm_smmu_ll_queue *llq = &q->llq;  in arm_smmu_evtq_thread()  local
1380 	} while (!queue_empty(llq));  in arm_smmu_evtq_thread()
1383 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in arm_smmu_evtq_thread()
1384 		    Q_IDX(llq, llq->cons);  in arm_smmu_evtq_thread()
1430 	struct arm_smmu_ll_queue *llq = &q->llq;  in arm_smmu_priq_thread()  local
1439 	} while (!queue_empty(llq));  in arm_smmu_priq_thread()
1442 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in arm_smmu_priq_thread()
1443 		      Q_IDX(llq, llq->cons);  in arm_smmu_priq_thread()
2599 		qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;  in arm_smmu_init_one_queue()
2605 		q->llq.max_n_shift--;  in arm_smmu_init_one_queue()
2617 			 1 << q->llq.max_n_shift, name);  in arm_smmu_init_one_queue()
2626 	q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);  in arm_smmu_init_one_queue()
2628 	q->llq.prod = q->llq.cons = 0;  in arm_smmu_init_one_queue()
2642 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;  in arm_smmu_cmdq_init()
3052 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);  in arm_smmu_device_reset()
3053 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);  in arm_smmu_device_reset()
3080 	writel_relaxed(smmu->evtq.q.llq.prod,  in arm_smmu_device_reset()
3082 	writel_relaxed(smmu->evtq.q.llq.cons,  in arm_smmu_device_reset()
3097 		writel_relaxed(smmu->priq.q.llq.prod,  in arm_smmu_device_reset()
3099 		writel_relaxed(smmu->priq.q.llq.cons,  in arm_smmu_device_reset()
3256 	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3258 	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {  in arm_smmu_device_hw_probe()
3270 	smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3272 	smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()