574 	struct xhci_ring *ep_ring;  in xhci_move_dequeue_past_td()  local
589 	ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,  in xhci_move_dequeue_past_td()
591 	if (!ep_ring) {  in xhci_move_dequeue_past_td()
604 		if (list_empty(&ep_ring->td_list)) {  in xhci_move_dequeue_past_td()
605 			new_seg = ep_ring->enq_seg;  in xhci_move_dequeue_past_td()
606 			new_deq = ep_ring->enqueue;  in xhci_move_dequeue_past_td()
607 			new_cycle = ep_ring->cycle_state;  in xhci_move_dequeue_past_td()
617 	new_seg = ep_ring->deq_seg;  in xhci_move_dequeue_past_td()
618 	new_deq = ep_ring->dequeue;  in xhci_move_dequeue_past_td()
661 		next_trb(xhci, ep_ring, &new_seg, &new_deq);  in xhci_move_dequeue_past_td()
725 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,  in td_to_noop()  argument
741 		next_trb(xhci, ep_ring, &seg, &trb);  in td_to_noop()
812 			   struct xhci_ring *ep_ring, int status)  in xhci_td_cleanup()  argument
820 	xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);  in xhci_td_cleanup()
1312 		struct xhci_ring *ep_ring,  in update_ring_for_set_deq_completion()  argument
1317 	dequeue_temp = ep_ring->dequeue;  in update_ring_for_set_deq_completion()
1325 	if (trb_is_link(ep_ring->dequeue)) {  in update_ring_for_set_deq_completion()
1326 		ep_ring->deq_seg = ep_ring->deq_seg->next;  in update_ring_for_set_deq_completion()
1327 		ep_ring->dequeue = ep_ring->deq_seg->trbs;  in update_ring_for_set_deq_completion()
1330 	while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {  in update_ring_for_set_deq_completion()
1331 		ep_ring->dequeue++;  in update_ring_for_set_deq_completion()
1332 		if (trb_is_link(ep_ring->dequeue)) {  in update_ring_for_set_deq_completion()
1333 			if (ep_ring->dequeue ==  in update_ring_for_set_deq_completion()
1336 			ep_ring->deq_seg = ep_ring->deq_seg->next;  in update_ring_for_set_deq_completion()
1337 			ep_ring->dequeue = ep_ring->deq_seg->trbs;  in update_ring_for_set_deq_completion()
1339 		if (ep_ring->dequeue == dequeue_temp) {  in update_ring_for_set_deq_completion()
1358 	struct xhci_ring *ep_ring;  in xhci_handle_cmd_set_deq()  local
1370 	ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id);  in xhci_handle_cmd_set_deq()
1371 	if (!ep_ring) {  in xhci_handle_cmd_set_deq()
1433 				ep_ring, ep_index);  in xhci_handle_cmd_set_deq()
1443 		ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);  in xhci_handle_cmd_set_deq()
1448 			xhci_td_cleanup(ep->xhci, td, ep_ring, td->status);  in xhci_handle_cmd_set_deq()
2197 		     struct xhci_ring *ep_ring, struct xhci_td *td,  in finish_td()  argument
2246 		xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,  in finish_td()
2263 		xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,  in finish_td()
2272 	ep_ring->dequeue = td->last_trb;  in finish_td()
2273 	ep_ring->deq_seg = td->last_trb_seg;  in finish_td()
2274 	ep_ring->num_trbs_free += td->num_trbs - 1;  in finish_td()
2275 	inc_deq(xhci, ep_ring);  in finish_td()
2277 	return xhci_td_cleanup(xhci, td, ep_ring, td->status);  in finish_td()
2299 		struct xhci_ring *ep_ring,  struct xhci_td *td,  in process_ctrl_td()  argument
2388 	return finish_td(xhci, ep, ep_ring, td, trb_comp_code);  in process_ctrl_td()
2395 		struct xhci_ring *ep_ring, struct xhci_td *td,  in process_isoc_td()  argument
2473 	return finish_td(xhci, ep, ep_ring, td, trb_comp_code);  in process_isoc_td()
2506 		struct xhci_ring *ep_ring, struct xhci_td *td,  in process_bulk_intr_td()  argument
2521 		ep_ring->err_count = 0;  in process_bulk_intr_td()
2547 		    (ep_ring->err_count++ > MAX_SOFT_RETRY) ||  in process_bulk_intr_td()
2553 		xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,  in process_bulk_intr_td()
2565 			sum_trb_lengths(xhci, ep_ring, ep_trb) +  in process_bulk_intr_td()
2574 	return finish_td(xhci, ep, ep_ring, td, trb_comp_code);  in process_bulk_intr_td()
2586 	struct xhci_ring *ep_ring;  in handle_tx_event()  local
2611 	ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);  in handle_tx_event()
2622 	if (!ep_ring) {  in handle_tx_event()
2644 		list_for_each(tmp, &ep_ring->td_list)  in handle_tx_event()
2657 		    ep_ring->last_td_was_short)  in handle_tx_event()
2732 		if (!list_empty(&ep_ring->td_list))  in handle_tx_event()
2740 		if (!list_empty(&ep_ring->td_list))  in handle_tx_event()
2787 		if (list_empty(&ep_ring->td_list)) {  in handle_tx_event()
2798 			      ep_ring->last_td_was_short)) {  in handle_tx_event()
2812 							    ep_ring->stream_id,  in handle_tx_event()
2827 		td = list_first_entry(&ep_ring->td_list, struct xhci_td,  in handle_tx_event()
2833 		ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,  in handle_tx_event()
2857 						ep_ring->last_td_was_short) {  in handle_tx_event()
2858 					ep_ring->last_td_was_short = false;  in handle_tx_event()
2867 				trb_in_td(xhci, ep_ring->deq_seg,  in handle_tx_event()
2868 					  ep_ring->dequeue, td->last_trb,  in handle_tx_event()
2877 			ep_ring->last_td_was_short = true;  in handle_tx_event()
2879 			ep_ring->last_td_was_short = false;  in handle_tx_event()
2891 		trace_xhci_handle_transfer(ep_ring,  in handle_tx_event()
2907 							    ep_ring->stream_id,  in handle_tx_event()
2916 			process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event);  in handle_tx_event()
2918 			process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event);  in handle_tx_event()
2920 			process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event);  in handle_tx_event()
3204 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,  in prepare_ring()  argument
3239 		if (room_on_ring(xhci, ep_ring, num_trbs))  in prepare_ring()
3242 		if (ep_ring == xhci->cmd_ring) {  in prepare_ring()
3249 		num_trbs_needed = num_trbs - ep_ring->num_trbs_free;  in prepare_ring()
3250 		if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,  in prepare_ring()
3257 	while (trb_is_link(ep_ring->enqueue)) {  in prepare_ring()
3262 		    !(ep_ring->type == TYPE_ISOC &&  in prepare_ring()
3264 			ep_ring->enqueue->link.control &=  in prepare_ring()
3267 			ep_ring->enqueue->link.control |=  in prepare_ring()
3271 		ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);  in prepare_ring()
3274 		if (link_trb_toggles_cycle(ep_ring->enqueue))  in prepare_ring()
3275 			ep_ring->cycle_state ^= 1;  in prepare_ring()
3277 		ep_ring->enq_seg = ep_ring->enq_seg->next;  in prepare_ring()
3278 		ep_ring->enqueue = ep_ring->enq_seg->trbs;  in prepare_ring()
3281 		if (link_trb_count++ > ep_ring->num_segs) {  in prepare_ring()
3287 	if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) {  in prepare_ring()
3307 	struct xhci_ring *ep_ring;  in prepare_transfer()  local
3310 	ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index,  in prepare_transfer()
3312 	if (!ep_ring) {  in prepare_transfer()
3318 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),  in prepare_transfer()
3337 	list_add_tail(&td->td_list, &ep_ring->td_list);  in prepare_transfer()
3338 	td->start_seg = ep_ring->enq_seg;  in prepare_transfer()
3339 	td->first_trb = ep_ring->enqueue;  in prepare_transfer()
3746 	struct xhci_ring *ep_ring;  in xhci_queue_ctrl_tx()  local
3756 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);  in xhci_queue_ctrl_tx()
3757 	if (!ep_ring)  in xhci_queue_ctrl_tx()
3791 	start_trb = &ep_ring->enqueue->generic;  in xhci_queue_ctrl_tx()
3792 	start_cycle = ep_ring->cycle_state;  in xhci_queue_ctrl_tx()
3812 	queue_trb(xhci, ep_ring, true,  in xhci_queue_ctrl_tx()
3848 		queue_trb(xhci, ep_ring, true,  in xhci_queue_ctrl_tx()
3852 				field | ep_ring->cycle_state);  in xhci_queue_ctrl_tx()
3856 	td->last_trb = ep_ring->enqueue;  in xhci_queue_ctrl_tx()
3857 	td->last_trb_seg = ep_ring->enq_seg;  in xhci_queue_ctrl_tx()
3865 	queue_trb(xhci, ep_ring, false,  in xhci_queue_ctrl_tx()
3870 			field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);  in xhci_queue_ctrl_tx()
4043 	struct xhci_ring *ep_ring;  in xhci_queue_isoc_tx()  local
4059 	ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;  in xhci_queue_isoc_tx()
4067 	start_trb = &ep_ring->enqueue->generic;  in xhci_queue_isoc_tx()
4068 	start_cycle = ep_ring->cycle_state;  in xhci_queue_isoc_tx()
4119 			(i ? ep_ring->cycle_state : !start_cycle);  in xhci_queue_isoc_tx()
4132 					ep_ring->cycle_state;  in xhci_queue_isoc_tx()
4144 				td->last_trb = ep_ring->enqueue;  in xhci_queue_isoc_tx()
4145 				td->last_trb_seg = ep_ring->enq_seg;  in xhci_queue_isoc_tx()
4170 			queue_trb(xhci, ep_ring, more_trbs_coming,  in xhci_queue_isoc_tx()
4213 	urb_priv->td[0].last_trb = ep_ring->enqueue;  in xhci_queue_isoc_tx()
4215 	td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);  in xhci_queue_isoc_tx()
4218 	ep_ring->enqueue = urb_priv->td[0].first_trb;  in xhci_queue_isoc_tx()
4219 	ep_ring->enq_seg = urb_priv->td[0].start_seg;  in xhci_queue_isoc_tx()
4220 	ep_ring->cycle_state = start_cycle;  in xhci_queue_isoc_tx()
4221 	ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp;  in xhci_queue_isoc_tx()
4237 	struct xhci_ring *ep_ring;  in xhci_queue_isoc_tx_prepare()  local
4247 	ep_ring = xdev->eps[ep_index].ring;  in xhci_queue_isoc_tx_prepare()
4258 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),  in xhci_queue_isoc_tx_prepare()
4270 	if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {  in xhci_queue_isoc_tx_prepare()
4303 	ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free;  in xhci_queue_isoc_tx_prepare()