411 		struct cdns3_request *priv_req;  in cdns3_start_all_request()  local
414 		priv_req = to_cdns3_request(request);  in cdns3_start_all_request()
415 		if ((priv_req->flags & REQUEST_INTERNAL) ||  in cdns3_start_all_request()
526 						     struct cdns3_request *priv_req)  in cdns3_wa2_gadget_giveback()  argument
529 	    priv_req->flags & REQUEST_INTERNAL) {  in cdns3_wa2_gadget_giveback()
561 	return &priv_req->request;  in cdns3_wa2_gadget_giveback()
566 				     struct cdns3_request *priv_req)  in cdns3_wa2_gadget_ep_queue()  argument
596 		    !(priv_req->flags & REQUEST_INTERNAL)) {  in cdns3_wa2_gadget_ep_queue()
598 						     &priv_req->request);  in cdns3_wa2_gadget_ep_queue()
602 			list_add_tail(&priv_req->request.list,  in cdns3_wa2_gadget_ep_queue()
604 			cdns3_gadget_giveback(priv_ep, priv_req,  in cdns3_wa2_gadget_ep_queue()
605 					      priv_req->request.status);  in cdns3_wa2_gadget_ep_queue()
624 		if (priv_req->flags & REQUEST_INTERNAL)  in cdns3_wa2_gadget_ep_queue()
625 			list_add_tail(&priv_req->list,  in cdns3_wa2_gadget_ep_queue()
634 	struct cdns3_request *priv_req;  in cdns3_wa2_remove_old_request()  local
639 		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);  in cdns3_wa2_remove_old_request()
640 		chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);  in cdns3_wa2_remove_old_request()
644 		kfree(priv_req->request.buf);  in cdns3_wa2_remove_old_request()
645 		list_del_init(&priv_req->list);  in cdns3_wa2_remove_old_request()
647 					     &priv_req->request);  in cdns3_wa2_remove_old_request()
664 	struct cdns3_request *priv_req;  in cdns3_wa2_descmissing_packet()  local
691 	priv_req = to_cdns3_request(request);  in cdns3_wa2_descmissing_packet()
692 	priv_req->flags |= REQUEST_INTERNAL;  in cdns3_wa2_descmissing_packet()
703 	priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,  in cdns3_wa2_descmissing_packet()
707 	if (!priv_req->request.buf) {  in cdns3_wa2_descmissing_packet()
712 	priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;  in cdns3_wa2_descmissing_packet()
713 	priv_ep->descmis_req = priv_req;  in cdns3_wa2_descmissing_packet()
809 			   struct cdns3_request *priv_req,  in cdns3_gadget_giveback()  argument
813 	struct usb_request *request = &priv_req->request;  in cdns3_gadget_giveback()
823 	if ((priv_req->flags & REQUEST_UNALIGNED) &&  in cdns3_gadget_giveback()
825 		memcpy(request->buf, priv_req->aligned_buf->buf,  in cdns3_gadget_giveback()
828 	priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);  in cdns3_gadget_giveback()
830 	priv_req->finished_trb = 0;  in cdns3_gadget_giveback()
831 	trace_cdns3_gadget_giveback(priv_req);  in cdns3_gadget_giveback()
835 						    priv_req);  in cdns3_gadget_giveback()
898 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)  in cdns3_prepare_aligned_request_buf()  argument
900 	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;  in cdns3_prepare_aligned_request_buf()
905 	if (!((uintptr_t)priv_req->request.buf & 0x7))  in cdns3_prepare_aligned_request_buf()
908 	buf = priv_req->aligned_buf;  in cdns3_prepare_aligned_request_buf()
910 	if (!buf || priv_req->request.length > buf->size) {  in cdns3_prepare_aligned_request_buf()
915 		buf->size = priv_req->request.length;  in cdns3_prepare_aligned_request_buf()
926 		if (priv_req->aligned_buf) {  in cdns3_prepare_aligned_request_buf()
927 			trace_cdns3_free_aligned_request(priv_req);  in cdns3_prepare_aligned_request_buf()
928 			priv_req->aligned_buf->in_use = 0;  in cdns3_prepare_aligned_request_buf()
934 		priv_req->aligned_buf = buf;  in cdns3_prepare_aligned_request_buf()
941 		memcpy(buf->buf, priv_req->request.buf,  in cdns3_prepare_aligned_request_buf()
942 		       priv_req->request.length);  in cdns3_prepare_aligned_request_buf()
945 	priv_req->flags |= REQUEST_UNALIGNED;  in cdns3_prepare_aligned_request_buf()
946 	trace_cdns3_prepare_aligned_request(priv_req);  in cdns3_prepare_aligned_request_buf()
990 	struct cdns3_request *priv_req;  in cdns3_ep_run_stream_transfer()  local
999 	priv_req = to_cdns3_request(request);  in cdns3_ep_run_stream_transfer()
1005 	if (priv_req->flags & REQUEST_UNALIGNED)  in cdns3_ep_run_stream_transfer()
1006 		trb_dma = priv_req->aligned_buf->dma;  in cdns3_ep_run_stream_transfer()
1012 	priv_req->start_trb = priv_ep->enqueue;  in cdns3_ep_run_stream_transfer()
1013 	priv_req->end_trb = priv_req->start_trb;  in cdns3_ep_run_stream_transfer()
1014 	priv_req->trb = trb;  in cdns3_ep_run_stream_transfer()
1019 		  TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;  in cdns3_ep_run_stream_transfer()
1042 	priv_req->flags |= REQUEST_PENDING;  in cdns3_ep_run_stream_transfer()
1046 	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);  in cdns3_ep_run_stream_transfer()
1076 		priv_ep->last_stream_id = priv_req->request.stream_id;  in cdns3_ep_run_stream_transfer()
1078 		writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |  in cdns3_ep_run_stream_transfer()
1115 	struct cdns3_request *priv_req;  in cdns3_ep_run_transfer()  local
1139 	priv_req = to_cdns3_request(request);  in cdns3_ep_run_transfer()
1145 	if (priv_req->flags & REQUEST_UNALIGNED)  in cdns3_ep_run_transfer()
1146 		trb_dma = priv_req->aligned_buf->dma;  in cdns3_ep_run_transfer()
1151 	priv_req->start_trb = priv_ep->enqueue;  in cdns3_ep_run_transfer()
1152 	priv_req->trb = trb;  in cdns3_ep_run_transfer()
1247 			priv_req->trb->control = cpu_to_le32(control);  in cdns3_ep_run_transfer()
1260 		priv_req->end_trb = priv_ep->enqueue;  in cdns3_ep_run_transfer()
1266 	trb = priv_req->trb;  in cdns3_ep_run_transfer()
1268 	priv_req->flags |= REQUEST_PENDING;  in cdns3_ep_run_transfer()
1269 	priv_req->num_of_trb = num_trb;  in cdns3_ep_run_transfer()
1317 		trace_cdns3_prepare_trb(priv_ep, priv_req->trb);  in cdns3_ep_run_transfer()
1344 					priv_req->start_trb * TRB_SIZE),  in cdns3_ep_run_transfer()
1431 				  struct cdns3_request *priv_req)  in cdns3_trb_handled()  argument
1443 	if (priv_req->start_trb < priv_req->end_trb) {  in cdns3_trb_handled()
1444 		if (priv_ep->dequeue > priv_req->end_trb)  in cdns3_trb_handled()
1447 		if (priv_ep->dequeue < priv_req->start_trb)  in cdns3_trb_handled()
1451 	if ((priv_req->start_trb > priv_req->end_trb) &&  in cdns3_trb_handled()
1452 		(priv_ep->dequeue > priv_req->end_trb) &&  in cdns3_trb_handled()
1453 		(priv_ep->dequeue < priv_req->start_trb))  in cdns3_trb_handled()
1456 	if ((priv_req->start_trb == priv_req->end_trb) &&  in cdns3_trb_handled()
1457 		(priv_ep->dequeue != priv_req->end_trb))  in cdns3_trb_handled()
1488 	trace_cdns3_request_handled(priv_req, current_index, handled);  in cdns3_trb_handled()
1496 	struct cdns3_request *priv_req;  in cdns3_transfer_completed()  local
1504 		priv_req = to_cdns3_request(request);  in cdns3_transfer_completed()
1521 			while (cdns3_trb_handled(priv_ep, priv_req)) {  in cdns3_transfer_completed()
1522 				priv_req->finished_trb++;  in cdns3_transfer_completed()
1523 				if (priv_req->finished_trb >= priv_req->num_of_trb)  in cdns3_transfer_completed()
1533 				if (priv_req->num_of_trb > 1 &&  in cdns3_transfer_completed()
1542 				cdns3_gadget_giveback(priv_ep, priv_req, 0);  in cdns3_transfer_completed()
1561 			if (trb != priv_req->trb)  in cdns3_transfer_completed()
1564 					 priv_req->trb, trb);  in cdns3_transfer_completed()
1571 				cdns3_gadget_giveback(priv_ep, priv_req, 0);  in cdns3_transfer_completed()
2237 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_alloc_request()  local
2239 	priv_req = kzalloc(sizeof(*priv_req), gfp_flags);  in cdns3_gadget_ep_alloc_request()
2240 	if (!priv_req)  in cdns3_gadget_ep_alloc_request()
2243 	priv_req->priv_ep = priv_ep;  in cdns3_gadget_ep_alloc_request()
2245 	trace_cdns3_alloc_request(priv_req);  in cdns3_gadget_ep_alloc_request()
2246 	return &priv_req->request;  in cdns3_gadget_ep_alloc_request()
2257 	struct cdns3_request *priv_req = to_cdns3_request(request);  in cdns3_gadget_ep_free_request()  local
2259 	if (priv_req->aligned_buf)  in cdns3_gadget_ep_free_request()
2260 		priv_req->aligned_buf->in_use = 0;  in cdns3_gadget_ep_free_request()
2262 	trace_cdns3_free_request(priv_req);  in cdns3_gadget_ep_free_request()
2263 	kfree(priv_req);  in cdns3_gadget_ep_free_request()
2415 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_disable()  local
2469 		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);  in cdns3_gadget_ep_disable()
2471 		kfree(priv_req->request.buf);  in cdns3_gadget_ep_disable()
2473 					     &priv_req->request);  in cdns3_gadget_ep_disable()
2474 		list_del_init(&priv_req->list);  in cdns3_gadget_ep_disable()
2510 	struct cdns3_request *priv_req;  in __cdns3_gadget_ep_queue()  local
2515 	priv_req = to_cdns3_request(request);  in __cdns3_gadget_ep_queue()
2516 	trace_cdns3_ep_queue(priv_req);  in __cdns3_gadget_ep_queue()
2520 						priv_req);  in __cdns3_gadget_ep_queue()
2526 	ret = cdns3_prepare_aligned_request_buf(priv_req);  in __cdns3_gadget_ep_queue()
2578 		struct cdns3_request *priv_req;  in cdns3_gadget_ep_queue()  local
2584 		priv_req = to_cdns3_request(zlp_request);  in cdns3_gadget_ep_queue()
2585 		priv_req->flags |= REQUEST_ZLP;  in cdns3_gadget_ep_queue()
2609 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_dequeue()  local
2622 	priv_req = to_cdns3_request(request);  in cdns3_gadget_ep_dequeue()
2624 	trace_cdns3_ep_dequeue(priv_req);  in cdns3_gadget_ep_dequeue()
2645 	link_trb = priv_req->trb;  in cdns3_gadget_ep_dequeue()
2650 			((priv_req->end_trb + 1) * TRB_SIZE)));  in cdns3_gadget_ep_dequeue()
2654 		if (priv_ep->wa1_trb == priv_req->trb)  in cdns3_gadget_ep_dequeue()
2658 	cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);  in cdns3_gadget_ep_dequeue()
2695 	struct cdns3_request *priv_req;  in __cdns3_gadget_ep_clear_halt()  local
2705 		priv_req = to_cdns3_request(request);  in __cdns3_gadget_ep_clear_halt()
2706 		trb = priv_req->trb;  in __cdns3_gadget_ep_clear_halt()