Lines Matching refs:dep
161 static void dwc3_ep_inc_enq(struct dwc3_ep *dep) in dwc3_ep_inc_enq() argument
163 dwc3_ep_inc_trb(&dep->trb_enqueue); in dwc3_ep_inc_enq()
170 static void dwc3_ep_inc_deq(struct dwc3_ep *dep) in dwc3_ep_inc_deq() argument
172 dwc3_ep_inc_trb(&dep->trb_dequeue); in dwc3_ep_inc_deq()
175 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep, in dwc3_gadget_del_and_unmap_request() argument
178 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_del_and_unmap_request()
194 if (dep->number > 1) in dwc3_gadget_del_and_unmap_request()
208 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, in dwc3_gadget_giveback() argument
211 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_giveback()
213 dwc3_gadget_del_and_unmap_request(dep, req, status); in dwc3_gadget_giveback()
217 usb_gadget_giveback_request(&dep->endpoint, &req->request); in dwc3_gadget_giveback()
272 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, in dwc3_send_gadget_ep_cmd() argument
275 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in dwc3_send_gadget_ep_cmd()
276 struct dwc3 *dwc = dep->dwc; in dwc3_send_gadget_ep_cmd()
342 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); in dwc3_send_gadget_ep_cmd()
343 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); in dwc3_send_gadget_ep_cmd()
344 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); in dwc3_send_gadget_ep_cmd()
368 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); in dwc3_send_gadget_ep_cmd()
378 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); in dwc3_send_gadget_ep_cmd()
388 dep->name); in dwc3_send_gadget_ep_cmd()
419 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); in dwc3_send_gadget_ep_cmd()
423 dep->flags |= DWC3_EP_TRANSFER_STARTED; in dwc3_send_gadget_ep_cmd()
426 dwc3_gadget_ep_get_transfer_index(dep); in dwc3_send_gadget_ep_cmd()
439 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep) in dwc3_send_clear_stall_ep_cmd() argument
441 struct dwc3 *dwc = dep->dwc; in dwc3_send_clear_stall_ep_cmd()
453 if (dep->direction && in dwc3_send_clear_stall_ep_cmd()
460 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in dwc3_send_clear_stall_ep_cmd()
463 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, in dwc3_trb_dma_offset() argument
466 u32 offset = (char *) trb - (char *) dep->trb_pool; in dwc3_trb_dma_offset()
468 return dep->trb_pool_dma + offset; in dwc3_trb_dma_offset()
471 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) in dwc3_alloc_trb_pool() argument
473 struct dwc3 *dwc = dep->dwc; in dwc3_alloc_trb_pool()
475 if (dep->trb_pool) in dwc3_alloc_trb_pool()
478 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, in dwc3_alloc_trb_pool()
480 &dep->trb_pool_dma, GFP_KERNEL); in dwc3_alloc_trb_pool()
481 if (!dep->trb_pool) { in dwc3_alloc_trb_pool()
482 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", in dwc3_alloc_trb_pool()
483 dep->name); in dwc3_alloc_trb_pool()
490 static void dwc3_free_trb_pool(struct dwc3_ep *dep) in dwc3_free_trb_pool() argument
492 struct dwc3 *dwc = dep->dwc; in dwc3_free_trb_pool()
495 dep->trb_pool, dep->trb_pool_dma); in dwc3_free_trb_pool()
497 dep->trb_pool = NULL; in dwc3_free_trb_pool()
498 dep->trb_pool_dma = 0; in dwc3_free_trb_pool()
501 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) in dwc3_gadget_set_xfer_resource() argument
509 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, in dwc3_gadget_set_xfer_resource()
546 static int dwc3_gadget_start_config(struct dwc3_ep *dep) in dwc3_gadget_start_config() argument
554 if (dep->number) in dwc3_gadget_start_config()
559 dwc = dep->dwc; in dwc3_gadget_start_config()
561 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in dwc3_gadget_start_config()
566 struct dwc3_ep *dep = dwc->eps[i]; in dwc3_gadget_start_config() local
568 if (!dep) in dwc3_gadget_start_config()
571 ret = dwc3_gadget_set_xfer_resource(dep); in dwc3_gadget_start_config()
579 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) in dwc3_gadget_set_ep_config() argument
584 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_set_ep_config()
586 comp_desc = dep->endpoint.comp_desc; in dwc3_gadget_set_ep_config()
587 desc = dep->endpoint.desc; in dwc3_gadget_set_ep_config()
596 u32 burst = dep->endpoint.maxburst; in dwc3_gadget_set_ep_config()
603 params.param2 |= dep->saved_state; in dwc3_gadget_set_ep_config()
608 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) in dwc3_gadget_set_ep_config()
615 dep->stream_capable = true; in dwc3_gadget_set_ep_config()
627 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); in dwc3_gadget_set_ep_config()
633 if (dep->direction) in dwc3_gadget_set_ep_config()
634 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); in dwc3_gadget_set_ep_config()
651 dep->interval = desc->bInterval; in dwc3_gadget_set_ep_config()
653 dep->interval = 1 << (desc->bInterval - 1); in dwc3_gadget_set_ep_config()
658 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); in dwc3_gadget_set_ep_config()
748 struct dwc3_ep *dep; in dwc3_gadget_clear_tx_fifos() local
757 dep = dwc->eps[1]; in dwc3_gadget_clear_tx_fifos()
768 dep = dwc->eps[num]; in dwc3_gadget_clear_tx_fifos()
775 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_clear_tx_fifos()
790 static int __dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) in __dwc3_gadget_resize_tx_fifos() argument
792 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_resize_tx_fifos()
797 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_resize_tx_fifos()
802 maxpacket = dep->endpoint.maxpacket; in __dwc3_gadget_resize_tx_fifos()
804 mult = dep->endpoint.mult * dep->endpoint.maxburst; in __dwc3_gadget_resize_tx_fifos()
806 mult = dep->endpoint.mult; in __dwc3_gadget_resize_tx_fifos()
811 } else if (usb_endpoint_xfer_bulk(dep->endpoint.desc)) { in __dwc3_gadget_resize_tx_fifos()
818 if (dep->endpoint.maxburst > mult) { in __dwc3_gadget_resize_tx_fifos()
819 mult = dep->endpoint.maxburst; in __dwc3_gadget_resize_tx_fifos()
827 } else if (usb_endpoint_xfer_int(dep->endpoint.desc)) { in __dwc3_gadget_resize_tx_fifos()
862 dep->endpoint.name, fifo_size & 0xfff); in __dwc3_gadget_resize_tx_fifos()
866 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); in __dwc3_gadget_resize_tx_fifos()
867 dep->flags |= DWC3_EP_TXFIFO_RESIZED; in __dwc3_gadget_resize_tx_fifos()
896 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) in dwc3_gadget_resize_tx_fifos() argument
898 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_resize_tx_fifos()
913 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) in dwc3_gadget_resize_tx_fifos()
917 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) in dwc3_gadget_resize_tx_fifos()
921 return __dwc3_gadget_resize_tx_fifos(dep); in dwc3_gadget_resize_tx_fifos()
925 if ((dep->endpoint.maxburst > 1 && in dwc3_gadget_resize_tx_fifos()
926 usb_endpoint_xfer_bulk(dep->endpoint.desc)) || in dwc3_gadget_resize_tx_fifos()
927 usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_resize_tx_fifos()
930 if (dep->endpoint.maxburst > 6 && in dwc3_gadget_resize_tx_fifos()
931 (usb_endpoint_xfer_bulk(dep->endpoint.desc) || in dwc3_gadget_resize_tx_fifos()
932 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31)) in dwc3_gadget_resize_tx_fifos()
973 dep->endpoint.name, fifo_size); in dwc3_gadget_resize_tx_fifos()
983 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); in dwc3_gadget_resize_tx_fifos()
984 dep->flags |= DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_resize_tx_fifos()
998 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) in __dwc3_gadget_ep_enable() argument
1000 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_ep_enable()
1001 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_enable()
1006 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
1007 ret = dwc3_gadget_resize_tx_fifos(dep); in __dwc3_gadget_ep_enable()
1011 ret = dwc3_gadget_start_config(dep); in __dwc3_gadget_ep_enable()
1016 ret = dwc3_gadget_set_ep_config(dep, action); in __dwc3_gadget_ep_enable()
1020 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
1024 dep->type = usb_endpoint_type(desc); in __dwc3_gadget_ep_enable()
1025 dep->flags |= DWC3_EP_ENABLED; in __dwc3_gadget_ep_enable()
1028 reg |= DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_enable()
1031 dep->trb_dequeue = 0; in __dwc3_gadget_ep_enable()
1032 dep->trb_enqueue = 0; in __dwc3_gadget_ep_enable()
1038 memset(dep->trb_pool, 0, in __dwc3_gadget_ep_enable()
1042 trb_st_hw = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
1044 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; in __dwc3_gadget_ep_enable()
1045 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
1046 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
1063 trb = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
1064 trb_dma = dwc3_trb_dma_offset(dep, trb); in __dwc3_gadget_ep_enable()
1071 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in __dwc3_gadget_ep_enable()
1075 if (dep->stream_capable) { in __dwc3_gadget_ep_enable()
1088 dwc3_stop_active_transfer(dep, true, true); in __dwc3_gadget_ep_enable()
1100 if (!dep->direction || in __dwc3_gadget_ep_enable()
1103 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; in __dwc3_gadget_ep_enable()
1108 trace_dwc3_gadget_ep_enable(dep); in __dwc3_gadget_ep_enable()
1113 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) in dwc3_remove_requests() argument
1117 dwc3_stop_active_transfer(dep, true, false); in dwc3_remove_requests()
1120 if (dep->flags & DWC3_EP_DELAY_STOP) in dwc3_remove_requests()
1124 while (!list_empty(&dep->started_list)) { in dwc3_remove_requests()
1125 req = next_request(&dep->started_list); in dwc3_remove_requests()
1127 dwc3_gadget_giveback(dep, req, status); in dwc3_remove_requests()
1130 while (!list_empty(&dep->pending_list)) { in dwc3_remove_requests()
1131 req = next_request(&dep->pending_list); in dwc3_remove_requests()
1133 dwc3_gadget_giveback(dep, req, status); in dwc3_remove_requests()
1136 while (!list_empty(&dep->cancelled_list)) { in dwc3_remove_requests()
1137 req = next_request(&dep->cancelled_list); in dwc3_remove_requests()
1139 dwc3_gadget_giveback(dep, req, status); in dwc3_remove_requests()
1153 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) in __dwc3_gadget_ep_disable() argument
1155 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_disable()
1159 trace_dwc3_gadget_ep_disable(dep); in __dwc3_gadget_ep_disable()
1162 if (dep->flags & DWC3_EP_STALL) in __dwc3_gadget_ep_disable()
1163 __dwc3_gadget_ep_set_halt(dep, 0, false); in __dwc3_gadget_ep_disable()
1166 reg &= ~DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_disable()
1169 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in __dwc3_gadget_ep_disable()
1171 dep->stream_capable = false; in __dwc3_gadget_ep_disable()
1172 dep->type = 0; in __dwc3_gadget_ep_disable()
1179 if (dep->flags & DWC3_EP_DELAY_STOP) in __dwc3_gadget_ep_disable()
1181 dep->flags &= mask; in __dwc3_gadget_ep_disable()
1184 if (dep->number > 1) { in __dwc3_gadget_ep_disable()
1185 dep->endpoint.comp_desc = NULL; in __dwc3_gadget_ep_disable()
1186 dep->endpoint.desc = NULL; in __dwc3_gadget_ep_disable()
1210 struct dwc3_ep *dep; in dwc3_gadget_ep_enable() local
1225 dep = to_dwc3_ep(ep); in dwc3_gadget_ep_enable()
1226 dwc = dep->dwc; in dwc3_gadget_ep_enable()
1228 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, in dwc3_gadget_ep_enable()
1230 dep->name)) in dwc3_gadget_ep_enable()
1234 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); in dwc3_gadget_ep_enable()
1242 struct dwc3_ep *dep; in dwc3_gadget_ep_disable() local
1252 dep = to_dwc3_ep(ep); in dwc3_gadget_ep_disable()
1253 dwc = dep->dwc; in dwc3_gadget_ep_disable()
1255 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), in dwc3_gadget_ep_disable()
1257 dep->name)) in dwc3_gadget_ep_disable()
1261 ret = __dwc3_gadget_ep_disable(dep); in dwc3_gadget_ep_disable()
1271 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_alloc_request() local
1277 req->direction = dep->direction; in dwc3_gadget_ep_alloc_request()
1278 req->epnum = dep->number; in dwc3_gadget_ep_alloc_request()
1279 req->dep = dep; in dwc3_gadget_ep_alloc_request()
1305 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) in dwc3_ep_prev_trb() argument
1312 return &dep->trb_pool[tmp - 1]; in dwc3_ep_prev_trb()
1315 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) in dwc3_calc_trbs_left() argument
1324 if (dep->trb_enqueue == dep->trb_dequeue) { in dwc3_calc_trbs_left()
1329 if (!list_empty(&dep->started_list)) in dwc3_calc_trbs_left()
1335 trbs_left = dep->trb_dequeue - dep->trb_enqueue; in dwc3_calc_trbs_left()
1338 if (dep->trb_dequeue < dep->trb_enqueue) in dwc3_calc_trbs_left()
1354 static void dwc3_prepare_one_trb(struct dwc3_ep *dep, in dwc3_prepare_one_trb() argument
1365 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_one_trb()
1370 dma = dep->dwc->bounce_addr; in dwc3_prepare_one_trb()
1376 trb = &dep->trb_pool[dep->trb_enqueue]; in dwc3_prepare_one_trb()
1381 req->trb_dma = dwc3_trb_dma_offset(dep, trb); in dwc3_prepare_one_trb()
1390 switch (usb_endpoint_type(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1422 struct usb_ep *ep = &dep->endpoint; in dwc3_prepare_one_trb()
1452 usb_endpoint_type(dep->endpoint.desc)); in dwc3_prepare_one_trb()
1459 if (usb_endpoint_dir_out(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1460 if (!dep->stream_capable) in dwc3_prepare_one_trb()
1472 else if (dep->stream_capable && is_last) in dwc3_prepare_one_trb()
1475 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) in dwc3_prepare_one_trb()
1493 dwc3_ep_inc_enq(dep); in dwc3_prepare_one_trb()
1495 trace_dwc3_prepare_trb(dep, trb); in dwc3_prepare_one_trb()
1498 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req) in dwc3_needs_extra_trb() argument
1500 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_needs_extra_trb()
1504 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || in dwc3_needs_extra_trb()
1520 static int dwc3_prepare_last_sg(struct dwc3_ep *dep, in dwc3_prepare_last_sg() argument
1524 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_prepare_last_sg()
1528 if (dwc3_needs_extra_trb(dep, req)) in dwc3_prepare_last_sg()
1531 if (dwc3_calc_trbs_left(dep) < num_trbs) in dwc3_prepare_last_sg()
1538 dwc3_prepare_one_trb(dep, req, entry_length, in dwc3_prepare_last_sg()
1543 dwc3_prepare_one_trb(dep, req, in dwc3_prepare_last_sg()
1550 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep, in dwc3_prepare_trbs_sg() argument
1560 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req); in dwc3_prepare_trbs_sg()
1570 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep); in dwc3_prepare_trbs_sg()
1593 if (!dwc3_prepare_last_sg(dep, req, trb_length, i)) in dwc3_prepare_trbs_sg()
1607 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false, in dwc3_prepare_trbs_sg()
1641 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep, in dwc3_prepare_trbs_linear() argument
1644 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); in dwc3_prepare_trbs_linear()
1657 static int dwc3_prepare_trbs(struct dwc3_ep *dep) in dwc3_prepare_trbs() argument
1674 list_for_each_entry(req, &dep->started_list, list) { in dwc3_prepare_trbs()
1676 ret = dwc3_prepare_trbs_sg(dep, req); in dwc3_prepare_trbs()
1681 if (!dwc3_calc_trbs_left(dep)) in dwc3_prepare_trbs()
1689 if (dep->stream_capable && req->request.is_last) in dwc3_prepare_trbs()
1693 list_for_each_entry_safe(req, n, &dep->pending_list, list) { in dwc3_prepare_trbs()
1694 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_trbs()
1697 dep->direction); in dwc3_prepare_trbs()
1707 ret = dwc3_prepare_trbs_sg(dep, req); in dwc3_prepare_trbs()
1711 ret = dwc3_prepare_trbs_linear(dep, req); in dwc3_prepare_trbs()
1714 if (!ret || !dwc3_calc_trbs_left(dep)) in dwc3_prepare_trbs()
1722 if (dep->stream_capable && req->request.is_last) in dwc3_prepare_trbs()
1729 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1731 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) in __dwc3_gadget_kick_transfer() argument
1744 ret = dwc3_prepare_trbs(dep); in __dwc3_gadget_kick_transfer()
1748 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); in __dwc3_gadget_kick_transfer()
1757 req = next_request(&dep->started_list); in __dwc3_gadget_kick_transfer()
1759 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_kick_transfer()
1770 if (dep->stream_capable) in __dwc3_gadget_kick_transfer()
1773 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_kick_transfer()
1774 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); in __dwc3_gadget_kick_transfer()
1777 DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_gadget_kick_transfer()
1780 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in __dwc3_gadget_kick_transfer()
1787 dwc3_stop_active_transfer(dep, true, true); in __dwc3_gadget_kick_transfer()
1789 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_kick_transfer()
1793 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in __dwc3_gadget_kick_transfer()
1794 dwc3_gadget_ep_cleanup_cancelled_requests(dep); in __dwc3_gadget_kick_transfer()
1799 if (dep->stream_capable && req->request.is_last) in __dwc3_gadget_kick_transfer()
1800 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; in __dwc3_gadget_kick_transfer()
1824 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) in __dwc3_stop_active_transfer() argument
1833 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_stop_active_transfer()
1835 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in __dwc3_stop_active_transfer()
1842 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { in __dwc3_stop_active_transfer()
1843 dep->flags |= DWC3_EP_DELAY_STOP; in __dwc3_stop_active_transfer()
1847 dep->resource_index = 0; in __dwc3_stop_active_transfer()
1850 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in __dwc3_stop_active_transfer()
1852 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; in __dwc3_stop_active_transfer()
1899 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) in dwc3_gadget_start_isoc_quirk() argument
1905 while (dep->combo_num < 2) { in dwc3_gadget_start_isoc_quirk()
1914 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1915 test_frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1916 test_frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1918 params.param0 = upper_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1919 params.param1 = lower_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1923 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); in dwc3_gadget_start_isoc_quirk()
1927 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1928 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1933 if (dep->combo_num == 0) in dwc3_gadget_start_isoc_quirk()
1934 dep->start_cmd_status = cmd_status; in dwc3_gadget_start_isoc_quirk()
1936 dep->combo_num++; in dwc3_gadget_start_isoc_quirk()
1943 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_start_isoc_quirk()
1949 test0 = (dep->start_cmd_status == 0); in dwc3_gadget_start_isoc_quirk()
1953 dep->combo_num = 1; in dwc3_gadget_start_isoc_quirk()
1955 dep->combo_num = 2; in dwc3_gadget_start_isoc_quirk()
1957 dep->combo_num = 3; in dwc3_gadget_start_isoc_quirk()
1959 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1961 dep->frame_number &= DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1962 dep->frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1963 dep->frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1966 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1967 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1969 return __dwc3_gadget_kick_transfer(dep); in dwc3_gadget_start_isoc_quirk()
1972 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) in __dwc3_gadget_start_isoc() argument
1974 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_start_isoc()
1975 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_start_isoc()
1979 if (list_empty(&dep->pending_list) && in __dwc3_gadget_start_isoc()
1980 list_empty(&dep->started_list)) { in __dwc3_gadget_start_isoc()
1981 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_start_isoc()
1988 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) in __dwc3_gadget_start_isoc()
1989 return dwc3_gadget_start_isoc_quirk(dep); in __dwc3_gadget_start_isoc()
1996 (dep->frame_number & DWC3_FRNUMBER_MASK); in __dwc3_gadget_start_isoc()
2007 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | in __dwc3_gadget_start_isoc()
2010 dep->frame_number += BIT(14); in __dwc3_gadget_start_isoc()
2020 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); in __dwc3_gadget_start_isoc()
2022 ret = __dwc3_gadget_kick_transfer(dep); in __dwc3_gadget_start_isoc()
2033 ret = __dwc3_stop_active_transfer(dep, false, true); in __dwc3_gadget_start_isoc()
2035 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; in __dwc3_gadget_start_isoc()
2041 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) in __dwc3_gadget_ep_queue() argument
2043 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_queue()
2045 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { in __dwc3_gadget_ep_queue()
2047 dep->name); in __dwc3_gadget_ep_queue()
2051 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", in __dwc3_gadget_ep_queue()
2052 &req->request, req->dep->name)) in __dwc3_gadget_ep_queue()
2057 dep->name, &req->request)) in __dwc3_gadget_ep_queue()
2067 list_add_tail(&req->list, &dep->pending_list); in __dwc3_gadget_ep_queue()
2070 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) in __dwc3_gadget_ep_queue()
2077 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || in __dwc3_gadget_ep_queue()
2078 (dep->flags & DWC3_EP_WEDGE) || in __dwc3_gadget_ep_queue()
2079 (dep->flags & DWC3_EP_DELAY_STOP) || in __dwc3_gadget_ep_queue()
2080 (dep->flags & DWC3_EP_STALL)) { in __dwc3_gadget_ep_queue()
2081 dep->flags |= DWC3_EP_DELAY_START; in __dwc3_gadget_ep_queue()
2093 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_queue()
2094 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { in __dwc3_gadget_ep_queue()
2095 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) in __dwc3_gadget_ep_queue()
2096 return __dwc3_gadget_start_isoc(dep); in __dwc3_gadget_ep_queue()
2102 __dwc3_gadget_kick_transfer(dep); in __dwc3_gadget_ep_queue()
2111 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_queue() local
2112 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_queue()
2119 ret = __dwc3_gadget_ep_queue(dep, req); in dwc3_gadget_ep_queue()
2125 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req) in dwc3_gadget_ep_skip_trbs() argument
2146 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_skip_trbs()
2148 dwc3_ep_inc_deq(dep); in dwc3_gadget_ep_skip_trbs()
2154 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) in dwc3_gadget_ep_cleanup_cancelled_requests() argument
2157 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_cleanup_cancelled_requests()
2159 while (!list_empty(&dep->cancelled_list)) { in dwc3_gadget_ep_cleanup_cancelled_requests()
2160 req = next_request(&dep->cancelled_list); in dwc3_gadget_ep_cleanup_cancelled_requests()
2161 dwc3_gadget_ep_skip_trbs(dep, req); in dwc3_gadget_ep_cleanup_cancelled_requests()
2164 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_gadget_ep_cleanup_cancelled_requests()
2167 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2170 dwc3_gadget_giveback(dep, req, -EPIPE); in dwc3_gadget_ep_cleanup_cancelled_requests()
2174 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2181 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_cancelled_requests()
2192 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_dequeue() local
2193 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_dequeue()
2202 list_for_each_entry(r, &dep->cancelled_list, list) { in dwc3_gadget_ep_dequeue()
2207 list_for_each_entry(r, &dep->pending_list, list) { in dwc3_gadget_ep_dequeue()
2209 dwc3_gadget_ep_skip_trbs(dep, req); in dwc3_gadget_ep_dequeue()
2210 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_dequeue()
2215 list_for_each_entry(r, &dep->started_list, list) { in dwc3_gadget_ep_dequeue()
2218 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_ep_dequeue()
2226 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_ep_dequeue()
2228 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { in dwc3_gadget_ep_dequeue()
2229 dwc3_gadget_ep_skip_trbs(dep, req); in dwc3_gadget_ep_dequeue()
2230 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_dequeue()
2246 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) in __dwc3_gadget_ep_set_halt() argument
2249 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_set_halt()
2253 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_set_halt()
2254 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); in __dwc3_gadget_ep_set_halt()
2266 if (dep->number > 1) in __dwc3_gadget_ep_set_halt()
2267 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); in __dwc3_gadget_ep_set_halt()
2269 trb = &dwc->ep0_trb[dep->trb_enqueue]; in __dwc3_gadget_ep_set_halt()
2272 started = !list_empty(&dep->started_list); in __dwc3_gadget_ep_set_halt()
2274 if (!protocol && ((dep->direction && transfer_in_flight) || in __dwc3_gadget_ep_set_halt()
2275 (!dep->direction && started))) { in __dwc3_gadget_ep_set_halt()
2279 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL, in __dwc3_gadget_ep_set_halt()
2283 dep->name); in __dwc3_gadget_ep_set_halt()
2285 dep->flags |= DWC3_EP_STALL; in __dwc3_gadget_ep_set_halt()
2292 if (dep->number <= 1) { in __dwc3_gadget_ep_set_halt()
2293 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2297 dwc3_stop_active_transfer(dep, true, true); in __dwc3_gadget_ep_set_halt()
2299 if (!list_empty(&dep->started_list)) in __dwc3_gadget_ep_set_halt()
2300 dep->flags |= DWC3_EP_DELAY_START; in __dwc3_gadget_ep_set_halt()
2302 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || in __dwc3_gadget_ep_set_halt()
2303 (dep->flags & DWC3_EP_DELAY_STOP)) { in __dwc3_gadget_ep_set_halt()
2304 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; in __dwc3_gadget_ep_set_halt()
2306 vdwc->clear_stall_protocol = dep->number; in __dwc3_gadget_ep_set_halt()
2311 ret = dwc3_send_clear_stall_ep_cmd(dep); in __dwc3_gadget_ep_set_halt()
2314 dep->name); in __dwc3_gadget_ep_set_halt()
2318 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2320 if ((dep->flags & DWC3_EP_DELAY_START) && in __dwc3_gadget_ep_set_halt()
2321 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_ep_set_halt()
2322 __dwc3_gadget_kick_transfer(dep); in __dwc3_gadget_ep_set_halt()
2324 dep->flags &= ~DWC3_EP_DELAY_START; in __dwc3_gadget_ep_set_halt()
2332 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_set_halt() local
2333 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_halt()
2340 ret = __dwc3_gadget_ep_set_halt(dep, value, false); in dwc3_gadget_ep_set_halt()
2348 struct dwc3_ep *dep = to_dwc3_ep(ep); in dwc3_gadget_ep_set_wedge() local
2349 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_wedge()
2354 dep->flags |= DWC3_EP_WEDGE; in dwc3_gadget_ep_set_wedge()
2356 if (dep->number == 0 || dep->number == 1) in dwc3_gadget_ep_set_wedge()
2359 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); in dwc3_gadget_ep_set_wedge()
2499 struct dwc3_ep *dep; in dwc3_stop_active_transfers() local
2501 dep = dwc->eps[epnum]; in dwc3_stop_active_transfers()
2502 if (!dep) in dwc3_stop_active_transfers()
2505 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in dwc3_stop_active_transfers()
2840 struct dwc3_ep *dep; in __dwc3_gadget_start() local
2895 dep = dwc->eps[0]; in __dwc3_gadget_start()
2896 dep->flags = 0; in __dwc3_gadget_start()
2897 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); in __dwc3_gadget_start()
2899 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2903 dep = dwc->eps[1]; in __dwc3_gadget_start()
2904 dep->flags = 0; in __dwc3_gadget_start()
2905 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); in __dwc3_gadget_start()
2907 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
3149 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) in dwc3_gadget_init_control_endpoint() argument
3151 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_control_endpoint()
3153 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); in dwc3_gadget_init_control_endpoint()
3154 dep->endpoint.maxburst = 1; in dwc3_gadget_init_control_endpoint()
3155 dep->endpoint.ops = &dwc3_gadget_ep0_ops; in dwc3_gadget_init_control_endpoint()
3156 if (!dep->direction) in dwc3_gadget_init_control_endpoint()
3157 dwc->gadget->ep0 = &dep->endpoint; in dwc3_gadget_init_control_endpoint()
3159 dep->endpoint.caps.type_control = true; in dwc3_gadget_init_control_endpoint()
3164 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) in dwc3_gadget_init_in_endpoint() argument
3166 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_in_endpoint()
3176 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); in dwc3_gadget_init_in_endpoint()
3217 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_in_endpoint()
3219 dep->endpoint.max_streams = 16; in dwc3_gadget_init_in_endpoint()
3220 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_in_endpoint()
3221 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_in_endpoint()
3223 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_in_endpoint()
3224 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_in_endpoint()
3225 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_in_endpoint()
3227 return dwc3_alloc_trb_pool(dep); in dwc3_gadget_init_in_endpoint()
3230 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) in dwc3_gadget_init_out_endpoint() argument
3232 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_out_endpoint()
3265 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_out_endpoint()
3266 dep->endpoint.max_streams = 16; in dwc3_gadget_init_out_endpoint()
3267 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_out_endpoint()
3268 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_out_endpoint()
3270 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_out_endpoint()
3271 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_out_endpoint()
3272 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_out_endpoint()
3274 return dwc3_alloc_trb_pool(dep); in dwc3_gadget_init_out_endpoint()
3279 struct dwc3_ep *dep; in dwc3_gadget_init_endpoint() local
3285 dep = kzalloc(sizeof(*dep), GFP_KERNEL); in dwc3_gadget_init_endpoint()
3286 if (!dep) in dwc3_gadget_init_endpoint()
3299 dep->dwc = dwc; in dwc3_gadget_init_endpoint()
3300 dep->number = num << 1 | direction; in dwc3_gadget_init_endpoint()
3301 dep->direction = direction; in dwc3_gadget_init_endpoint()
3302 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); in dwc3_gadget_init_endpoint()
3303 dwc->eps[epnum] = dep; in dwc3_gadget_init_endpoint()
3304 dep->combo_num = 0; in dwc3_gadget_init_endpoint()
3305 dep->start_cmd_status = 0; in dwc3_gadget_init_endpoint()
3307 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, in dwc3_gadget_init_endpoint()
3310 dep->endpoint.name = dep->name; in dwc3_gadget_init_endpoint()
3312 if (!(dep->number > 1)) { in dwc3_gadget_init_endpoint()
3313 dep->endpoint.desc = &dwc3_gadget_ep0_desc; in dwc3_gadget_init_endpoint()
3314 dep->endpoint.comp_desc = NULL; in dwc3_gadget_init_endpoint()
3318 ret = dwc3_gadget_init_control_endpoint(dep); in dwc3_gadget_init_endpoint()
3320 ret = dwc3_gadget_init_in_endpoint(dep); in dwc3_gadget_init_endpoint()
3322 ret = dwc3_gadget_init_out_endpoint(dep); in dwc3_gadget_init_endpoint()
3327 dep->endpoint.caps.dir_in = direction; in dwc3_gadget_init_endpoint()
3328 dep->endpoint.caps.dir_out = !direction; in dwc3_gadget_init_endpoint()
3330 INIT_LIST_HEAD(&dep->pending_list); in dwc3_gadget_init_endpoint()
3331 INIT_LIST_HEAD(&dep->started_list); in dwc3_gadget_init_endpoint()
3332 INIT_LIST_HEAD(&dep->cancelled_list); in dwc3_gadget_init_endpoint()
3334 dwc3_debugfs_create_endpoint_dir(dep); in dwc3_gadget_init_endpoint()
3358 struct dwc3_ep *dep; in dwc3_gadget_free_endpoints() local
3362 dep = dwc->eps[epnum]; in dwc3_gadget_free_endpoints()
3363 if (!dep) in dwc3_gadget_free_endpoints()
3375 dwc3_free_trb_pool(dep); in dwc3_gadget_free_endpoints()
3376 list_del(&dep->endpoint.ep_list); in dwc3_gadget_free_endpoints()
3379 debugfs_remove_recursive(debugfs_lookup(dep->name, dwc->root)); in dwc3_gadget_free_endpoints()
3380 kfree(dep); in dwc3_gadget_free_endpoints()
3386 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, in dwc3_gadget_ep_reclaim_completed_trb() argument
3392 dwc3_ep_inc_deq(dep); in dwc3_gadget_ep_reclaim_completed_trb()
3394 trace_dwc3_complete_trb(dep, trb); in dwc3_gadget_ep_reclaim_completed_trb()
3414 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_ep_reclaim_completed_trb()
3419 frame_number &= ~(dep->interval - 1); in dwc3_gadget_ep_reclaim_completed_trb()
3428 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && in dwc3_gadget_ep_reclaim_completed_trb()
3429 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { in dwc3_gadget_ep_reclaim_completed_trb()
3454 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, in dwc3_gadget_ep_reclaim_trb_sg() argument
3458 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3466 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3471 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, in dwc3_gadget_ep_reclaim_trb_sg()
3480 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep, in dwc3_gadget_ep_reclaim_trb_linear() argument
3484 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_linear()
3486 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb, in dwc3_gadget_ep_reclaim_trb_linear()
3495 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, in dwc3_gadget_ep_cleanup_completed_request() argument
3499 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_cleanup_completed_request()
3504 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, in dwc3_gadget_ep_cleanup_completed_request()
3507 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, in dwc3_gadget_ep_cleanup_completed_request()
3516 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, in dwc3_gadget_ep_cleanup_completed_request()
3528 if (status == -EXDEV && usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in dwc3_gadget_ep_cleanup_completed_request()
3552 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); in dwc3_gadget_ep_cleanup_completed_request()
3571 dwc3_gadget_giveback(dep, req, request_status); in dwc3_gadget_ep_cleanup_completed_request()
3577 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, in dwc3_gadget_ep_cleanup_completed_requests() argument
3582 while (!list_empty(&dep->started_list)) { in dwc3_gadget_ep_cleanup_completed_requests()
3585 req = next_request(&dep->started_list); in dwc3_gadget_ep_cleanup_completed_requests()
3586 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, in dwc3_gadget_ep_cleanup_completed_requests()
3594 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_completed_requests()
3599 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep) in dwc3_gadget_ep_should_continue() argument
3602 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_should_continue()
3604 if (!dep->endpoint.desc || !dwc->pullups_connected || in dwc3_gadget_ep_should_continue()
3608 if (!list_empty(&dep->pending_list)) in dwc3_gadget_ep_should_continue()
3615 req = next_request(&dep->started_list); in dwc3_gadget_ep_should_continue()
3622 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, in dwc3_gadget_endpoint_frame_from_event() argument
3625 dep->frame_number = event->parameters; in dwc3_gadget_endpoint_frame_from_event()
3628 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, in dwc3_gadget_endpoint_trbs_complete() argument
3631 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_trbs_complete()
3635 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); in dwc3_gadget_endpoint_trbs_complete()
3637 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_trbs_complete()
3640 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_trbs_complete()
3653 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && status == -EXDEV && in dwc3_gadget_endpoint_trbs_complete()
3654 !list_empty(&dep->cancelled_list) && in dwc3_gadget_endpoint_trbs_complete()
3655 !list_empty(&dep->pending_list)) { in dwc3_gadget_endpoint_trbs_complete()
3656 list_for_each_entry_safe(req, tmp, &dep->pending_list, list) in dwc3_gadget_endpoint_trbs_complete()
3660 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && status == -EXDEV && in dwc3_gadget_endpoint_trbs_complete()
3661 !list_empty(&dep->cancelled_list)) { in dwc3_gadget_endpoint_trbs_complete()
3662 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) in dwc3_gadget_endpoint_trbs_complete()
3666 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_endpoint_trbs_complete()
3667 list_empty(&dep->started_list) && in dwc3_gadget_endpoint_trbs_complete()
3668 (list_empty(&dep->pending_list) || status == -EXDEV)) in dwc3_gadget_endpoint_trbs_complete()
3669 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_endpoint_trbs_complete()
3670 else if (dwc3_gadget_ep_should_continue(dep)) in dwc3_gadget_endpoint_trbs_complete()
3671 if (__dwc3_gadget_kick_transfer(dep) == 0) in dwc3_gadget_endpoint_trbs_complete()
3684 dep = dwc->eps[i]; in dwc3_gadget_endpoint_trbs_complete()
3686 if (!(dep->flags & DWC3_EP_ENABLED)) in dwc3_gadget_endpoint_trbs_complete()
3689 if (!list_empty(&dep->started_list)) in dwc3_gadget_endpoint_trbs_complete()
3703 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, in dwc3_gadget_endpoint_transfer_in_progress() argument
3708 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_transfer_in_progress()
3711 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_transfer_in_progress()
3712 dwc3_gadget_endpoint_frame_from_event(dep, event); in dwc3_gadget_endpoint_transfer_in_progress()
3720 dwc3_gadget_endpoint_trbs_complete(dep, event, status); in dwc3_gadget_endpoint_transfer_in_progress()
3723 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep, in dwc3_gadget_endpoint_transfer_complete() argument
3728 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_transfer_complete()
3733 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status)) in dwc3_gadget_endpoint_transfer_complete()
3734 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_endpoint_transfer_complete()
3737 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, in dwc3_gadget_endpoint_transfer_not_ready() argument
3740 dwc3_gadget_endpoint_frame_from_event(dep, event); in dwc3_gadget_endpoint_transfer_not_ready()
3750 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_transfer_not_ready()
3753 (void) __dwc3_gadget_start_isoc(dep); in dwc3_gadget_endpoint_transfer_not_ready()
3756 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep, in dwc3_gadget_endpoint_command_complete() argument
3769 if (dep->stream_capable) in dwc3_gadget_endpoint_command_complete()
3770 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_command_complete()
3772 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; in dwc3_gadget_endpoint_command_complete()
3773 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_command_complete()
3774 dwc3_gadget_ep_cleanup_cancelled_requests(dep); in dwc3_gadget_endpoint_command_complete()
3776 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { in dwc3_gadget_endpoint_command_complete()
3777 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_command_complete()
3780 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; in dwc3_gadget_endpoint_command_complete()
3781 if (dwc3_send_clear_stall_ep_cmd(dep)) { in dwc3_gadget_endpoint_command_complete()
3784 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); in dwc3_gadget_endpoint_command_complete()
3790 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in dwc3_gadget_endpoint_command_complete()
3791 if (vdwc->clear_stall_protocol == dep->number) in dwc3_gadget_endpoint_command_complete()
3795 if ((dep->flags & DWC3_EP_DELAY_START) && in dwc3_gadget_endpoint_command_complete()
3796 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_command_complete()
3797 __dwc3_gadget_kick_transfer(dep); in dwc3_gadget_endpoint_command_complete()
3799 dep->flags &= ~DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_command_complete()
3802 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep, in dwc3_gadget_endpoint_stream_event() argument
3805 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_stream_event()
3808 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3822 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) { in dwc3_gadget_endpoint_stream_event()
3823 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED) in dwc3_gadget_endpoint_stream_event()
3824 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM; in dwc3_gadget_endpoint_stream_event()
3826 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3831 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || in dwc3_gadget_endpoint_stream_event()
3832 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) || in dwc3_gadget_endpoint_stream_event()
3833 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)) in dwc3_gadget_endpoint_stream_event()
3853 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); in dwc3_gadget_endpoint_stream_event()
3855 dep->flags |= DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_stream_event()
3856 dwc3_stop_active_transfer(dep, true, true); in dwc3_gadget_endpoint_stream_event()
3863 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3869 struct dwc3_ep *dep; in dwc3_endpoint_interrupt() local
3872 dep = dwc->eps[epnum]; in dwc3_endpoint_interrupt()
3874 if (!(dep->flags & DWC3_EP_ENABLED)) { in dwc3_endpoint_interrupt()
3875 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) in dwc3_endpoint_interrupt()
3891 dwc3_gadget_endpoint_transfer_in_progress(dep, event); in dwc3_endpoint_interrupt()
3894 dwc3_gadget_endpoint_transfer_not_ready(dep, event); in dwc3_endpoint_interrupt()
3897 dwc3_gadget_endpoint_command_complete(dep, event); in dwc3_endpoint_interrupt()
3900 dwc3_gadget_endpoint_transfer_complete(dep, event); in dwc3_endpoint_interrupt()
3903 dwc3_gadget_endpoint_stream_event(dep, event); in dwc3_endpoint_interrupt()
3949 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, in dwc3_stop_active_transfer() argument
3952 struct dwc3 *dwc = dep->dwc; in dwc3_stop_active_transfer()
3960 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) in dwc3_stop_active_transfer()
3963 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || in dwc3_stop_active_transfer()
3964 (dep->flags & DWC3_EP_DELAY_STOP) || in dwc3_stop_active_transfer()
3965 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in dwc3_stop_active_transfer()
3976 dep->flags |= DWC3_EP_DELAY_STOP; in dwc3_stop_active_transfer()
4007 __dwc3_stop_active_transfer(dep, force, interrupt); in dwc3_stop_active_transfer()
4016 struct dwc3_ep *dep; in dwc3_clear_stall_all_ep() local
4019 dep = dwc->eps[epnum]; in dwc3_clear_stall_all_ep()
4020 if (!dep) in dwc3_clear_stall_all_ep()
4023 if (!(dep->flags & DWC3_EP_STALL)) in dwc3_clear_stall_all_ep()
4026 dep->flags &= ~DWC3_EP_STALL; in dwc3_clear_stall_all_ep()
4028 ret = dwc3_send_clear_stall_ep_cmd(dep); in dwc3_clear_stall_all_ep()
4153 struct dwc3_ep *dep; in dwc3_gadget_conndone_interrupt() local
4278 dep = dwc->eps[0]; in dwc3_gadget_conndone_interrupt()
4279 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); in dwc3_gadget_conndone_interrupt()
4281 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
4285 dep = dwc->eps[1]; in dwc3_gadget_conndone_interrupt()
4286 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); in dwc3_gadget_conndone_interrupt()
4288 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()