38 #define CHECK(ep, expr, fmt...)					\  argument
40 		if (!(expr)) EPDBG(ep, "CHECK:" fmt);		\
43 #define CHECK(ep, expr, fmt...)	do { } while(0)  argument
46 static void ast_vhub_epn_kick(struct ast_vhub_ep *ep, struct ast_vhub_req *req)  in ast_vhub_epn_kick()  argument
57 	if (chunk > ep->ep.maxpacket)  in ast_vhub_epn_kick()
58 		chunk = ep->ep.maxpacket;  in ast_vhub_epn_kick()
59 	else if ((chunk < ep->ep.maxpacket) || !req->req.zero)  in ast_vhub_epn_kick()
62 	EPVDBG(ep, "kick req %p act=%d/%d chunk=%d last=%d\n",  in ast_vhub_epn_kick()
69 		if (ep->epn.is_in) {  in ast_vhub_epn_kick()
70 			memcpy(ep->buf, req->req.buf + act, chunk);  in ast_vhub_epn_kick()
71 			vhub_dma_workaround(ep->buf);  in ast_vhub_epn_kick()
73 		writel(ep->buf_dma, ep->epn.regs + AST_VHUB_EP_DESC_BASE);  in ast_vhub_epn_kick()
75 		if (ep->epn.is_in)  in ast_vhub_epn_kick()
77 		writel(req->req.dma + act, ep->epn.regs + AST_VHUB_EP_DESC_BASE);  in ast_vhub_epn_kick()
83 	       ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_kick()
85 	       ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_kick()
88 static void ast_vhub_epn_handle_ack(struct ast_vhub_ep *ep)  in ast_vhub_epn_handle_ack()  argument
95 	stat = readl(ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_handle_ack()
98 	req = list_first_entry_or_null(&ep->queue, struct ast_vhub_req, queue);  in ast_vhub_epn_handle_ack()
100 	EPVDBG(ep, "ACK status=%08x is_in=%d, req=%p (active=%d)\n",  in ast_vhub_epn_handle_ack()
101 	       stat, ep->epn.is_in, req, req ? req->active : 0);  in ast_vhub_epn_handle_ack()
116 		EPDBG(ep, "DMA read pointer not 0 !\n");  in ast_vhub_epn_handle_ack()
127 	if (!req->req.dma && !ep->epn.is_in && len)  in ast_vhub_epn_handle_ack()
128 		memcpy(req->req.buf + req->req.actual, ep->buf, len);  in ast_vhub_epn_handle_ack()
134 	if (len < ep->ep.maxpacket)  in ast_vhub_epn_handle_ack()
139 		ast_vhub_done(ep, req, 0);  in ast_vhub_epn_handle_ack()
140 		req = list_first_entry_or_null(&ep->queue, struct ast_vhub_req,  in ast_vhub_epn_handle_ack()
152 	ast_vhub_epn_kick(ep, req);  in ast_vhub_epn_handle_ack()
155 static inline unsigned int ast_vhub_count_free_descs(struct ast_vhub_ep *ep)  in ast_vhub_count_free_descs()  argument
162 	return (ep->epn.d_last + AST_VHUB_DESCS_COUNT - ep->epn.d_next - 1) &  in ast_vhub_count_free_descs()
166 static void ast_vhub_epn_kick_desc(struct ast_vhub_ep *ep,  in ast_vhub_epn_kick_desc()  argument
181 	EPVDBG(ep, "kick act=%d/%d chunk_max=%d free_descs=%d\n",  in ast_vhub_epn_kick_desc()
182 	       act, len, ep->epn.chunk_max, ast_vhub_count_free_descs(ep));  in ast_vhub_epn_kick_desc()
185 	while (ast_vhub_count_free_descs(ep) && req->last_desc < 0) {  in ast_vhub_epn_kick_desc()
189 		d_num = ep->epn.d_next;  in ast_vhub_epn_kick_desc()
190 		desc = &ep->epn.descs[d_num];  in ast_vhub_epn_kick_desc()
191 		ep->epn.d_next = (d_num + 1) & (AST_VHUB_DESCS_COUNT - 1);  in ast_vhub_epn_kick_desc()
195 		if (chunk <= ep->epn.chunk_max) {  in ast_vhub_epn_kick_desc()
204 			if (!chunk || !req->req.zero || (chunk % ep->ep.maxpacket) != 0)  in ast_vhub_epn_kick_desc()
207 			chunk = ep->epn.chunk_max;  in ast_vhub_epn_kick_desc()
210 		EPVDBG(ep, " chunk: act=%d/%d chunk=%d last=%d desc=%d free=%d\n",  in ast_vhub_epn_kick_desc()
212 		       ast_vhub_count_free_descs(ep));  in ast_vhub_epn_kick_desc()
228 		if (req->last_desc >= 0 || !ast_vhub_count_free_descs(ep))  in ast_vhub_epn_kick_desc()
239 	writel(VHUB_EP_DMA_SET_CPU_WPTR(ep->epn.d_next),  in ast_vhub_epn_kick_desc()
240 	       ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_kick_desc()
242 	EPVDBG(ep, "HW kicked, d_next=%d dstat=%08x\n",  in ast_vhub_epn_kick_desc()
243 	       ep->epn.d_next, readl(ep->epn.regs + AST_VHUB_EP_DESC_STATUS));  in ast_vhub_epn_kick_desc()
246 static void ast_vhub_epn_handle_ack_desc(struct ast_vhub_ep *ep)  in ast_vhub_epn_handle_ack_desc()  argument
254 		stat = readl(ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_handle_ack_desc()
255 		stat1 = readl(ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_handle_ack_desc()
262 	req = list_first_entry_or_null(&ep->queue, struct ast_vhub_req, queue);  in ast_vhub_epn_handle_ack_desc()
264 	EPVDBG(ep, "ACK status=%08x is_in=%d ep->d_last=%d..%d\n",  in ast_vhub_epn_handle_ack_desc()
265 	       stat, ep->epn.is_in, ep->epn.d_last, d_last);  in ast_vhub_epn_handle_ack_desc()
268 	while (ep->epn.d_last != d_last) {  in ast_vhub_epn_handle_ack_desc()
274 		d_num = ep->epn.d_last;  in ast_vhub_epn_handle_ack_desc()
275 		desc = &ep->epn.descs[d_num];  in ast_vhub_epn_handle_ack_desc()
276 		ep->epn.d_last = (d_num + 1) & (AST_VHUB_DESCS_COUNT - 1);  in ast_vhub_epn_handle_ack_desc()
281 		EPVDBG(ep, " desc %d len=%d req=%p (act=%d)\n",  in ast_vhub_epn_handle_ack_desc()
293 		CHECK(ep, is_last_desc == (len < ep->ep.maxpacket ||  in ast_vhub_epn_handle_ack_desc()
299 		      req->req.zero, ep->ep.maxpacket);  in ast_vhub_epn_handle_ack_desc()
307 			CHECK(ep, d_last == ep->epn.d_last,  in ast_vhub_epn_handle_ack_desc()
309 			      d_last, ep->epn.d_last);  in ast_vhub_epn_handle_ack_desc()
312 			ast_vhub_done(ep, req, 0);  in ast_vhub_epn_handle_ack_desc()
313 			req = list_first_entry_or_null(&ep->queue,  in ast_vhub_epn_handle_ack_desc()
322 		ast_vhub_epn_kick_desc(ep, req);  in ast_vhub_epn_handle_ack_desc()
325 void ast_vhub_epn_ack_irq(struct ast_vhub_ep *ep)  in ast_vhub_epn_ack_irq()  argument
327 	if (ep->epn.desc_mode)  in ast_vhub_epn_ack_irq()
328 		ast_vhub_epn_handle_ack_desc(ep);  in ast_vhub_epn_ack_irq()
330 		ast_vhub_epn_handle_ack(ep);  in ast_vhub_epn_ack_irq()
337 	struct ast_vhub_ep *ep = to_ast_ep(u_ep);  in ast_vhub_epn_queue()  local
338 	struct ast_vhub *vhub = ep->vhub;  in ast_vhub_epn_queue()
354 	if (!ep->epn.enabled || !u_ep->desc || !ep->dev || !ep->d_idx ||  in ast_vhub_epn_queue()
355 	    !ep->dev->enabled) {  in ast_vhub_epn_queue()
356 		EPDBG(ep, "Enqueuing request on wrong or disabled EP\n");  in ast_vhub_epn_queue()
376 	if (ep->epn.desc_mode ||  in ast_vhub_epn_queue()
378 	     (ep->epn.is_in || !(u_req->length & (u_ep->maxpacket - 1))))) {  in ast_vhub_epn_queue()
380 					    ep->epn.is_in);  in ast_vhub_epn_queue()
389 	EPVDBG(ep, "enqueue req @%p\n", req);  in ast_vhub_epn_queue()
390 	EPVDBG(ep, " l=%d dma=0x%x zero=%d noshort=%d noirq=%d is_in=%d\n",  in ast_vhub_epn_queue()
393 	       ep->epn.is_in);  in ast_vhub_epn_queue()
402 	empty = list_empty(&ep->queue);  in ast_vhub_epn_queue()
405 	list_add_tail(&req->queue, &ep->queue);  in ast_vhub_epn_queue()
407 		if (ep->epn.desc_mode)  in ast_vhub_epn_queue()
408 			ast_vhub_epn_kick_desc(ep, req);  in ast_vhub_epn_queue()
410 			ast_vhub_epn_kick(ep, req);  in ast_vhub_epn_queue()
417 static void ast_vhub_stop_active_req(struct ast_vhub_ep *ep,  in ast_vhub_stop_active_req()  argument
423 	if (ep->epn.desc_mode)  in ast_vhub_stop_active_req()
424 		writel(VHUB_EP_DMA_CTRL_RESET, ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_stop_active_req()
426 		writel(0, ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_stop_active_req()
430 		state = readl(ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_stop_active_req()
438 		dev_warn(&ep->vhub->pdev->dev, "Timeout waiting for DMA\n");  in ast_vhub_stop_active_req()
445 	if (ep->epn.desc_mode) {  in ast_vhub_stop_active_req()
456 		reg = VHUB_EP_DMA_SET_RPTR(ep->epn.d_next) |  in ast_vhub_stop_active_req()
457 			VHUB_EP_DMA_SET_CPU_WPTR(ep->epn.d_next);  in ast_vhub_stop_active_req()
458 		writel(reg, ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_stop_active_req()
461 		writel(ep->epn.dma_conf,  in ast_vhub_stop_active_req()
462 		       ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_stop_active_req()
465 		writel(ep->epn.dma_conf,  in ast_vhub_stop_active_req()
466 		       ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_stop_active_req()
472 	struct ast_vhub_ep *ep = to_ast_ep(u_ep);  in ast_vhub_epn_dequeue()  local
473 	struct ast_vhub *vhub = ep->vhub;  in ast_vhub_epn_dequeue()
481 	list_for_each_entry (req, &ep->queue, queue) {  in ast_vhub_epn_dequeue()
487 		EPVDBG(ep, "dequeue req @%p active=%d\n",  in ast_vhub_epn_dequeue()
490 			ast_vhub_stop_active_req(ep, true);  in ast_vhub_epn_dequeue()
491 		ast_vhub_done(ep, req, -ECONNRESET);  in ast_vhub_epn_dequeue()
499 void ast_vhub_update_epn_stall(struct ast_vhub_ep *ep)  in ast_vhub_update_epn_stall()  argument
503 	if (WARN_ON(ep->d_idx == 0))  in ast_vhub_update_epn_stall()
505 	reg = readl(ep->epn.regs + AST_VHUB_EP_CONFIG);  in ast_vhub_update_epn_stall()
506 	if (ep->epn.stalled || ep->epn.wedged)  in ast_vhub_update_epn_stall()
510 	writel(reg, ep->epn.regs + AST_VHUB_EP_CONFIG);  in ast_vhub_update_epn_stall()
512 	if (!ep->epn.stalled && !ep->epn.wedged)  in ast_vhub_update_epn_stall()
513 		writel(VHUB_EP_TOGGLE_SET_EPNUM(ep->epn.g_idx),  in ast_vhub_update_epn_stall()
514 		       ep->vhub->regs + AST_VHUB_EP_TOGGLE);  in ast_vhub_update_epn_stall()
520 	struct ast_vhub_ep *ep = to_ast_ep(u_ep);  in ast_vhub_set_halt_and_wedge()  local
521 	struct ast_vhub *vhub = ep->vhub;  in ast_vhub_set_halt_and_wedge()
524 	EPDBG(ep, "Set halt (%d) & wedge (%d)\n", halt, wedge);  in ast_vhub_set_halt_and_wedge()
528 	if (ep->d_idx == 0)  in ast_vhub_set_halt_and_wedge()
530 	if (ep->epn.is_iso)  in ast_vhub_set_halt_and_wedge()
536 	if (halt && ep->epn.is_in && !list_empty(&ep->queue)) {  in ast_vhub_set_halt_and_wedge()
540 	ep->epn.stalled = halt;  in ast_vhub_set_halt_and_wedge()
541 	ep->epn.wedged = wedge;  in ast_vhub_set_halt_and_wedge()
542 	ast_vhub_update_epn_stall(ep);  in ast_vhub_set_halt_and_wedge()
561 	struct ast_vhub_ep *ep = to_ast_ep(u_ep);  in ast_vhub_epn_disable()  local
562 	struct ast_vhub *vhub = ep->vhub;  in ast_vhub_epn_disable()
566 	EPDBG(ep, "Disabling !\n");  in ast_vhub_epn_disable()
570 	ep->epn.enabled = false;  in ast_vhub_epn_disable()
573 	ast_vhub_stop_active_req(ep, false);  in ast_vhub_epn_disable()
576 	writel(0, ep->epn.regs + AST_VHUB_EP_CONFIG);  in ast_vhub_epn_disable()
579 	imask = VHUB_EP_IRQ(ep->epn.g_idx);  in ast_vhub_epn_disable()
586 	ast_vhub_nuke(ep, -ESHUTDOWN);  in ast_vhub_epn_disable()
589 	ep->ep.desc = NULL;  in ast_vhub_epn_disable()
599 	struct ast_vhub_ep *ep = to_ast_ep(u_ep);  in ast_vhub_epn_enable()  local
611 	if (!ep->d_idx || !ep->dev ||  in ast_vhub_epn_enable()
613 	    maxpacket == 0 || maxpacket > ep->ep.maxpacket) {  in ast_vhub_epn_enable()
614 		EPDBG(ep, "Invalid EP enable,d_idx=%d,dev=%p,type=%d,mp=%d/%d\n",  in ast_vhub_epn_enable()
615 		      ep->d_idx, ep->dev, desc->bDescriptorType,  in ast_vhub_epn_enable()
616 		      maxpacket, ep->ep.maxpacket);  in ast_vhub_epn_enable()
619 	if (ep->d_idx != usb_endpoint_num(desc)) {  in ast_vhub_epn_enable()
620 		EPDBG(ep, "EP number mismatch !\n");  in ast_vhub_epn_enable()
624 	if (ep->epn.enabled) {  in ast_vhub_epn_enable()
625 		EPDBG(ep, "Already enabled\n");  in ast_vhub_epn_enable()
628 	dev = ep->dev;  in ast_vhub_epn_enable()
629 	vhub = ep->vhub;  in ast_vhub_epn_enable()
633 		EPDBG(ep, "Bogus device state: driver=%p speed=%d\n",  in ast_vhub_epn_enable()
639 	ep->epn.is_in = usb_endpoint_dir_in(desc);  in ast_vhub_epn_enable()
640 	ep->ep.maxpacket = maxpacket;  in ast_vhub_epn_enable()
642 	ep->epn.d_next = ep->epn.d_last = 0;  in ast_vhub_epn_enable()
643 	ep->epn.is_iso = false;  in ast_vhub_epn_enable()
644 	ep->epn.stalled = false;  in ast_vhub_epn_enable()
645 	ep->epn.wedged = false;  in ast_vhub_epn_enable()
647 	EPDBG(ep, "Enabling [%s] %s num %d maxpacket=%d\n",  in ast_vhub_epn_enable()
648 	      ep->epn.is_in ? "in" : "out", usb_ep_type_string(type),  in ast_vhub_epn_enable()
652 	ep->epn.desc_mode = ep->epn.descs && ep->epn.is_in;  in ast_vhub_epn_enable()
653 	if (ep->epn.desc_mode)  in ast_vhub_epn_enable()
654 		memset(ep->epn.descs, 0, 8 * AST_VHUB_DESCS_COUNT);  in ast_vhub_epn_enable()
660 	ep->epn.chunk_max = ep->ep.maxpacket;  in ast_vhub_epn_enable()
661 	if (ep->epn.is_in) {  in ast_vhub_epn_enable()
662 		ep->epn.chunk_max <<= 3;  in ast_vhub_epn_enable()
663 		while (ep->epn.chunk_max > 4095)  in ast_vhub_epn_enable()
664 			ep->epn.chunk_max -= ep->ep.maxpacket;  in ast_vhub_epn_enable()
669 		EPDBG(ep, "Only one control endpoint\n");  in ast_vhub_epn_enable()
679 		ep->epn.is_iso = true;  in ast_vhub_epn_enable()
688 	if (!ep->epn.is_in)  in ast_vhub_epn_enable()
693 	EPVDBG(ep, "config=%08x\n", ep_conf);  in ast_vhub_epn_enable()
698 	writel(0, ep->epn.regs + AST_VHUB_EP_CONFIG);  in ast_vhub_epn_enable()
700 	       ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_epn_enable()
703 	writel(ep_conf, ep->epn.regs + AST_VHUB_EP_CONFIG);  in ast_vhub_epn_enable()
705 	if (ep->epn.desc_mode) {  in ast_vhub_epn_enable()
707 		writel(0, ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_enable()
710 		writel(ep->epn.descs_dma,  in ast_vhub_epn_enable()
711 		       ep->epn.regs + AST_VHUB_EP_DESC_BASE);  in ast_vhub_epn_enable()
714 		ep->epn.dma_conf = VHUB_EP_DMA_DESC_MODE;  in ast_vhub_epn_enable()
715 		if (ep->epn.is_in)  in ast_vhub_epn_enable()
716 			ep->epn.dma_conf |= VHUB_EP_DMA_IN_LONG_MODE;  in ast_vhub_epn_enable()
719 		writel(ep->epn.dma_conf | VHUB_EP_DMA_CTRL_RESET,  in ast_vhub_epn_enable()
720 		       ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_epn_enable()
723 		writel(ep->epn.dma_conf,  in ast_vhub_epn_enable()
724 		       ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_epn_enable()
727 		ep->epn.dma_conf = VHUB_EP_DMA_SINGLE_STAGE;  in ast_vhub_epn_enable()
730 		writel(ep->epn.dma_conf | VHUB_EP_DMA_CTRL_RESET,  in ast_vhub_epn_enable()
731 		       ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_epn_enable()
732 		writel(ep->epn.dma_conf,  in ast_vhub_epn_enable()
733 		       ep->epn.regs + AST_VHUB_EP_DMA_CTLSTAT);  in ast_vhub_epn_enable()
734 		writel(0, ep->epn.regs + AST_VHUB_EP_DESC_STATUS);  in ast_vhub_epn_enable()
738 	writel(VHUB_EP_TOGGLE_SET_EPNUM(ep->epn.g_idx),  in ast_vhub_epn_enable()
742 	imask = VHUB_EP_IRQ(ep->epn.g_idx);  in ast_vhub_epn_enable()
749 	ep->epn.enabled = true;  in ast_vhub_epn_enable()
758 	struct ast_vhub_ep *ep = to_ast_ep(u_ep);  in ast_vhub_epn_dispose()  local
760 	if (WARN_ON(!ep->dev || !ep->d_idx))  in ast_vhub_epn_dispose()
763 	EPDBG(ep, "Releasing endpoint\n");  in ast_vhub_epn_dispose()
766 	list_del_init(&ep->ep.ep_list);  in ast_vhub_epn_dispose()
769 	ep->dev->epns[ep->d_idx - 1] = NULL;  in ast_vhub_epn_dispose()
772 	kfree(ep->ep.name);  in ast_vhub_epn_dispose()
773 	ep->ep.name = NULL;  in ast_vhub_epn_dispose()
774 	dma_free_coherent(&ep->vhub->pdev->dev,  in ast_vhub_epn_dispose()
777 			  ep->buf, ep->buf_dma);  in ast_vhub_epn_dispose()
778 	ep->buf = NULL;  in ast_vhub_epn_dispose()
779 	ep->epn.descs = NULL;  in ast_vhub_epn_dispose()
782 	ep->dev = NULL;  in ast_vhub_epn_dispose()
800 	struct ast_vhub_ep *ep;  in ast_vhub_alloc_epn()  local
815 	ep = &vhub->epns[i];  in ast_vhub_alloc_epn()
816 	ep->dev = d;  in ast_vhub_alloc_epn()
820 	INIT_LIST_HEAD(&ep->queue);  in ast_vhub_alloc_epn()
821 	ep->d_idx = addr;  in ast_vhub_alloc_epn()
822 	ep->vhub = vhub;  in ast_vhub_alloc_epn()
823 	ep->ep.ops = &ast_vhub_epn_ops;  in ast_vhub_alloc_epn()
824 	ep->ep.name = kasprintf(GFP_KERNEL, "ep%d", addr);  in ast_vhub_alloc_epn()
825 	d->epns[addr-1] = ep;  in ast_vhub_alloc_epn()
826 	ep->epn.g_idx = i;  in ast_vhub_alloc_epn()
827 	ep->epn.regs = vhub->regs + 0x200 + (i * 0x10);  in ast_vhub_alloc_epn()
829 	ep->buf = dma_alloc_coherent(&vhub->pdev->dev,  in ast_vhub_alloc_epn()
832 				     &ep->buf_dma, GFP_KERNEL);  in ast_vhub_alloc_epn()
833 	if (!ep->buf) {  in ast_vhub_alloc_epn()
834 		kfree(ep->ep.name);  in ast_vhub_alloc_epn()
835 		ep->ep.name = NULL;  in ast_vhub_alloc_epn()
838 	ep->epn.descs = ep->buf + AST_VHUB_EPn_MAX_PACKET;  in ast_vhub_alloc_epn()
839 	ep->epn.descs_dma = ep->buf_dma + AST_VHUB_EPn_MAX_PACKET;  in ast_vhub_alloc_epn()
841 	usb_ep_set_maxpacket_limit(&ep->ep, AST_VHUB_EPn_MAX_PACKET);  in ast_vhub_alloc_epn()
842 	list_add_tail(&ep->ep.ep_list, &d->gadget.ep_list);  in ast_vhub_alloc_epn()
843 	ep->ep.caps.type_iso = true;  in ast_vhub_alloc_epn()
844 	ep->ep.caps.type_bulk = true;  in ast_vhub_alloc_epn()
845 	ep->ep.caps.type_int = true;  in ast_vhub_alloc_epn()
846 	ep->ep.caps.dir_in = true;  in ast_vhub_alloc_epn()
847 	ep->ep.caps.dir_out = true;  in ast_vhub_alloc_epn()
849 	return ep;  in ast_vhub_alloc_epn()