1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
3  * hcd_queue.c - DesignWare HS OTG Controller host queuing routines
5  * Copyright (C) 2004-2013 Synopsys, Inc.
16  * 3. The names of the above-listed copyright holders may not be used
47 #include <linux/dma-mapping.h>
65  * dwc2_periodic_channel_available() - Checks that a channel is available for a
77 	 * non-periodic transactions  in dwc2_periodic_channel_available()
82 	num_channels = hsotg->params.host_channels;  in dwc2_periodic_channel_available()
83 	if ((hsotg->periodic_channels + hsotg->non_periodic_channels <  in dwc2_periodic_channel_available()
84 	     num_channels) && (hsotg->periodic_channels < num_channels - 1)) {  in dwc2_periodic_channel_available()
87 		dev_dbg(hsotg->dev,  in dwc2_periodic_channel_available()
88 			"%s: Total channels: %d, Periodic: %d, Non-periodic: %d\n",  in dwc2_periodic_channel_available()
90 			hsotg->periodic_channels, hsotg->non_periodic_channels);  in dwc2_periodic_channel_available()
91 		status = -ENOSPC;  in dwc2_periodic_channel_available()
98  * dwc2_check_periodic_bandwidth() - Checks that there is sufficient bandwidth
107  * periodic schedule may occur in the same (micro)frame
117 	if (qh->dev_speed == USB_SPEED_HIGH || qh->do_split) {  in dwc2_check_periodic_bandwidth()
122 		max_claimed_usecs = 100 - qh->host_us;  in dwc2_check_periodic_bandwidth()
125 		 * Full speed mode  in dwc2_check_periodic_bandwidth()
128 		max_claimed_usecs = 900 - qh->host_us;  in dwc2_check_periodic_bandwidth()
131 	if (hsotg->periodic_usecs > max_claimed_usecs) {  in dwc2_check_periodic_bandwidth()
132 		dev_err(hsotg->dev,  in dwc2_check_periodic_bandwidth()
134 			__func__, hsotg->periodic_usecs, qh->host_us);  in dwc2_check_periodic_bandwidth()
135 		status = -ENOSPC;  in dwc2_check_periodic_bandwidth()
142  * pmap_schedule() - Schedule time in a periodic bitmap (pmap).
164  * want the same resource.  The resource is divided into fixed-sized periods
170  * but does mean that we need to handle things specially (and non-ideally) if
175  * - If interval is 1, we need to take up space in each of the 8
177  * - If interval is 2, we need to take up space in half of the
179  * - If interval is 3, we actually need to fall back to interval 1.
183  *   0, 3, and 6.  Since we could be in any frame we need to reserve
187  * - If interval is 4, easy.
188  * - If interval is 5, we again need interval 1.  The schedule will be
190  * - If interval is 6, we need interval 2.  0, 6, 4, 2.
191  * - If interval is 7, we need interval 1.
192  * - If interval is 8, we need interval 8.
197  * Note that at the moment this function tends to front-pack the schedule.
198  * In some cases that's really non-ideal (it's hard to schedule things that
229  * Returns either -ENOSPC or a >= 0 start bit which should be passed to the
230  * unschedule routine.  The map bitmap will be updated on a non-error result.
242 		return -ENOSPC;  in pmap_schedule()
252 		return -ENOSPC;  in pmap_schedule()
309 				start = ith_start - interval_bits * i;  in pmap_schedule()
319 		return -ENOSPC;  in pmap_schedule()
331  * pmap_unschedule() - Undo work done by pmap_schedule()
362  * dwc2_get_ls_map() - Get the map used for the given qh
368  * running the host straight in low speed / full speed mode it appears as if
370  * add logic here to get a map out of "hsotg" if !qh->do_split.
380 	if (WARN_ON(!qh->dwc_tt))  in dwc2_get_ls_map()
384 	map = qh->dwc_tt->periodic_bitmaps;  in dwc2_get_ls_map()
385 	if (qh->dwc_tt->usb_tt->multi)  in dwc2_get_ls_map()
386 		map += DWC2_ELEMENTS_PER_LS_BITMAP * (qh->ttport - 1);  in dwc2_get_ls_map()
393  * cat_printf() - A printf() + strcat() helper
418 		(*buf)[*size - 1] = '\0';  in cat_printf()
423 		*size -= i;  in cat_printf()
428  * pmap_print() - Print the given periodic map
465 					start = i - period_start;  in pmap_print()
481 			cat_printf(&buf, &buf_size, "%d %s -%3d %s", start,  in pmap_print()
482 				   units, start + count - 1, units);  in pmap_print()
497  * dwc2_qh_print() - Helper function for dwc2_qh_schedule_print()
506 	dwc2_sch_dbg(print_data->hsotg, "QH=%p ...%s\n", print_data->qh, str);  in dwc2_qh_print()
510  * dwc2_qh_schedule_print() - Print the periodic schedule
527 	if (qh->schedule_low_speed) {  in dwc2_qh_schedule_print()
531 			     qh, qh->device_us,  in dwc2_qh_schedule_print()
532 			     DWC2_ROUND_US_TO_SLICE(qh->device_us),  in dwc2_qh_schedule_print()
533 			     DWC2_US_PER_SLICE * qh->ls_start_schedule_slice);  in dwc2_qh_schedule_print()
537 				     "QH=%p Whole low/full speed map %p now:\n",  in dwc2_qh_schedule_print()
540 				   DWC2_LS_SCHEDULE_FRAMES, "Frame ", "slices",  in dwc2_qh_schedule_print()
545 	for (i = 0; i < qh->num_hs_transfers; i++) {  in dwc2_qh_schedule_print()
546 		struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + i;  in dwc2_qh_schedule_print()
547 		int uframe = trans_time->start_schedule_us /  in dwc2_qh_schedule_print()
549 		int rel_us = trans_time->start_schedule_us %  in dwc2_qh_schedule_print()
554 			     qh, i, trans_time->duration_us, uframe, rel_us);  in dwc2_qh_schedule_print()
556 	if (qh->num_hs_transfers) {  in dwc2_qh_schedule_print()
558 		pmap_print(hsotg->hs_periodic_bitmap,  in dwc2_qh_schedule_print()
570  * dwc2_ls_pmap_schedule() - Schedule a low speed QH
587 	int slices = DIV_ROUND_UP(qh->device_us, DWC2_US_PER_SLICE);  in dwc2_ls_pmap_schedule()
592 		return -EINVAL;  in dwc2_ls_pmap_schedule()
608 			      qh->device_interval, search_slice, false);  in dwc2_ls_pmap_schedule()
613 	qh->ls_start_schedule_slice = slice;  in dwc2_ls_pmap_schedule()
618  * dwc2_ls_pmap_unschedule() - Undo work done by dwc2_ls_pmap_schedule()
626 	int slices = DIV_ROUND_UP(qh->device_us, DWC2_US_PER_SLICE);  in dwc2_ls_pmap_unschedule()
634 			DWC2_LS_SCHEDULE_FRAMES, slices, qh->device_interval,  in dwc2_ls_pmap_unschedule()
635 			qh->ls_start_schedule_slice);  in dwc2_ls_pmap_unschedule()
639  * dwc2_hs_pmap_schedule - Schedule in the main high speed schedule
643  * We'll start looking in qh->hs_transfers[index].start_schedule_us.  We'll
653  * @index:           The index into qh->hs_transfers that we're working with.
661 	struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + index;  in dwc2_hs_pmap_schedule()
664 	us = pmap_schedule(hsotg->hs_periodic_bitmap,  in dwc2_hs_pmap_schedule()
666 			   DWC2_HS_SCHEDULE_UFRAMES, trans_time->duration_us,  in dwc2_hs_pmap_schedule()
667 			   qh->host_interval, trans_time->start_schedule_us,  in dwc2_hs_pmap_schedule()
673 	trans_time->start_schedule_us = us;  in dwc2_hs_pmap_schedule()
678  * dwc2_ls_pmap_unschedule() - Undo work done by dwc2_hs_pmap_schedule()
687 	struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + index;  in dwc2_hs_pmap_unschedule()
689 	pmap_unschedule(hsotg->hs_periodic_bitmap,  in dwc2_hs_pmap_unschedule()
691 			DWC2_HS_SCHEDULE_UFRAMES, trans_time->duration_us,  in dwc2_hs_pmap_unschedule()
692 			qh->host_interval, trans_time->start_schedule_us);  in dwc2_hs_pmap_unschedule()
696  * dwc2_uframe_schedule_split - Schedule a QH for a periodic split xfer.
703  * device is in low or full speed mode (and behind a hub).
711 	int bytecount = qh->maxp_mult * qh->maxp;  in dwc2_uframe_schedule_split()
720 	host_interval_in_sched = gcd(qh->host_interval,  in dwc2_uframe_schedule_split()
730 	 * Note that this will tend to front-load the high speed schedule.  in dwc2_uframe_schedule_split()
735 	 * to transfer SSPLIT-begin OUT transaction like EHCI controller.  in dwc2_uframe_schedule_split()
737 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC && !qh->ep_is_in)  in dwc2_uframe_schedule_split()
754 		if (qh->schedule_low_speed) {  in dwc2_uframe_schedule_split()
771 		 * This will give us a number 0 - 7 if  in dwc2_uframe_schedule_split()
772 		 * DWC2_LS_SCHEDULE_FRAMES == 1, or 0 - 15 if == 2, or ...  in dwc2_uframe_schedule_split()
774 		start_s_uframe = qh->ls_start_schedule_slice /  in dwc2_uframe_schedule_split()
777 		/* Get a number that's always 0 - 7 */  in dwc2_uframe_schedule_split()
783 		 * Move on to the next full frame (assuming there is one).  in dwc2_uframe_schedule_split()
789 			if (qh->schedule_low_speed)  in dwc2_uframe_schedule_split()
792 				(qh->ls_start_schedule_slice /  in dwc2_uframe_schedule_split()
800 		 * - start split            (frame -1)  in dwc2_uframe_schedule_split()
801 		 * - complete split w/ data (frame +1)  in dwc2_uframe_schedule_split()
802 		 * - complete split w/ data (frame +2)  in dwc2_uframe_schedule_split()
803 		 * - ...  in dwc2_uframe_schedule_split()
804 		 * - complete split w/ data (frame +num_data_packets)  in dwc2_uframe_schedule_split()
805 		 * - complete split w/ data (frame +num_data_packets+1)  in dwc2_uframe_schedule_split()
806 		 * - complete split w/ data (frame +num_data_packets+2, max 8)  in dwc2_uframe_schedule_split()
807 		 *   ...though if frame was "0" then max is 7...  in dwc2_uframe_schedule_split()
810 		 * - start split w/ data    (frame -1)  in dwc2_uframe_schedule_split()
811 		 * - start split w/ data    (frame +0)  in dwc2_uframe_schedule_split()
812 		 * - ...  in dwc2_uframe_schedule_split()
813 		 * - start split w/ data    (frame +num_data_packets-2)  in dwc2_uframe_schedule_split()
816 		 * - start split            (frame -1)  in dwc2_uframe_schedule_split()
817 		 * - complete split w/ data (frame +1)  in dwc2_uframe_schedule_split()
818 		 * - complete split w/ data (frame +2)  in dwc2_uframe_schedule_split()
819 		 * - complete split w/ data (frame +3, max 8)  in dwc2_uframe_schedule_split()
822 		 * - start split w/ data    (frame -1)  in dwc2_uframe_schedule_split()
823 		 * - complete split         (frame +1)  in dwc2_uframe_schedule_split()
824 		 * - complete split         (frame +2)  in dwc2_uframe_schedule_split()
825 		 * - complete split         (frame +3, max 8)  in dwc2_uframe_schedule_split()
830 				   host_interval_in_sched - 1) %  in dwc2_uframe_schedule_split()
832 		if (qh->ep_type == USB_ENDPOINT_XFER_ISOC && !qh->ep_is_in)  in dwc2_uframe_schedule_split()
838 		first_data_bytes = 188 -  in dwc2_uframe_schedule_split()
839 			DIV_ROUND_UP(188 * (qh->ls_start_schedule_slice %  in dwc2_uframe_schedule_split()
844 		other_data_bytes = bytecount - first_data_bytes;  in dwc2_uframe_schedule_split()
850 		 * - INT transfers never get split in two.  in dwc2_uframe_schedule_split()
851 		 * - ISOC transfers can always transfer 188 bytes the first  in dwc2_uframe_schedule_split()
861 		if (!qh->ep_is_in &&  in dwc2_uframe_schedule_split()
866 			if (qh->schedule_low_speed)  in dwc2_uframe_schedule_split()
874 		qh->num_hs_transfers = 1 + DIV_ROUND_UP(other_data_bytes, 188);  in dwc2_uframe_schedule_split()
881 		if (qh->ep_type == USB_ENDPOINT_XFER_INT) {  in dwc2_uframe_schedule_split()
883 				qh->num_hs_transfers += 2;  in dwc2_uframe_schedule_split()
885 				qh->num_hs_transfers += 3;  in dwc2_uframe_schedule_split()
887 			if (qh->ep_is_in) {  in dwc2_uframe_schedule_split()
890 				 * Allocate full data bytes for all data.  in dwc2_uframe_schedule_split()
906 			if (qh->ep_is_in) {  in dwc2_uframe_schedule_split()
910 				qh->num_hs_transfers++;  in dwc2_uframe_schedule_split()
913 				last = rel_uframe + qh->num_hs_transfers + 1;  in dwc2_uframe_schedule_split()
917 					qh->num_hs_transfers += 2;  in dwc2_uframe_schedule_split()
919 					qh->num_hs_transfers += 1;  in dwc2_uframe_schedule_split()
923 					qh->num_hs_transfers--;  in dwc2_uframe_schedule_split()
939 		qh->hs_transfers[0].duration_us = HS_USECS_ISO(first_count);  in dwc2_uframe_schedule_split()
940 		for (i = 1; i < qh->num_hs_transfers - 1; i++)  in dwc2_uframe_schedule_split()
941 			qh->hs_transfers[i].duration_us =  in dwc2_uframe_schedule_split()
943 		if (qh->num_hs_transfers > 1)  in dwc2_uframe_schedule_split()
944 			qh->hs_transfers[qh->num_hs_transfers - 1].duration_us =  in dwc2_uframe_schedule_split()
952 		qh->hs_transfers[0].start_schedule_us =  in dwc2_uframe_schedule_split()
954 		for (i = 1; i < qh->num_hs_transfers; i++)  in dwc2_uframe_schedule_split()
955 			qh->hs_transfers[i].start_schedule_us =  in dwc2_uframe_schedule_split()
956 				((second_s_uframe + i - 1) %  in dwc2_uframe_schedule_split()
961 		for (i = 0; i < qh->num_hs_transfers; i++) {  in dwc2_uframe_schedule_split()
968 		if (i == qh->num_hs_transfers)  in dwc2_uframe_schedule_split()
971 		for (; i >= 0; i--)  in dwc2_uframe_schedule_split()
974 		if (qh->schedule_low_speed)  in dwc2_uframe_schedule_split()
982 		return -ENOSPC;  in dwc2_uframe_schedule_split()
988  * dwc2_uframe_schedule_hs - Schedule a QH for a periodic high speed xfer.
998 	/* In non-split host and device time are the same */  in dwc2_uframe_schedule_hs()
999 	WARN_ON(qh->host_us != qh->device_us);  in dwc2_uframe_schedule_hs()
1000 	WARN_ON(qh->host_interval != qh->device_interval);  in dwc2_uframe_schedule_hs()
1001 	WARN_ON(qh->num_hs_transfers != 1);  in dwc2_uframe_schedule_hs()
1004 	qh->hs_transfers[0].start_schedule_us = 0;  in dwc2_uframe_schedule_hs()
1005 	qh->hs_transfers[0].duration_us = qh->host_us;  in dwc2_uframe_schedule_hs()
1011  * dwc2_uframe_schedule_ls - Schedule a QH for a periodic low/full speed xfer.
1021 	/* In non-split host and device time are the same */  in dwc2_uframe_schedule_ls()
1022 	WARN_ON(qh->host_us != qh->device_us);  in dwc2_uframe_schedule_ls()
1023 	WARN_ON(qh->host_interval != qh->device_interval);  in dwc2_uframe_schedule_ls()
1024 	WARN_ON(!qh->schedule_low_speed);  in dwc2_uframe_schedule_ls()
1031  * dwc2_uframe_schedule - Schedule a QH for a periodic xfer.
1033  * Calls one of the 3 sub-function depending on what type of transfer this QH
1043 	if (qh->dev_speed == USB_SPEED_HIGH)  in dwc2_uframe_schedule()
1045 	else if (!qh->do_split)  in dwc2_uframe_schedule()
1059  * dwc2_uframe_unschedule - Undoes dwc2_uframe_schedule().
1068 	for (i = 0; i < qh->num_hs_transfers; i++)  in dwc2_uframe_unschedule()
1071 	if (qh->schedule_low_speed)  in dwc2_uframe_unschedule()
1078  * dwc2_pick_first_frame() - Choose 1st frame for qh that's already scheduled
1085  * running.  It just picks the next frame that we can fit into without any
1101 	 * Use the real frame number rather than the cached value as of the  in dwc2_pick_first_frame()
1107 	 * We wouldn't want to start any earlier than the next frame just in  in dwc2_pick_first_frame()
1108 	 * case the frame number ticks as we're doing this calculation.  in dwc2_pick_first_frame()
1118 	if (!hsotg->params.uframe_sched) {  in dwc2_pick_first_frame()
1119 		if (qh->do_split)  in dwc2_pick_first_frame()
1125 	if (qh->dev_speed == USB_SPEED_HIGH || qh->do_split) {  in dwc2_pick_first_frame()
1129 		 * the first frame should be based on when the first scheduled  in dwc2_pick_first_frame()
1132 		WARN_ON(qh->num_hs_transfers < 1);  in dwc2_pick_first_frame()
1134 		relative_frame = qh->hs_transfers[0].start_schedule_us /  in dwc2_pick_first_frame()
1138 		interval = gcd(qh->host_interval, DWC2_HS_SCHEDULE_UFRAMES);  in dwc2_pick_first_frame()
1142 		 * Low or full speed directly on dwc2.  Just about the same  in dwc2_pick_first_frame()
1148 		relative_frame = qh->ls_start_schedule_slice /  in dwc2_pick_first_frame()
1150 		interval = gcd(qh->host_interval, DWC2_LS_SCHEDULE_FRAMES);  in dwc2_pick_first_frame()
1153 	/* Scheduler messed up if frame is past interval */  in dwc2_pick_first_frame()
1167 	 * Actually choose to start at the frame number we've been  in dwc2_pick_first_frame()
1174 	 * We actually need 1 frame before since the next_active_frame is  in dwc2_pick_first_frame()
1175 	 * the frame number we'll be put on the ready list and we won't be on  in dwc2_pick_first_frame()
1176 	 * the bus until 1 frame later.  in dwc2_pick_first_frame()
1189 	qh->next_active_frame = next_active_frame;  in dwc2_pick_first_frame()
1190 	qh->start_active_frame = next_active_frame;  in dwc2_pick_first_frame()
1193 		      qh, frame_number, qh->next_active_frame);  in dwc2_pick_first_frame()
1197  * dwc2_do_reserve() - Make a periodic reservation
1211 	if (hsotg->params.uframe_sched) {  in dwc2_do_reserve()
1216 			dev_info(hsotg->dev,  in dwc2_do_reserve()
1226 		dev_dbg(hsotg->dev,  in dwc2_do_reserve()
1232 	if (!hsotg->params.uframe_sched)  in dwc2_do_reserve()
1234 		hsotg->periodic_channels++;  in dwc2_do_reserve()
1236 	/* Update claimed usecs per (micro)frame */  in dwc2_do_reserve()
1237 	hsotg->periodic_usecs += qh->host_us;  in dwc2_do_reserve()
1245  * dwc2_do_unreserve() - Actually release the periodic reservation
1255 	assert_spin_locked(&hsotg->lock);  in dwc2_do_unreserve()
1257 	WARN_ON(!qh->unreserve_pending);  in dwc2_do_unreserve()
1259 	/* No more unreserve pending--we're doing it */  in dwc2_do_unreserve()
1260 	qh->unreserve_pending = false;  in dwc2_do_unreserve()
1262 	if (WARN_ON(!list_empty(&qh->qh_list_entry)))  in dwc2_do_unreserve()
1263 		list_del_init(&qh->qh_list_entry);  in dwc2_do_unreserve()
1265 	/* Update claimed usecs per (micro)frame */  in dwc2_do_unreserve()
1266 	hsotg->periodic_usecs -= qh->host_us;  in dwc2_do_unreserve()
1268 	if (hsotg->params.uframe_sched) {  in dwc2_do_unreserve()
1272 		hsotg->periodic_channels--;  in dwc2_do_unreserve()
1277  * dwc2_unreserve_timer_fn() - Timer function to release periodic reservation
1291 	struct dwc2_hsotg *hsotg = qh->hsotg;  in dwc2_unreserve_timer_fn()
1297 	 * - We started executing but didn't get the lock yet.  in dwc2_unreserve_timer_fn()
1298 	 * - A new reservation came in, but cancel didn't take effect  in dwc2_unreserve_timer_fn()
1300 	 * - The timer has been kicked again.  in dwc2_unreserve_timer_fn()
1303 	while (!spin_trylock_irqsave(&hsotg->lock, flags)) {  in dwc2_unreserve_timer_fn()
1304 		if (timer_pending(&qh->unreserve_timer))  in dwc2_unreserve_timer_fn()
1310 	 * - We started executing but didn't get the lock yet.  in dwc2_unreserve_timer_fn()
1311 	 * - A new reservation came in, but cancel didn't take effect  in dwc2_unreserve_timer_fn()
1318 	if (qh->unreserve_pending)  in dwc2_unreserve_timer_fn()
1321 	spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_unreserve_timer_fn()
1325  * dwc2_check_max_xfer_size() - Checks that the max transfer size allowed in a
1327  * (micro)frame for a periodic transfer
1341 	max_xfer_size = qh->maxp * qh->maxp_mult;  in dwc2_check_max_xfer_size()
1342 	max_channel_xfer_size = hsotg->params.max_transfer_size;  in dwc2_check_max_xfer_size()
1345 		dev_err(hsotg->dev,  in dwc2_check_max_xfer_size()
1348 		status = -ENOSPC;  in dwc2_check_max_xfer_size()
1355  * dwc2_schedule_periodic() - Schedules an interrupt or isochronous transfer in
1370 		dev_dbg(hsotg->dev,  in dwc2_schedule_periodic()
1377 	if (del_timer(&qh->unreserve_timer))  in dwc2_schedule_periodic()
1378 		WARN_ON(!qh->unreserve_pending);  in dwc2_schedule_periodic()
1387 	if (!qh->unreserve_pending) {  in dwc2_schedule_periodic()
1398 		if (dwc2_frame_num_le(qh->next_active_frame,  in dwc2_schedule_periodic()
1399 				      hsotg->frame_number))  in dwc2_schedule_periodic()
1403 	qh->unreserve_pending = 0;  in dwc2_schedule_periodic()
1405 	if (hsotg->params.dma_desc_enable)  in dwc2_schedule_periodic()
1407 		list_add_tail(&qh->qh_list_entry, &hsotg->periodic_sched_ready);  in dwc2_schedule_periodic()
1410 		list_add_tail(&qh->qh_list_entry,  in dwc2_schedule_periodic()
1411 			      &hsotg->periodic_sched_inactive);  in dwc2_schedule_periodic()
1417  * dwc2_deschedule_periodic() - Removes an interrupt or isochronous transfer
1428 	assert_spin_locked(&hsotg->lock);  in dwc2_deschedule_periodic()
1432 	 * - Unreserve worker might be sitting there waiting to grab the lock.  in dwc2_deschedule_periodic()
1435 	 * - Unreserve worker might not be scheduled.  in dwc2_deschedule_periodic()
1445 	did_modify = mod_timer(&qh->unreserve_timer,  in dwc2_deschedule_periodic()
1448 	qh->unreserve_pending = 1;  in dwc2_deschedule_periodic()
1450 	list_del_init(&qh->qh_list_entry);  in dwc2_deschedule_periodic()
1454  * dwc2_wait_timer_fn() - Timer function to re-queue after waiting
1479 	struct dwc2_hsotg *hsotg = qh->hsotg;  in dwc2_wait_timer_fn()
1482 	spin_lock_irqsave(&hsotg->lock, flags);  in dwc2_wait_timer_fn()
1488 	if (!qh->wait_timer_cancel) {  in dwc2_wait_timer_fn()
1491 		qh->want_wait = false;  in dwc2_wait_timer_fn()
1493 		list_move(&qh->qh_list_entry,  in dwc2_wait_timer_fn()
1494 			  &hsotg->non_periodic_sched_inactive);  in dwc2_wait_timer_fn()
1501 	spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_wait_timer_fn()
1506  * dwc2_qh_init() - Initializes a QH structure
1517 	int dev_speed = dwc2_host_get_speed(hsotg, urb->priv);  in dwc2_qh_init()
1518 	u8 ep_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);  in dwc2_qh_init()
1519 	bool ep_is_in = !!dwc2_hcd_is_pipe_in(&urb->pipe_info);  in dwc2_qh_init()
1526 	int maxp = dwc2_hcd_get_maxp(&urb->pipe_info);  in dwc2_qh_init()
1527 	int maxp_mult = dwc2_hcd_get_maxp_mult(&urb->pipe_info);  in dwc2_qh_init()
1532 	qh->hsotg = hsotg;  in dwc2_qh_init()
1533 	timer_setup(&qh->unreserve_timer, dwc2_unreserve_timer_fn, 0);  in dwc2_qh_init()
1534 	hrtimer_init(&qh->wait_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in dwc2_qh_init()
1535 	qh->wait_timer.function = &dwc2_wait_timer_fn;  in dwc2_qh_init()
1536 	qh->ep_type = ep_type;  in dwc2_qh_init()
1537 	qh->ep_is_in = ep_is_in;  in dwc2_qh_init()
1539 	qh->data_toggle = DWC2_HC_PID_DATA0;  in dwc2_qh_init()
1540 	qh->maxp = maxp;  in dwc2_qh_init()
1541 	qh->maxp_mult = maxp_mult;  in dwc2_qh_init()
1542 	INIT_LIST_HEAD(&qh->qtd_list);  in dwc2_qh_init()
1543 	INIT_LIST_HEAD(&qh->qh_list_entry);  in dwc2_qh_init()
1545 	qh->do_split = do_split;  in dwc2_qh_init()
1546 	qh->dev_speed = dev_speed;  in dwc2_qh_init()
1551 		struct dwc2_tt *dwc_tt = dwc2_host_get_tt_info(hsotg, urb->priv,  in dwc2_qh_init()
1553 							       &qh->ttport);  in dwc2_qh_init()
1556 		qh->dwc_tt = dwc_tt;  in dwc2_qh_init()
1558 		qh->host_us = NS_TO_US(usb_calc_bus_time(host_speed, ep_is_in,  in dwc2_qh_init()
1564 			device_ns += dwc_tt->usb_tt->think_time;  in dwc2_qh_init()
1565 		qh->device_us = NS_TO_US(device_ns);  in dwc2_qh_init()
1567 		qh->device_interval = urb->interval;  in dwc2_qh_init()
1568 		qh->host_interval = urb->interval * (do_split ? 8 : 1);  in dwc2_qh_init()
1572 		 * full speed OR if we've got a "TT" to deal with to access this  in dwc2_qh_init()
1575 		qh->schedule_low_speed = prtspd != HPRT0_SPD_HIGH_SPEED ||  in dwc2_qh_init()
1580 			qh->num_hs_transfers = -1;  in dwc2_qh_init()
1582 			qh->num_hs_transfers = 1;  in dwc2_qh_init()
1584 			qh->num_hs_transfers = 0;  in dwc2_qh_init()
1595 		speed = "full";  in dwc2_qh_init()
1605 	switch (qh->ep_type) {  in dwc2_qh_init()
1626 		     dwc2_hcd_get_dev_addr(&urb->pipe_info),  in dwc2_qh_init()
1627 		     dwc2_hcd_get_ep_num(&urb->pipe_info),  in dwc2_qh_init()
1632 			     qh, qh->host_us, qh->device_us);  in dwc2_qh_init()
1634 			     qh, qh->host_interval, qh->device_interval);  in dwc2_qh_init()
1635 		if (qh->schedule_low_speed)  in dwc2_qh_init()
1642  * dwc2_hcd_qh_create() - Allocates and initializes a QH
1657 	if (!urb->priv)  in dwc2_hcd_qh_create()
1667 	if (hsotg->params.dma_desc_enable &&  in dwc2_hcd_qh_create()
1677  * dwc2_hcd_qh_free() - Frees the QH
1690 	if (del_timer_sync(&qh->unreserve_timer)) {  in dwc2_hcd_qh_free()
1693 		spin_lock_irqsave(&hsotg->lock, flags);  in dwc2_hcd_qh_free()
1695 		spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_hcd_qh_free()
1705 	hrtimer_cancel(&qh->wait_timer);  in dwc2_hcd_qh_free()
1707 	dwc2_host_put_tt_info(hsotg, qh->dwc_tt);  in dwc2_hcd_qh_free()
1709 	if (qh->desc_list)  in dwc2_hcd_qh_free()
1711 	else if (hsotg->unaligned_cache && qh->dw_align_buf)  in dwc2_hcd_qh_free()
1712 		kmem_cache_free(hsotg->unaligned_cache, qh->dw_align_buf);  in dwc2_hcd_qh_free()
1718  * dwc2_hcd_qh_add() - Adds a QH to either the non periodic or periodic
1734 		dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_add()
1736 	if (!list_empty(&qh->qh_list_entry))  in dwc2_hcd_qh_add()
1743 		qh->start_active_frame = hsotg->frame_number;  in dwc2_hcd_qh_add()
1744 		qh->next_active_frame = qh->start_active_frame;  in dwc2_hcd_qh_add()
1746 		if (qh->want_wait) {  in dwc2_hcd_qh_add()
1747 			list_add_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_add()
1748 				      &hsotg->non_periodic_sched_waiting);  in dwc2_hcd_qh_add()
1749 			qh->wait_timer_cancel = false;  in dwc2_hcd_qh_add()
1751 			hrtimer_start(&qh->wait_timer, delay, HRTIMER_MODE_REL);  in dwc2_hcd_qh_add()
1753 			list_add_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_add()
1754 				      &hsotg->non_periodic_sched_inactive);  in dwc2_hcd_qh_add()
1762 	if (!hsotg->periodic_qh_count) {  in dwc2_hcd_qh_add()
1767 	hsotg->periodic_qh_count++;  in dwc2_hcd_qh_add()
1773  * dwc2_hcd_qh_unlink() - Removes a QH from either the non-periodic or periodic
1783 	dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_unlink()
1786 	qh->wait_timer_cancel = true;  in dwc2_hcd_qh_unlink()
1788 	if (list_empty(&qh->qh_list_entry))  in dwc2_hcd_qh_unlink()
1793 		if (hsotg->non_periodic_qh_ptr == &qh->qh_list_entry)  in dwc2_hcd_qh_unlink()
1794 			hsotg->non_periodic_qh_ptr =  in dwc2_hcd_qh_unlink()
1795 					hsotg->non_periodic_qh_ptr->next;  in dwc2_hcd_qh_unlink()
1796 		list_del_init(&qh->qh_list_entry);  in dwc2_hcd_qh_unlink()
1801 	hsotg->periodic_qh_count--;  in dwc2_hcd_qh_unlink()
1802 	if (!hsotg->periodic_qh_count &&  in dwc2_hcd_qh_unlink()
1803 	    !hsotg->params.dma_desc_enable) {  in dwc2_hcd_qh_unlink()
1811  * dwc2_next_for_periodic_split() - Set next_active_frame midway thru a split.
1816  * Periodic splits are single low/full speed transfers that we end up splitting
1817  * up into several high speed transfers.  They always fit into one full (1 ms)
1818  * frame but might be split over several microframes (125 us each).  We to put
1819  * each of the parts on a very specific high speed frame.
1825  * @frame_number: The current frame number.
1832 	u16 old_frame = qh->next_active_frame;  in dwc2_next_for_periodic_split()
1843 	if (old_frame == qh->start_active_frame &&  in dwc2_next_for_periodic_split()
1844 	    !(qh->ep_type == USB_ENDPOINT_XFER_ISOC && !qh->ep_is_in))  in dwc2_next_for_periodic_split()
1849 	qh->next_active_frame = dwc2_frame_num_inc(old_frame, incr);  in dwc2_next_for_periodic_split()
1852 	 * Note that it's OK for frame_number to be 1 frame past  in dwc2_next_for_periodic_split()
1854 	 * be 1 frame _before_ when we want to be scheduled.  If we're 1 frame  in dwc2_next_for_periodic_split()
1857 	 * It's _not_ OK, however, if we're more than one frame past.  in dwc2_next_for_periodic_split()
1859 	if (dwc2_frame_num_gt(prev_frame_number, qh->next_active_frame)) {  in dwc2_next_for_periodic_split()
1865 					    qh->next_active_frame);  in dwc2_next_for_periodic_split()
1866 		qh->next_active_frame = frame_number;  in dwc2_next_for_periodic_split()
1873  * dwc2_next_periodic_start() - Set next_active_frame for next transfer start
1888  * @frame_number: The current frame number.
1896 	u16 interval = qh->host_interval;  in dwc2_next_periodic_start()
1899 	qh->start_active_frame = dwc2_frame_num_inc(qh->start_active_frame,  in dwc2_next_periodic_start()
1905 	 * frame counter only goes to 0x3fff.  It's terribly unlikely that we  in dwc2_next_periodic_start()
1917 	 * - We compare against prev_frame_number since start_active_frame  in dwc2_next_periodic_start()
1918 	 *   and next_active_frame are always 1 frame before we want things  in dwc2_next_periodic_start()
1920 	 *   current frame number.  in dwc2_next_periodic_start()
1921 	 * - It's possible for start_active_frame (now incremented) to be  in dwc2_next_periodic_start()
1926 	 *   another transfer for the same frame.  My test webcam doesn't seem  in dwc2_next_periodic_start()
1928 	 *   we do two transfers in the same frame.  in dwc2_next_periodic_start()
1929 	 * - Some misses are expected.  Specifically, in order to work  in dwc2_next_periodic_start()
1939 	if (qh->start_active_frame == qh->next_active_frame ||  in dwc2_next_periodic_start()
1940 	    dwc2_frame_num_gt(prev_frame_number, qh->start_active_frame)) {  in dwc2_next_periodic_start()
1941 		u16 ideal_start = qh->start_active_frame;  in dwc2_next_periodic_start()
1948 		if (qh->do_split || qh->dev_speed == USB_SPEED_HIGH)  in dwc2_next_periodic_start()
1955 			qh->start_active_frame = dwc2_frame_num_inc(  in dwc2_next_periodic_start()
1956 				qh->start_active_frame, interval);  in dwc2_next_periodic_start()
1958 					   qh->start_active_frame));  in dwc2_next_periodic_start()
1960 		missed = dwc2_frame_num_dec(qh->start_active_frame,  in dwc2_next_periodic_start()
1965 	qh->next_active_frame = qh->start_active_frame;  in dwc2_next_periodic_start()
1971  * Deactivates a QH. For non-periodic QHs, removes the QH from the active
1972  * non-periodic schedule. The QH is added to the inactive non-periodic
1978  * scheduled frame is calculated. The QH is placed in the ready schedule if
1979  * the scheduled frame has been reached already. Otherwise it's placed in the
1986 	u16 old_frame = qh->next_active_frame;  in dwc2_hcd_qh_deactivate()
1991 		dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_deactivate()
1995 		if (!list_empty(&qh->qtd_list))  in dwc2_hcd_qh_deactivate()
1996 			/* Add back to inactive/waiting non-periodic schedule */  in dwc2_hcd_qh_deactivate()
2002 	 * Use the real frame number rather than the cached value as of the  in dwc2_hcd_qh_deactivate()
2005 	 * interrupt for this frame.  in dwc2_hcd_qh_deactivate()
2017 		     qh->next_active_frame,  in dwc2_hcd_qh_deactivate()
2018 		     dwc2_frame_num_dec(qh->next_active_frame, old_frame),  in dwc2_hcd_qh_deactivate()
2021 	if (list_empty(&qh->qtd_list)) {  in dwc2_hcd_qh_deactivate()
2033 	if (dwc2_frame_num_le(qh->next_active_frame, hsotg->frame_number))  in dwc2_hcd_qh_deactivate()
2034 		list_move_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_deactivate()
2035 			       &hsotg->periodic_sched_ready);  in dwc2_hcd_qh_deactivate()
2037 		list_move_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_deactivate()
2038 			       &hsotg->periodic_sched_inactive);  in dwc2_hcd_qh_deactivate()
2042  * dwc2_hcd_qtd_init() - Initializes a QTD structure
2049 	qtd->urb = urb;  in dwc2_hcd_qtd_init()
2050 	if (dwc2_hcd_get_pipe_type(&urb->pipe_info) ==  in dwc2_hcd_qtd_init()
2057 		qtd->data_toggle = DWC2_HC_PID_DATA1;  in dwc2_hcd_qtd_init()
2058 		qtd->control_phase = DWC2_CONTROL_SETUP;  in dwc2_hcd_qtd_init()
2062 	qtd->complete_split = 0;  in dwc2_hcd_qtd_init()
2063 	qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;  in dwc2_hcd_qtd_init()
2064 	qtd->isoc_split_offset = 0;  in dwc2_hcd_qtd_init()
2065 	qtd->in_process = 0;  in dwc2_hcd_qtd_init()
2068 	urb->qtd = qtd;  in dwc2_hcd_qtd_init()
2072  * dwc2_hcd_qtd_add() - Adds a QTD to the QTD-list of a QH
2090 		dev_err(hsotg->dev, "%s: Invalid QH\n", __func__);  in dwc2_hcd_qtd_add()
2091 		retval = -EINVAL;  in dwc2_hcd_qtd_add()
2099 	qtd->qh = qh;  in dwc2_hcd_qtd_add()
2100 	list_add_tail(&qtd->qtd_list_entry, &qh->qtd_list);  in dwc2_hcd_qtd_add()