Lines Matching full:xhci
3 * xHCI host controller driver
17 #include "xhci.h"
18 #include "xhci-trace.h"
19 #include "xhci-debugfs.h"
28 static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, in xhci_segment_alloc() argument
36 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_segment_alloc()
42 seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma); in xhci_segment_alloc()
52 dma_pool_free(xhci->segment_pool, seg->trbs, dma); in xhci_segment_alloc()
68 void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) in xhci_segment_free() argument
71 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma); in xhci_segment_free()
79 void xhci_free_segments_for_ring(struct xhci_hcd *xhci, in xhci_free_segments_for_ring() argument
87 xhci_segment_free(xhci, seg); in xhci_free_segments_for_ring()
90 xhci_segment_free(xhci, first); in xhci_free_segments_for_ring()
128 static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_link_rings() argument
139 chain_links = !!(xhci_link_trb_quirk(xhci) || in xhci_link_rings()
141 (xhci->quirks & XHCI_AMD_0x96_HOST))); in xhci_link_rings()
282 void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_ring_free() argument
292 xhci_free_segments_for_ring(xhci, ring->first_seg); in xhci_ring_free()
325 static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, in xhci_alloc_segments_for_ring() argument
334 chain_links = !!(xhci_link_trb_quirk(xhci) || in xhci_alloc_segments_for_ring()
336 (xhci->quirks & XHCI_AMD_0x96_HOST))); in xhci_alloc_segments_for_ring()
338 prev = xhci_segment_alloc(xhci, cycle_state, max_packet, flags); in xhci_alloc_segments_for_ring()
347 next = xhci_segment_alloc(xhci, cycle_state, max_packet, flags); in xhci_alloc_segments_for_ring()
352 xhci_segment_free(xhci, prev); in xhci_alloc_segments_for_ring()
368 static void xhci_vendor_free_container_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx) in xhci_vendor_free_container_ctx() argument
370 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_free_container_ctx()
373 ops->free_container_ctx(xhci, ctx); in xhci_vendor_free_container_ctx()
376 static void xhci_vendor_alloc_container_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx, in xhci_vendor_alloc_container_ctx() argument
379 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_alloc_container_ctx()
382 ops->alloc_container_ctx(xhci, ctx, type, flags); in xhci_vendor_alloc_container_ctx()
385 static struct xhci_ring *xhci_vendor_alloc_transfer_ring(struct xhci_hcd *xhci, in xhci_vendor_alloc_transfer_ring() argument
389 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_alloc_transfer_ring()
392 return ops->alloc_transfer_ring(xhci, endpoint_type, ring_type, in xhci_vendor_alloc_transfer_ring()
397 void xhci_vendor_free_transfer_ring(struct xhci_hcd *xhci, in xhci_vendor_free_transfer_ring() argument
400 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_free_transfer_ring()
403 ops->free_transfer_ring(xhci, virt_dev, ep_index); in xhci_vendor_free_transfer_ring()
406 bool xhci_vendor_is_usb_offload_enabled(struct xhci_hcd *xhci, in xhci_vendor_is_usb_offload_enabled() argument
409 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_is_usb_offload_enabled()
412 return ops->is_usb_offload_enabled(xhci, virt_dev, ep_index); in xhci_vendor_is_usb_offload_enabled()
423 struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, in xhci_ring_alloc() argument
429 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_ring_alloc()
442 ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg, in xhci_ring_alloc()
464 void xhci_free_endpoint_ring(struct xhci_hcd *xhci, in xhci_free_endpoint_ring() argument
468 if (xhci_vendor_is_usb_offload_enabled(xhci, virt_dev, ep_index)) in xhci_free_endpoint_ring()
469 xhci_vendor_free_transfer_ring(xhci, virt_dev, ep_index); in xhci_free_endpoint_ring()
471 xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); in xhci_free_endpoint_ring()
480 int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_ring_expansion() argument
496 ret = xhci_alloc_segments_for_ring(xhci, &first, &last, in xhci_ring_expansion()
509 xhci_segment_free(xhci, first); in xhci_ring_expansion()
517 xhci_link_rings(xhci, ring, first, last, num_segs); in xhci_ring_expansion()
519 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in xhci_ring_expansion()
526 struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, in xhci_alloc_container_ctx() argument
530 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_container_ctx()
531 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_alloc_container_ctx()
541 ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024; in xhci_alloc_container_ctx()
543 ctx->size += CTX_SIZE(xhci->hcc_params); in xhci_alloc_container_ctx()
545 if (xhci_vendor_is_usb_offload_enabled(xhci, NULL, 0) && in xhci_alloc_container_ctx()
547 xhci_vendor_alloc_container_ctx(xhci, ctx, type, flags); in xhci_alloc_container_ctx()
549 ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma); in xhci_alloc_container_ctx()
558 void xhci_free_container_ctx(struct xhci_hcd *xhci, in xhci_free_container_ctx() argument
561 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_free_container_ctx()
565 if (xhci_vendor_is_usb_offload_enabled(xhci, NULL, 0) && in xhci_free_container_ctx()
567 xhci_vendor_free_container_ctx(xhci, ctx); in xhci_free_container_ctx()
569 dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma); in xhci_free_container_ctx()
583 struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci, in xhci_get_slot_ctx() argument
590 (ctx->bytes + CTX_SIZE(xhci->hcc_params)); in xhci_get_slot_ctx()
594 struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, in xhci_get_ep_ctx() argument
604 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params))); in xhci_get_ep_ctx()
611 static void xhci_free_stream_ctx(struct xhci_hcd *xhci, in xhci_free_stream_ctx() argument
615 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_stream_ctx()
622 return dma_pool_free(xhci->small_streams_pool, in xhci_free_stream_ctx()
625 return dma_pool_free(xhci->medium_streams_pool, in xhci_free_stream_ctx()
639 static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci, in xhci_alloc_stream_ctx() argument
643 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_ctx()
650 return dma_pool_alloc(xhci->small_streams_pool, in xhci_alloc_stream_ctx()
653 return dma_pool_alloc(xhci->medium_streams_pool, in xhci_alloc_stream_ctx()
676 struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, in xhci_alloc_stream_info() argument
686 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_info()
688 xhci_dbg(xhci, "Allocating %u streams and %u " in xhci_alloc_stream_info()
691 if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) { in xhci_alloc_stream_info()
692 xhci_dbg(xhci, "Command ring has no reserved TRBs available\n"); in xhci_alloc_stream_info()
695 xhci->cmd_ring_reserved_trbs++; in xhci_alloc_stream_info()
713 stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci, in xhci_alloc_stream_info()
723 xhci_alloc_command_with_ctx(xhci, true, mem_flags); in xhci_alloc_stream_info()
736 xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, max_packet, in xhci_alloc_stream_info()
749 xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", in xhci_alloc_stream_info()
754 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
772 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
776 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_alloc_stream_info()
778 xhci_free_stream_ctx(xhci, in xhci_alloc_stream_info()
787 xhci->cmd_ring_reserved_trbs--; in xhci_alloc_stream_info()
794 void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, in xhci_setup_streams_ep_input_ctx() argument
804 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, in xhci_setup_streams_ep_input_ctx()
831 void xhci_free_stream_info(struct xhci_hcd *xhci, in xhci_free_stream_info() argument
844 xhci_ring_free(xhci, cur_ring); in xhci_free_stream_info()
848 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_free_stream_info()
849 xhci->cmd_ring_reserved_trbs--; in xhci_free_stream_info()
851 xhci_free_stream_ctx(xhci, in xhci_free_stream_info()
863 static void xhci_init_endpoint_timer(struct xhci_hcd *xhci, in xhci_init_endpoint_timer() argument
868 ep->xhci = xhci; in xhci_init_endpoint_timer()
871 static void xhci_free_tt_info(struct xhci_hcd *xhci, in xhci_free_tt_info() argument
883 virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { in xhci_free_tt_info()
884 xhci_dbg(xhci, "Bad real port.\n"); in xhci_free_tt_info()
888 tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts); in xhci_free_tt_info()
901 int xhci_alloc_tt_info(struct xhci_hcd *xhci, in xhci_alloc_tt_info() argument
909 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_tt_info()
925 &xhci->rh_bw[virt_dev->real_port - 1].tts); in xhci_alloc_tt_info()
936 xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id); in xhci_alloc_tt_info()
942 * Should be called with xhci->lock held if there is any chance the TT lists
946 void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) in xhci_free_virt_device() argument
953 if (slot_id == 0 || !xhci->devs[slot_id]) in xhci_free_virt_device()
956 dev = xhci->devs[slot_id]; in xhci_free_virt_device()
958 xhci->dcbaa->dev_context_ptrs[slot_id] = 0; in xhci_free_virt_device()
969 xhci_free_endpoint_ring(xhci, dev, i); in xhci_free_virt_device()
971 xhci_free_stream_info(xhci, in xhci_free_virt_device()
983 xhci_dbg(xhci, "Slot %u endpoint %u not removed from BW list!\n", in xhci_free_virt_device()
988 xhci_free_tt_info(xhci, dev, slot_id); in xhci_free_virt_device()
990 xhci_update_tt_active_eps(xhci, dev, old_active_eps); in xhci_free_virt_device()
993 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_free_virt_device()
995 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_free_virt_device()
999 kfree(xhci->devs[slot_id]); in xhci_free_virt_device()
1000 xhci->devs[slot_id] = NULL; in xhci_free_virt_device()
1009 static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id) in xhci_free_virt_devices_depth_first() argument
1016 vdev = xhci->devs[slot_id]; in xhci_free_virt_devices_depth_first()
1021 vdev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { in xhci_free_virt_devices_depth_first()
1022 xhci_dbg(xhci, "Bad vdev->real_port.\n"); in xhci_free_virt_devices_depth_first()
1026 tt_list_head = &(xhci->rh_bw[vdev->real_port - 1].tts); in xhci_free_virt_devices_depth_first()
1031 for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_free_virt_devices_depth_first()
1032 vdev = xhci->devs[i]; in xhci_free_virt_devices_depth_first()
1035 xhci, i); in xhci_free_virt_devices_depth_first()
1041 xhci_debugfs_remove_slot(xhci, slot_id); in xhci_free_virt_devices_depth_first()
1042 xhci_free_virt_device(xhci, slot_id); in xhci_free_virt_devices_depth_first()
1045 int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, in xhci_alloc_virt_device() argument
1052 if (slot_id == 0 || xhci->devs[slot_id]) { in xhci_alloc_virt_device()
1053 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id); in xhci_alloc_virt_device()
1064 dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags); in xhci_alloc_virt_device()
1068 xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id, in xhci_alloc_virt_device()
1072 dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags); in xhci_alloc_virt_device()
1076 xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id, in xhci_alloc_virt_device()
1083 xhci_init_endpoint_timer(xhci, &dev->eps[i]); in xhci_alloc_virt_device()
1089 dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, 0, flags); in xhci_alloc_virt_device()
1096 xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma); in xhci_alloc_virt_device()
1097 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n", in xhci_alloc_virt_device()
1099 &xhci->dcbaa->dev_context_ptrs[slot_id], in xhci_alloc_virt_device()
1100 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id])); in xhci_alloc_virt_device()
1104 xhci->devs[slot_id] = dev; in xhci_alloc_virt_device()
1110 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_alloc_virt_device()
1112 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_alloc_virt_device()
1118 void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, in xhci_copy_ep0_dequeue_into_input_ctx() argument
1125 virt_dev = xhci->devs[udev->slot_id]; in xhci_copy_ep0_dequeue_into_input_ctx()
1126 ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0); in xhci_copy_ep0_dequeue_into_input_ctx()
1141 * The xHCI roothub may have ports of differing speeds in any order in the port
1144 * The xHCI hardware wants to know the roothub port number that the USB device
1150 static u32 xhci_find_real_port_number(struct xhci_hcd *xhci, in xhci_find_real_port_number() argument
1157 hcd = xhci->shared_hcd; in xhci_find_real_port_number()
1159 hcd = xhci->main_hcd; in xhci_find_real_port_number()
1168 /* Setup an xHCI virtual device for a Set Address command */
1169 int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev) in xhci_setup_addressable_virt_dev() argument
1178 dev = xhci->devs[udev->slot_id]; in xhci_setup_addressable_virt_dev()
1181 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n", in xhci_setup_addressable_virt_dev()
1185 ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0); in xhci_setup_addressable_virt_dev()
1186 slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx); in xhci_setup_addressable_virt_dev()
1213 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n"); in xhci_setup_addressable_virt_dev()
1221 port_num = xhci_find_real_port_number(xhci, udev); in xhci_setup_addressable_virt_dev()
1231 xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num); in xhci_setup_addressable_virt_dev()
1232 xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port); in xhci_setup_addressable_virt_dev()
1241 dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table; in xhci_setup_addressable_virt_dev()
1246 rh_bw = &xhci->rh_bw[port_num - 1]; in xhci_setup_addressable_virt_dev()
1261 xhci_warn(xhci, "WARN: Didn't find a matching TT\n"); in xhci_setup_addressable_virt_dev()
1271 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt); in xhci_setup_addressable_virt_dev()
1272 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport); in xhci_setup_addressable_virt_dev()
1363 * The polling interval is expressed in "microframes". If xHCI's Interval field
1498 int xhci_endpoint_init(struct xhci_hcd *xhci, in xhci_endpoint_init() argument
1518 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_init()
1530 * set it to max available. See xHCI 1.1 spec 4.14.1.1 for details. in xhci_endpoint_init()
1538 if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) && in xhci_endpoint_init()
1564 /* xHCI 1.0 and 1.1 indicates that ctrl ep avg TRB Length should be 8 */ in xhci_endpoint_init()
1565 if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100) in xhci_endpoint_init()
1567 /* xhci 1.1 with LEC support doesn't use mult field, use RsvdZ */ in xhci_endpoint_init()
1568 if ((xhci->hci_version > 0x100) && HCC2_LEC(xhci->hcc_params2)) in xhci_endpoint_init()
1572 if (xhci_vendor_is_usb_offload_enabled(xhci, virt_dev, ep_index) && in xhci_endpoint_init()
1575 xhci_vendor_alloc_transfer_ring(xhci, endpoint_type, ring_type, in xhci_endpoint_init()
1579 xhci_ring_alloc(xhci, 2, 1, ring_type, max_packet, mem_flags); in xhci_endpoint_init()
1605 void xhci_endpoint_zero(struct xhci_hcd *xhci, in xhci_endpoint_zero() argument
1613 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_zero()
1634 void xhci_update_bw_info(struct xhci_hcd *xhci, in xhci_update_bw_info() argument
1659 ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i); in xhci_update_bw_info()
1692 void xhci_endpoint_copy(struct xhci_hcd *xhci, in xhci_endpoint_copy() argument
1700 out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index); in xhci_endpoint_copy()
1701 in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index); in xhci_endpoint_copy()
1707 if (xhci->quirks & XHCI_MTK_HOST) { in xhci_endpoint_copy()
1718 void xhci_slot_copy(struct xhci_hcd *xhci, in xhci_slot_copy() argument
1725 in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx); in xhci_slot_copy()
1726 out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx); in xhci_slot_copy()
1735 static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) in scratchpad_alloc() argument
1738 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_alloc()
1739 int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_alloc()
1741 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in scratchpad_alloc()
1747 xhci->scratchpad = kzalloc_node(sizeof(*xhci->scratchpad), flags, in scratchpad_alloc()
1749 if (!xhci->scratchpad) in scratchpad_alloc()
1752 xhci->scratchpad->sp_array = dma_alloc_coherent(dev, in scratchpad_alloc()
1754 &xhci->scratchpad->sp_dma, flags); in scratchpad_alloc()
1755 if (!xhci->scratchpad->sp_array) in scratchpad_alloc()
1758 xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *), in scratchpad_alloc()
1760 if (!xhci->scratchpad->sp_buffers) in scratchpad_alloc()
1763 xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma); in scratchpad_alloc()
1766 void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma, in scratchpad_alloc()
1771 xhci->scratchpad->sp_array[i] = dma; in scratchpad_alloc()
1772 xhci->scratchpad->sp_buffers[i] = buf; in scratchpad_alloc()
1779 dma_free_coherent(dev, xhci->page_size, in scratchpad_alloc()
1780 xhci->scratchpad->sp_buffers[i], in scratchpad_alloc()
1781 xhci->scratchpad->sp_array[i]); in scratchpad_alloc()
1784 kfree(xhci->scratchpad->sp_buffers); in scratchpad_alloc()
1788 xhci->scratchpad->sp_array, in scratchpad_alloc()
1789 xhci->scratchpad->sp_dma); in scratchpad_alloc()
1792 kfree(xhci->scratchpad); in scratchpad_alloc()
1793 xhci->scratchpad = NULL; in scratchpad_alloc()
1799 static void scratchpad_free(struct xhci_hcd *xhci) in scratchpad_free() argument
1803 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_free()
1805 if (!xhci->scratchpad) in scratchpad_free()
1808 num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_free()
1811 dma_free_coherent(dev, xhci->page_size, in scratchpad_free()
1812 xhci->scratchpad->sp_buffers[i], in scratchpad_free()
1813 xhci->scratchpad->sp_array[i]); in scratchpad_free()
1815 kfree(xhci->scratchpad->sp_buffers); in scratchpad_free()
1817 xhci->scratchpad->sp_array, in scratchpad_free()
1818 xhci->scratchpad->sp_dma); in scratchpad_free()
1819 kfree(xhci->scratchpad); in scratchpad_free()
1820 xhci->scratchpad = NULL; in scratchpad_free()
1823 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, in xhci_alloc_command() argument
1827 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_command()
1850 struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci, in xhci_alloc_command_with_ctx() argument
1855 command = xhci_alloc_command(xhci, allocate_completion, mem_flags); in xhci_alloc_command_with_ctx()
1859 command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, in xhci_alloc_command_with_ctx()
1874 void xhci_free_command(struct xhci_hcd *xhci, in xhci_free_command() argument
1877 xhci_free_container_ctx(xhci, in xhci_free_command()
1884 int xhci_alloc_erst(struct xhci_hcd *xhci, in xhci_alloc_erst() argument
1895 erst->entries = dma_alloc_coherent(xhci_to_hcd(xhci)->self.sysdev, in xhci_alloc_erst()
1915 void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst) in xhci_free_erst() argument
1918 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_erst()
1930 struct xhci_hcd *xhci, gfp_t flags) in xhci_vendor_alloc_dcbaa() argument
1932 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_alloc_dcbaa()
1935 return ops->alloc_dcbaa(xhci, flags); in xhci_vendor_alloc_dcbaa()
1939 static void xhci_vendor_free_dcbaa(struct xhci_hcd *xhci) in xhci_vendor_free_dcbaa() argument
1941 struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci); in xhci_vendor_free_dcbaa()
1944 ops->free_dcbaa(xhci); in xhci_vendor_free_dcbaa()
1947 void xhci_mem_cleanup(struct xhci_hcd *xhci) in xhci_mem_cleanup() argument
1949 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_cleanup()
1952 cancel_delayed_work_sync(&xhci->cmd_timer); in xhci_mem_cleanup()
1954 xhci_free_erst(xhci, &xhci->erst); in xhci_mem_cleanup()
1956 if (xhci->event_ring) in xhci_mem_cleanup()
1957 xhci_ring_free(xhci, xhci->event_ring); in xhci_mem_cleanup()
1958 xhci->event_ring = NULL; in xhci_mem_cleanup()
1959 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring"); in xhci_mem_cleanup()
1961 if (xhci->lpm_command) in xhci_mem_cleanup()
1962 xhci_free_command(xhci, xhci->lpm_command); in xhci_mem_cleanup()
1963 xhci->lpm_command = NULL; in xhci_mem_cleanup()
1964 if (xhci->cmd_ring) in xhci_mem_cleanup()
1965 xhci_ring_free(xhci, xhci->cmd_ring); in xhci_mem_cleanup()
1966 xhci->cmd_ring = NULL; in xhci_mem_cleanup()
1967 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); in xhci_mem_cleanup()
1968 xhci_cleanup_command_queue(xhci); in xhci_mem_cleanup()
1970 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); in xhci_mem_cleanup()
1971 for (i = 0; i < num_ports && xhci->rh_bw; i++) { in xhci_mem_cleanup()
1972 struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; in xhci_mem_cleanup()
1980 for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--) in xhci_mem_cleanup()
1981 xhci_free_virt_devices_depth_first(xhci, i); in xhci_mem_cleanup()
1983 dma_pool_destroy(xhci->segment_pool); in xhci_mem_cleanup()
1984 xhci->segment_pool = NULL; in xhci_mem_cleanup()
1985 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool"); in xhci_mem_cleanup()
1987 dma_pool_destroy(xhci->device_pool); in xhci_mem_cleanup()
1988 xhci->device_pool = NULL; in xhci_mem_cleanup()
1989 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool"); in xhci_mem_cleanup()
1991 dma_pool_destroy(xhci->small_streams_pool); in xhci_mem_cleanup()
1992 xhci->small_streams_pool = NULL; in xhci_mem_cleanup()
1993 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1996 dma_pool_destroy(xhci->medium_streams_pool); in xhci_mem_cleanup()
1997 xhci->medium_streams_pool = NULL; in xhci_mem_cleanup()
1998 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
2001 if (xhci_vendor_is_usb_offload_enabled(xhci, NULL, 0)) { in xhci_mem_cleanup()
2002 xhci_vendor_free_dcbaa(xhci); in xhci_mem_cleanup()
2004 if (xhci->dcbaa) in xhci_mem_cleanup()
2005 dma_free_coherent(dev, sizeof(*xhci->dcbaa), in xhci_mem_cleanup()
2006 xhci->dcbaa, xhci->dcbaa->dma); in xhci_mem_cleanup()
2008 xhci->dcbaa = NULL; in xhci_mem_cleanup()
2010 scratchpad_free(xhci); in xhci_mem_cleanup()
2012 if (!xhci->rh_bw) in xhci_mem_cleanup()
2017 list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { in xhci_mem_cleanup()
2024 xhci->cmd_ring_reserved_trbs = 0; in xhci_mem_cleanup()
2025 xhci->usb2_rhub.num_ports = 0; in xhci_mem_cleanup()
2026 xhci->usb3_rhub.num_ports = 0; in xhci_mem_cleanup()
2027 xhci->num_active_eps = 0; in xhci_mem_cleanup()
2028 kfree(xhci->usb2_rhub.ports); in xhci_mem_cleanup()
2029 kfree(xhci->usb3_rhub.ports); in xhci_mem_cleanup()
2030 kfree(xhci->hw_ports); in xhci_mem_cleanup()
2031 kfree(xhci->rh_bw); in xhci_mem_cleanup()
2032 kfree(xhci->ext_caps); in xhci_mem_cleanup()
2033 for (i = 0; i < xhci->num_port_caps; i++) in xhci_mem_cleanup()
2034 kfree(xhci->port_caps[i].psi); in xhci_mem_cleanup()
2035 kfree(xhci->port_caps); in xhci_mem_cleanup()
2036 xhci->num_port_caps = 0; in xhci_mem_cleanup()
2038 xhci->usb2_rhub.ports = NULL; in xhci_mem_cleanup()
2039 xhci->usb3_rhub.ports = NULL; in xhci_mem_cleanup()
2040 xhci->hw_ports = NULL; in xhci_mem_cleanup()
2041 xhci->rh_bw = NULL; in xhci_mem_cleanup()
2042 xhci->ext_caps = NULL; in xhci_mem_cleanup()
2043 xhci->port_caps = NULL; in xhci_mem_cleanup()
2045 xhci->page_size = 0; in xhci_mem_cleanup()
2046 xhci->page_shift = 0; in xhci_mem_cleanup()
2047 xhci->usb2_rhub.bus_state.bus_suspended = 0; in xhci_mem_cleanup()
2048 xhci->usb3_rhub.bus_state.bus_suspended = 0; in xhci_mem_cleanup()
2051 static int xhci_test_trb_in_td(struct xhci_hcd *xhci, in xhci_test_trb_in_td() argument
2066 seg = trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, false); in xhci_test_trb_in_td()
2068 xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n", in xhci_test_trb_in_td()
2070 xhci_warn(xhci, "Tested TRB math w/ seg %p and " in xhci_test_trb_in_td()
2074 xhci_warn(xhci, "starting TRB %p (0x%llx DMA), " in xhci_test_trb_in_td()
2078 xhci_warn(xhci, "Expected seg %p, got seg %p\n", in xhci_test_trb_in_td()
2080 trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, in xhci_test_trb_in_td()
2088 int xhci_check_trb_in_td_math(struct xhci_hcd *xhci) in xhci_check_trb_in_td_math() argument
2097 { xhci->event_ring->first_seg->dma - 16, NULL }, in xhci_check_trb_in_td_math()
2099 { xhci->event_ring->first_seg->dma - 1, NULL }, in xhci_check_trb_in_td_math()
2101 { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg }, in xhci_check_trb_in_td_math()
2103 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16, in xhci_check_trb_in_td_math()
2104 xhci->event_ring->first_seg }, in xhci_check_trb_in_td_math()
2106 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL }, in xhci_check_trb_in_td_math()
2108 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL }, in xhci_check_trb_in_td_math()
2120 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2121 .start_trb = xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2122 .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2123 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2127 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2128 .start_trb = xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2129 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2130 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2134 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2135 .start_trb = xhci->cmd_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2136 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2137 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2141 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2142 .start_trb = &xhci->event_ring->first_seg->trbs[0], in xhci_check_trb_in_td_math()
2143 .end_trb = &xhci->event_ring->first_seg->trbs[3], in xhci_check_trb_in_td_math()
2144 .input_dma = xhci->event_ring->first_seg->dma + 4*16, in xhci_check_trb_in_td_math()
2148 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2149 .start_trb = &xhci->event_ring->first_seg->trbs[3], in xhci_check_trb_in_td_math()
2150 .end_trb = &xhci->event_ring->first_seg->trbs[6], in xhci_check_trb_in_td_math()
2151 .input_dma = xhci->event_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2155 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2156 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2157 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2158 .input_dma = xhci->event_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2162 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2163 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2164 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2165 .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16, in xhci_check_trb_in_td_math()
2169 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2170 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2171 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2172 .input_dma = xhci->cmd_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2182 ret = xhci_test_trb_in_td(xhci, in xhci_check_trb_in_td_math()
2183 xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2184 xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2185 &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2195 ret = xhci_test_trb_in_td(xhci, in xhci_check_trb_in_td_math()
2205 xhci_dbg(xhci, "TRB math tests passed.\n"); in xhci_check_trb_in_td_math()
2210 static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) in xhci_set_hc_event_deq() argument
2215 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, in xhci_set_hc_event_deq()
2216 xhci->event_ring->dequeue); in xhci_set_hc_event_deq()
2218 xhci_warn(xhci, "WARN something wrong with SW event ring " in xhci_set_hc_event_deq()
2221 temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_set_hc_event_deq()
2227 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_set_hc_event_deq()
2230 xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, in xhci_set_hc_event_deq()
2231 &xhci->ir_set->erst_dequeue); in xhci_set_hc_event_deq()
2234 static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, in xhci_add_in_port() argument
2241 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_add_in_port()
2249 rhub = &xhci->usb3_rhub; in xhci_add_in_port()
2260 rhub = &xhci->usb2_rhub; in xhci_add_in_port()
2262 xhci_warn(xhci, "Ignoring unknown port speed, " in xhci_add_in_port()
2277 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2286 port_cap = &xhci->port_caps[xhci->num_port_caps++]; in xhci_add_in_port()
2287 if (xhci->num_port_caps > max_caps) in xhci_add_in_port()
2312 xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n", in xhci_add_in_port()
2322 if (major_revision < 0x03 && xhci->num_ext_caps < max_caps) in xhci_add_in_port()
2323 xhci->ext_caps[xhci->num_ext_caps++] = temp; in xhci_add_in_port()
2325 if ((xhci->hci_version >= 0x100) && (major_revision != 0x03) && in xhci_add_in_port()
2327 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2328 "xHCI 1.0: support USB2 hardware lpm"); in xhci_add_in_port()
2329 xhci->hw_lpm_support = 1; in xhci_add_in_port()
2334 struct xhci_port *hw_port = &xhci->hw_ports[i]; in xhci_add_in_port()
2337 xhci_warn(xhci, "Duplicate port entry, Ext Cap %p," in xhci_add_in_port()
2339 xhci_warn(xhci, "Port was marked as USB %u, " in xhci_add_in_port()
2359 static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, in xhci_create_rhub_port_array() argument
2364 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_create_rhub_port_array()
2373 for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { in xhci_create_rhub_port_array()
2374 if (xhci->hw_ports[i].rhub != rhub || in xhci_create_rhub_port_array()
2375 xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) in xhci_create_rhub_port_array()
2377 xhci->hw_ports[i].hcd_portnum = port_index; in xhci_create_rhub_port_array()
2378 rhub->ports[port_index] = &xhci->hw_ports[i]; in xhci_create_rhub_port_array()
2392 static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) in xhci_setup_port_arrays() argument
2400 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_setup_port_arrays()
2402 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); in xhci_setup_port_arrays()
2403 xhci->hw_ports = kcalloc_node(num_ports, sizeof(*xhci->hw_ports), in xhci_setup_port_arrays()
2405 if (!xhci->hw_ports) in xhci_setup_port_arrays()
2409 xhci->hw_ports[i].addr = &xhci->op_regs->port_status_base + in xhci_setup_port_arrays()
2411 xhci->hw_ports[i].hw_portnum = i; in xhci_setup_port_arrays()
2414 xhci->rh_bw = kcalloc_node(num_ports, sizeof(*xhci->rh_bw), flags, in xhci_setup_port_arrays()
2416 if (!xhci->rh_bw) in xhci_setup_port_arrays()
2421 INIT_LIST_HEAD(&xhci->rh_bw[i].tts); in xhci_setup_port_arrays()
2422 bw_table = &xhci->rh_bw[i].bw_table; in xhci_setup_port_arrays()
2426 base = &xhci->cap_regs->hc_capbase; in xhci_setup_port_arrays()
2430 xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n"); in xhci_setup_port_arrays()
2442 xhci->ext_caps = kcalloc_node(cap_count, sizeof(*xhci->ext_caps), in xhci_setup_port_arrays()
2444 if (!xhci->ext_caps) in xhci_setup_port_arrays()
2447 xhci->port_caps = kcalloc_node(cap_count, sizeof(*xhci->port_caps), in xhci_setup_port_arrays()
2449 if (!xhci->port_caps) in xhci_setup_port_arrays()
2455 xhci_add_in_port(xhci, num_ports, base + offset, cap_count); in xhci_setup_port_arrays()
2456 if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == in xhci_setup_port_arrays()
2462 if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) { in xhci_setup_port_arrays()
2463 xhci_warn(xhci, "No ports on the roothubs?\n"); in xhci_setup_port_arrays()
2466 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2468 xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports); in xhci_setup_port_arrays()
2473 if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) { in xhci_setup_port_arrays()
2474 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2477 xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS; in xhci_setup_port_arrays()
2479 if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) { in xhci_setup_port_arrays()
2480 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2483 xhci->usb2_rhub.num_ports = USB_MAXCHILDREN; in xhci_setup_port_arrays()
2491 xhci_create_rhub_port_array(xhci, &xhci->usb2_rhub, flags); in xhci_setup_port_arrays()
2492 xhci_create_rhub_port_array(xhci, &xhci->usb3_rhub, flags); in xhci_setup_port_arrays()
2497 int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) in xhci_mem_init() argument
2500 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_init()
2506 INIT_LIST_HEAD(&xhci->cmd_list); in xhci_mem_init()
2509 INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout); in xhci_mem_init()
2510 init_completion(&xhci->cmd_ring_stop_completion); in xhci_mem_init()
2512 page_size = readl(&xhci->op_regs->page_size); in xhci_mem_init()
2513 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2521 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2524 xhci_warn(xhci, "WARN: no supported page size\n"); in xhci_mem_init()
2526 xhci->page_shift = 12; in xhci_mem_init()
2527 xhci->page_size = 1 << xhci->page_shift; in xhci_mem_init()
2528 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2529 "HCD page size set to %iK", xhci->page_size / 1024); in xhci_mem_init()
2535 val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1)); in xhci_mem_init()
2536 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2538 val2 = readl(&xhci->op_regs->config_reg); in xhci_mem_init()
2540 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2542 writel(val, &xhci->op_regs->config_reg); in xhci_mem_init()
2545 * xHCI section 5.4.6 - doorbell array must be in xhci_mem_init()
2548 if (xhci_vendor_is_usb_offload_enabled(xhci, NULL, 0)) { in xhci_mem_init()
2549 xhci->dcbaa = xhci_vendor_alloc_dcbaa(xhci, flags); in xhci_mem_init()
2550 if (!xhci->dcbaa) in xhci_mem_init()
2553 xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, in xhci_mem_init()
2555 if (!xhci->dcbaa) in xhci_mem_init()
2557 xhci->dcbaa->dma = dma; in xhci_mem_init()
2559 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2561 (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa); in xhci_mem_init()
2562 xhci_write_64(xhci, xhci->dcbaa->dma, &xhci->op_regs->dcbaa_ptr); in xhci_mem_init()
2571 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, in xhci_mem_init()
2572 TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size); in xhci_mem_init()
2575 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, in xhci_mem_init()
2576 2112, 64, xhci->page_size); in xhci_mem_init()
2577 if (!xhci->segment_pool || !xhci->device_pool) in xhci_mem_init()
2583 xhci->small_streams_pool = in xhci_mem_init()
2584 dma_pool_create("xHCI 256 byte stream ctx arrays", in xhci_mem_init()
2586 xhci->medium_streams_pool = in xhci_mem_init()
2587 dma_pool_create("xHCI 1KB stream ctx arrays", in xhci_mem_init()
2593 if (!xhci->small_streams_pool || !xhci->medium_streams_pool) in xhci_mem_init()
2597 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, 0, flags); in xhci_mem_init()
2598 if (!xhci->cmd_ring) in xhci_mem_init()
2600 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2601 "Allocated command ring at %p", xhci->cmd_ring); in xhci_mem_init()
2602 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%llx", in xhci_mem_init()
2603 (unsigned long long)xhci->cmd_ring->first_seg->dma); in xhci_mem_init()
2606 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_mem_init()
2608 (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) | in xhci_mem_init()
2609 xhci->cmd_ring->cycle_state; in xhci_mem_init()
2610 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2612 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring); in xhci_mem_init()
2614 xhci->lpm_command = xhci_alloc_command_with_ctx(xhci, true, flags); in xhci_mem_init()
2615 if (!xhci->lpm_command) in xhci_mem_init()
2622 xhci->cmd_ring_reserved_trbs++; in xhci_mem_init()
2624 val = readl(&xhci->cap_regs->db_off); in xhci_mem_init()
2626 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2629 xhci->dba = (void __iomem *) xhci->cap_regs + val; in xhci_mem_init()
2631 xhci->ir_set = &xhci->run_regs->ir_set[0]; in xhci_mem_init()
2637 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); in xhci_mem_init()
2638 xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, in xhci_mem_init()
2640 if (!xhci->event_ring) in xhci_mem_init()
2642 if (xhci_check_trb_in_td_math(xhci) < 0) in xhci_mem_init()
2645 ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags); in xhci_mem_init()
2650 val = readl(&xhci->ir_set->erst_size); in xhci_mem_init()
2653 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2656 writel(val, &xhci->ir_set->erst_size); in xhci_mem_init()
2658 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2661 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2663 (unsigned long long)xhci->erst.erst_dma_addr); in xhci_mem_init()
2664 val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); in xhci_mem_init()
2666 val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK); in xhci_mem_init()
2667 xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base); in xhci_mem_init()
2670 xhci_set_hc_event_deq(xhci); in xhci_mem_init()
2671 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2680 xhci->devs[i] = NULL; in xhci_mem_init()
2682 xhci->usb2_rhub.bus_state.resume_done[i] = 0; in xhci_mem_init()
2683 xhci->usb3_rhub.bus_state.resume_done[i] = 0; in xhci_mem_init()
2685 init_completion(&xhci->usb2_rhub.bus_state.rexit_done[i]); in xhci_mem_init()
2686 init_completion(&xhci->usb3_rhub.bus_state.u3exit_done[i]); in xhci_mem_init()
2689 if (scratchpad_alloc(xhci, flags)) in xhci_mem_init()
2691 if (xhci_setup_port_arrays(xhci, flags)) in xhci_mem_init()
2698 temp = readl(&xhci->op_regs->dev_notification); in xhci_mem_init()
2701 writel(temp, &xhci->op_regs->dev_notification); in xhci_mem_init()
2706 xhci_halt(xhci); in xhci_mem_init()
2707 xhci_reset(xhci, XHCI_RESET_SHORT_USEC); in xhci_mem_init()
2708 xhci_mem_cleanup(xhci); in xhci_mem_init()