86 	struct dma_chan *chan;  in dmaengine_dbg_summary_show()  local
88 	list_for_each_entry(chan, &dma_dev->channels, device_node) {  in dmaengine_dbg_summary_show()
89 		if (chan->client_count) {  in dmaengine_dbg_summary_show()
90 			seq_printf(s, " %-13s| %s", dma_chan_name(chan),  in dmaengine_dbg_summary_show()
91 				   chan->dbg_client_name ?: "in-use");  in dmaengine_dbg_summary_show()
93 			if (chan->router)  in dmaengine_dbg_summary_show()
95 					dev_name(chan->router->dev));  in dmaengine_dbg_summary_show()
159 	return chan_dev->chan;  in dev_to_dma_chan()
165 	struct dma_chan *chan;  in memcpy_count_show()  local
171 	chan = dev_to_dma_chan(dev);  in memcpy_count_show()
172 	if (chan) {  in memcpy_count_show()
174 			count += per_cpu_ptr(chan->local, i)->memcpy_count;  in memcpy_count_show()
187 	struct dma_chan *chan;  in bytes_transferred_show()  local
193 	chan = dev_to_dma_chan(dev);  in bytes_transferred_show()
194 	if (chan) {  in bytes_transferred_show()
196 			count += per_cpu_ptr(chan->local, i)->bytes_transferred;  in bytes_transferred_show()
209 	struct dma_chan *chan;  in in_use_show()  local
213 	chan = dev_to_dma_chan(dev);  in in_use_show()
214 	if (chan)  in in_use_show()
215 		err = sprintf(buf, "%d\n", chan->client_count);  in in_use_show()
256 	struct dma_chan *chan;  member
302 static bool dma_chan_is_local(struct dma_chan *chan, int cpu)  in dma_chan_is_local()  argument
304 	int node = dev_to_node(chan->device->dev);  in dma_chan_is_local()
323 	struct dma_chan *chan;  in min_chan()  local
331 		list_for_each_entry(chan, &device->channels, device_node) {  in min_chan()
332 			if (!chan->client_count)  in min_chan()
334 			if (!min || chan->table_count < min->table_count)  in min_chan()
335 				min = chan;  in min_chan()
337 			if (dma_chan_is_local(chan, cpu))  in min_chan()
339 				    chan->table_count < localmin->table_count)  in min_chan()
340 					localmin = chan;  in min_chan()
344 	chan = localmin ? localmin : min;  in min_chan()
346 	if (chan)  in min_chan()
347 		chan->table_count++;  in min_chan()
349 	return chan;  in min_chan()
363 	struct dma_chan *chan;  in dma_channel_rebalance()  local
371 			per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;  in dma_channel_rebalance()
376 		list_for_each_entry(chan, &device->channels, device_node)  in dma_channel_rebalance()
377 			chan->table_count = 0;  in dma_channel_rebalance()
387 			chan = min_chan(cap, cpu);  in dma_channel_rebalance()
388 			per_cpu_ptr(channel_table[cap], cpu)->chan = chan;  in dma_channel_rebalance()
402 static struct module *dma_chan_to_owner(struct dma_chan *chan)  in dma_chan_to_owner()  argument
404 	return chan->device->owner;  in dma_chan_to_owner()
413 static void balance_ref_count(struct dma_chan *chan)  in balance_ref_count()  argument
415 	struct module *owner = dma_chan_to_owner(chan);  in balance_ref_count()
417 	while (chan->client_count < dmaengine_ref_count) {  in balance_ref_count()
419 		chan->client_count++;  in balance_ref_count()
446 static int dma_chan_get(struct dma_chan *chan)  in dma_chan_get()  argument
448 	struct module *owner = dma_chan_to_owner(chan);  in dma_chan_get()
452 	if (chan->client_count) {  in dma_chan_get()
460 	ret = kref_get_unless_zero(&chan->device->ref);  in dma_chan_get()
467 	if (chan->device->device_alloc_chan_resources) {  in dma_chan_get()
468 		ret = chan->device->device_alloc_chan_resources(chan);  in dma_chan_get()
473 	if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))  in dma_chan_get()
474 		balance_ref_count(chan);  in dma_chan_get()
477 	chan->client_count++;  in dma_chan_get()
481 	dma_device_put(chan->device);  in dma_chan_get()
493 static void dma_chan_put(struct dma_chan *chan)  in dma_chan_put()  argument
496 	if (!chan->client_count)  in dma_chan_put()
499 	chan->client_count--;  in dma_chan_put()
502 	if (!chan->client_count && chan->device->device_free_chan_resources) {  in dma_chan_put()
504 		dmaengine_synchronize(chan);  in dma_chan_put()
505 		chan->device->device_free_chan_resources(chan);  in dma_chan_put()
509 	if (chan->router && chan->router->route_free) {  in dma_chan_put()
510 		chan->router->route_free(chan->router->dev, chan->route_data);  in dma_chan_put()
511 		chan->router = NULL;  in dma_chan_put()
512 		chan->route_data = NULL;  in dma_chan_put()
515 	dma_device_put(chan->device);  in dma_chan_put()
516 	module_put(dma_chan_to_owner(chan));  in dma_chan_put()
519 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)  in dma_sync_wait()  argument
524 	dma_async_issue_pending(chan);  in dma_sync_wait()
526 		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);  in dma_sync_wait()
528 			dev_err(chan->device->dev, "%s: timeout!\n", __func__);  in dma_sync_wait()
546 	return this_cpu_read(channel_table[tx_type]->chan);  in dma_find_channel()
556 	struct dma_chan *chan;  in dma_issue_pending_all()  local
562 		list_for_each_entry(chan, &device->channels, device_node)  in dma_issue_pending_all()
563 			if (chan->client_count)  in dma_issue_pending_all()
564 				device->device_issue_pending(chan);  in dma_issue_pending_all()
570 int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)  in dma_get_slave_caps()  argument
574 	if (!chan || !caps)  in dma_get_slave_caps()
577 	device = chan->device;  in dma_get_slave_caps()
612 		device->device_caps(chan, caps);  in dma_get_slave_caps()
622 	struct dma_chan *chan;  in private_candidate()  local
632 		list_for_each_entry(chan, &dev->channels, device_node) {  in private_candidate()
634 			if (chan->client_count)  in private_candidate()
638 	list_for_each_entry(chan, &dev->channels, device_node) {  in private_candidate()
639 		if (chan->client_count) {  in private_candidate()
641 				 __func__, dma_chan_name(chan));  in private_candidate()
644 		if (fn && !fn(chan, fn_param)) {  in private_candidate()
646 				 __func__, dma_chan_name(chan));  in private_candidate()
649 		return chan;  in private_candidate()
659 	struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);  in find_candidate()  local
662 	if (chan) {  in find_candidate()
670 		err = dma_chan_get(chan);  in find_candidate()
675 					__func__, dma_chan_name(chan));  in find_candidate()
680 					 __func__, dma_chan_name(chan), err);  in find_candidate()
685 			chan = ERR_PTR(err);  in find_candidate()
689 	return chan ? chan : ERR_PTR(-EPROBE_DEFER);  in find_candidate()
696 struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)  in dma_get_slave_channel()  argument
703 	if (chan->client_count == 0) {  in dma_get_slave_channel()
704 		struct dma_device *device = chan->device;  in dma_get_slave_channel()
708 		err = dma_chan_get(chan);  in dma_get_slave_channel()
710 			dev_dbg(chan->device->dev,  in dma_get_slave_channel()
712 				__func__, dma_chan_name(chan), err);  in dma_get_slave_channel()
713 			chan = NULL;  in dma_get_slave_channel()
718 		chan = NULL;  in dma_get_slave_channel()
723 	return chan;  in dma_get_slave_channel()
730 	struct dma_chan *chan;  in dma_get_any_slave_channel()  local
738 	chan = find_candidate(device, &mask, NULL, NULL);  in dma_get_any_slave_channel()
742 	return IS_ERR(chan) ? NULL : chan;  in dma_get_any_slave_channel()
760 	struct dma_chan *chan = NULL;  in __dma_request_channel()  local
769 		chan = find_candidate(device, mask, fn, fn_param);  in __dma_request_channel()
770 		if (!IS_ERR(chan))  in __dma_request_channel()
773 		chan = NULL;  in __dma_request_channel()
779 		 chan ? "success" : "fail",  in __dma_request_channel()
780 		 chan ? dma_chan_name(chan) : NULL);  in __dma_request_channel()
782 	return chan;  in __dma_request_channel()
816 	struct dma_chan *chan = NULL;  in dma_request_chan()  local
820 		chan = of_dma_request_slave_channel(dev->of_node, name);  in dma_request_chan()
823 	if (has_acpi_companion(dev) && !chan)  in dma_request_chan()
824 		chan = acpi_dma_request_slave_chan_by_name(dev, name);  in dma_request_chan()
826 	if (PTR_ERR(chan) == -EPROBE_DEFER)  in dma_request_chan()
827 		return chan;  in dma_request_chan()
829 	if (!IS_ERR_OR_NULL(chan))  in dma_request_chan()
844 		chan = find_candidate(d, &mask, d->filter.fn, map->param);  in dma_request_chan()
845 		if (!IS_ERR(chan))  in dma_request_chan()
850 	if (IS_ERR(chan))  in dma_request_chan()
851 		return chan;  in dma_request_chan()
852 	if (!chan)  in dma_request_chan()
857 	chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev),  in dma_request_chan()
861 	chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);  in dma_request_chan()
862 	if (!chan->name)  in dma_request_chan()
863 		return chan;  in dma_request_chan()
864 	chan->slave = dev;  in dma_request_chan()
866 	if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,  in dma_request_chan()
869 	if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))  in dma_request_chan()
870 		dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);  in dma_request_chan()
872 	return chan;  in dma_request_chan()
884 	struct dma_chan *chan;  in dma_request_chan_by_mask()  local
889 	chan = __dma_request_channel(mask, NULL, NULL, NULL);  in dma_request_chan_by_mask()
890 	if (!chan) {  in dma_request_chan_by_mask()
893 			chan = ERR_PTR(-EPROBE_DEFER);  in dma_request_chan_by_mask()
895 			chan = ERR_PTR(-ENODEV);  in dma_request_chan_by_mask()
899 	return chan;  in dma_request_chan_by_mask()
903 void dma_release_channel(struct dma_chan *chan)  in dma_release_channel()  argument
906 	WARN_ONCE(chan->client_count != 1,  in dma_release_channel()
907 		  "chan reference count %d != 1\n", chan->client_count);  in dma_release_channel()
908 	dma_chan_put(chan);  in dma_release_channel()
910 	if (--chan->device->privatecnt == 0)  in dma_release_channel()
911 		dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);  in dma_release_channel()
913 	if (chan->slave) {  in dma_release_channel()
914 		sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);  in dma_release_channel()
915 		sysfs_remove_link(&chan->slave->kobj, chan->name);  in dma_release_channel()
916 		kfree(chan->name);  in dma_release_channel()
917 		chan->name = NULL;  in dma_release_channel()
918 		chan->slave = NULL;  in dma_release_channel()
922 	kfree(chan->dbg_client_name);  in dma_release_channel()
923 	chan->dbg_client_name = NULL;  in dma_release_channel()
935 	struct dma_chan *chan;  in dmaengine_get()  local
945 		list_for_each_entry(chan, &device->channels, device_node) {  in dmaengine_get()
946 			err = dma_chan_get(chan);  in dmaengine_get()
952 				dev_dbg(chan->device->dev,  in dmaengine_get()
954 					__func__, dma_chan_name(chan), err);  in dmaengine_get()
974 	struct dma_chan *chan;  in dmaengine_put()  local
983 		list_for_each_entry(chan, &device->channels, device_node)  in dmaengine_put()
984 			dma_chan_put(chan);  in dmaengine_put()
1040 					       struct dma_chan *chan)  in __dma_async_device_channel_register()  argument
1044 	chan->local = alloc_percpu(typeof(*chan->local));  in __dma_async_device_channel_register()
1045 	if (!chan->local)  in __dma_async_device_channel_register()
1047 	chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);  in __dma_async_device_channel_register()
1048 	if (!chan->dev) {  in __dma_async_device_channel_register()
1058 	chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);  in __dma_async_device_channel_register()
1060 	if (chan->chan_id < 0) {  in __dma_async_device_channel_register()
1062 		       __func__, chan->chan_id);  in __dma_async_device_channel_register()
1063 		rc = chan->chan_id;  in __dma_async_device_channel_register()
1067 	chan->dev->device.class = &dma_devclass;  in __dma_async_device_channel_register()
1068 	chan->dev->device.parent = device->dev;  in __dma_async_device_channel_register()
1069 	chan->dev->chan = chan;  in __dma_async_device_channel_register()
1070 	chan->dev->dev_id = device->dev_id;  in __dma_async_device_channel_register()
1071 	dev_set_name(&chan->dev->device, "dma%dchan%d",  in __dma_async_device_channel_register()
1072 		     device->dev_id, chan->chan_id);  in __dma_async_device_channel_register()
1073 	rc = device_register(&chan->dev->device);  in __dma_async_device_channel_register()
1076 	chan->client_count = 0;  in __dma_async_device_channel_register()
1083 	ida_free(&device->chan_ida, chan->chan_id);  in __dma_async_device_channel_register()
1086 	kfree(chan->dev);  in __dma_async_device_channel_register()
1088 	free_percpu(chan->local);  in __dma_async_device_channel_register()
1089 	chan->local = NULL;  in __dma_async_device_channel_register()
1094 				      struct dma_chan *chan)  in dma_async_device_channel_register()  argument
1098 	rc = __dma_async_device_channel_register(device, chan);  in dma_async_device_channel_register()
1108 						  struct dma_chan *chan)  in __dma_async_device_channel_unregister()  argument
1110 	WARN_ONCE(!device->device_release && chan->client_count,  in __dma_async_device_channel_unregister()
1112 		  __func__, chan->client_count);  in __dma_async_device_channel_unregister()
1115 	chan->dev->chan = NULL;  in __dma_async_device_channel_unregister()
1118 	ida_free(&device->chan_ida, chan->chan_id);  in __dma_async_device_channel_unregister()
1120 	device_unregister(&chan->dev->device);  in __dma_async_device_channel_unregister()
1121 	free_percpu(chan->local);  in __dma_async_device_channel_unregister()
1125 					 struct dma_chan *chan)  in dma_async_device_channel_unregister()  argument
1127 	__dma_async_device_channel_unregister(device, chan);  in dma_async_device_channel_unregister()
1143 	struct dma_chan* chan;  in dma_async_device_register()  local
1251 	list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1252 		rc = __dma_async_device_channel_register(device, chan);  in dma_async_device_register()
1260 		list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1264 			if (dma_chan_get(chan) == -ENODEV) {  in dma_async_device_register()
1291 	list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1292 		if (chan->local == NULL)  in dma_async_device_register()
1295 		chan->dev->chan = NULL;  in dma_async_device_register()
1297 		device_unregister(&chan->dev->device);  in dma_async_device_register()
1298 		free_percpu(chan->local);  in dma_async_device_register()
1313 	struct dma_chan *chan, *n;  in dma_async_device_unregister()  local
1317 	list_for_each_entry_safe(chan, n, &device->channels, device_node)  in dma_async_device_unregister()
1318 		__dma_async_device_channel_unregister(device, chan);  in dma_async_device_unregister()
1498 	struct dma_chan *chan)  in dma_async_tx_descriptor_init()  argument
1500 	tx->chan = chan;  in dma_async_tx_descriptor_init()
1512 		if (dmaengine_is_metadata_mode_supported(desc->chan, mode))  in desc_check_and_set_metadata_mode()
1594 			dev_err(tx->chan->device->dev,  in dma_wait_for_async_tx()
1601 	return dma_sync_wait(tx->chan, tx->cookie);  in dma_wait_for_async_tx()
1616 	struct dma_chan *chan;  in dma_run_dependencies()  local
1623 	chan = dep->chan;  in dma_run_dependencies()
1633 		if (dep_next && dep_next->chan == chan)  in dma_run_dependencies()
1642 	chan->device->device_issue_pending(chan);  in dma_run_dependencies()