47 int get_each_dmabuf(int (*callback)(const struct dma_buf *dmabuf,  in get_each_dmabuf()  argument
105 	struct dma_buf *dmabuf;  in dmabuffs_dname()  local
109 	dmabuf = dentry->d_fsdata;  in dmabuffs_dname()
110 	spin_lock(&dmabuf->name_lock);  in dmabuffs_dname()
111 	if (dmabuf->name)  in dmabuffs_dname()
112 		ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN);  in dmabuffs_dname()
113 	spin_unlock(&dmabuf->name_lock);  in dmabuffs_dname()
121 	struct dma_buf *dmabuf;  in dma_buf_release()  local
126 	dmabuf = dentry->d_fsdata;  in dma_buf_release()
127 	if (unlikely(!dmabuf))  in dma_buf_release()
130 	BUG_ON(dmabuf->vmapping_counter);  in dma_buf_release()
140 	BUG_ON(dmabuf->cb_shared.active || dmabuf->cb_excl.active);  in dma_buf_release()
142 	dma_buf_stats_teardown(dmabuf);  in dma_buf_release()
144 	if (dmabuf->dtor)  in dma_buf_release()
145 		dtor_ret = dmabuf->dtor(dmabuf, dmabuf->dtor_data);  in dma_buf_release()
149 		dmabuf->ops->release(dmabuf);  in dma_buf_release()
151 	if (dmabuf->resv == (struct dma_resv *)&dmabuf[1])  in dma_buf_release()
152 		dma_resv_fini(dmabuf->resv);  in dma_buf_release()
154 	WARN_ON(!list_empty(&dmabuf->attachments));  in dma_buf_release()
155 	module_put(dmabuf->owner);  in dma_buf_release()
156 	kfree(dmabuf->name);  in dma_buf_release()
157 	kfree(dmabuf);  in dma_buf_release()
162 	struct dma_buf *dmabuf;  in dma_buf_file_release()  local
167 	dmabuf = file->private_data;  in dma_buf_file_release()
171 	db_total_size -= dmabuf->size;  in dma_buf_file_release()
173 	list_del(&dmabuf->list_node);  in dma_buf_file_release()
205 	struct dma_buf *dmabuf;  in dma_buf_mmap_internal()  local
210 	dmabuf = file->private_data;  in dma_buf_mmap_internal()
213 	if (!dmabuf->ops->mmap)  in dma_buf_mmap_internal()
218 	    dmabuf->size >> PAGE_SHIFT)  in dma_buf_mmap_internal()
221 	return dmabuf->ops->mmap(dmabuf, vma);  in dma_buf_mmap_internal()
226 	struct dma_buf *dmabuf;  in dma_buf_llseek()  local
232 	dmabuf = file->private_data;  in dma_buf_llseek()
238 		base = dmabuf->size;  in dma_buf_llseek()
285 	struct dma_buf *dmabuf;  in dma_buf_poll()  local
292 	dmabuf = file->private_data;  in dma_buf_poll()
293 	if (!dmabuf || !dmabuf->resv)  in dma_buf_poll()
296 	resv = dmabuf->resv;  in dma_buf_poll()
298 	poll_wait(file, &dmabuf->poll, poll);  in dma_buf_poll()
320 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl;  in dma_buf_poll()
326 		spin_lock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
332 		spin_unlock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
355 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_shared;  in dma_buf_poll()
359 		spin_lock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
364 		spin_unlock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
402 static long _dma_buf_set_name(struct dma_buf *dmabuf, const char *name)  in _dma_buf_set_name()  argument
404 	spin_lock(&dmabuf->name_lock);  in _dma_buf_set_name()
405 	kfree(dmabuf->name);  in _dma_buf_set_name()
406 	dmabuf->name = name;  in _dma_buf_set_name()
407 	spin_unlock(&dmabuf->name_lock);  in _dma_buf_set_name()
425 long dma_buf_set_name(struct dma_buf *dmabuf, const char *name)  in dma_buf_set_name()  argument
433 	ret = _dma_buf_set_name(dmabuf, buf);  in dma_buf_set_name()
441 static long dma_buf_set_name_user(struct dma_buf *dmabuf, const char __user *buf)  in dma_buf_set_name_user()  argument
449 	ret = _dma_buf_set_name(dmabuf, name);  in dma_buf_set_name_user()
459 	struct dma_buf *dmabuf;  in dma_buf_ioctl()  local
465 	dmabuf = file->private_data;  in dma_buf_ioctl()
490 			ret = dma_buf_end_cpu_access(dmabuf, direction);  in dma_buf_ioctl()
492 			ret = dma_buf_begin_cpu_access(dmabuf, direction);  in dma_buf_ioctl()
498 		return dma_buf_set_name_user(dmabuf, (const char __user *)arg);  in dma_buf_ioctl()
507 		if (sync_p.len > dmabuf->size || sync_p.offset > dmabuf->size - sync_p.len)  in dma_buf_ioctl()
528 			ret = dma_buf_end_cpu_access_partial(dmabuf, direction,  in dma_buf_ioctl()
532 			ret = dma_buf_begin_cpu_access_partial(dmabuf, direction,  in dma_buf_ioctl()
545 	struct dma_buf *dmabuf = file->private_data;  in dma_buf_show_fdinfo()  local
547 	seq_printf(m, "size:\t%zu\n", dmabuf->size);  in dma_buf_show_fdinfo()
549 	seq_printf(m, "count:\t%ld\n", file_count(dmabuf->file) - 1);  in dma_buf_show_fdinfo()
550 	seq_printf(m, "exp_name:\t%s\n", dmabuf->exp_name);  in dma_buf_show_fdinfo()
551 	spin_lock(&dmabuf->name_lock);  in dma_buf_show_fdinfo()
552 	if (dmabuf->name)  in dma_buf_show_fdinfo()
553 		seq_printf(m, "name:\t%s\n", dmabuf->name);  in dma_buf_show_fdinfo()
554 	spin_unlock(&dmabuf->name_lock);  in dma_buf_show_fdinfo()
576 static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags)  in dma_buf_getfile()  argument
585 	inode->i_size = dmabuf->size;  in dma_buf_getfile()
586 	inode_set_bytes(inode, dmabuf->size);  in dma_buf_getfile()
600 	file->private_data = dmabuf;  in dma_buf_getfile()
601 	file->f_path.dentry->d_fsdata = dmabuf;  in dma_buf_getfile()
610 static void dma_buf_set_default_name(struct dma_buf *dmabuf)  in dma_buf_set_default_name()  argument
617 	dma_buf_set_name(dmabuf, name);  in dma_buf_set_default_name()
671 	struct dma_buf *dmabuf;  in dma_buf_export()  local
701 	dmabuf = kzalloc(alloc_size, GFP_KERNEL);  in dma_buf_export()
702 	if (!dmabuf) {  in dma_buf_export()
707 	dmabuf->priv = exp_info->priv;  in dma_buf_export()
708 	dmabuf->ops = exp_info->ops;  in dma_buf_export()
709 	dmabuf->size = exp_info->size;  in dma_buf_export()
710 	dmabuf->exp_name = exp_info->exp_name;  in dma_buf_export()
711 	dmabuf->owner = exp_info->owner;  in dma_buf_export()
712 	spin_lock_init(&dmabuf->name_lock);  in dma_buf_export()
714 	mutex_init(&dmabuf->cache_lock);  in dma_buf_export()
716 	init_waitqueue_head(&dmabuf->poll);  in dma_buf_export()
717 	dmabuf->cb_excl.poll = dmabuf->cb_shared.poll = &dmabuf->poll;  in dma_buf_export()
718 	dmabuf->cb_excl.active = dmabuf->cb_shared.active = 0;  in dma_buf_export()
721 		resv = (struct dma_resv *)&dmabuf[1];  in dma_buf_export()
724 	dmabuf->resv = resv;  in dma_buf_export()
726 	file = dma_buf_getfile(dmabuf, exp_info->flags);  in dma_buf_export()
733 	dmabuf->file = file;  in dma_buf_export()
735 	mutex_init(&dmabuf->lock);  in dma_buf_export()
736 	INIT_LIST_HEAD(&dmabuf->attachments);  in dma_buf_export()
739 	list_add(&dmabuf->list_node, &db_list.head);  in dma_buf_export()
741 	db_total_size += dmabuf->size;  in dma_buf_export()
746 	ret = dma_buf_stats_setup(dmabuf);  in dma_buf_export()
751 		dma_buf_set_default_name(dmabuf);  in dma_buf_export()
753 	return dmabuf;  in dma_buf_export()
764 	kfree(dmabuf);  in dma_buf_export()
778 int dma_buf_fd(struct dma_buf *dmabuf, int flags)  in dma_buf_fd()  argument
782 	if (!dmabuf || !dmabuf->file)  in dma_buf_fd()
789 	fd_install(fd, dmabuf->file);  in dma_buf_fd()
831 void dma_buf_put(struct dma_buf *dmabuf)  in dma_buf_put()  argument
833 	if (WARN_ON(!dmabuf || !dmabuf->file))  in dma_buf_put()
836 	fput(dmabuf->file);  in dma_buf_put()
861 dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,  in dma_buf_dynamic_attach()  argument
868 	if (WARN_ON(!dmabuf || !dev))  in dma_buf_dynamic_attach()
879 	attach->dmabuf = dmabuf;  in dma_buf_dynamic_attach()
885 	if (dmabuf->ops->attach) {  in dma_buf_dynamic_attach()
886 		ret = dmabuf->ops->attach(dmabuf, attach);  in dma_buf_dynamic_attach()
890 	dma_resv_lock(dmabuf->resv, NULL);  in dma_buf_dynamic_attach()
891 	list_add(&attach->node, &dmabuf->attachments);  in dma_buf_dynamic_attach()
892 	dma_resv_unlock(dmabuf->resv);  in dma_buf_dynamic_attach()
899 	    dma_buf_is_dynamic(dmabuf)) {  in dma_buf_dynamic_attach()
902 		if (dma_buf_is_dynamic(attach->dmabuf)) {  in dma_buf_dynamic_attach()
903 			dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_dynamic_attach()
909 		sgt = dmabuf->ops->map_dma_buf(attach, DMA_BIDIRECTIONAL);  in dma_buf_dynamic_attach()
916 		if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_dynamic_attach()
917 			dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_dynamic_attach()
929 	if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_dynamic_attach()
933 	if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_dynamic_attach()
934 		dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_dynamic_attach()
936 	dma_buf_detach(dmabuf, attach);  in dma_buf_dynamic_attach()
949 struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,  in dma_buf_attach()  argument
952 	return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL);  in dma_buf_attach()
964 void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)  in dma_buf_detach()  argument
966 	if (WARN_ON(!dmabuf || !attach))  in dma_buf_detach()
970 		if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_detach()
971 			dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_detach()
973 		dmabuf->ops->unmap_dma_buf(attach, attach->sgt, attach->dir);  in dma_buf_detach()
975 		if (dma_buf_is_dynamic(attach->dmabuf)) {  in dma_buf_detach()
977 			dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_detach()
981 	dma_resv_lock(dmabuf->resv, NULL);  in dma_buf_detach()
983 	dma_resv_unlock(dmabuf->resv);  in dma_buf_detach()
984 	if (dmabuf->ops->detach)  in dma_buf_detach()
985 		dmabuf->ops->detach(dmabuf, attach);  in dma_buf_detach()
1001 	struct dma_buf *dmabuf = attach->dmabuf;  in dma_buf_pin()  local
1004 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_pin()
1006 	if (dmabuf->ops->pin)  in dma_buf_pin()
1007 		ret = dmabuf->ops->pin(attach);  in dma_buf_pin()
1020 	struct dma_buf *dmabuf = attach->dmabuf;  in dma_buf_unpin()  local
1022 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_unpin()
1024 	if (dmabuf->ops->unpin)  in dma_buf_unpin()
1025 		dmabuf->ops->unpin(attach);  in dma_buf_unpin()
1052 	if (WARN_ON(!attach || !attach->dmabuf))  in dma_buf_map_attachment()
1056 		dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_map_attachment()
1070 	if (dma_buf_is_dynamic(attach->dmabuf)) {  in dma_buf_map_attachment()
1071 		dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_map_attachment()
1079 	sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);  in dma_buf_map_attachment()
1083 	if (IS_ERR(sg_table) && dma_buf_is_dynamic(attach->dmabuf) &&  in dma_buf_map_attachment()
1087 	if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) {  in dma_buf_map_attachment()
1112 	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))  in dma_buf_unmap_attachment()
1116 		dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_unmap_attachment()
1121 	if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_unmap_attachment()
1122 		dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_unmap_attachment()
1124 	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);  in dma_buf_unmap_attachment()
1126 	if (dma_buf_is_dynamic(attach->dmabuf) &&  in dma_buf_unmap_attachment()
1140 void dma_buf_move_notify(struct dma_buf *dmabuf)  in dma_buf_move_notify()  argument
1144 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_move_notify()
1146 	list_for_each_entry(attach, &dmabuf->attachments, node)  in dma_buf_move_notify()
1236 static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf,  in __dma_buf_begin_cpu_access()  argument
1241 	struct dma_resv *resv = dmabuf->resv;  in __dma_buf_begin_cpu_access()
1267 int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,  in dma_buf_begin_cpu_access()  argument
1272 	if (WARN_ON(!dmabuf))  in dma_buf_begin_cpu_access()
1275 	if (dmabuf->ops->begin_cpu_access)  in dma_buf_begin_cpu_access()
1276 		ret = dmabuf->ops->begin_cpu_access(dmabuf, direction);  in dma_buf_begin_cpu_access()
1283 		ret = __dma_buf_begin_cpu_access(dmabuf, direction);  in dma_buf_begin_cpu_access()
1289 int dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf,  in dma_buf_begin_cpu_access_partial()  argument
1295 	if (WARN_ON(!dmabuf))  in dma_buf_begin_cpu_access_partial()
1298 	if (dmabuf->ops->begin_cpu_access_partial)  in dma_buf_begin_cpu_access_partial()
1299 		ret = dmabuf->ops->begin_cpu_access_partial(dmabuf, direction,  in dma_buf_begin_cpu_access_partial()
1307 		ret = __dma_buf_begin_cpu_access(dmabuf, direction);  in dma_buf_begin_cpu_access_partial()
1325 int dma_buf_end_cpu_access(struct dma_buf *dmabuf,  in dma_buf_end_cpu_access()  argument
1330 	WARN_ON(!dmabuf);  in dma_buf_end_cpu_access()
1332 	if (dmabuf->ops->end_cpu_access)  in dma_buf_end_cpu_access()
1333 		ret = dmabuf->ops->end_cpu_access(dmabuf, direction);  in dma_buf_end_cpu_access()
1339 int dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf,  in dma_buf_end_cpu_access_partial()  argument
1345 	WARN_ON(!dmabuf);  in dma_buf_end_cpu_access_partial()
1347 	if (dmabuf->ops->end_cpu_access_partial)  in dma_buf_end_cpu_access_partial()
1348 		ret = dmabuf->ops->end_cpu_access_partial(dmabuf, direction,  in dma_buf_end_cpu_access_partial()
1369 int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,  in dma_buf_mmap()  argument
1375 	if (WARN_ON(!dmabuf || !vma))  in dma_buf_mmap()
1379 	if (!dmabuf->ops->mmap)  in dma_buf_mmap()
1388 	    dmabuf->size >> PAGE_SHIFT)  in dma_buf_mmap()
1392 	get_file(dmabuf->file);  in dma_buf_mmap()
1394 	vma->vm_file = dmabuf->file;  in dma_buf_mmap()
1397 	ret = dmabuf->ops->mmap(dmabuf, vma);  in dma_buf_mmap()
1401 		fput(dmabuf->file);  in dma_buf_mmap()
1423 void *dma_buf_vmap(struct dma_buf *dmabuf)  in dma_buf_vmap()  argument
1427 	if (WARN_ON(!dmabuf))  in dma_buf_vmap()
1430 	if (!dmabuf->ops->vmap)  in dma_buf_vmap()
1433 	mutex_lock(&dmabuf->lock);  in dma_buf_vmap()
1434 	if (dmabuf->vmapping_counter) {  in dma_buf_vmap()
1435 		dmabuf->vmapping_counter++;  in dma_buf_vmap()
1436 		BUG_ON(!dmabuf->vmap_ptr);  in dma_buf_vmap()
1437 		ptr = dmabuf->vmap_ptr;  in dma_buf_vmap()
1441 	BUG_ON(dmabuf->vmap_ptr);  in dma_buf_vmap()
1443 	ptr = dmabuf->ops->vmap(dmabuf);  in dma_buf_vmap()
1449 	dmabuf->vmap_ptr = ptr;  in dma_buf_vmap()
1450 	dmabuf->vmapping_counter = 1;  in dma_buf_vmap()
1453 	mutex_unlock(&dmabuf->lock);  in dma_buf_vmap()
1463 void dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr)  in dma_buf_vunmap()  argument
1465 	if (WARN_ON(!dmabuf))  in dma_buf_vunmap()
1468 	BUG_ON(!dmabuf->vmap_ptr);  in dma_buf_vunmap()
1469 	BUG_ON(dmabuf->vmapping_counter == 0);  in dma_buf_vunmap()
1470 	BUG_ON(dmabuf->vmap_ptr != vaddr);  in dma_buf_vunmap()
1472 	mutex_lock(&dmabuf->lock);  in dma_buf_vunmap()
1473 	if (--dmabuf->vmapping_counter == 0) {  in dma_buf_vunmap()
1474 		if (dmabuf->ops->vunmap)  in dma_buf_vunmap()
1475 			dmabuf->ops->vunmap(dmabuf, vaddr);  in dma_buf_vunmap()
1476 		dmabuf->vmap_ptr = NULL;  in dma_buf_vunmap()
1478 	mutex_unlock(&dmabuf->lock);  in dma_buf_vunmap()
1482 int dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags)  in dma_buf_get_flags()  argument
1486 	if (WARN_ON(!dmabuf) || !flags)  in dma_buf_get_flags()
1489 	if (dmabuf->ops->get_flags)  in dma_buf_get_flags()
1490 		ret = dmabuf->ops->get_flags(dmabuf, flags);  in dma_buf_get_flags()
1496 int dma_buf_get_uuid(struct dma_buf *dmabuf, uuid_t *uuid)  in dma_buf_get_uuid()  argument
1498 	if (WARN_ON(!dmabuf) || !uuid)  in dma_buf_get_uuid()
1501 	if (!dmabuf->ops->get_uuid)  in dma_buf_get_uuid()
1504 	return dmabuf->ops->get_uuid(dmabuf, uuid);  in dma_buf_get_uuid()