47 static int vb2_cma_sg_alloc_compacted(struct vb2_cma_sg_buf *buf,  in vb2_cma_sg_alloc_compacted()  argument
51 	unsigned long size = buf->size;  in vb2_cma_sg_alloc_compacted()
72 					__free_page(buf->pages[last_page]);  in vb2_cma_sg_alloc_compacted()
80 			buf->pages[last_page++] = &pages[i];  in vb2_cma_sg_alloc_compacted()
88 static void vb2_cma_sg_free_compacted(struct vb2_cma_sg_buf *buf)  in vb2_cma_sg_free_compacted()  argument
90 	int num_pages = buf->num_pages;  in vb2_cma_sg_free_compacted()
93 		__free_page(buf->pages[num_pages]);  in vb2_cma_sg_free_compacted()
94 		buf->pages[num_pages] = NULL;  in vb2_cma_sg_free_compacted()
98 static int vb2_cma_sg_alloc_contiguous(struct vb2_cma_sg_buf *buf)  in vb2_cma_sg_alloc_contiguous()  argument
110 			page = rk_dma_heap_alloc_contig_pages(heap, buf->size,  in vb2_cma_sg_alloc_contiguous()
111 							      dev_name(buf->dev));  in vb2_cma_sg_alloc_contiguous()
113 			page = cma_alloc(dev_get_cma_area(buf->dev), buf->num_pages,  in vb2_cma_sg_alloc_contiguous()
114 					 get_order(buf->size), GFP_KERNEL);  in vb2_cma_sg_alloc_contiguous()
120 	for (i = 0; i < buf->num_pages; i++)  in vb2_cma_sg_alloc_contiguous()
121 		buf->pages[i] = page + i;  in vb2_cma_sg_alloc_contiguous()
126 static void vb2_cma_sg_free_contiguous(struct vb2_cma_sg_buf *buf)  in vb2_cma_sg_free_contiguous()  argument
132 			rk_dma_heap_free_contig_pages(heap, buf->pages[0],  in vb2_cma_sg_free_contiguous()
133 						      buf->size, dev_name(buf->dev));  in vb2_cma_sg_free_contiguous()
135 			cma_release(dev_get_cma_area(buf->dev),  in vb2_cma_sg_free_contiguous()
136 				    buf->pages[0], buf->num_pages);  in vb2_cma_sg_free_contiguous()
145 	struct vb2_cma_sg_buf *buf;  in vb2_cma_sg_alloc()  local
152 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);  in vb2_cma_sg_alloc()
153 	if (!buf)  in vb2_cma_sg_alloc()
156 	buf->vaddr = NULL;  in vb2_cma_sg_alloc()
157 	buf->dma_attrs = dma_attrs;  in vb2_cma_sg_alloc()
158 	buf->dma_dir = dma_dir;  in vb2_cma_sg_alloc()
159 	buf->offset = 0;  in vb2_cma_sg_alloc()
160 	buf->size = size;  in vb2_cma_sg_alloc()
162 	buf->num_pages = size >> PAGE_SHIFT;  in vb2_cma_sg_alloc()
163 	buf->dma_sgt = &buf->sg_table;  in vb2_cma_sg_alloc()
165 	buf->dev = get_device(dev);  in vb2_cma_sg_alloc()
167 	buf->pages = kvmalloc_array(buf->num_pages, sizeof(struct page *),  in vb2_cma_sg_alloc()
169 	if (!buf->pages)  in vb2_cma_sg_alloc()
173 		ret = vb2_cma_sg_alloc_contiguous(buf);  in vb2_cma_sg_alloc()
175 		ret = vb2_cma_sg_alloc_compacted(buf, gfp_flags);  in vb2_cma_sg_alloc()
179 	ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,  in vb2_cma_sg_alloc()
180 			buf->num_pages, 0, size, GFP_KERNEL);  in vb2_cma_sg_alloc()
184 	sgt = &buf->sg_table;  in vb2_cma_sg_alloc()
189 	if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,  in vb2_cma_sg_alloc()
193 	buf->handler.refcount = &buf->refcount;  in vb2_cma_sg_alloc()
194 	buf->handler.put = vb2_cma_sg_put;  in vb2_cma_sg_alloc()
195 	buf->handler.arg = buf;  in vb2_cma_sg_alloc()
197 	refcount_set(&buf->refcount, 1);  in vb2_cma_sg_alloc()
199 	return buf;  in vb2_cma_sg_alloc()
202 	sg_free_table(buf->dma_sgt);  in vb2_cma_sg_alloc()
205 		vb2_cma_sg_free_contiguous(buf);  in vb2_cma_sg_alloc()
207 		vb2_cma_sg_free_compacted(buf);  in vb2_cma_sg_alloc()
209 	kvfree(buf->pages);  in vb2_cma_sg_alloc()
211 	put_device(buf->dev);  in vb2_cma_sg_alloc()
212 	kfree(buf);  in vb2_cma_sg_alloc()
218 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_put()  local
219 	struct sg_table *sgt = &buf->sg_table;  in vb2_cma_sg_put()
221 	if (refcount_dec_and_test(&buf->refcount)) {  in vb2_cma_sg_put()
222 		dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir,  in vb2_cma_sg_put()
224 		if (buf->vaddr)  in vb2_cma_sg_put()
225 			vm_unmap_ram(buf->vaddr, buf->num_pages);  in vb2_cma_sg_put()
226 		sg_free_table(buf->dma_sgt);  in vb2_cma_sg_put()
227 		if (buf->dma_attrs & DMA_ATTR_FORCE_CONTIGUOUS)  in vb2_cma_sg_put()
228 			vb2_cma_sg_free_contiguous(buf);  in vb2_cma_sg_put()
230 			vb2_cma_sg_free_compacted(buf);  in vb2_cma_sg_put()
231 		kvfree(buf->pages);  in vb2_cma_sg_put()
232 		buf->pages = NULL;  in vb2_cma_sg_put()
233 		put_device(buf->dev);  in vb2_cma_sg_put()
234 		kfree(buf);  in vb2_cma_sg_put()
240 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_prepare()  local
241 	struct sg_table *sgt = buf->dma_sgt;  in vb2_cma_sg_prepare()
243 	dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir);  in vb2_cma_sg_prepare()
248 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_finish()  local
249 	struct sg_table *sgt = buf->dma_sgt;  in vb2_cma_sg_finish()
251 	dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);  in vb2_cma_sg_finish()
258 	struct vb2_cma_sg_buf *buf;  in vb2_cma_sg_get_userptr()  local
265 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);  in vb2_cma_sg_get_userptr()
266 	if (!buf)  in vb2_cma_sg_get_userptr()
269 	buf->vaddr = NULL;  in vb2_cma_sg_get_userptr()
270 	buf->dev = dev;  in vb2_cma_sg_get_userptr()
271 	buf->dma_dir = dma_dir;  in vb2_cma_sg_get_userptr()
272 	buf->offset = vaddr & ~PAGE_MASK;  in vb2_cma_sg_get_userptr()
273 	buf->size = size;  in vb2_cma_sg_get_userptr()
274 	buf->dma_sgt = &buf->sg_table;  in vb2_cma_sg_get_userptr()
278 	buf->vec = vec;  in vb2_cma_sg_get_userptr()
280 	buf->pages = frame_vector_pages(vec);  in vb2_cma_sg_get_userptr()
281 	if (IS_ERR(buf->pages))  in vb2_cma_sg_get_userptr()
283 	buf->num_pages = frame_vector_count(vec);  in vb2_cma_sg_get_userptr()
285 	if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,  in vb2_cma_sg_get_userptr()
286 			buf->num_pages, buf->offset, size, 0))  in vb2_cma_sg_get_userptr()
289 	sgt = &buf->sg_table;  in vb2_cma_sg_get_userptr()
294 	if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,  in vb2_cma_sg_get_userptr()
298 	return buf;  in vb2_cma_sg_get_userptr()
301 	sg_free_table(&buf->sg_table);  in vb2_cma_sg_get_userptr()
305 	kfree(buf);  in vb2_cma_sg_get_userptr()
315 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_put_userptr()  local
316 	struct sg_table *sgt = &buf->sg_table;  in vb2_cma_sg_put_userptr()
317 	int i = buf->num_pages;  in vb2_cma_sg_put_userptr()
319 	dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);  in vb2_cma_sg_put_userptr()
320 	if (buf->vaddr)  in vb2_cma_sg_put_userptr()
321 		vm_unmap_ram(buf->vaddr, buf->num_pages);  in vb2_cma_sg_put_userptr()
322 	sg_free_table(buf->dma_sgt);  in vb2_cma_sg_put_userptr()
323 	if (buf->dma_dir == DMA_FROM_DEVICE ||  in vb2_cma_sg_put_userptr()
324 	    buf->dma_dir == DMA_BIDIRECTIONAL)  in vb2_cma_sg_put_userptr()
326 			set_page_dirty_lock(buf->pages[i]);  in vb2_cma_sg_put_userptr()
327 	vb2_destroy_framevec(buf->vec);  in vb2_cma_sg_put_userptr()
328 	kfree(buf);  in vb2_cma_sg_put_userptr()
333 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_vaddr()  local
335 	WARN_ON(!buf);  in vb2_cma_sg_vaddr()
337 	if (!buf->vaddr) {  in vb2_cma_sg_vaddr()
338 		if (buf->db_attach)  in vb2_cma_sg_vaddr()
339 			buf->vaddr = dma_buf_vmap(buf->db_attach->dmabuf);  in vb2_cma_sg_vaddr()
341 			buf->vaddr = vm_map_ram(buf->pages, buf->num_pages, -1);  in vb2_cma_sg_vaddr()
345 	return buf->vaddr ? buf->vaddr + buf->offset : NULL;  in vb2_cma_sg_vaddr()
350 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_num_users()  local
352 	return refcount_read(&buf->refcount);  in vb2_cma_sg_num_users()
357 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_mmap()  local
360 	if (!buf) {  in vb2_cma_sg_mmap()
365 	err = vm_map_pages(vma, buf->pages, buf->num_pages);  in vb2_cma_sg_mmap()
374 	vma->vm_private_data	= &buf->handler;  in vb2_cma_sg_mmap()
398 	struct vb2_cma_sg_buf *buf = dbuf->priv;  in vb2_cma_sg_dmabuf_ops_attach()  local
409 	ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL);  in vb2_cma_sg_dmabuf_ops_attach()
415 	rd = buf->dma_sgt->sgl;  in vb2_cma_sg_dmabuf_ops_attach()
501 	struct vb2_cma_sg_buf *buf = dbuf->priv;  in vb2_cma_sg_dmabuf_ops_begin_cpu_access()  local
502 	struct sg_table *sgt = buf->dma_sgt;  in vb2_cma_sg_dmabuf_ops_begin_cpu_access()
504 	dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);  in vb2_cma_sg_dmabuf_ops_begin_cpu_access()
512 	struct vb2_cma_sg_buf *buf = dbuf->priv;  in vb2_cma_sg_dmabuf_ops_end_cpu_access()  local
513 	struct sg_table *sgt = buf->dma_sgt;  in vb2_cma_sg_dmabuf_ops_end_cpu_access()
515 	dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir);  in vb2_cma_sg_dmabuf_ops_end_cpu_access()
521 	struct vb2_cma_sg_buf *buf = dbuf->priv;  in vb2_cma_sg_dmabuf_ops_vmap()  local
523 	return vb2_cma_sg_vaddr(buf);  in vb2_cma_sg_dmabuf_ops_vmap()
546 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_get_dmabuf()  local
551 	exp_info.size = buf->size;  in vb2_cma_sg_get_dmabuf()
553 	exp_info.priv = buf;  in vb2_cma_sg_get_dmabuf()
555 	if (WARN_ON(!buf->dma_sgt))  in vb2_cma_sg_get_dmabuf()
563 	refcount_inc(&buf->refcount);  in vb2_cma_sg_get_dmabuf()
574 	struct vb2_cma_sg_buf *buf = mem_priv;  in vb2_cma_sg_map_dmabuf()  local
577 	if (WARN_ON(!buf->db_attach)) {  in vb2_cma_sg_map_dmabuf()
582 	if (WARN_ON(buf->dma_sgt)) {  in vb2_cma_sg_map_dmabuf()
588 	sgt = dma_buf_map_attachment(buf->db_attach, buf->dma_dir);  in vb2_cma_sg_map_dmabuf()
594 	buf->dma_sgt = sgt;  in vb2_cma_sg_map_dmabuf()
595 	buf->vaddr = NULL;  in vb2_cma_sg_map_dmabuf()
602 	struct vb2_cma_sg_buf *buf = mem_priv;  in vb2_cma_sg_unmap_dmabuf()  local
603 	struct sg_table *sgt = buf->dma_sgt;  in vb2_cma_sg_unmap_dmabuf()
605 	if (WARN_ON(!buf->db_attach)) {  in vb2_cma_sg_unmap_dmabuf()
615 	if (buf->vaddr) {  in vb2_cma_sg_unmap_dmabuf()
616 		dma_buf_vunmap(buf->db_attach->dmabuf, buf->vaddr);  in vb2_cma_sg_unmap_dmabuf()
617 		buf->vaddr = NULL;  in vb2_cma_sg_unmap_dmabuf()
619 	dma_buf_unmap_attachment(buf->db_attach, sgt, buf->dma_dir);  in vb2_cma_sg_unmap_dmabuf()
621 	buf->dma_sgt = NULL;  in vb2_cma_sg_unmap_dmabuf()
626 	struct vb2_cma_sg_buf *buf = mem_priv;  in vb2_cma_sg_detach_dmabuf()  local
629 	if (WARN_ON(buf->dma_sgt))  in vb2_cma_sg_detach_dmabuf()
630 		vb2_cma_sg_unmap_dmabuf(buf);  in vb2_cma_sg_detach_dmabuf()
633 	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);  in vb2_cma_sg_detach_dmabuf()
634 	kfree(buf);  in vb2_cma_sg_detach_dmabuf()
640 	struct vb2_cma_sg_buf *buf;  in vb2_cma_sg_attach_dmabuf()  local
649 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);  in vb2_cma_sg_attach_dmabuf()
650 	if (!buf)  in vb2_cma_sg_attach_dmabuf()
653 	buf->dev = dev;  in vb2_cma_sg_attach_dmabuf()
655 	dba = dma_buf_attach(dbuf, buf->dev);  in vb2_cma_sg_attach_dmabuf()
658 		kfree(buf);  in vb2_cma_sg_attach_dmabuf()
662 	buf->dma_dir = dma_dir;  in vb2_cma_sg_attach_dmabuf()
663 	buf->size = size;  in vb2_cma_sg_attach_dmabuf()
664 	buf->db_attach = dba;  in vb2_cma_sg_attach_dmabuf()
666 	return buf;  in vb2_cma_sg_attach_dmabuf()
671 	struct vb2_cma_sg_buf *buf = buf_priv;  in vb2_cma_sg_cookie()  local
673 	return buf->dma_sgt;  in vb2_cma_sg_cookie()