60 			      struct ttm_tt *ttm,
147 	if (radeon_ttm_tt_has_userptr(rdev, bo->ttm))  in radeon_verify_access()
235 	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);  in radeon_move_vram_ram()
240 	r = ttm_tt_populate(bo->bdev, bo->ttm, &ctx);  in radeon_move_vram_ram()
245 	r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, &tmp_mem);  in radeon_move_vram_ram()
317 	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {  in radeon_bo_move()
421 	struct ttm_dma_tt		ttm;  member
431 static int radeon_ttm_tt_pin_userptr(struct ttm_bo_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_tt_pin_userptr()  argument
434 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_tt_pin_userptr()
448 		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;  in radeon_ttm_tt_pin_userptr()
456 		unsigned num_pages = ttm->num_pages - pinned;  in radeon_ttm_tt_pin_userptr()
458 		struct page **pages = ttm->pages + pinned;  in radeon_ttm_tt_pin_userptr()
467 	} while (pinned < ttm->num_pages);  in radeon_ttm_tt_pin_userptr()
469 	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,  in radeon_ttm_tt_pin_userptr()
470 				      ttm->num_pages << PAGE_SHIFT,  in radeon_ttm_tt_pin_userptr()
475 	r = dma_map_sgtable(rdev->dev, ttm->sg, direction, 0);  in radeon_ttm_tt_pin_userptr()
479 	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in radeon_ttm_tt_pin_userptr()
480 					 gtt->ttm.dma_address, ttm->num_pages);  in radeon_ttm_tt_pin_userptr()
485 	kfree(ttm->sg);  in radeon_ttm_tt_pin_userptr()
488 	release_pages(ttm->pages, pinned);  in radeon_ttm_tt_pin_userptr()
492 static void radeon_ttm_tt_unpin_userptr(struct ttm_bo_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_tt_unpin_userptr()  argument
495 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_tt_unpin_userptr()
503 	if (!ttm->sg->sgl)  in radeon_ttm_tt_unpin_userptr()
507 	dma_unmap_sgtable(rdev->dev, ttm->sg, direction, 0);  in radeon_ttm_tt_unpin_userptr()
509 	for_each_sgtable_page(ttm->sg, &sg_iter, 0) {  in radeon_ttm_tt_unpin_userptr()
518 	sg_free_table(ttm->sg);  in radeon_ttm_tt_unpin_userptr()
521 static bool radeon_ttm_backend_is_bound(struct ttm_tt *ttm)  in radeon_ttm_backend_is_bound()  argument
523 	struct radeon_ttm_tt *gtt = (void*)ttm;  in radeon_ttm_backend_is_bound()
529 				   struct ttm_tt *ttm,  in radeon_ttm_backend_bind()  argument
532 	struct radeon_ttm_tt *gtt = (void*)ttm;  in radeon_ttm_backend_bind()
542 		radeon_ttm_tt_pin_userptr(bdev, ttm);  in radeon_ttm_backend_bind()
547 	if (!ttm->num_pages) {  in radeon_ttm_backend_bind()
549 		     ttm->num_pages, bo_mem, ttm);  in radeon_ttm_backend_bind()
551 	if (ttm->caching_state == tt_cached)  in radeon_ttm_backend_bind()
553 	r = radeon_gart_bind(rdev, gtt->offset, ttm->num_pages,  in radeon_ttm_backend_bind()
554 			     ttm->pages, gtt->ttm.dma_address, flags);  in radeon_ttm_backend_bind()
557 			  ttm->num_pages, (unsigned)gtt->offset);  in radeon_ttm_backend_bind()
564 static void radeon_ttm_backend_unbind(struct ttm_bo_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_backend_unbind()  argument
566 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_backend_unbind()
570 		radeon_ttm_tt_unpin_userptr(bdev, ttm);  in radeon_ttm_backend_unbind()
575 	radeon_gart_unbind(rdev, gtt->offset, ttm->num_pages);  in radeon_ttm_backend_unbind()
580 static void radeon_ttm_backend_destroy(struct ttm_bo_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_backend_destroy()  argument
582 	struct radeon_ttm_tt *gtt = (void *)ttm;  in radeon_ttm_backend_destroy()
584 	radeon_ttm_backend_unbind(bdev, ttm);  in radeon_ttm_backend_destroy()
585 	ttm_tt_destroy_common(bdev, ttm);  in radeon_ttm_backend_destroy()
587 	ttm_dma_tt_fini(&gtt->ttm);  in radeon_ttm_backend_destroy()
609 	if (ttm_dma_tt_init(&gtt->ttm, bo, page_flags)) {  in radeon_ttm_tt_create()
613 	return &gtt->ttm.ttm;  in radeon_ttm_tt_create()
617 						  struct ttm_tt *ttm)  in radeon_ttm_tt_to_gtt()  argument
624 	if (!ttm)  in radeon_ttm_tt_to_gtt()
626 	return container_of(ttm, struct radeon_ttm_tt, ttm.ttm);  in radeon_ttm_tt_to_gtt()
630 				  struct ttm_tt *ttm,  in radeon_ttm_tt_populate()  argument
634 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_populate()
635 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);  in radeon_ttm_tt_populate()
638 		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);  in radeon_ttm_tt_populate()
639 		if (!ttm->sg)  in radeon_ttm_tt_populate()
642 		ttm->page_flags |= TTM_PAGE_FLAG_SG;  in radeon_ttm_tt_populate()
643 		ttm_tt_set_populated(ttm);  in radeon_ttm_tt_populate()
647 	if (slave && ttm->sg) {  in radeon_ttm_tt_populate()
648 		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,  in radeon_ttm_tt_populate()
649 						 gtt->ttm.dma_address, ttm->num_pages);  in radeon_ttm_tt_populate()
650 		ttm_tt_set_populated(ttm);  in radeon_ttm_tt_populate()
656 		return ttm_pool_populate(ttm, ctx);  in radeon_ttm_tt_populate()
662 		return ttm_dma_populate(&gtt->ttm, rdev->dev, ctx);  in radeon_ttm_tt_populate()
666 	return ttm_populate_and_map_pages(rdev->dev, &gtt->ttm, ctx);  in radeon_ttm_tt_populate()
669 static void radeon_ttm_tt_unpopulate(struct ttm_bo_device *bdev, struct ttm_tt *ttm)  in radeon_ttm_tt_unpopulate()  argument
672 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_unpopulate()
673 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);  in radeon_ttm_tt_unpopulate()
676 		kfree(ttm->sg);  in radeon_ttm_tt_unpopulate()
677 		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;  in radeon_ttm_tt_unpopulate()
686 		ttm_pool_unpopulate(ttm);  in radeon_ttm_tt_unpopulate()
693 		ttm_dma_unpopulate(&gtt->ttm, rdev->dev);  in radeon_ttm_tt_unpopulate()
698 	ttm_unmap_and_unpopulate_pages(rdev->dev, &gtt->ttm);  in radeon_ttm_tt_unpopulate()
702 			      struct ttm_tt *ttm, uint64_t addr,  in radeon_ttm_tt_set_userptr()  argument
705 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_set_userptr()
717 			    struct ttm_tt *ttm)  in radeon_ttm_tt_is_bound()  argument
722 		return ttm_agp_is_bound(ttm);  in radeon_ttm_tt_is_bound()
724 	return radeon_ttm_backend_is_bound(ttm);  in radeon_ttm_tt_is_bound()
728 			      struct ttm_tt *ttm,  in radeon_ttm_tt_bind()  argument
739 		return ttm_agp_bind(ttm, bo_mem);  in radeon_ttm_tt_bind()
742 	return radeon_ttm_backend_bind(bdev, ttm, bo_mem);  in radeon_ttm_tt_bind()
746 				 struct ttm_tt *ttm)  in radeon_ttm_tt_unbind()  argument
752 		ttm_agp_unbind(ttm);  in radeon_ttm_tt_unbind()
756 	radeon_ttm_backend_unbind(bdev, ttm);  in radeon_ttm_tt_unbind()
760 				  struct ttm_tt *ttm)  in radeon_ttm_tt_destroy()  argument
766 		ttm_agp_unbind(ttm);  in radeon_ttm_tt_destroy()
767 		ttm_tt_destroy_common(bdev, ttm);  in radeon_ttm_tt_destroy()
768 		ttm_agp_destroy(ttm);  in radeon_ttm_tt_destroy()
772 	radeon_ttm_backend_destroy(bdev, ttm);  in radeon_ttm_tt_destroy()
776 			       struct ttm_tt *ttm)  in radeon_ttm_tt_has_userptr()  argument
778 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_has_userptr()
787 			       struct ttm_tt *ttm)  in radeon_ttm_tt_is_readonly()  argument
789 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm);  in radeon_ttm_tt_is_readonly()