556 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_get_root_entry()
583 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_set_root_entry()
605 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_guest_entry()
617 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_guest_entry()
628 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_host_entry()
638 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_host_entry()
655 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_get_entry()
684 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_set_entry()
747 	radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);  in ppgtt_free_spt()
768 	radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) {  in ppgtt_free_all_spt()
816 	return radix_tree_lookup(&vgpu->gtt.spt_tree, mfn);  in intel_vgpu_find_spt_by_mfn()
858 	ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);  in ppgtt_alloc_spt()
912 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
918 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
947 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_spt_by_shadow_entry()
968 			vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)  in ppgtt_invalidate_spt_by_shadow_entry()
984 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_pte()
992 	if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn)  in ppgtt_invalidate_pte()
1072 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_spt_by_guest_entry()
1136 	struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;  in ppgtt_generate_shadow_entry()
1159 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in is_2MB_gtt_possible()
1176 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_2MB_gtt_entry()
1223 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_64KB_gtt_entry()
1254 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_shadow_entry()
1308 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_populate_spt()
1330 				ops->set_pfn(&se, gvt->gtt.scratch_mfn);  in ppgtt_populate_spt()
1351 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_entry_removal()
1364 	    vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn)  in ppgtt_handle_guest_entry_removal()
1432 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in sync_oos_page()
1484 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);  in detach_oos_page()
1504 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);  in attach_oos_page()
1530 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in ppgtt_allocate_oos_page()  local
1536 	if (list_empty(&gtt->oos_page_free_list_head)) {  in ppgtt_allocate_oos_page()
1537 		oos_page = container_of(gtt->oos_page_use_list_head.next,  in ppgtt_allocate_oos_page()
1546 		oos_page = container_of(gtt->oos_page_free_list_head.next,  in ppgtt_allocate_oos_page()
1561 	list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head);  in ppgtt_set_guest_page_oos()
1584 	list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {  in intel_vgpu_sync_oos_pages()
1603 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table()
1635 					vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1642 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1646 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1675 			&spt->vgpu->gtt.post_shadow_list_head);  in ppgtt_set_post_shadow()
1696 	list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {  in intel_vgpu_flush_post_shadow()
1720 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table_bytes()
1754 			ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table_bytes()
1784 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in invalidate_ppgtt_mm()  local
1785 	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in invalidate_ppgtt_mm()
1814 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in shadow_ppgtt_mm()  local
1815 	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in shadow_ppgtt_mm()
1918 	list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);  in intel_vgpu_create_ppgtt_mm()
1920 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
1921 	list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_create_ppgtt_mm()
1922 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
1967 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
1969 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
2012 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2014 			       &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_pin_mm()
2015 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2026 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2028 	list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) {  in reclaim_one_ppgtt_mm()
2035 		mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2039 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2050 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_get_next_level_entry()
2079 	struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;  in intel_vgpu_gma_to_gpa()
2080 	struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;  in intel_vgpu_gma_to_gpa()
2152 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_read()
2204 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ggtt_invalidate_pte()
2208 	if (pfn != vgpu->gvt->gtt.scratch_mfn)  in ggtt_invalidate_pte()
2218 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_write()
2219 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in emulate_ggtt_mmio_write()
2297 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2309 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2313 		ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2373 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in alloc_scratch_pages()  local
2374 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in alloc_scratch_pages()
2399 	gtt->scratch_pt[type].page_mfn =  in alloc_scratch_pages()
2401 	gtt->scratch_pt[type].page = virt_to_page(scratch_pt);  in alloc_scratch_pages()
2403 			vgpu->id, type, gtt->scratch_pt[type].page_mfn);  in alloc_scratch_pages()
2418 		ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn);  in alloc_scratch_pages()
2441 		if (vgpu->gtt.scratch_pt[i].page != NULL) {  in release_scratch_page_tree()
2442 			daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<  in release_scratch_page_tree()
2445 			__free_page(vgpu->gtt.scratch_pt[i].page);  in release_scratch_page_tree()
2446 			vgpu->gtt.scratch_pt[i].page = NULL;  in release_scratch_page_tree()
2447 			vgpu->gtt.scratch_pt[i].page_mfn = 0;  in release_scratch_page_tree()
2483 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in intel_vgpu_init_gtt()  local
2485 	INIT_RADIX_TREE(&gtt->spt_tree, GFP_KERNEL);  in intel_vgpu_init_gtt()
2487 	INIT_LIST_HEAD(&gtt->ppgtt_mm_list_head);  in intel_vgpu_init_gtt()
2488 	INIT_LIST_HEAD(&gtt->oos_page_list_head);  in intel_vgpu_init_gtt()
2489 	INIT_LIST_HEAD(&gtt->post_shadow_list_head);  in intel_vgpu_init_gtt()
2491 	gtt->ggtt_mm = intel_vgpu_create_ggtt_mm(vgpu);  in intel_vgpu_init_gtt()
2492 	if (IS_ERR(gtt->ggtt_mm)) {  in intel_vgpu_init_gtt()
2494 		return PTR_ERR(gtt->ggtt_mm);  in intel_vgpu_init_gtt()
2499 	INIT_LIST_HEAD(&gtt->ggtt_mm->ggtt_mm.partial_pte_list);  in intel_vgpu_init_gtt()
2509 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_destroy_all_ppgtt_mm()
2514 	if (GEM_WARN_ON(!list_empty(&vgpu->gtt.ppgtt_mm_list_head)))  in intel_vgpu_destroy_all_ppgtt_mm()
2517 	if (GEM_WARN_ON(!radix_tree_empty(&vgpu->gtt.spt_tree))) {  in intel_vgpu_destroy_all_ppgtt_mm()
2528 				 &vgpu->gtt.ggtt_mm->ggtt_mm.partial_pte_list,  in intel_vgpu_destroy_ggtt_mm()
2534 	intel_vgpu_destroy_mm(vgpu->gtt.ggtt_mm);  in intel_vgpu_destroy_ggtt_mm()
2535 	vgpu->gtt.ggtt_mm = NULL;  in intel_vgpu_destroy_ggtt_mm()
2557 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in clean_spt_oos()  local
2561 	WARN(!list_empty(&gtt->oos_page_use_list_head),  in clean_spt_oos()
2564 	list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {  in clean_spt_oos()
2574 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in setup_spt_oos()  local
2579 	INIT_LIST_HEAD(&gtt->oos_page_free_list_head);  in setup_spt_oos()
2580 	INIT_LIST_HEAD(&gtt->oos_page_use_list_head);  in setup_spt_oos()
2598 		list_add_tail(&oos_page->list, &gtt->oos_page_free_list_head);  in setup_spt_oos()
2625 	list_for_each(pos, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_find_ppgtt_mm()
2714 	gvt->gtt.pte_ops = &gen8_gtt_pte_ops;  in intel_gvt_init_gtt()
2715 	gvt->gtt.gma_ops = &gen8_gtt_gma_ops;  in intel_gvt_init_gtt()
2731 	gvt->gtt.scratch_page = virt_to_page(page);  in intel_gvt_init_gtt()
2732 	gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);  in intel_gvt_init_gtt()
2739 			__free_page(gvt->gtt.scratch_page);  in intel_gvt_init_gtt()
2743 	INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head);  in intel_gvt_init_gtt()
2744 	mutex_init(&gvt->gtt.ppgtt_mm_lock);  in intel_gvt_init_gtt()
2759 	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<  in intel_gvt_clean_gtt()
2764 	__free_page(gvt->gtt.scratch_page);  in intel_gvt_clean_gtt()
2782 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_invalidate_ppgtt()
2785 			mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2787 			mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2806 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in intel_vgpu_reset_ggtt()
2812 	pte_ops->set_pfn(&entry, gvt->gtt.scratch_mfn);  in intel_vgpu_reset_ggtt()
2819 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2822 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2829 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2832 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()