54 bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size)  in intel_gvt_ggtt_validate_range()  argument
57 		return vgpu_gmadr_is_valid(vgpu, addr);  in intel_gvt_ggtt_validate_range()
59 	if (vgpu_gmadr_is_aperture(vgpu, addr) &&  in intel_gvt_ggtt_validate_range()
60 	    vgpu_gmadr_is_aperture(vgpu, addr + size - 1))  in intel_gvt_ggtt_validate_range()
62 	else if (vgpu_gmadr_is_hidden(vgpu, addr) &&  in intel_gvt_ggtt_validate_range()
63 		 vgpu_gmadr_is_hidden(vgpu, addr + size - 1))  in intel_gvt_ggtt_validate_range()
72 int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr)  in intel_gvt_ggtt_gmadr_g2h()  argument
74 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in intel_gvt_ggtt_gmadr_g2h()
76 	if (drm_WARN(&i915->drm, !vgpu_gmadr_is_valid(vgpu, g_addr),  in intel_gvt_ggtt_gmadr_g2h()
80 	if (vgpu_gmadr_is_aperture(vgpu, g_addr))  in intel_gvt_ggtt_gmadr_g2h()
81 		*h_addr = vgpu_aperture_gmadr_base(vgpu)  in intel_gvt_ggtt_gmadr_g2h()
82 			  + (g_addr - vgpu_aperture_offset(vgpu));  in intel_gvt_ggtt_gmadr_g2h()
84 		*h_addr = vgpu_hidden_gmadr_base(vgpu)  in intel_gvt_ggtt_gmadr_g2h()
85 			  + (g_addr - vgpu_hidden_offset(vgpu));  in intel_gvt_ggtt_gmadr_g2h()
90 int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr)  in intel_gvt_ggtt_gmadr_h2g()  argument
92 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in intel_gvt_ggtt_gmadr_h2g()
94 	if (drm_WARN(&i915->drm, !gvt_gmadr_is_valid(vgpu->gvt, h_addr),  in intel_gvt_ggtt_gmadr_h2g()
98 	if (gvt_gmadr_is_aperture(vgpu->gvt, h_addr))  in intel_gvt_ggtt_gmadr_h2g()
99 		*g_addr = vgpu_aperture_gmadr_base(vgpu)  in intel_gvt_ggtt_gmadr_h2g()
100 			+ (h_addr - gvt_aperture_gmadr_base(vgpu->gvt));  in intel_gvt_ggtt_gmadr_h2g()
102 		*g_addr = vgpu_hidden_gmadr_base(vgpu)  in intel_gvt_ggtt_gmadr_h2g()
103 			+ (h_addr - gvt_hidden_gmadr_base(vgpu->gvt));  in intel_gvt_ggtt_gmadr_h2g()
107 int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,  in intel_gvt_ggtt_index_g2h()  argument
113 	ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << I915_GTT_PAGE_SHIFT,  in intel_gvt_ggtt_index_g2h()
122 int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,  in intel_gvt_ggtt_h2g_index()  argument
128 	ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << I915_GTT_PAGE_SHIFT,  in intel_gvt_ggtt_h2g_index()
306 		struct intel_vgpu *vgpu)  in gtt_get_entry64()  argument
308 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in gtt_get_entry64()
315 		ret = intel_gvt_hypervisor_read_gpa(vgpu, gpa +  in gtt_get_entry64()
321 		e->val64 = read_pte64(vgpu->gvt->gt->ggtt, index);  in gtt_get_entry64()
331 		struct intel_vgpu *vgpu)  in gtt_set_entry64()  argument
333 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in gtt_set_entry64()
340 		ret = intel_gvt_hypervisor_write_gpa(vgpu, gpa +  in gtt_set_entry64()
346 		write_pte64(vgpu->gvt->gt->ggtt, index, e->val64);  in gtt_set_entry64()
556 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_get_root_entry()
563 			   entry, index, false, 0, mm->vgpu);  in _ppgtt_get_root_entry()
583 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_set_root_entry()
587 			   entry, index, false, 0, mm->vgpu);  in _ppgtt_set_root_entry()
605 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_guest_entry()
611 			   false, 0, mm->vgpu);  in ggtt_get_guest_entry()
617 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_guest_entry()
622 			   false, 0, mm->vgpu);  in ggtt_set_guest_entry()
628 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_host_entry()
632 	pte_ops->get_entry(NULL, entry, index, false, 0, mm->vgpu);  in ggtt_get_host_entry()
638 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_host_entry()
642 	pte_ops->set_entry(NULL, entry, index, false, 0, mm->vgpu);  in ggtt_set_host_entry()
654 	struct intel_gvt *gvt = spt->vgpu->gvt;  in ppgtt_spt_get_entry()
665 			spt->vgpu);  in ppgtt_spt_get_entry()
683 	struct intel_gvt *gvt = spt->vgpu->gvt;  in ppgtt_spt_set_entry()
694 			spt->vgpu);  in ppgtt_spt_set_entry()
735 static int detach_oos_page(struct intel_vgpu *vgpu,
740 	struct device *kdev = &spt->vgpu->gvt->gt->i915->drm.pdev->dev;  in ppgtt_free_spt()
742 	trace_spt_free(spt->vgpu->id, spt, spt->guest_page.type);  in ppgtt_free_spt()
747 	radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);  in ppgtt_free_spt()
751 			detach_oos_page(spt->vgpu, spt->guest_page.oos_page);  in ppgtt_free_spt()
753 		intel_vgpu_unregister_page_track(spt->vgpu, spt->guest_page.gfn);  in ppgtt_free_spt()
760 static void ppgtt_free_all_spt(struct intel_vgpu *vgpu)  in ppgtt_free_all_spt()  argument
768 	radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) {  in ppgtt_free_all_spt()
801 		struct intel_vgpu *vgpu, unsigned long gfn)  in intel_vgpu_find_spt_by_gfn()  argument
805 	track = intel_vgpu_find_page_track(vgpu, gfn);  in intel_vgpu_find_spt_by_gfn()
814 		struct intel_vgpu *vgpu, unsigned long mfn)  in intel_vgpu_find_spt_by_mfn()  argument
816 	return radix_tree_lookup(&vgpu->gtt.spt_tree, mfn);  in intel_vgpu_find_spt_by_mfn()
823 		struct intel_vgpu *vgpu, enum intel_gvt_gtt_type type)  in ppgtt_alloc_spt()  argument
825 	struct device *kdev = &vgpu->gvt->gt->i915->drm.pdev->dev;  in ppgtt_alloc_spt()
833 		if (reclaim_one_ppgtt_mm(vgpu->gvt))  in ppgtt_alloc_spt()
840 	spt->vgpu = vgpu;  in ppgtt_alloc_spt()
858 	ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);  in ppgtt_alloc_spt()
873 		struct intel_vgpu *vgpu, enum intel_gvt_gtt_type type,  in ppgtt_alloc_spt_gfn()  argument
879 	spt = ppgtt_alloc_spt(vgpu, type);  in ppgtt_alloc_spt_gfn()
886 	ret = intel_vgpu_register_page_track(vgpu, gfn,  in ppgtt_alloc_spt_gfn()
897 	trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);  in ppgtt_alloc_spt_gfn()
903 	((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
912 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
918 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
929 	trace_spt_refcount(spt->vgpu->id, "inc", spt, v, (v + 1));  in ppgtt_get_spt()
937 	trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));  in ppgtt_put_spt()
943 static int ppgtt_invalidate_spt_by_shadow_entry(struct intel_vgpu *vgpu,  in ppgtt_invalidate_spt_by_shadow_entry()  argument
946 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in ppgtt_invalidate_spt_by_shadow_entry()
947 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_spt_by_shadow_entry()
968 			vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)  in ppgtt_invalidate_spt_by_shadow_entry()
971 	s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e));  in ppgtt_invalidate_spt_by_shadow_entry()
983 	struct intel_vgpu *vgpu = spt->vgpu;  in ppgtt_invalidate_pte()  local
984 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_pte()
992 	if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn)  in ppgtt_invalidate_pte()
995 	intel_gvt_hypervisor_dma_unmap_guest_page(vgpu, pfn << PAGE_SHIFT);  in ppgtt_invalidate_pte()
1000 	struct intel_vgpu *vgpu = spt->vgpu;  in ppgtt_invalidate_spt()  local
1005 	trace_spt_change(spt->vgpu->id, "die", spt,  in ppgtt_invalidate_spt()
1032 					spt->vgpu, &e);  in ppgtt_invalidate_spt()
1041 	trace_spt_change(spt->vgpu->id, "release", spt,  in ppgtt_invalidate_spt()
1051 static bool vgpu_ips_enabled(struct intel_vgpu *vgpu)  in vgpu_ips_enabled()  argument
1053 	struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;  in vgpu_ips_enabled()
1056 		u32 ips = vgpu_vreg_t(vgpu, GEN8_GAMW_ECO_DEV_RW_IA) &  in vgpu_ips_enabled()
1070 		struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we)  in ppgtt_populate_spt_by_guest_entry()  argument
1072 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_spt_by_guest_entry()
1080 		ips = vgpu_ips_enabled(vgpu) && ops->test_ips(we);  in ppgtt_populate_spt_by_guest_entry()
1082 	spt = intel_vgpu_find_spt_by_gfn(vgpu, ops->get_pfn(we));  in ppgtt_populate_spt_by_guest_entry()
1105 		spt = ppgtt_alloc_spt_gfn(vgpu, type, ops->get_pfn(we), ips);  in ppgtt_populate_spt_by_guest_entry()
1111 		ret = intel_vgpu_enable_page_track(vgpu, spt->guest_page.gfn);  in ppgtt_populate_spt_by_guest_entry()
1119 		trace_spt_change(vgpu->id, "new", spt, spt->guest_page.gfn,  in ppgtt_populate_spt_by_guest_entry()
1136 	struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;  in ppgtt_generate_shadow_entry()
1156 static int is_2MB_gtt_possible(struct intel_vgpu *vgpu,  in is_2MB_gtt_possible()  argument
1159 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in is_2MB_gtt_possible()
1162 	if (!HAS_PAGE_SIZES(vgpu->gvt->gt->i915, I915_GTT_PAGE_SIZE_2M))  in is_2MB_gtt_possible()
1165 	pfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, ops->get_pfn(entry));  in is_2MB_gtt_possible()
1172 static int split_2MB_gtt_entry(struct intel_vgpu *vgpu,  in split_2MB_gtt_entry()  argument
1176 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_2MB_gtt_entry()
1188 	sub_spt = ppgtt_alloc_spt(vgpu, GTT_TYPE_PPGTT_PTE_PT);  in split_2MB_gtt_entry()
1193 		ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu,  in split_2MB_gtt_entry()
1219 static int split_64KB_gtt_entry(struct intel_vgpu *vgpu,  in split_64KB_gtt_entry()  argument
1223 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_64KB_gtt_entry()
1239 		ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu,  in split_64KB_gtt_entry()
1250 static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu,  in ppgtt_populate_shadow_entry()  argument
1254 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_shadow_entry()
1276 		return split_64KB_gtt_entry(vgpu, spt, index, &se);  in ppgtt_populate_shadow_entry()
1279 		ret = is_2MB_gtt_possible(vgpu, ge);  in ppgtt_populate_shadow_entry()
1281 			return split_2MB_gtt_entry(vgpu, spt, index, &se);  in ppgtt_populate_shadow_entry()
1294 	ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn, page_size,  in ppgtt_populate_shadow_entry()
1306 	struct intel_vgpu *vgpu = spt->vgpu;  in ppgtt_populate_spt()  local
1307 	struct intel_gvt *gvt = vgpu->gvt;  in ppgtt_populate_spt()
1314 	trace_spt_change(spt->vgpu->id, "born", spt,  in ppgtt_populate_spt()
1319 			s = ppgtt_populate_spt_by_guest_entry(vgpu, &ge);  in ppgtt_populate_spt()
1329 			if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) {  in ppgtt_populate_spt()
1335 			ret = ppgtt_populate_shadow_entry(vgpu, spt, i, &ge);  in ppgtt_populate_spt()
1350 	struct intel_vgpu *vgpu = spt->vgpu;  in ppgtt_handle_guest_entry_removal()  local
1351 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_entry_removal()
1354 	trace_spt_guest_change(spt->vgpu->id, "remove", spt,  in ppgtt_handle_guest_entry_removal()
1364 	    vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn)  in ppgtt_handle_guest_entry_removal()
1369 			intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(se));  in ppgtt_handle_guest_entry_removal()
1395 	struct intel_vgpu *vgpu = spt->vgpu;  in ppgtt_handle_guest_entry_add()  local
1400 	trace_spt_guest_change(spt->vgpu->id, "add", spt, spt->shadow_page.type,  in ppgtt_handle_guest_entry_add()
1407 		s = ppgtt_populate_spt_by_guest_entry(vgpu, we);  in ppgtt_handle_guest_entry_add()
1416 		ret = ppgtt_populate_shadow_entry(vgpu, spt, index, we);  in ppgtt_handle_guest_entry_add()
1427 static int sync_oos_page(struct intel_vgpu *vgpu,  in sync_oos_page()  argument
1430 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in sync_oos_page()
1431 	struct intel_gvt *gvt = vgpu->gvt;  in sync_oos_page()
1438 	trace_oos_change(vgpu->id, "sync", oos_page->id,  in sync_oos_page()
1446 		ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu);  in sync_oos_page()
1448 			       spt->guest_page.gfn << PAGE_SHIFT, vgpu);  in sync_oos_page()
1454 		trace_oos_sync(vgpu->id, oos_page->id,  in sync_oos_page()
1458 		ret = ppgtt_populate_shadow_entry(vgpu, spt, index, &new);  in sync_oos_page()
1462 		ops->set_entry(oos_page->mem, &new, index, false, 0, vgpu);  in sync_oos_page()
1470 static int detach_oos_page(struct intel_vgpu *vgpu,  in detach_oos_page()  argument
1473 	struct intel_gvt *gvt = vgpu->gvt;  in detach_oos_page()
1476 	trace_oos_change(vgpu->id, "detach", oos_page->id,  in detach_oos_page()
1492 	struct intel_gvt *gvt = spt->vgpu->gvt;  in attach_oos_page()
1495 	ret = intel_gvt_hypervisor_read_gpa(spt->vgpu,  in attach_oos_page()
1506 	trace_oos_change(spt->vgpu->id, "attach", oos_page->id,  in attach_oos_page()
1516 	ret = intel_vgpu_enable_page_track(spt->vgpu, spt->guest_page.gfn);  in ppgtt_set_guest_page_sync()
1520 	trace_oos_change(spt->vgpu->id, "set page sync", oos_page->id,  in ppgtt_set_guest_page_sync()
1524 	return sync_oos_page(spt->vgpu, oos_page);  in ppgtt_set_guest_page_sync()
1529 	struct intel_gvt *gvt = spt->vgpu->gvt;  in ppgtt_allocate_oos_page()
1542 		ret = detach_oos_page(spt->vgpu, oos_page);  in ppgtt_allocate_oos_page()
1558 	trace_oos_change(spt->vgpu->id, "set page out of sync", oos_page->id,  in ppgtt_set_guest_page_oos()
1561 	list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head);  in ppgtt_set_guest_page_oos()
1562 	return intel_vgpu_disable_page_track(spt->vgpu, spt->guest_page.gfn);  in ppgtt_set_guest_page_oos()
1575 int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu)  in intel_vgpu_sync_oos_pages()  argument
1584 	list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {  in intel_vgpu_sync_oos_pages()
1601 	struct intel_vgpu *vgpu = spt->vgpu;  in ppgtt_handle_guest_write_page_table()  local
1603 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table()
1635 					vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1642 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1646 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1675 			&spt->vgpu->gtt.post_shadow_list_head);  in ppgtt_set_post_shadow()
1688 int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)  in intel_vgpu_flush_post_shadow()  argument
1696 	list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {  in intel_vgpu_flush_post_shadow()
1719 	struct intel_vgpu *vgpu = spt->vgpu;  in ppgtt_handle_guest_write_page_table_bytes()  local
1720 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table_bytes()
1721 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in ppgtt_handle_guest_write_page_table_bytes()
1754 			ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table_bytes()
1767 				false, 0, vgpu);  in ppgtt_handle_guest_write_page_table_bytes()
1782 	struct intel_vgpu *vgpu = mm->vgpu;  in invalidate_ppgtt_mm()  local
1783 	struct intel_gvt *gvt = vgpu->gvt;  in invalidate_ppgtt_mm()
1798 		ppgtt_invalidate_spt_by_shadow_entry(vgpu, &se);  in invalidate_ppgtt_mm()
1802 		trace_spt_guest_change(vgpu->id, "destroy root pointer",  in invalidate_ppgtt_mm()
1812 	struct intel_vgpu *vgpu = mm->vgpu;  in shadow_ppgtt_mm()  local
1813 	struct intel_gvt *gvt = vgpu->gvt;  in shadow_ppgtt_mm()
1831 		trace_spt_guest_change(vgpu->id, __func__, NULL,  in shadow_ppgtt_mm()
1834 		spt = ppgtt_populate_spt_by_guest_entry(vgpu, &ge);  in shadow_ppgtt_mm()
1843 		trace_spt_guest_change(vgpu->id, "populate root pointer",  in shadow_ppgtt_mm()
1853 static struct intel_vgpu_mm *vgpu_alloc_mm(struct intel_vgpu *vgpu)  in vgpu_alloc_mm()  argument
1861 	mm->vgpu = vgpu;  in vgpu_alloc_mm()
1884 struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu,  in intel_vgpu_create_ppgtt_mm()  argument
1887 	struct intel_gvt *gvt = vgpu->gvt;  in intel_vgpu_create_ppgtt_mm()
1891 	mm = vgpu_alloc_mm(vgpu);  in intel_vgpu_create_ppgtt_mm()
1918 	list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);  in intel_vgpu_create_ppgtt_mm()
1927 static struct intel_vgpu_mm *intel_vgpu_create_ggtt_mm(struct intel_vgpu *vgpu)  in intel_vgpu_create_ggtt_mm()  argument
1932 	mm = vgpu_alloc_mm(vgpu);  in intel_vgpu_create_ggtt_mm()
1938 	nr_entries = gvt_ggtt_gm_sz(vgpu->gvt) >> I915_GTT_PAGE_SHIFT;  in intel_vgpu_create_ggtt_mm()
1941 				   vgpu->gvt->device_info.gtt_entry_size));  in intel_vgpu_create_ggtt_mm()
1967 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
1969 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
2012 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2014 			       &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_pin_mm()
2015 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2049 	struct intel_vgpu *vgpu = mm->vgpu;  in ppgtt_get_next_level_entry()  local
2050 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_get_next_level_entry()
2053 	s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e));  in ppgtt_get_next_level_entry()
2077 	struct intel_vgpu *vgpu = mm->vgpu;  in intel_vgpu_gma_to_gpa()  local
2078 	struct intel_gvt *gvt = vgpu->gvt;  in intel_vgpu_gma_to_gpa()
2091 		if (!vgpu_gmadr_is_valid(vgpu, gma))  in intel_vgpu_gma_to_gpa()
2100 		trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa);  in intel_vgpu_gma_to_gpa()
2139 		trace_gma_translate(vgpu->id, "ppgtt", 0,  in intel_vgpu_gma_to_gpa()
2149 static int emulate_ggtt_mmio_read(struct intel_vgpu *vgpu,  in emulate_ggtt_mmio_read()  argument
2152 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_read()
2153 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in emulate_ggtt_mmio_read()
2162 	if (!intel_gvt_ggtt_validate_range(vgpu,  in emulate_ggtt_mmio_read()
2187 int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off,  in intel_vgpu_emulate_ggtt_mmio_read()  argument
2190 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in intel_vgpu_emulate_ggtt_mmio_read()
2197 	ret = emulate_ggtt_mmio_read(vgpu, off, p_data, bytes);  in intel_vgpu_emulate_ggtt_mmio_read()
2201 static void ggtt_invalidate_pte(struct intel_vgpu *vgpu,  in ggtt_invalidate_pte()  argument
2204 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ggtt_invalidate_pte()
2208 	if (pfn != vgpu->gvt->gtt.scratch_mfn)  in ggtt_invalidate_pte()
2209 		intel_gvt_hypervisor_dma_unmap_guest_page(vgpu,  in ggtt_invalidate_pte()
2213 static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,  in emulate_ggtt_mmio_write()  argument
2216 	struct intel_gvt *gvt = vgpu->gvt;  in emulate_ggtt_mmio_write()
2218 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_write()
2235 	if (!vgpu_gmadr_is_valid(vgpu, gma))  in emulate_ggtt_mmio_write()
2296 		if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) {  in emulate_ggtt_mmio_write()
2301 		ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn,  in emulate_ggtt_mmio_write()
2321 	ggtt_invalidate_pte(vgpu, &e);  in emulate_ggtt_mmio_write()
2340 int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu,  in intel_vgpu_emulate_ggtt_mmio_write()  argument
2343 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in intel_vgpu_emulate_ggtt_mmio_write()
2345 	struct intel_vgpu_submission *s = &vgpu->submission;  in intel_vgpu_emulate_ggtt_mmio_write()
2353 	ret = emulate_ggtt_mmio_write(vgpu, off, p_data, bytes);  in intel_vgpu_emulate_ggtt_mmio_write()
2359 	for_each_engine(engine, vgpu->gvt->gt, i) {  in intel_vgpu_emulate_ggtt_mmio_write()
2369 static int alloc_scratch_pages(struct intel_vgpu *vgpu,  in alloc_scratch_pages()  argument
2372 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in alloc_scratch_pages()
2373 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in alloc_scratch_pages()
2374 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in alloc_scratch_pages()
2376 				vgpu->gvt->device_info.gtt_entry_size_shift;  in alloc_scratch_pages()
2379 	struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;  in alloc_scratch_pages()
2403 			vgpu->id, type, gtt->scratch_pt[type].page_mfn);  in alloc_scratch_pages()
2428 			ops->set_entry(scratch_pt, &se, i, false, 0, vgpu);  in alloc_scratch_pages()
2434 static int release_scratch_page_tree(struct intel_vgpu *vgpu)  in release_scratch_page_tree()  argument
2437 	struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;  in release_scratch_page_tree()
2441 		if (vgpu->gtt.scratch_pt[i].page != NULL) {  in release_scratch_page_tree()
2442 			daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<  in release_scratch_page_tree()
2445 			__free_page(vgpu->gtt.scratch_pt[i].page);  in release_scratch_page_tree()
2446 			vgpu->gtt.scratch_pt[i].page = NULL;  in release_scratch_page_tree()
2447 			vgpu->gtt.scratch_pt[i].page_mfn = 0;  in release_scratch_page_tree()
2454 static int create_scratch_page_tree(struct intel_vgpu *vgpu)  in create_scratch_page_tree()  argument
2459 		ret = alloc_scratch_pages(vgpu, i);  in create_scratch_page_tree()
2467 	release_scratch_page_tree(vgpu);  in create_scratch_page_tree()
2481 int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)  in intel_vgpu_init_gtt()  argument
2483 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in intel_vgpu_init_gtt()
2491 	gtt->ggtt_mm = intel_vgpu_create_ggtt_mm(vgpu);  in intel_vgpu_init_gtt()
2497 	intel_vgpu_reset_ggtt(vgpu, false);  in intel_vgpu_init_gtt()
2501 	return create_scratch_page_tree(vgpu);  in intel_vgpu_init_gtt()
2504 void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu)  in intel_vgpu_destroy_all_ppgtt_mm()  argument
2509 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_destroy_all_ppgtt_mm()
2514 	if (GEM_WARN_ON(!list_empty(&vgpu->gtt.ppgtt_mm_list_head)))  in intel_vgpu_destroy_all_ppgtt_mm()
2517 	if (GEM_WARN_ON(!radix_tree_empty(&vgpu->gtt.spt_tree))) {  in intel_vgpu_destroy_all_ppgtt_mm()
2519 		ppgtt_free_all_spt(vgpu);  in intel_vgpu_destroy_all_ppgtt_mm()
2523 static void intel_vgpu_destroy_ggtt_mm(struct intel_vgpu *vgpu)  in intel_vgpu_destroy_ggtt_mm()  argument
2528 				 &vgpu->gtt.ggtt_mm->ggtt_mm.partial_pte_list,  in intel_vgpu_destroy_ggtt_mm()
2534 	intel_vgpu_destroy_mm(vgpu->gtt.ggtt_mm);  in intel_vgpu_destroy_ggtt_mm()
2535 	vgpu->gtt.ggtt_mm = NULL;  in intel_vgpu_destroy_ggtt_mm()
2548 void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu)  in intel_vgpu_clean_gtt()  argument
2550 	intel_vgpu_destroy_all_ppgtt_mm(vgpu);  in intel_vgpu_clean_gtt()
2551 	intel_vgpu_destroy_ggtt_mm(vgpu);  in intel_vgpu_clean_gtt()
2552 	release_scratch_page_tree(vgpu);  in intel_vgpu_clean_gtt()
2619 struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,  in intel_vgpu_find_ppgtt_mm()  argument
2625 	list_for_each(pos, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_find_ppgtt_mm()
2656 struct intel_vgpu_mm *intel_vgpu_get_ppgtt_mm(struct intel_vgpu *vgpu,  in intel_vgpu_get_ppgtt_mm()  argument
2661 	mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps);  in intel_vgpu_get_ppgtt_mm()
2665 		mm = intel_vgpu_create_ppgtt_mm(vgpu, root_entry_type, pdps);  in intel_vgpu_get_ppgtt_mm()
2682 int intel_vgpu_put_ppgtt_mm(struct intel_vgpu *vgpu, u64 pdps[])  in intel_vgpu_put_ppgtt_mm()  argument
2686 	mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps);  in intel_vgpu_put_ppgtt_mm()
2777 void intel_vgpu_invalidate_ppgtt(struct intel_vgpu *vgpu)  in intel_vgpu_invalidate_ppgtt()  argument
2782 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_invalidate_ppgtt()
2785 			mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2787 			mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2803 void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu, bool invalidate_old)  in intel_vgpu_reset_ggtt()  argument
2805 	struct intel_gvt *gvt = vgpu->gvt;  in intel_vgpu_reset_ggtt()
2806 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in intel_vgpu_reset_ggtt()
2815 	index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT;  in intel_vgpu_reset_ggtt()
2816 	num_entries = vgpu_aperture_sz(vgpu) >> PAGE_SHIFT;  in intel_vgpu_reset_ggtt()
2819 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2820 			ggtt_invalidate_pte(vgpu, &old_entry);  in intel_vgpu_reset_ggtt()
2822 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2825 	index = vgpu_hidden_gmadr_base(vgpu) >> PAGE_SHIFT;  in intel_vgpu_reset_ggtt()
2826 	num_entries = vgpu_hidden_sz(vgpu) >> PAGE_SHIFT;  in intel_vgpu_reset_ggtt()
2829 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2830 			ggtt_invalidate_pte(vgpu, &old_entry);  in intel_vgpu_reset_ggtt()
2832 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2846 void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)  in intel_vgpu_reset_gtt()  argument
2852 	intel_vgpu_destroy_all_ppgtt_mm(vgpu);  in intel_vgpu_reset_gtt()
2853 	intel_vgpu_reset_ggtt(vgpu, true);  in intel_vgpu_reset_gtt()