74 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in intel_gvt_ggtt_gmadr_g2h()
92 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in intel_gvt_ggtt_gmadr_h2g()
94 	if (drm_WARN(&i915->drm, !gvt_gmadr_is_valid(vgpu->gvt, h_addr),  in intel_gvt_ggtt_gmadr_h2g()
98 	if (gvt_gmadr_is_aperture(vgpu->gvt, h_addr))  in intel_gvt_ggtt_gmadr_h2g()
100 			+ (h_addr - gvt_aperture_gmadr_base(vgpu->gvt));  in intel_gvt_ggtt_gmadr_h2g()
103 			+ (h_addr - gvt_hidden_gmadr_base(vgpu->gvt));  in intel_gvt_ggtt_gmadr_h2g()
308 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in gtt_get_entry64()
321 		e->val64 = read_pte64(vgpu->gvt->gt->ggtt, index);  in gtt_get_entry64()
333 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in gtt_set_entry64()
346 		write_pte64(vgpu->gvt->gt->ggtt, index, e->val64);  in gtt_set_entry64()
556 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_get_root_entry()
583 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_set_root_entry()
605 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_guest_entry()
617 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_guest_entry()
628 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_host_entry()
638 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_host_entry()
654 	struct intel_gvt *gvt = spt->vgpu->gvt;  in ppgtt_spt_get_entry()  local
655 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_get_entry()
683 	struct intel_gvt *gvt = spt->vgpu->gvt;  in ppgtt_spt_set_entry()  local
684 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_set_entry()
740 	struct device *kdev = &spt->vgpu->gvt->gt->i915->drm.pdev->dev;  in ppgtt_free_spt()
819 static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt);
825 	struct device *kdev = &vgpu->gvt->gt->i915->drm.pdev->dev;  in ppgtt_alloc_spt()
833 		if (reclaim_one_ppgtt_mm(vgpu->gvt))  in ppgtt_alloc_spt()
903 	((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
912 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
918 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
946 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in ppgtt_invalidate_spt_by_shadow_entry()
947 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_spt_by_shadow_entry()
984 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_pte()
1053 	struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;  in vgpu_ips_enabled()
1072 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_spt_by_guest_entry()
1136 	struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;  in ppgtt_generate_shadow_entry()
1159 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in is_2MB_gtt_possible()
1162 	if (!HAS_PAGE_SIZES(vgpu->gvt->gt->i915, I915_GTT_PAGE_SIZE_2M))  in is_2MB_gtt_possible()
1176 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_2MB_gtt_entry()
1223 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_64KB_gtt_entry()
1254 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_shadow_entry()
1307 	struct intel_gvt *gvt = vgpu->gvt;  in ppgtt_populate_spt()  local
1308 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_populate_spt()
1330 				ops->set_pfn(&se, gvt->gtt.scratch_mfn);  in ppgtt_populate_spt()
1351 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_entry_removal()
1430 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in sync_oos_page()
1431 	struct intel_gvt *gvt = vgpu->gvt;  in sync_oos_page()  local
1432 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in sync_oos_page()
1473 	struct intel_gvt *gvt = vgpu->gvt;  in detach_oos_page()  local
1484 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);  in detach_oos_page()
1492 	struct intel_gvt *gvt = spt->vgpu->gvt;  in attach_oos_page()  local
1504 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);  in attach_oos_page()
1529 	struct intel_gvt *gvt = spt->vgpu->gvt;  in ppgtt_allocate_oos_page()  local
1530 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in ppgtt_allocate_oos_page()
1603 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table()
1720 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table_bytes()
1721 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in ppgtt_handle_guest_write_page_table_bytes()
1783 	struct intel_gvt *gvt = vgpu->gvt;  in invalidate_ppgtt_mm()  local
1784 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in invalidate_ppgtt_mm()
1813 	struct intel_gvt *gvt = vgpu->gvt;  in shadow_ppgtt_mm()  local
1814 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in shadow_ppgtt_mm()
1887 	struct intel_gvt *gvt = vgpu->gvt;  in intel_vgpu_create_ppgtt_mm()  local
1920 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
1921 	list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_create_ppgtt_mm()
1922 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
1938 	nr_entries = gvt_ggtt_gm_sz(vgpu->gvt) >> I915_GTT_PAGE_SHIFT;  in intel_vgpu_create_ggtt_mm()
1941 				   vgpu->gvt->device_info.gtt_entry_size));  in intel_vgpu_create_ggtt_mm()
1967 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
1969 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
2012 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2014 			       &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_pin_mm()
2015 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2021 static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt)  in reclaim_one_ppgtt_mm()  argument
2026 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2028 	list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) {  in reclaim_one_ppgtt_mm()
2035 		mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2039 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2050 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_get_next_level_entry()
2078 	struct intel_gvt *gvt = vgpu->gvt;  in intel_vgpu_gma_to_gpa()  local
2079 	struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;  in intel_vgpu_gma_to_gpa()
2080 	struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;  in intel_vgpu_gma_to_gpa()
2153 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in emulate_ggtt_mmio_read()
2190 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in intel_vgpu_emulate_ggtt_mmio_read()
2204 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ggtt_invalidate_pte()
2208 	if (pfn != vgpu->gvt->gtt.scratch_mfn)  in ggtt_invalidate_pte()
2216 	struct intel_gvt *gvt = vgpu->gvt;  in emulate_ggtt_mmio_write()  local
2217 	const struct intel_gvt_device_info *info = &gvt->device_info;  in emulate_ggtt_mmio_write()
2219 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in emulate_ggtt_mmio_write()
2297 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2309 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2313 		ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2324 	ggtt_invalidate(gvt->gt);  in emulate_ggtt_mmio_write()
2343 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;  in intel_vgpu_emulate_ggtt_mmio_write()
2359 	for_each_engine(engine, vgpu->gvt->gt, i) {  in intel_vgpu_emulate_ggtt_mmio_write()
2372 	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;  in alloc_scratch_pages()
2374 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in alloc_scratch_pages()
2376 				vgpu->gvt->device_info.gtt_entry_size_shift;  in alloc_scratch_pages()
2379 	struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;  in alloc_scratch_pages()
2437 	struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;  in release_scratch_page_tree()
2555 static void clean_spt_oos(struct intel_gvt *gvt)  in clean_spt_oos()  argument
2557 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in clean_spt_oos()
2572 static int setup_spt_oos(struct intel_gvt *gvt)  in setup_spt_oos()  argument
2574 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in setup_spt_oos()
2605 	clean_spt_oos(gvt);  in setup_spt_oos()
2705 int intel_gvt_init_gtt(struct intel_gvt *gvt)  in intel_gvt_init_gtt()  argument
2709 	struct device *dev = &gvt->gt->i915->drm.pdev->dev;  in intel_gvt_init_gtt()
2714 	gvt->gtt.pte_ops = &gen8_gtt_pte_ops;  in intel_gvt_init_gtt()
2715 	gvt->gtt.gma_ops = &gen8_gtt_gma_ops;  in intel_gvt_init_gtt()
2731 	gvt->gtt.scratch_page = virt_to_page(page);  in intel_gvt_init_gtt()
2732 	gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);  in intel_gvt_init_gtt()
2735 		ret = setup_spt_oos(gvt);  in intel_gvt_init_gtt()
2739 			__free_page(gvt->gtt.scratch_page);  in intel_gvt_init_gtt()
2743 	INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head);  in intel_gvt_init_gtt()
2744 	mutex_init(&gvt->gtt.ppgtt_mm_lock);  in intel_gvt_init_gtt()
2756 void intel_gvt_clean_gtt(struct intel_gvt *gvt)  in intel_gvt_clean_gtt()  argument
2758 	struct device *dev = &gvt->gt->i915->drm.pdev->dev;  in intel_gvt_clean_gtt()
2759 	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<  in intel_gvt_clean_gtt()
2764 	__free_page(gvt->gtt.scratch_page);  in intel_gvt_clean_gtt()
2767 		clean_spt_oos(gvt);  in intel_gvt_clean_gtt()
2785 			mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2787 			mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2805 	struct intel_gvt *gvt = vgpu->gvt;  in intel_vgpu_reset_ggtt()  local
2806 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in intel_vgpu_reset_ggtt()
2812 	pte_ops->set_pfn(&entry, gvt->gtt.scratch_mfn);  in intel_vgpu_reset_ggtt()
2835 	ggtt_invalidate(gvt->gt);  in intel_vgpu_reset_ggtt()