103 					  struct domain_pgtable *pgtable);
156 					 struct domain_pgtable *pgtable)  in amd_iommu_domain_get_pgtable()  argument
160 	pgtable->root = (u64 *)(pt_root & PAGE_MASK);  in amd_iommu_domain_get_pgtable()
161 	pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */  in amd_iommu_domain_get_pgtable()
1482 static void free_pagetable(struct domain_pgtable *pgtable)  in free_pagetable()  argument
1487 	if (pgtable->mode == PAGE_MODE_NONE)  in free_pagetable()
1490 	BUG_ON(pgtable->mode < PAGE_MODE_NONE ||  in free_pagetable()
1491 	       pgtable->mode > PAGE_MODE_6_LEVEL);  in free_pagetable()
1493 	root = (unsigned long)pgtable->root;  in free_pagetable()
1494 	freelist = free_sub_pt(root, pgtable->mode, freelist);  in free_pagetable()
1508 	struct domain_pgtable pgtable;  in increase_address_space()  local
1519 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in increase_address_space()
1521 	if (address <= PM_LEVEL_SIZE(pgtable.mode))  in increase_address_space()
1525 	if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))  in increase_address_space()
1528 	*pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));  in increase_address_space()
1530 	pgtable.root  = pte;  in increase_address_space()
1531 	pgtable.mode += 1;  in increase_address_space()
1532 	update_and_flush_device_table(domain, &pgtable);  in increase_address_space()
1539 	amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);  in increase_address_space()
1558 	struct domain_pgtable pgtable;  in alloc_pte()  local
1564 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in alloc_pte()
1566 	while (address > PM_LEVEL_SIZE(pgtable.mode)) {  in alloc_pte()
1575 		amd_iommu_domain_get_pgtable(domain, &pgtable);  in alloc_pte()
1579 	level   = pgtable.mode - 1;  in alloc_pte()
1580 	pte     = &pgtable.root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1656 	struct domain_pgtable pgtable;  in fetch_pte()  local
1662 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in fetch_pte()
1664 	if (address > PM_LEVEL_SIZE(pgtable.mode))  in fetch_pte()
1667 	level	   =  pgtable.mode - 1;  in fetch_pte()
1668 	pte	   = &pgtable.root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1909 			  struct domain_pgtable *pgtable,  in set_dte_entry()  argument
1916 	if (pgtable->mode != PAGE_MODE_NONE)  in set_dte_entry()
1917 		pte_root = iommu_virt_to_phys(pgtable->root);  in set_dte_entry()
1919 	pte_root |= (pgtable->mode & DEV_ENTRY_MODE_MASK)  in set_dte_entry()
1992 	struct domain_pgtable pgtable;  in do_attach()  local
2008 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in do_attach()
2009 	set_dte_entry(dev_data->devid, domain, &pgtable,  in do_attach()
2317 				struct domain_pgtable *pgtable)  in update_device_table()  argument
2322 		set_dte_entry(dev_data->devid, domain, pgtable,  in update_device_table()
2329 					  struct domain_pgtable *pgtable)  in update_and_flush_device_table()  argument
2331 	update_device_table(domain, pgtable);  in update_and_flush_device_table()
2337 	struct domain_pgtable pgtable;  in update_domain()  local
2340 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in update_domain()
2341 	update_and_flush_device_table(domain, &pgtable);  in update_domain()
2413 	struct domain_pgtable pgtable;  in protection_domain_free()  local
2421 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in protection_domain_free()
2423 	free_pagetable(&pgtable);  in protection_domain_free()
2593 	struct domain_pgtable pgtable;  in amd_iommu_map()  local
2597 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_map()
2598 	if (pgtable.mode == PAGE_MODE_NONE)  in amd_iommu_map()
2618 	struct domain_pgtable pgtable;  in amd_iommu_unmap()  local
2620 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_unmap()
2621 	if (pgtable.mode == PAGE_MODE_NONE)  in amd_iommu_unmap()
2632 	struct domain_pgtable pgtable;  in amd_iommu_iova_to_phys()  local
2635 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_iova_to_phys()
2636 	if (pgtable.mode == PAGE_MODE_NONE)  in amd_iommu_iova_to_phys()
2812 	struct domain_pgtable pgtable;  in amd_iommu_domain_direct_map()  local
2818 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_domain_direct_map()
2827 	free_pagetable(&pgtable);  in amd_iommu_domain_direct_map()
3016 	struct domain_pgtable pgtable;  in __set_gcr3()  local
3019 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in __set_gcr3()
3020 	if (pgtable.mode != PAGE_MODE_NONE)  in __set_gcr3()
3034 	struct domain_pgtable pgtable;  in __clear_gcr3()  local
3037 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in __clear_gcr3()
3038 	if (pgtable.mode != PAGE_MODE_NONE)  in __clear_gcr3()