64 static u64 kvm_granule_shift(u32 level)  in kvm_granule_shift()  argument
67 	return ARM64_HW_PGTABLE_LEVEL_SHIFT(level);  in kvm_granule_shift()
70 static u64 kvm_granule_size(u32 level)  in kvm_granule_size()  argument
72 	return BIT(kvm_granule_shift(level));  in kvm_granule_size()
82 static bool kvm_level_supports_block_mapping(u32 level)  in kvm_level_supports_block_mapping()  argument
88 	return !(level == 0 || (PAGE_SIZE != SZ_4K && level == 1));  in kvm_level_supports_block_mapping()
91 static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level)  in kvm_block_mapping_supported()  argument
93 	u64 granule = kvm_granule_size(level);  in kvm_block_mapping_supported()
95 	if (!kvm_level_supports_block_mapping(level))  in kvm_block_mapping_supported()
107 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level)  in kvm_pgtable_idx()  argument
109 	u64 shift = kvm_granule_shift(level);  in kvm_pgtable_idx()
143 static bool kvm_pte_table(kvm_pte_t pte, u32 level)  in kvm_pte_table()  argument
145 	if (level == KVM_PGTABLE_MAX_LEVELS - 1)  in kvm_pte_table()
196 static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)  in kvm_init_valid_leaf_pte()  argument
199 	u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE :  in kvm_init_valid_leaf_pte()
215 				  u32 level, kvm_pte_t *ptep,  in kvm_pgtable_visitor_cb()  argument
219 	return walker->cb(addr, data->end, level, ptep, flag, walker->arg);  in kvm_pgtable_visitor_cb()
223 			      kvm_pte_t *pgtable, u32 level);
226 				      kvm_pte_t *ptep, u32 level)  in __kvm_pgtable_visit()  argument
231 	bool table = kvm_pte_table(pte, level);  in __kvm_pgtable_visit()
235 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
240 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
243 		table = kvm_pte_table(pte, level);  in __kvm_pgtable_visit()
250 		data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level));  in __kvm_pgtable_visit()
251 		data->addr += kvm_granule_size(level);  in __kvm_pgtable_visit()
256 	ret = __kvm_pgtable_walk(data, childp, level + 1);  in __kvm_pgtable_visit()
261 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
270 			      kvm_pte_t *pgtable, u32 level)  in __kvm_pgtable_walk()  argument
275 	if (WARN_ON_ONCE(level >= KVM_PGTABLE_MAX_LEVELS))  in __kvm_pgtable_walk()
278 	for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) {  in __kvm_pgtable_walk()
284 		ret = __kvm_pgtable_visit(data, ptep, level);  in __kvm_pgtable_walk()
365 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,  in hyp_map_walker_try_leaf()  argument
369 	u64 granule = kvm_granule_size(level), phys = data->phys;  in hyp_map_walker_try_leaf()
371 	if (!kvm_block_mapping_supported(addr, end, phys, level))  in hyp_map_walker_try_leaf()
375 	new = kvm_init_valid_leaf_pte(phys, data->attr, level);  in hyp_map_walker_try_leaf()
383 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_map_walker()  argument
390 	if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))  in hyp_map_walker()
393 	if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))  in hyp_map_walker()
444 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_free_walker()  argument
566 			   u32 level, struct kvm_pgtable_mm_ops *mm_ops)  in stage2_put_pte()  argument
574 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level);  in stage2_put_pte()
580 static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,  in stage2_map_walker_try_leaf()  argument
585 	u64 granule = kvm_granule_size(level), phys = data->phys;  in stage2_map_walker_try_leaf()
588 	if (!kvm_block_mapping_supported(addr, end, phys, level))  in stage2_map_walker_try_leaf()
592 		new = kvm_init_valid_leaf_pte(phys, data->attr, level);  in stage2_map_walker_try_leaf()
606 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walker_try_leaf()
617 static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,  in stage2_map_walk_table_pre()  argument
624 	if (!kvm_block_mapping_supported(addr, end, data->phys, level))  in stage2_map_walk_table_pre()
640 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walk_leaf()  argument
654 	ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);  in stage2_map_walk_leaf()
658 	if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))  in stage2_map_walk_leaf()
674 		stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);  in stage2_map_walk_leaf()
682 static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level,  in stage2_map_walk_table_post()  argument
697 		ret = stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walk_table_post()
727 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walker()  argument
734 		return stage2_map_walk_table_pre(addr, end, level, ptep, data);  in stage2_map_walker()
736 		return stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walker()
738 		return stage2_map_walk_table_post(addr, end, level, ptep, data);  in stage2_map_walker()
807 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_unmap_walker()  argument
825 	if (kvm_pte_table(pte, level)) {  in stage2_unmap_walker()
839 	stage2_put_pte(ptep, mmu, addr, level, mm_ops);  in stage2_unmap_walker()
843 				    kvm_granule_size(level));  in stage2_unmap_walker()
867 	u32		level;  member
870 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_attr_walker()  argument
880 	data->level = level;  in stage2_attr_walker()
899 				    u32 *level)  in stage2_update_leaf_attrs()  argument
920 	if (level)  in stage2_update_leaf_attrs()
921 		*level = data.level;  in stage2_update_leaf_attrs()
966 	u32 level;  in kvm_pgtable_stage2_relax_perms()  local
978 	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level);  in kvm_pgtable_stage2_relax_perms()
980 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level);  in kvm_pgtable_stage2_relax_perms()
984 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_flush_walker()  argument
995 	__flush_dcache_area(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level));  in stage2_flush_walker()
1039 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_free_walker()  argument
1051 	if (kvm_pte_table(pte, level))  in stage2_free_walker()
1077 static int stage2_check_permission_walker(u64 addr, u64 end, u32 level,  in stage2_check_permission_walker()  argument
1110 	u32 level;  in kvm_pgtable_stage2_find_range()  local
1118 	for (level = pgt->start_level; level < KVM_PGTABLE_MAX_LEVELS; level++) {  in kvm_pgtable_stage2_find_range()
1119 		granule = kvm_granule_size(level);  in kvm_pgtable_stage2_find_range()
1123 		if (!kvm_level_supports_block_mapping(level))  in kvm_pgtable_stage2_find_range()