Lines Matching +full:data +full:- +full:addr
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Stand-alone page-table allocator for hyp stage-1 and guest stage-2.
60 u64 addr; member
75 #define KVM_PHYS_INVALID (-1ULL)
86 * 52-bit PAs. in kvm_level_supports_block_mapping()
91 static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level) in kvm_block_mapping_supported() argument
98 if (granule > (end - addr)) in kvm_block_mapping_supported()
104 return IS_ALIGNED(addr, granule); in kvm_block_mapping_supported()
107 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) in kvm_pgtable_idx() argument
110 u64 mask = BIT(PAGE_SHIFT - 3) - 1; in kvm_pgtable_idx()
112 return (data->addr >> shift) & mask; in kvm_pgtable_idx()
115 static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) in __kvm_pgd_page_idx() argument
117 u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */ in __kvm_pgd_page_idx()
118 u64 mask = BIT(pgt->ia_bits) - 1; in __kvm_pgd_page_idx()
120 return (addr & mask) >> shift; in __kvm_pgd_page_idx()
123 static u32 kvm_pgd_page_idx(struct kvm_pgtable_walk_data *data) in kvm_pgd_page_idx() argument
125 return __kvm_pgd_page_idx(data->pgt, data->addr); in kvm_pgd_page_idx()
135 return __kvm_pgd_page_idx(&pgt, -1ULL) + 1; in kvm_pgd_pages()
145 if (level == KVM_PGTABLE_MAX_LEVELS - 1) in kvm_pte_table()
176 return mm_ops->phys_to_virt(kvm_pte_to_phys(pte)); in kvm_pte_follow()
187 kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp)); in kvm_set_table_pte()
199 u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE : in kvm_init_valid_leaf_pte()
214 static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr, in kvm_pgtable_visitor_cb() argument
218 struct kvm_pgtable_walker *walker = data->walker; in kvm_pgtable_visitor_cb()
219 return walker->cb(addr, data->end, level, ptep, flag, walker->arg); in kvm_pgtable_visitor_cb()
222 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
225 static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, in __kvm_pgtable_visit() argument
229 u64 addr = data->addr; in __kvm_pgtable_visit() local
232 enum kvm_pgtable_walk_flags flags = data->walker->flags; in __kvm_pgtable_visit()
235 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
240 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
250 data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level)); in __kvm_pgtable_visit()
251 data->addr += kvm_granule_size(level); in __kvm_pgtable_visit()
255 childp = kvm_pte_follow(pte, data->pgt->mm_ops); in __kvm_pgtable_visit()
256 ret = __kvm_pgtable_walk(data, childp, level + 1); in __kvm_pgtable_visit()
261 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
269 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, in __kvm_pgtable_walk() argument
276 return -EINVAL; in __kvm_pgtable_walk()
278 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) { in __kvm_pgtable_walk()
281 if (data->addr >= data->end) in __kvm_pgtable_walk()
284 ret = __kvm_pgtable_visit(data, ptep, level); in __kvm_pgtable_walk()
292 static int _kvm_pgtable_walk(struct kvm_pgtable_walk_data *data) in _kvm_pgtable_walk() argument
296 struct kvm_pgtable *pgt = data->pgt; in _kvm_pgtable_walk()
297 u64 limit = BIT(pgt->ia_bits); in _kvm_pgtable_walk()
299 if (data->addr > limit || data->end > limit) in _kvm_pgtable_walk()
300 return -ERANGE; in _kvm_pgtable_walk()
302 if (!pgt->pgd) in _kvm_pgtable_walk()
303 return -EINVAL; in _kvm_pgtable_walk()
305 for (idx = kvm_pgd_page_idx(data); data->addr < data->end; ++idx) { in _kvm_pgtable_walk()
306 kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE]; in _kvm_pgtable_walk()
308 ret = __kvm_pgtable_walk(data, ptep, pgt->start_level); in _kvm_pgtable_walk()
316 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, in kvm_pgtable_walk() argument
321 .addr = ALIGN_DOWN(addr, PAGE_SIZE), in kvm_pgtable_walk()
322 .end = PAGE_ALIGN(walk_data.addr + size), in kvm_pgtable_walk()
345 return -EINVAL; in hyp_set_prot_attr()
349 return -EINVAL; in hyp_set_prot_attr()
352 return -EINVAL; in hyp_set_prot_attr()
365 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level, in hyp_map_walker_try_leaf() argument
366 kvm_pte_t *ptep, struct hyp_map_data *data) in hyp_map_walker_try_leaf() argument
369 u64 granule = kvm_granule_size(level), phys = data->phys; in hyp_map_walker_try_leaf()
371 if (!kvm_block_mapping_supported(addr, end, phys, level)) in hyp_map_walker_try_leaf()
375 new = kvm_init_valid_leaf_pte(phys, data->attr, level); in hyp_map_walker_try_leaf()
379 data->phys += granule; in hyp_map_walker_try_leaf()
383 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_map_walker() argument
387 struct hyp_map_data *data = arg; in hyp_map_walker() local
388 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; in hyp_map_walker()
390 if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg)) in hyp_map_walker()
393 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in hyp_map_walker()
394 return -EINVAL; in hyp_map_walker()
396 childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL); in hyp_map_walker()
398 return -ENOMEM; in hyp_map_walker()
404 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, in kvm_pgtable_hyp_map() argument
410 .mm_ops = pgt->mm_ops, in kvm_pgtable_hyp_map()
422 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_hyp_map()
433 pgt->pgd = (kvm_pte_t *)mm_ops->zalloc_page(NULL); in kvm_pgtable_hyp_init()
434 if (!pgt->pgd) in kvm_pgtable_hyp_init()
435 return -ENOMEM; in kvm_pgtable_hyp_init()
437 pgt->ia_bits = va_bits; in kvm_pgtable_hyp_init()
438 pgt->start_level = KVM_PGTABLE_MAX_LEVELS - levels; in kvm_pgtable_hyp_init()
439 pgt->mm_ops = mm_ops; in kvm_pgtable_hyp_init()
440 pgt->mmu = NULL; in kvm_pgtable_hyp_init()
444 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_free_walker() argument
449 mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops)); in hyp_free_walker()
458 .arg = pgt->mm_ops, in kvm_pgtable_hyp_destroy()
461 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); in kvm_pgtable_hyp_destroy()
462 pgt->mm_ops->put_page(pgt->pgd); in kvm_pgtable_hyp_destroy()
463 pgt->pgd = NULL; in kvm_pgtable_hyp_destroy()
516 return !(pgt->flags & KVM_PGTABLE_S2_NOFWB); in stage2_has_fwb()
532 return -EINVAL; in stage2_set_prot_attr()
559 * encode ownership of a page to another entity than the page-table in stage2_pte_is_counted()
565 static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr, in stage2_put_pte() argument
569 * Clear the existing PTE, and perform break-before-make with in stage2_put_pte()
574 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level); in stage2_put_pte()
577 mm_ops->put_page(ptep); in stage2_put_pte()
580 static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, in stage2_map_walker_try_leaf() argument
582 struct stage2_map_data *data) in stage2_map_walker_try_leaf() argument
585 u64 granule = kvm_granule_size(level), phys = data->phys; in stage2_map_walker_try_leaf()
586 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; in stage2_map_walker_try_leaf()
588 if (!kvm_block_mapping_supported(addr, end, phys, level)) in stage2_map_walker_try_leaf()
589 return -E2BIG; in stage2_map_walker_try_leaf()
592 new = kvm_init_valid_leaf_pte(phys, data->attr, level); in stage2_map_walker_try_leaf()
594 new = kvm_init_invalid_leaf_owner(data->owner_id); in stage2_map_walker_try_leaf()
604 return -EAGAIN; in stage2_map_walker_try_leaf()
606 stage2_put_pte(ptep, data->mmu, addr, level, mm_ops); in stage2_map_walker_try_leaf()
611 mm_ops->get_page(ptep); in stage2_map_walker_try_leaf()
613 data->phys += granule; in stage2_map_walker_try_leaf()
617 static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, in stage2_map_walk_table_pre() argument
619 struct stage2_map_data *data) in stage2_map_walk_table_pre() argument
621 if (data->anchor) in stage2_map_walk_table_pre()
624 if (!kvm_block_mapping_supported(addr, end, data->phys, level)) in stage2_map_walk_table_pre()
627 data->childp = kvm_pte_follow(*ptep, data->mm_ops); in stage2_map_walk_table_pre()
631 * Invalidate the whole stage-2, as we may have numerous leaf in stage2_map_walk_table_pre()
635 kvm_call_hyp(__kvm_tlb_flush_vmid, data->mmu); in stage2_map_walk_table_pre()
636 data->anchor = ptep; in stage2_map_walk_table_pre()
640 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walk_leaf() argument
641 struct stage2_map_data *data) in stage2_map_walk_leaf() argument
643 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; in stage2_map_walk_leaf()
647 if (data->anchor) { in stage2_map_walk_leaf()
649 mm_ops->put_page(ptep); in stage2_map_walk_leaf()
654 ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data); in stage2_map_walk_leaf()
655 if (ret != -E2BIG) in stage2_map_walk_leaf()
658 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in stage2_map_walk_leaf()
659 return -EINVAL; in stage2_map_walk_leaf()
661 if (!data->memcache) in stage2_map_walk_leaf()
662 return -ENOMEM; in stage2_map_walk_leaf()
664 childp = mm_ops->zalloc_page(data->memcache); in stage2_map_walk_leaf()
666 return -ENOMEM; in stage2_map_walk_leaf()
674 stage2_put_pte(ptep, data->mmu, addr, level, mm_ops); in stage2_map_walk_leaf()
677 mm_ops->get_page(ptep); in stage2_map_walk_leaf()
682 static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level, in stage2_map_walk_table_post() argument
684 struct stage2_map_data *data) in stage2_map_walk_table_post() argument
686 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; in stage2_map_walk_table_post()
690 if (!data->anchor) in stage2_map_walk_table_post()
693 if (data->anchor == ptep) { in stage2_map_walk_table_post()
694 childp = data->childp; in stage2_map_walk_table_post()
695 data->anchor = NULL; in stage2_map_walk_table_post()
696 data->childp = NULL; in stage2_map_walk_table_post()
697 ret = stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walk_table_post()
702 mm_ops->put_page(childp); in stage2_map_walk_table_post()
703 mm_ops->put_page(ptep); in stage2_map_walk_table_post()
723 * been set, but otherwise frees the page-table pages while walking back up
724 * the page-table, installing the block entry when it revisits the anchor
727 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walker() argument
730 struct stage2_map_data *data = arg; in stage2_map_walker() local
734 return stage2_map_walk_table_pre(addr, end, level, ptep, data); in stage2_map_walker()
736 return stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walker()
738 return stage2_map_walk_table_post(addr, end, level, ptep, data); in stage2_map_walker()
741 return -EINVAL; in stage2_map_walker()
744 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, in kvm_pgtable_stage2_map() argument
751 .mmu = pgt->mmu, in kvm_pgtable_stage2_map()
753 .mm_ops = pgt->mm_ops, in kvm_pgtable_stage2_map()
763 if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys))) in kvm_pgtable_stage2_map()
764 return -EINVAL; in kvm_pgtable_stage2_map()
770 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_map()
775 int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, in kvm_pgtable_stage2_set_owner() argument
781 .mmu = pgt->mmu, in kvm_pgtable_stage2_set_owner()
783 .mm_ops = pgt->mm_ops, in kvm_pgtable_stage2_set_owner()
795 return -EINVAL; in kvm_pgtable_stage2_set_owner()
797 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_set_owner()
807 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_unmap_walker() argument
812 struct kvm_s2_mmu *mmu = pgt->mmu; in stage2_unmap_walker()
813 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; in stage2_unmap_walker()
820 mm_ops->put_page(ptep); in stage2_unmap_walker()
828 if (mm_ops->page_count(childp) != 1) in stage2_unmap_walker()
839 stage2_put_pte(ptep, mmu, addr, level, mm_ops); in stage2_unmap_walker()
847 mm_ops->put_page(childp); in stage2_unmap_walker()
852 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_unmap() argument
860 return kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_unmap()
870 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_attr_walker() argument
875 struct stage2_attr_data *data = arg; in stage2_attr_walker() local
880 data->level = level; in stage2_attr_walker()
881 data->pte = pte; in stage2_attr_walker()
882 pte &= ~data->attr_clr; in stage2_attr_walker()
883 pte |= data->attr_set; in stage2_attr_walker()
887 * but worst-case the access flag update gets lost and will be in stage2_attr_walker()
890 if (data->pte != pte) in stage2_attr_walker()
896 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, in stage2_update_leaf_attrs() argument
903 struct stage2_attr_data data = { in stage2_update_leaf_attrs() local
909 .arg = &data, in stage2_update_leaf_attrs()
913 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in stage2_update_leaf_attrs()
918 *orig_pte = data.pte; in stage2_update_leaf_attrs()
921 *level = data.level; in stage2_update_leaf_attrs()
925 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_wrprotect() argument
927 return stage2_update_leaf_attrs(pgt, addr, size, 0, in kvm_pgtable_stage2_wrprotect()
932 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_mkyoung() argument
935 stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0, in kvm_pgtable_stage2_mkyoung()
941 kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_mkold() argument
944 stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF, in kvm_pgtable_stage2_mkold()
950 * See the '->clear_flush_young()' callback on the KVM mmu notifier. in kvm_pgtable_stage2_mkold()
955 bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_is_young() argument
958 stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL); in kvm_pgtable_stage2_is_young()
962 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, in kvm_pgtable_stage2_relax_perms() argument
978 ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level); in kvm_pgtable_stage2_relax_perms()
980 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level); in kvm_pgtable_stage2_relax_perms()
984 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_flush_walker() argument
989 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; in stage2_flush_walker()
999 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_flush() argument
1010 return kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_flush()
1018 u64 vtcr = arch->vtcr; in kvm_pgtable_stage2_init_flags()
1021 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; in kvm_pgtable_stage2_init_flags()
1024 pgt->pgd = mm_ops->zalloc_pages_exact(pgd_sz); in kvm_pgtable_stage2_init_flags()
1025 if (!pgt->pgd) in kvm_pgtable_stage2_init_flags()
1026 return -ENOMEM; in kvm_pgtable_stage2_init_flags()
1028 pgt->ia_bits = ia_bits; in kvm_pgtable_stage2_init_flags()
1029 pgt->start_level = start_level; in kvm_pgtable_stage2_init_flags()
1030 pgt->mm_ops = mm_ops; in kvm_pgtable_stage2_init_flags()
1031 pgt->mmu = &arch->mmu; in kvm_pgtable_stage2_init_flags()
1032 pgt->flags = flags; in kvm_pgtable_stage2_init_flags()
1039 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_free_walker() argument
1049 mm_ops->put_page(ptep); in stage2_free_walker()
1052 mm_ops->put_page(kvm_pte_follow(pte, mm_ops)); in stage2_free_walker()
1064 .arg = pgt->mm_ops, in kvm_pgtable_stage2_destroy()
1067 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); in kvm_pgtable_stage2_destroy()
1068 pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE; in kvm_pgtable_stage2_destroy()
1069 pgt->mm_ops->free_pages_exact(pgt->pgd, pgd_sz); in kvm_pgtable_stage2_destroy()
1070 pgt->pgd = NULL; in kvm_pgtable_stage2_destroy()
1077 static int stage2_check_permission_walker(u64 addr, u64 end, u32 level, in stage2_check_permission_walker() argument
1085 * Compatible mappings are either invalid and owned by the page-table in stage2_check_permission_walker()
1091 return -EEXIST; in stage2_check_permission_walker()
1093 return -EEXIST; in stage2_check_permission_walker()
1099 int kvm_pgtable_stage2_find_range(struct kvm_pgtable *pgt, u64 addr, in kvm_pgtable_stage2_find_range() argument
1118 for (level = pgt->start_level; level < KVM_PGTABLE_MAX_LEVELS; level++) { in kvm_pgtable_stage2_find_range()
1120 start = ALIGN_DOWN(addr, granule); in kvm_pgtable_stage2_find_range()
1126 if (start < range->start || range->end < end) in kvm_pgtable_stage2_find_range()
1135 if (ret != -EEXIST) in kvm_pgtable_stage2_find_range()
1140 range->start = start; in kvm_pgtable_stage2_find_range()
1141 range->end = end; in kvm_pgtable_stage2_find_range()