Lines Matching refs:iova
126 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) in __cached_rbnode_insert_update()
135 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) in __cached_rbnode_delete_update()
137 struct iova *cached_iova; in __cached_rbnode_delete_update()
139 cached_iova = rb_entry(iovad->cached32_node, struct iova, node); in __cached_rbnode_delete_update()
148 cached_iova = rb_entry(iovad->cached_node, struct iova, node); in __cached_rbnode_delete_update()
155 iova_insert_rbtree(struct rb_root *root, struct iova *iova, in iova_insert_rbtree() argument
163 struct iova *this = rb_entry(*new, struct iova, node); in iova_insert_rbtree()
167 if (iova->pfn_lo < this->pfn_lo) in iova_insert_rbtree()
169 else if (iova->pfn_lo > this->pfn_lo) in iova_insert_rbtree()
177 rb_link_node(&iova->node, parent, new); in iova_insert_rbtree()
178 rb_insert_color(&iova->node, root); in iova_insert_rbtree()
201 struct iova *new, bool size_aligned) in __alloc_and_insert_iova_range()
204 struct iova *curr_iova; in __alloc_and_insert_iova_range()
220 curr_iova = rb_entry(curr, struct iova, node); in __alloc_and_insert_iova_range()
229 curr_iova = rb_entry(curr, struct iova, node); in __alloc_and_insert_iova_range()
237 curr_iova = rb_entry(curr, struct iova, node); in __alloc_and_insert_iova_range()
263 struct iova *new, bool size_aligned) in __alloc_and_insert_iova_best_fit()
266 struct iova *curr_iova, *prev_iova; in __alloc_and_insert_iova_best_fit()
281 curr_iova = rb_entry(curr, struct iova, node); in __alloc_and_insert_iova_best_fit()
282 prev_iova = rb_entry(prev, struct iova, node); in __alloc_and_insert_iova_best_fit()
297 curr_iova = rb_entry(curr, struct iova, node); in __alloc_and_insert_iova_best_fit()
328 struct iova *alloc_iova_mem(void) in alloc_iova_mem()
334 void free_iova_mem(struct iova *iova) in free_iova_mem() argument
336 if (iova->pfn_lo != IOVA_ANCHOR) in free_iova_mem()
337 kmem_cache_free(iova_cache, iova); in free_iova_mem()
346 "iommu_iova", sizeof(struct iova), 0, in iova_cache_get()
387 struct iova *
392 struct iova *new_iova; in alloc_iova()
416 static struct iova *
424 struct iova *iova = rb_entry(node, struct iova, node); in private_find_iova() local
426 if (pfn < iova->pfn_lo) in private_find_iova()
428 else if (pfn > iova->pfn_hi) in private_find_iova()
431 return iova; /* pfn falls within iova's range */ in private_find_iova()
437 static void private_free_iova(struct iova_domain *iovad, struct iova *iova) in private_free_iova() argument
440 __cached_rbnode_delete_update(iovad, iova); in private_free_iova()
441 rb_erase(&iova->node, &iovad->rbroot); in private_free_iova()
442 free_iova_mem(iova); in private_free_iova()
452 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) in find_iova()
455 struct iova *iova; in find_iova() local
459 iova = private_find_iova(iovad, pfn); in find_iova()
461 return iova; in find_iova()
472 __free_iova(struct iova_domain *iovad, struct iova *iova) in __free_iova() argument
477 private_free_iova(iovad, iova); in __free_iova()
492 struct iova *iova = find_iova(iovad, pfn); in free_iova() local
494 if (iova) in free_iova()
495 __free_iova(iovad, iova); in free_iova()
515 struct iova *new_iova; in alloc_iova_fast()
695 struct iova *iova, *tmp; in put_iova_domain() local
699 rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node) in put_iova_domain()
700 free_iova_mem(iova); in put_iova_domain()
708 struct iova *iova = rb_entry(node, struct iova, node); in __is_range_overlap() local
710 if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo)) in __is_range_overlap()
715 static inline struct iova *
718 struct iova *iova; in alloc_and_init_iova() local
720 iova = alloc_iova_mem(); in alloc_and_init_iova()
721 if (iova) { in alloc_and_init_iova()
722 iova->pfn_lo = pfn_lo; in alloc_and_init_iova()
723 iova->pfn_hi = pfn_hi; in alloc_and_init_iova()
726 return iova; in alloc_and_init_iova()
729 static struct iova *
733 struct iova *iova; in __insert_new_range() local
735 iova = alloc_and_init_iova(pfn_lo, pfn_hi); in __insert_new_range()
736 if (iova) in __insert_new_range()
737 iova_insert_rbtree(&iovad->rbroot, iova, NULL); in __insert_new_range()
739 return iova; in __insert_new_range()
743 __adjust_overlap_range(struct iova *iova, in __adjust_overlap_range() argument
746 if (*pfn_lo < iova->pfn_lo) in __adjust_overlap_range()
747 iova->pfn_lo = *pfn_lo; in __adjust_overlap_range()
748 if (*pfn_hi > iova->pfn_hi) in __adjust_overlap_range()
749 *pfn_lo = iova->pfn_hi + 1; in __adjust_overlap_range()
760 struct iova *
766 struct iova *iova; in reserve_iova() local
776 iova = rb_entry(node, struct iova, node); in reserve_iova()
777 __adjust_overlap_range(iova, &pfn_lo, &pfn_hi); in reserve_iova()
778 if ((pfn_lo >= iova->pfn_lo) && in reserve_iova()
779 (pfn_hi <= iova->pfn_hi)) in reserve_iova()
790 iova = __insert_new_range(iovad, pfn_lo, pfn_hi); in reserve_iova()
794 return iova; in reserve_iova()
813 struct iova *iova = rb_entry(node, struct iova, node); in copy_reserved_iova() local
814 struct iova *new_iova; in copy_reserved_iova()
816 if (iova->pfn_lo == IOVA_ANCHOR) in copy_reserved_iova()
819 new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi); in copy_reserved_iova()
822 iova->pfn_lo, iova->pfn_lo); in copy_reserved_iova()
828 struct iova *
829 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, in split_and_remove_iova() argument
833 struct iova *prev = NULL, *next = NULL; in split_and_remove_iova()
836 if (iova->pfn_lo < pfn_lo) { in split_and_remove_iova()
837 prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1); in split_and_remove_iova()
841 if (iova->pfn_hi > pfn_hi) { in split_and_remove_iova()
842 next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi); in split_and_remove_iova()
847 __cached_rbnode_delete_update(iovad, iova); in split_and_remove_iova()
848 rb_erase(&iova->node, &iovad->rbroot); in split_and_remove_iova()
852 iova->pfn_lo = pfn_lo; in split_and_remove_iova()
856 iova->pfn_hi = pfn_hi; in split_and_remove_iova()
860 return iova; in split_and_remove_iova()
912 struct iova *iova = private_find_iova(iovad, mag->pfns[i]); in iova_magazine_free_pfns() local
914 if (WARN_ON(!iova)) in iova_magazine_free_pfns()
917 private_free_iova(iovad, iova); in iova_magazine_free_pfns()