Lines Matching refs:rmap_item
123 struct rmap_item *rmap_list;
139 struct rmap_item **rmap_list;
195 struct rmap_item { struct
196 struct rmap_item *rmap_list; argument
308 rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0); in ksm_slab_init()
377 static inline struct rmap_item *alloc_rmap_item(void) in alloc_rmap_item()
379 struct rmap_item *rmap_item; in alloc_rmap_item() local
381 rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL | in alloc_rmap_item()
383 if (rmap_item) in alloc_rmap_item()
385 return rmap_item; in alloc_rmap_item()
388 static inline void free_rmap_item(struct rmap_item *rmap_item) in free_rmap_item() argument
391 rmap_item->mm = NULL; /* debug safety */ in free_rmap_item()
392 kmem_cache_free(rmap_item_cache, rmap_item); in free_rmap_item()
534 static void break_cow(struct rmap_item *rmap_item) in break_cow() argument
536 struct mm_struct *mm = rmap_item->mm; in break_cow()
537 unsigned long addr = rmap_item->address; in break_cow()
544 put_anon_vma(rmap_item->anon_vma); in break_cow()
553 static struct page *get_mergeable_page(struct rmap_item *rmap_item) in get_mergeable_page() argument
555 struct mm_struct *mm = rmap_item->mm; in get_mergeable_page()
556 unsigned long addr = rmap_item->address; in get_mergeable_page()
634 struct rmap_item *rmap_item; in remove_node_from_stable_tree() local
639 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in remove_node_from_stable_tree()
640 if (rmap_item->hlist.next) in remove_node_from_stable_tree()
646 put_anon_vma(rmap_item->anon_vma); in remove_node_from_stable_tree()
647 rmap_item->address &= PAGE_MASK; in remove_node_from_stable_tree()
774 static void remove_rmap_item_from_tree(struct rmap_item *rmap_item) in remove_rmap_item_from_tree() argument
776 if (rmap_item->address & STABLE_FLAG) { in remove_rmap_item_from_tree()
780 stable_node = rmap_item->head; in remove_rmap_item_from_tree()
785 hlist_del(&rmap_item->hlist); in remove_rmap_item_from_tree()
796 put_anon_vma(rmap_item->anon_vma); in remove_rmap_item_from_tree()
797 rmap_item->head = NULL; in remove_rmap_item_from_tree()
798 rmap_item->address &= PAGE_MASK; in remove_rmap_item_from_tree()
800 } else if (rmap_item->address & UNSTABLE_FLAG) { in remove_rmap_item_from_tree()
809 age = (unsigned char)(ksm_scan.seqnr - rmap_item->address); in remove_rmap_item_from_tree()
812 rb_erase(&rmap_item->node, in remove_rmap_item_from_tree()
813 root_unstable_tree + NUMA(rmap_item->nid)); in remove_rmap_item_from_tree()
815 rmap_item->address &= PAGE_MASK; in remove_rmap_item_from_tree()
822 struct rmap_item **rmap_list) in remove_trailing_rmap_items()
825 struct rmap_item *rmap_item = *rmap_list; in remove_trailing_rmap_items() local
826 *rmap_list = rmap_item->rmap_list; in remove_trailing_rmap_items()
827 remove_rmap_item_from_tree(rmap_item); in remove_trailing_rmap_items()
828 free_rmap_item(rmap_item); in remove_trailing_rmap_items()
1278 static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, in try_to_merge_with_ksm_page() argument
1281 struct mm_struct *mm = rmap_item->mm; in try_to_merge_with_ksm_page()
1286 vma = find_mergeable_vma(mm, rmap_item->address); in try_to_merge_with_ksm_page()
1295 remove_rmap_item_from_tree(rmap_item); in try_to_merge_with_ksm_page()
1298 rmap_item->anon_vma = vma->anon_vma; in try_to_merge_with_ksm_page()
1315 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, in try_to_merge_two_pages() argument
1317 struct rmap_item *tree_rmap_item, in try_to_merge_two_pages()
1322 err = try_to_merge_with_ksm_page(rmap_item, page, NULL); in try_to_merge_two_pages()
1331 break_cow(rmap_item); in try_to_merge_two_pages()
1925 struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item, in unstable_tree_search_insert() argument
1939 struct rmap_item *tree_rmap_item; in unstable_tree_search_insert()
1944 tree_rmap_item = rb_entry(*new, struct rmap_item, node); in unstable_tree_search_insert()
1981 rmap_item->address |= UNSTABLE_FLAG; in unstable_tree_search_insert()
1982 rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK); in unstable_tree_search_insert()
1983 DO_NUMA(rmap_item->nid = nid); in unstable_tree_search_insert()
1984 rb_link_node(&rmap_item->node, parent, new); in unstable_tree_search_insert()
1985 rb_insert_color(&rmap_item->node, root); in unstable_tree_search_insert()
1996 static void stable_tree_append(struct rmap_item *rmap_item, in stable_tree_append() argument
2018 rmap_item->head = stable_node; in stable_tree_append()
2019 rmap_item->address |= STABLE_FLAG; in stable_tree_append()
2020 hlist_add_head(&rmap_item->hlist, &stable_node->hlist); in stable_tree_append()
2022 if (rmap_item->hlist.next) in stable_tree_append()
2037 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) in cmp_and_merge_page() argument
2039 struct mm_struct *mm = rmap_item->mm; in cmp_and_merge_page()
2040 struct rmap_item *tree_rmap_item; in cmp_and_merge_page()
2058 rmap_item->head == stable_node) in cmp_and_merge_page()
2070 if (kpage == page && rmap_item->head == stable_node) { in cmp_and_merge_page()
2075 remove_rmap_item_from_tree(rmap_item); in cmp_and_merge_page()
2081 err = try_to_merge_with_ksm_page(rmap_item, page, kpage); in cmp_and_merge_page()
2088 stable_tree_append(rmap_item, page_stable_node(kpage), in cmp_and_merge_page()
2103 if (rmap_item->oldchecksum != checksum) { in cmp_and_merge_page()
2104 rmap_item->oldchecksum = checksum; in cmp_and_merge_page()
2116 vma = find_mergeable_vma(mm, rmap_item->address); in cmp_and_merge_page()
2119 ZERO_PAGE(rmap_item->address)); in cmp_and_merge_page()
2136 unstable_tree_search_insert(rmap_item, page, &tree_page); in cmp_and_merge_page()
2140 kpage = try_to_merge_two_pages(rmap_item, page, in cmp_and_merge_page()
2165 stable_tree_append(rmap_item, stable_node, in cmp_and_merge_page()
2178 break_cow(rmap_item); in cmp_and_merge_page()
2198 static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, in get_next_rmap_item()
2199 struct rmap_item **rmap_list, in get_next_rmap_item()
2202 struct rmap_item *rmap_item; in get_next_rmap_item() local
2205 rmap_item = *rmap_list; in get_next_rmap_item()
2206 if ((rmap_item->address & PAGE_MASK) == addr) in get_next_rmap_item()
2207 return rmap_item; in get_next_rmap_item()
2208 if (rmap_item->address > addr) in get_next_rmap_item()
2210 *rmap_list = rmap_item->rmap_list; in get_next_rmap_item()
2211 remove_rmap_item_from_tree(rmap_item); in get_next_rmap_item()
2212 free_rmap_item(rmap_item); in get_next_rmap_item()
2215 rmap_item = alloc_rmap_item(); in get_next_rmap_item()
2216 if (rmap_item) { in get_next_rmap_item()
2218 rmap_item->mm = mm_slot->mm; in get_next_rmap_item()
2219 rmap_item->address = addr; in get_next_rmap_item()
2220 rmap_item->rmap_list = *rmap_list; in get_next_rmap_item()
2221 *rmap_list = rmap_item; in get_next_rmap_item()
2223 return rmap_item; in get_next_rmap_item()
2226 static struct rmap_item *scan_get_next_rmap_item(struct page **page) in scan_get_next_rmap_item()
2231 struct rmap_item *rmap_item; in scan_get_next_rmap_item() local
2316 rmap_item = get_next_rmap_item(slot, in scan_get_next_rmap_item()
2318 if (rmap_item) { in scan_get_next_rmap_item()
2320 &rmap_item->rmap_list; in scan_get_next_rmap_item()
2325 return rmap_item; in scan_get_next_rmap_item()
2391 struct rmap_item *rmap_item; in ksm_do_scan() local
2396 rmap_item = scan_get_next_rmap_item(&page); in ksm_do_scan()
2397 if (!rmap_item) in ksm_do_scan()
2399 cmp_and_merge_page(page, rmap_item); in ksm_do_scan()
2608 struct rmap_item *rmap_item; in rmap_walk_ksm() local
2623 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in rmap_walk_ksm()
2624 struct anon_vma *anon_vma = rmap_item->anon_vma; in rmap_walk_ksm()
2644 addr = rmap_item->address & ~KSM_FLAG_MASK; in rmap_walk_ksm()
2654 if ((rmap_item->mm == vma->vm_mm) == search_new_forks) in rmap_walk_ksm()