94 struct mm_slot {  struct
114 	struct mm_slot *mm_slot;  member
387 					  sizeof(struct mm_slot),  in khugepaged_init()
388 					  __alignof__(struct mm_slot), 0, NULL);  in khugepaged_init()
405 static inline struct mm_slot *alloc_mm_slot(void)  in alloc_mm_slot()
412 static inline void free_mm_slot(struct mm_slot *mm_slot)  in free_mm_slot()  argument
414 	kmem_cache_free(mm_slot_cache, mm_slot);  in free_mm_slot()
417 static struct mm_slot *get_mm_slot(struct mm_struct *mm)  in get_mm_slot()
419 	struct mm_slot *mm_slot;  in get_mm_slot()  local
421 	hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)  in get_mm_slot()
422 		if (mm == mm_slot->mm)  in get_mm_slot()
423 			return mm_slot;  in get_mm_slot()
429 				    struct mm_slot *mm_slot)  in insert_to_mm_slots_hash()  argument
431 	mm_slot->mm = mm;  in insert_to_mm_slots_hash()
432 	hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);  in insert_to_mm_slots_hash()
476 	struct mm_slot *mm_slot;  in __khugepaged_enter()  local
479 	mm_slot = alloc_mm_slot();  in __khugepaged_enter()
480 	if (!mm_slot)  in __khugepaged_enter()
486 		free_mm_slot(mm_slot);  in __khugepaged_enter()
491 	insert_to_mm_slots_hash(mm, mm_slot);  in __khugepaged_enter()
497 	list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);  in __khugepaged_enter()
529 	struct mm_slot *mm_slot;  in __khugepaged_exit()  local
533 	mm_slot = get_mm_slot(mm);  in __khugepaged_exit()
534 	if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {  in __khugepaged_exit()
535 		hash_del(&mm_slot->hash);  in __khugepaged_exit()
536 		list_del(&mm_slot->mm_node);  in __khugepaged_exit()
543 		free_mm_slot(mm_slot);  in __khugepaged_exit()
545 	} else if (mm_slot) {  in __khugepaged_exit()
1392 static void collect_mm_slot(struct mm_slot *mm_slot)  in collect_mm_slot()  argument
1394 	struct mm_struct *mm = mm_slot->mm;  in collect_mm_slot()
1400 		hash_del(&mm_slot->hash);  in collect_mm_slot()
1401 		list_del(&mm_slot->mm_node);  in collect_mm_slot()
1410 		free_mm_slot(mm_slot);  in collect_mm_slot()
1423 	struct mm_slot *mm_slot;  in khugepaged_add_pte_mapped_thp()  local
1428 	mm_slot = get_mm_slot(mm);  in khugepaged_add_pte_mapped_thp()
1429 	if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP))  in khugepaged_add_pte_mapped_thp()
1430 		mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr;  in khugepaged_add_pte_mapped_thp()
1572 static int khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)  in khugepaged_collapse_pte_mapped_thps()  argument
1574 	struct mm_struct *mm = mm_slot->mm;  in khugepaged_collapse_pte_mapped_thps()
1577 	if (likely(mm_slot->nr_pte_mapped_thp == 0))  in khugepaged_collapse_pte_mapped_thps()
1586 	for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++)  in khugepaged_collapse_pte_mapped_thps()
1587 		collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i]);  in khugepaged_collapse_pte_mapped_thps()
1590 	mm_slot->nr_pte_mapped_thp = 0;  in khugepaged_collapse_pte_mapped_thps()
2116 static int khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot)  in khugepaged_collapse_pte_mapped_thps()  argument
2127 	struct mm_slot *mm_slot;  in khugepaged_scan_mm_slot()  local
2135 	if (khugepaged_scan.mm_slot)  in khugepaged_scan_mm_slot()
2136 		mm_slot = khugepaged_scan.mm_slot;  in khugepaged_scan_mm_slot()
2138 		mm_slot = list_entry(khugepaged_scan.mm_head.next,  in khugepaged_scan_mm_slot()
2139 				     struct mm_slot, mm_node);  in khugepaged_scan_mm_slot()
2141 		khugepaged_scan.mm_slot = mm_slot;  in khugepaged_scan_mm_slot()
2144 	khugepaged_collapse_pte_mapped_thps(mm_slot);  in khugepaged_scan_mm_slot()
2146 	mm = mm_slot->mm;  in khugepaged_scan_mm_slot()
2221 	VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);  in khugepaged_scan_mm_slot()
2232 		if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {  in khugepaged_scan_mm_slot()
2233 			khugepaged_scan.mm_slot = list_entry(  in khugepaged_scan_mm_slot()
2234 				mm_slot->mm_node.next,  in khugepaged_scan_mm_slot()
2235 				struct mm_slot, mm_node);  in khugepaged_scan_mm_slot()
2238 			khugepaged_scan.mm_slot = NULL;  in khugepaged_scan_mm_slot()
2242 		collect_mm_slot(mm_slot);  in khugepaged_scan_mm_slot()
2281 		if (!khugepaged_scan.mm_slot)  in khugepaged_do_scan()
2324 	struct mm_slot *mm_slot;  in khugepaged()  local
2335 	mm_slot = khugepaged_scan.mm_slot;  in khugepaged()
2336 	khugepaged_scan.mm_slot = NULL;  in khugepaged()
2337 	if (mm_slot)  in khugepaged()
2338 		collect_mm_slot(mm_slot);  in khugepaged()