279 struct zspage {  struct
304 static void migrate_lock_init(struct zspage *zspage);  argument
305 static void migrate_read_lock(struct zspage *zspage);
306 static void migrate_read_unlock(struct zspage *zspage);
309 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
315 static void migrate_lock_init(struct zspage *zspage) {}  in migrate_lock_init()  argument
316 static void migrate_read_lock(struct zspage *zspage) {}  in migrate_read_lock()  argument
317 static void migrate_read_unlock(struct zspage *zspage) {}  in migrate_read_unlock()  argument
320 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}  in SetZsPageMovable()  argument
330 	pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),  in create_cache()
358 static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)  in cache_alloc_zspage()
364 static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)  in cache_free_zspage()  argument
366 	kmem_cache_free(pool->zspage_cachep, zspage);  in cache_free_zspage()
460 static bool is_zspage_isolated(struct zspage *zspage)  in is_zspage_isolated()  argument
462 	return zspage->isolated;  in is_zspage_isolated()
471 static inline int get_zspage_inuse(struct zspage *zspage)  in get_zspage_inuse()  argument
473 	return zspage->inuse;  in get_zspage_inuse()
477 static inline void mod_zspage_inuse(struct zspage *zspage, int val)  in mod_zspage_inuse()  argument
479 	zspage->inuse += val;  in mod_zspage_inuse()
482 static inline struct page *get_first_page(struct zspage *zspage)  in get_first_page()  argument
484 	struct page *first_page = zspage->first_page;  in get_first_page()
500 static inline unsigned int get_freeobj(struct zspage *zspage)  in get_freeobj()  argument
502 	return zspage->freeobj;  in get_freeobj()
505 static inline void set_freeobj(struct zspage *zspage, unsigned int obj)  in set_freeobj()  argument
507 	zspage->freeobj = obj;  in set_freeobj()
510 static void get_zspage_mapping(struct zspage *zspage,  in get_zspage_mapping()  argument
514 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage_mapping()
516 	*fullness = zspage->fullness;  in get_zspage_mapping()
517 	*class_idx = zspage->class;  in get_zspage_mapping()
520 static void set_zspage_mapping(struct zspage *zspage,  in set_zspage_mapping()  argument
524 	zspage->class = class_idx;  in set_zspage_mapping()
525 	zspage->fullness = fullness;  in set_zspage_mapping()
690 						struct zspage *zspage)  in get_fullness_group()  argument
695 	inuse = get_zspage_inuse(zspage);  in get_fullness_group()
717 				struct zspage *zspage,  in insert_zspage()  argument
720 	struct zspage *head;  in insert_zspage()
724 					struct zspage, list);  in insert_zspage()
730 		if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {  in insert_zspage()
731 			list_add(&zspage->list, &head->list);  in insert_zspage()
735 	list_add(&zspage->list, &class->fullness_list[fullness]);  in insert_zspage()
743 				struct zspage *zspage,  in remove_zspage()  argument
747 	VM_BUG_ON(is_zspage_isolated(zspage));  in remove_zspage()
749 	list_del_init(&zspage->list);  in remove_zspage()
763 						struct zspage *zspage)  in fix_fullness_group()  argument
768 	get_zspage_mapping(zspage, &class_idx, &currfg);  in fix_fullness_group()
769 	newfg = get_fullness_group(class, zspage);  in fix_fullness_group()
773 	if (!is_zspage_isolated(zspage)) {  in fix_fullness_group()
774 		remove_zspage(class, zspage, currfg);  in fix_fullness_group()
775 		insert_zspage(class, zspage, newfg);  in fix_fullness_group()
778 	set_zspage_mapping(zspage, class_idx, newfg);  in fix_fullness_group()
820 static struct zspage *get_zspage(struct page *page)  in get_zspage()
822 	struct zspage *zspage = (struct zspage *)page->private;  in get_zspage()  local
824 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage()
825 	return zspage;  in get_zspage()
910 static int trylock_zspage(struct zspage *zspage)  in trylock_zspage()  argument
914 	for (cursor = get_first_page(zspage); cursor != NULL; cursor =  in trylock_zspage()
924 	for (cursor = get_first_page(zspage); cursor != fail; cursor =  in trylock_zspage()
932 				struct zspage *zspage)  in __free_zspage()  argument
938 	get_zspage_mapping(zspage, &class_idx, &fg);  in __free_zspage()
942 	VM_BUG_ON(get_zspage_inuse(zspage));  in __free_zspage()
945 	next = page = get_first_page(zspage);  in __free_zspage()
956 	cache_free_zspage(pool, zspage);  in __free_zspage()
964 				struct zspage *zspage)  in free_zspage()  argument
966 	VM_BUG_ON(get_zspage_inuse(zspage));  in free_zspage()
967 	VM_BUG_ON(list_empty(&zspage->list));  in free_zspage()
969 	if (!trylock_zspage(zspage)) {  in free_zspage()
974 	remove_zspage(class, zspage, ZS_EMPTY);  in free_zspage()
975 	__free_zspage(pool, class, zspage);  in free_zspage()
979 static void init_zspage(struct size_class *class, struct zspage *zspage)  in init_zspage()  argument
983 	struct page *page = get_first_page(zspage);  in init_zspage()
1020 	set_freeobj(zspage, 0);  in init_zspage()
1023 static void create_page_chain(struct size_class *class, struct zspage *zspage,  in create_page_chain()  argument
1041 		set_page_private(page, (unsigned long)zspage);  in create_page_chain()
1044 			zspage->first_page = page;  in create_page_chain()
1059 static struct zspage *alloc_zspage(struct zs_pool *pool,  in alloc_zspage()
1065 	struct zspage *zspage = cache_alloc_zspage(pool, gfp);  in alloc_zspage()  local
1067 	if (!zspage)  in alloc_zspage()
1070 	memset(zspage, 0, sizeof(struct zspage));  in alloc_zspage()
1071 	zspage->magic = ZSPAGE_MAGIC;  in alloc_zspage()
1072 	migrate_lock_init(zspage);  in alloc_zspage()
1083 			cache_free_zspage(pool, zspage);  in alloc_zspage()
1091 	create_page_chain(class, zspage, pages);  in alloc_zspage()
1092 	init_zspage(class, zspage);  in alloc_zspage()
1094 	return zspage;  in alloc_zspage()
1097 static struct zspage *find_get_zspage(struct size_class *class)  in find_get_zspage()
1100 	struct zspage *zspage;  in find_get_zspage()  local
1103 		zspage = list_first_entry_or_null(&class->fullness_list[i],  in find_get_zspage()
1104 				struct zspage, list);  in find_get_zspage()
1105 		if (zspage)  in find_get_zspage()
1109 	return zspage;  in find_get_zspage()
1219 static bool zspage_full(struct size_class *class, struct zspage *zspage)  in zspage_full()  argument
1221 	return get_zspage_inuse(zspage) == class->objs_per_zspage;  in zspage_full()
1248 	struct zspage *zspage;  in zs_map_object()  local
1272 	zspage = get_zspage(page);  in zs_map_object()
1275 	migrate_read_lock(zspage);  in zs_map_object()
1277 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_map_object()
1306 	struct zspage *zspage;  in zs_unmap_object()  local
1318 	zspage = get_zspage(page);  in zs_unmap_object()
1319 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_unmap_object()
1337 	migrate_read_unlock(zspage);  in zs_unmap_object()
1362 				struct zspage *zspage, unsigned long handle)  in obj_malloc()  argument
1373 	obj = get_freeobj(zspage);  in obj_malloc()
1378 	m_page = get_first_page(zspage);  in obj_malloc()
1385 	set_freeobj(zspage, link->next >> OBJ_TAG_BITS);  in obj_malloc()
1391 		zspage->first_page->index = handle;  in obj_malloc()
1394 	mod_zspage_inuse(zspage, 1);  in obj_malloc()
1418 	struct zspage *zspage;  in zs_malloc()  local
1432 	zspage = find_get_zspage(class);  in zs_malloc()
1433 	if (likely(zspage)) {  in zs_malloc()
1434 		obj = obj_malloc(class, zspage, handle);  in zs_malloc()
1436 		fix_fullness_group(class, zspage);  in zs_malloc()
1445 	zspage = alloc_zspage(pool, class, gfp);  in zs_malloc()
1446 	if (!zspage) {  in zs_malloc()
1452 	obj = obj_malloc(class, zspage, handle);  in zs_malloc()
1453 	newfg = get_fullness_group(class, zspage);  in zs_malloc()
1454 	insert_zspage(class, zspage, newfg);  in zs_malloc()
1455 	set_zspage_mapping(zspage, class->index, newfg);  in zs_malloc()
1462 	SetZsPageMovable(pool, zspage);  in zs_malloc()
1472 	struct zspage *zspage;  in obj_free()  local
1481 	zspage = get_zspage(f_page);  in obj_free()
1487 	link->next = get_freeobj(zspage) << OBJ_TAG_BITS;  in obj_free()
1489 	set_freeobj(zspage, f_objidx);  in obj_free()
1490 	mod_zspage_inuse(zspage, -1);  in obj_free()
1496 	struct zspage *zspage;  in zs_free()  local
1511 	zspage = get_zspage(f_page);  in zs_free()
1513 	migrate_read_lock(zspage);  in zs_free()
1515 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_free()
1520 	fullness = fix_fullness_group(class, zspage);  in zs_free()
1522 		migrate_read_unlock(zspage);  in zs_free()
1526 	isolated = is_zspage_isolated(zspage);  in zs_free()
1527 	migrate_read_unlock(zspage);  in zs_free()
1530 		free_zspage(pool, class, zspage);  in zs_free()
1699 static struct zspage *isolate_zspage(struct size_class *class, bool source)  in isolate_zspage()
1702 	struct zspage *zspage;  in isolate_zspage()  local
1711 		zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],  in isolate_zspage()
1712 							struct zspage, list);  in isolate_zspage()
1713 		if (zspage) {  in isolate_zspage()
1714 			VM_BUG_ON(is_zspage_isolated(zspage));  in isolate_zspage()
1715 			remove_zspage(class, zspage, fg[i]);  in isolate_zspage()
1716 			return zspage;  in isolate_zspage()
1720 	return zspage;  in isolate_zspage()
1731 			struct zspage *zspage)  in putback_zspage()  argument
1735 	VM_BUG_ON(is_zspage_isolated(zspage));  in putback_zspage()
1737 	fullness = get_fullness_group(class, zspage);  in putback_zspage()
1738 	insert_zspage(class, zspage, fullness);  in putback_zspage()
1739 	set_zspage_mapping(zspage, class->index, fullness);  in putback_zspage()
1749 static void lock_zspage(struct zspage *zspage)  in lock_zspage()  argument
1762 		migrate_read_lock(zspage);  in lock_zspage()
1763 		page = get_first_page(zspage);  in lock_zspage()
1767 		migrate_read_unlock(zspage);  in lock_zspage()
1778 			migrate_read_unlock(zspage);  in lock_zspage()
1781 			migrate_read_lock(zspage);  in lock_zspage()
1784 	migrate_read_unlock(zspage);  in lock_zspage()
1814 static void migrate_lock_init(struct zspage *zspage)  in migrate_lock_init()  argument
1816 	rwlock_init(&zspage->lock);  in migrate_lock_init()
1819 static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock)  in migrate_read_lock()  argument
1821 	read_lock(&zspage->lock);  in migrate_read_lock()
1824 static void migrate_read_unlock(struct zspage *zspage) __releases(&zspage->lock)  in migrate_read_unlock()  argument
1826 	read_unlock(&zspage->lock);  in migrate_read_unlock()
1829 static void migrate_write_lock(struct zspage *zspage)  in migrate_write_lock()  argument
1831 	write_lock(&zspage->lock);  in migrate_write_lock()
1834 static void migrate_write_unlock(struct zspage *zspage)  in migrate_write_unlock()  argument
1836 	write_unlock(&zspage->lock);  in migrate_write_unlock()
1840 static void inc_zspage_isolation(struct zspage *zspage)  in inc_zspage_isolation()  argument
1842 	zspage->isolated++;  in inc_zspage_isolation()
1845 static void dec_zspage_isolation(struct zspage *zspage)  in dec_zspage_isolation()  argument
1847 	zspage->isolated--;  in dec_zspage_isolation()
1852 				    struct zspage *zspage)  in putback_zspage_deferred()  argument
1856 	fg = putback_zspage(class, zspage);  in putback_zspage_deferred()
1876 static void replace_sub_page(struct size_class *class, struct zspage *zspage,  in replace_sub_page()  argument
1883 	page = get_first_page(zspage);  in replace_sub_page()
1892 	create_page_chain(class, zspage, pages);  in replace_sub_page()
1905 	struct zspage *zspage;  in zs_page_isolate()  local
1915 	zspage = get_zspage(page);  in zs_page_isolate()
1922 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_isolate()
1928 	if (get_zspage_inuse(zspage) == 0) {  in zs_page_isolate()
1934 	if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {  in zs_page_isolate()
1943 	if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {  in zs_page_isolate()
1944 		get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_isolate()
1946 		remove_zspage(class, zspage, fullness);  in zs_page_isolate()
1949 	inc_zspage_isolation(zspage);  in zs_page_isolate()
1962 	struct zspage *zspage;  in zs_page_migrate()  local
1982 	zspage = get_zspage(page);  in zs_page_migrate()
1985 	migrate_write_lock(zspage);  in zs_page_migrate()
1986 	get_zspage_mapping(zspage, &class_idx, &fullness);  in zs_page_migrate()
1992 	if (!get_zspage_inuse(zspage)) {  in zs_page_migrate()
2036 	replace_sub_page(class, zspage, newpage, page);  in zs_page_migrate()
2039 	dec_zspage_isolation(zspage);  in zs_page_migrate()
2045 	if (!is_zspage_isolated(zspage)) {  in zs_page_migrate()
2052 		putback_zspage_deferred(pool, class, zspage);  in zs_page_migrate()
2079 	migrate_write_unlock(zspage);  in zs_page_migrate()
2091 	struct zspage *zspage;  in zs_page_putback()  local
2096 	zspage = get_zspage(page);  in zs_page_putback()
2097 	get_zspage_mapping(zspage, &class_idx, &fg);  in zs_page_putback()
2103 	dec_zspage_isolation(zspage);  in zs_page_putback()
2104 	if (!is_zspage_isolated(zspage)) {  in zs_page_putback()
2109 		putback_zspage_deferred(pool, class, zspage);  in zs_page_putback()
2178 	struct zspage *zspage, *tmp;  in async_free_zspage()  local
2194 	list_for_each_entry_safe(zspage, tmp, &free_pages, list) {  in async_free_zspage()
2195 		list_del(&zspage->list);  in async_free_zspage()
2196 		lock_zspage(zspage);  in async_free_zspage()
2198 		get_zspage_mapping(zspage, &class_idx, &fullness);  in async_free_zspage()
2202 		__free_zspage(pool, pool->size_class[class_idx], zspage);  in async_free_zspage()
2217 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)  in SetZsPageMovable()  argument
2219 	struct page *page = get_first_page(zspage);  in SetZsPageMovable()
2253 	struct zspage *src_zspage;  in __zs_compact()
2254 	struct zspage *dst_zspage = NULL;  in __zs_compact()