Lines Matching refs:zram
58 static void zram_free_page(struct zram *zram, size_t index);
59 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
63 static int zram_slot_trylock(struct zram *zram, u32 index) in zram_slot_trylock() argument
65 return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags); in zram_slot_trylock()
68 static void zram_slot_lock(struct zram *zram, u32 index) in zram_slot_lock() argument
70 bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags); in zram_slot_lock()
73 static void zram_slot_unlock(struct zram *zram, u32 index) in zram_slot_unlock() argument
75 bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags); in zram_slot_unlock()
78 static inline bool init_done(struct zram *zram) in init_done() argument
80 return zram->disksize; in init_done()
83 static inline struct zram *dev_to_zram(struct device *dev) in dev_to_zram()
85 return (struct zram *)dev_to_disk(dev)->private_data; in dev_to_zram()
88 static unsigned long zram_get_handle(struct zram *zram, u32 index) in zram_get_handle() argument
90 return zram->table[index].handle; in zram_get_handle()
93 static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle) in zram_set_handle() argument
95 zram->table[index].handle = handle; in zram_set_handle()
99 static bool zram_test_flag(struct zram *zram, u32 index, in zram_test_flag() argument
102 return zram->table[index].flags & BIT(flag); in zram_test_flag()
105 static void zram_set_flag(struct zram *zram, u32 index, in zram_set_flag() argument
108 zram->table[index].flags |= BIT(flag); in zram_set_flag()
111 static void zram_clear_flag(struct zram *zram, u32 index, in zram_clear_flag() argument
114 zram->table[index].flags &= ~BIT(flag); in zram_clear_flag()
117 static inline void zram_set_element(struct zram *zram, u32 index, in zram_set_element() argument
120 zram->table[index].element = element; in zram_set_element()
123 static unsigned long zram_get_element(struct zram *zram, u32 index) in zram_get_element() argument
125 return zram->table[index].element; in zram_get_element()
128 static size_t zram_get_obj_size(struct zram *zram, u32 index) in zram_get_obj_size() argument
130 return zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1); in zram_get_obj_size()
133 static void zram_set_obj_size(struct zram *zram, in zram_set_obj_size() argument
136 unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT; in zram_set_obj_size()
138 zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; in zram_set_obj_size()
141 static inline bool zram_allocated(struct zram *zram, u32 index) in zram_allocated() argument
143 return zram_get_obj_size(zram, index) || in zram_allocated()
144 zram_test_flag(zram, index, ZRAM_SAME) || in zram_allocated()
145 zram_test_flag(zram, index, ZRAM_WB); in zram_allocated()
163 static inline bool valid_io_request(struct zram *zram, in valid_io_request() argument
175 bound = zram->disksize >> SECTOR_SHIFT; in valid_io_request()
190 static inline void update_used_max(struct zram *zram, in update_used_max() argument
195 old_max = atomic_long_read(&zram->stats.max_used_pages); in update_used_max()
201 &zram->stats.max_used_pages, cur_max, pages); in update_used_max()
238 struct zram *zram = dev_to_zram(dev); in initstate_show() local
240 down_read(&zram->init_lock); in initstate_show()
241 val = init_done(zram); in initstate_show()
242 up_read(&zram->init_lock); in initstate_show()
250 struct zram *zram = dev_to_zram(dev); in disksize_show() local
252 return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize); in disksize_show()
260 struct zram *zram = dev_to_zram(dev); in mem_limit_store() local
266 down_write(&zram->init_lock); in mem_limit_store()
267 zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT; in mem_limit_store()
268 up_write(&zram->init_lock); in mem_limit_store()
278 struct zram *zram = dev_to_zram(dev); in mem_used_max_store() local
284 down_read(&zram->init_lock); in mem_used_max_store()
285 if (init_done(zram)) { in mem_used_max_store()
286 atomic_long_set(&zram->stats.max_used_pages, in mem_used_max_store()
287 zs_get_total_pages(zram->mem_pool)); in mem_used_max_store()
289 up_read(&zram->init_lock); in mem_used_max_store()
297 struct zram *zram = dev_to_zram(dev); in idle_store() local
298 unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; in idle_store()
304 down_read(&zram->init_lock); in idle_store()
305 if (!init_done(zram)) { in idle_store()
306 up_read(&zram->init_lock); in idle_store()
315 zram_slot_lock(zram, index); in idle_store()
316 if (zram_allocated(zram, index) && in idle_store()
317 !zram_test_flag(zram, index, ZRAM_UNDER_WB)) in idle_store()
318 zram_set_flag(zram, index, ZRAM_IDLE); in idle_store()
319 zram_slot_unlock(zram, index); in idle_store()
322 up_read(&zram->init_lock); in idle_store()
331 struct zram *zram = dev_to_zram(dev); in writeback_limit_enable_store() local
338 down_read(&zram->init_lock); in writeback_limit_enable_store()
339 spin_lock(&zram->wb_limit_lock); in writeback_limit_enable_store()
340 zram->wb_limit_enable = val; in writeback_limit_enable_store()
341 spin_unlock(&zram->wb_limit_lock); in writeback_limit_enable_store()
342 up_read(&zram->init_lock); in writeback_limit_enable_store()
352 struct zram *zram = dev_to_zram(dev); in writeback_limit_enable_show() local
354 down_read(&zram->init_lock); in writeback_limit_enable_show()
355 spin_lock(&zram->wb_limit_lock); in writeback_limit_enable_show()
356 val = zram->wb_limit_enable; in writeback_limit_enable_show()
357 spin_unlock(&zram->wb_limit_lock); in writeback_limit_enable_show()
358 up_read(&zram->init_lock); in writeback_limit_enable_show()
366 struct zram *zram = dev_to_zram(dev); in writeback_limit_store() local
373 down_read(&zram->init_lock); in writeback_limit_store()
374 spin_lock(&zram->wb_limit_lock); in writeback_limit_store()
375 zram->bd_wb_limit = val; in writeback_limit_store()
376 spin_unlock(&zram->wb_limit_lock); in writeback_limit_store()
377 up_read(&zram->init_lock); in writeback_limit_store()
387 struct zram *zram = dev_to_zram(dev); in writeback_limit_show() local
389 down_read(&zram->init_lock); in writeback_limit_show()
390 spin_lock(&zram->wb_limit_lock); in writeback_limit_show()
391 val = zram->bd_wb_limit; in writeback_limit_show()
392 spin_unlock(&zram->wb_limit_lock); in writeback_limit_show()
393 up_read(&zram->init_lock); in writeback_limit_show()
398 static void reset_bdev(struct zram *zram) in reset_bdev() argument
402 if (!zram->backing_dev) in reset_bdev()
405 bdev = zram->bdev; in reset_bdev()
406 if (zram->old_block_size) in reset_bdev()
407 set_blocksize(bdev, zram->old_block_size); in reset_bdev()
410 filp_close(zram->backing_dev, NULL); in reset_bdev()
411 zram->backing_dev = NULL; in reset_bdev()
412 zram->old_block_size = 0; in reset_bdev()
413 zram->bdev = NULL; in reset_bdev()
414 zram->disk->fops = &zram_devops; in reset_bdev()
415 kvfree(zram->bitmap); in reset_bdev()
416 zram->bitmap = NULL; in reset_bdev()
423 struct zram *zram = dev_to_zram(dev); in backing_dev_show() local
427 down_read(&zram->init_lock); in backing_dev_show()
428 file = zram->backing_dev; in backing_dev_show()
431 up_read(&zram->init_lock); in backing_dev_show()
445 up_read(&zram->init_lock); in backing_dev_show()
461 struct zram *zram = dev_to_zram(dev); in backing_dev_store() local
467 down_write(&zram->init_lock); in backing_dev_store()
468 if (init_done(zram)) { in backing_dev_store()
497 FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram); in backing_dev_store()
517 reset_bdev(zram); in backing_dev_store()
519 zram->old_block_size = old_block_size; in backing_dev_store()
520 zram->bdev = bdev; in backing_dev_store()
521 zram->backing_dev = backing_dev; in backing_dev_store()
522 zram->bitmap = bitmap; in backing_dev_store()
523 zram->nr_pages = nr_pages; in backing_dev_store()
534 zram->disk->fops = &zram_wb_devops; in backing_dev_store()
535 up_write(&zram->init_lock); in backing_dev_store()
551 up_write(&zram->init_lock); in backing_dev_store()
558 static unsigned long alloc_block_bdev(struct zram *zram) in alloc_block_bdev() argument
563 blk_idx = find_next_zero_bit(zram->bitmap, zram->nr_pages, blk_idx); in alloc_block_bdev()
564 if (blk_idx == zram->nr_pages) in alloc_block_bdev()
567 if (test_and_set_bit(blk_idx, zram->bitmap)) in alloc_block_bdev()
570 atomic64_inc(&zram->stats.bd_count); in alloc_block_bdev()
574 static void free_block_bdev(struct zram *zram, unsigned long blk_idx) in free_block_bdev() argument
578 was_set = test_and_clear_bit(blk_idx, zram->bitmap); in free_block_bdev()
580 atomic64_dec(&zram->stats.bd_count); in free_block_bdev()
595 static int read_from_bdev_async(struct zram *zram, struct bio_vec *bvec, in read_from_bdev_async() argument
605 bio_set_dev(bio, zram->bdev); in read_from_bdev_async()
633 struct zram *zram = dev_to_zram(dev); in writeback_store() local
634 unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; in writeback_store()
659 down_read(&zram->init_lock); in writeback_store()
660 if (!init_done(zram)) { in writeback_store()
665 if (!zram->backing_dev) { in writeback_store()
683 spin_lock(&zram->wb_limit_lock); in writeback_store()
684 if (zram->wb_limit_enable && !zram->bd_wb_limit) { in writeback_store()
685 spin_unlock(&zram->wb_limit_lock); in writeback_store()
689 spin_unlock(&zram->wb_limit_lock); in writeback_store()
692 blk_idx = alloc_block_bdev(zram); in writeback_store()
699 zram_slot_lock(zram, index); in writeback_store()
700 if (!zram_allocated(zram, index)) in writeback_store()
703 if (zram_test_flag(zram, index, ZRAM_WB) || in writeback_store()
704 zram_test_flag(zram, index, ZRAM_SAME) || in writeback_store()
705 zram_test_flag(zram, index, ZRAM_UNDER_WB)) in writeback_store()
709 !zram_test_flag(zram, index, ZRAM_IDLE)) in writeback_store()
712 !zram_test_flag(zram, index, ZRAM_HUGE)) in writeback_store()
718 zram_set_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
720 zram_set_flag(zram, index, ZRAM_IDLE); in writeback_store()
721 zram_slot_unlock(zram, index); in writeback_store()
722 if (zram_bvec_read(zram, &bvec, index, 0, NULL)) { in writeback_store()
723 zram_slot_lock(zram, index); in writeback_store()
724 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
725 zram_clear_flag(zram, index, ZRAM_IDLE); in writeback_store()
726 zram_slot_unlock(zram, index); in writeback_store()
731 bio_set_dev(&bio, zram->bdev); in writeback_store()
743 zram_slot_lock(zram, index); in writeback_store()
744 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
745 zram_clear_flag(zram, index, ZRAM_IDLE); in writeback_store()
746 zram_slot_unlock(zram, index); in writeback_store()
755 atomic64_inc(&zram->stats.bd_writes); in writeback_store()
765 zram_slot_lock(zram, index); in writeback_store()
766 if (!zram_allocated(zram, index) || in writeback_store()
767 !zram_test_flag(zram, index, ZRAM_IDLE)) { in writeback_store()
768 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
769 zram_clear_flag(zram, index, ZRAM_IDLE); in writeback_store()
773 zram_free_page(zram, index); in writeback_store()
774 zram_clear_flag(zram, index, ZRAM_UNDER_WB); in writeback_store()
775 zram_set_flag(zram, index, ZRAM_WB); in writeback_store()
776 zram_set_element(zram, index, blk_idx); in writeback_store()
778 atomic64_inc(&zram->stats.pages_stored); in writeback_store()
779 spin_lock(&zram->wb_limit_lock); in writeback_store()
780 if (zram->wb_limit_enable && zram->bd_wb_limit > 0) in writeback_store()
781 zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12); in writeback_store()
782 spin_unlock(&zram->wb_limit_lock); in writeback_store()
784 zram_slot_unlock(zram, index); in writeback_store()
788 free_block_bdev(zram, blk_idx); in writeback_store()
791 up_read(&zram->init_lock); in writeback_store()
798 struct zram *zram; member
808 struct zram *zram = zw->zram; in zram_sync_read() local
812 read_from_bdev_async(zram, &zw->bvec, entry, bio); in zram_sync_read()
820 static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec, in read_from_bdev_sync() argument
826 work.zram = zram; in read_from_bdev_sync()
838 static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec, in read_from_bdev_sync() argument
846 static int read_from_bdev(struct zram *zram, struct bio_vec *bvec, in read_from_bdev() argument
849 atomic64_inc(&zram->stats.bd_reads); in read_from_bdev()
851 return read_from_bdev_sync(zram, bvec, entry, parent); in read_from_bdev()
853 return read_from_bdev_async(zram, bvec, entry, parent); in read_from_bdev()
856 static inline void reset_bdev(struct zram *zram) {}; in reset_bdev() argument
857 static int read_from_bdev(struct zram *zram, struct bio_vec *bvec, in read_from_bdev() argument
863 static void free_block_bdev(struct zram *zram, unsigned long blk_idx) {}; in free_block_bdev() argument
880 static void zram_accessed(struct zram *zram, u32 index) in zram_accessed() argument
882 zram_clear_flag(zram, index, ZRAM_IDLE); in zram_accessed()
883 zram->table[index].ac_time = ktime_get_boottime(); in zram_accessed()
891 struct zram *zram = file->private_data; in read_block_state() local
892 unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; in read_block_state()
899 down_read(&zram->init_lock); in read_block_state()
900 if (!init_done(zram)) { in read_block_state()
901 up_read(&zram->init_lock); in read_block_state()
909 zram_slot_lock(zram, index); in read_block_state()
910 if (!zram_allocated(zram, index)) in read_block_state()
913 ts = ktime_to_timespec64(zram->table[index].ac_time); in read_block_state()
918 zram_test_flag(zram, index, ZRAM_SAME) ? 's' : '.', in read_block_state()
919 zram_test_flag(zram, index, ZRAM_WB) ? 'w' : '.', in read_block_state()
920 zram_test_flag(zram, index, ZRAM_HUGE) ? 'h' : '.', in read_block_state()
921 zram_test_flag(zram, index, ZRAM_IDLE) ? 'i' : '.'); in read_block_state()
924 zram_slot_unlock(zram, index); in read_block_state()
930 zram_slot_unlock(zram, index); in read_block_state()
934 up_read(&zram->init_lock); in read_block_state()
948 static void zram_debugfs_register(struct zram *zram) in zram_debugfs_register() argument
953 zram->debugfs_dir = debugfs_create_dir(zram->disk->disk_name, in zram_debugfs_register()
955 debugfs_create_file("block_state", 0400, zram->debugfs_dir, in zram_debugfs_register()
956 zram, &proc_zram_block_state_op); in zram_debugfs_register()
959 static void zram_debugfs_unregister(struct zram *zram) in zram_debugfs_unregister() argument
961 debugfs_remove_recursive(zram->debugfs_dir); in zram_debugfs_unregister()
966 static void zram_accessed(struct zram *zram, u32 index) in zram_accessed() argument
968 zram_clear_flag(zram, index, ZRAM_IDLE); in zram_accessed()
970 static void zram_debugfs_register(struct zram *zram) {}; in zram_debugfs_register() argument
971 static void zram_debugfs_unregister(struct zram *zram) {}; in zram_debugfs_unregister() argument
999 struct zram *zram = dev_to_zram(dev); in comp_algorithm_show() local
1001 down_read(&zram->init_lock); in comp_algorithm_show()
1002 sz = zcomp_available_show(zram->compressor, buf); in comp_algorithm_show()
1003 up_read(&zram->init_lock); in comp_algorithm_show()
1011 struct zram *zram = dev_to_zram(dev); in comp_algorithm_store() local
1012 char compressor[ARRAY_SIZE(zram->compressor)]; in comp_algorithm_store()
1024 down_write(&zram->init_lock); in comp_algorithm_store()
1025 if (init_done(zram)) { in comp_algorithm_store()
1026 up_write(&zram->init_lock); in comp_algorithm_store()
1031 strcpy(zram->compressor, compressor); in comp_algorithm_store()
1032 up_write(&zram->init_lock); in comp_algorithm_store()
1039 struct zram *zram = dev_to_zram(dev); in compact_store() local
1041 down_read(&zram->init_lock); in compact_store()
1042 if (!init_done(zram)) { in compact_store()
1043 up_read(&zram->init_lock); in compact_store()
1047 zs_compact(zram->mem_pool); in compact_store()
1048 up_read(&zram->init_lock); in compact_store()
1056 struct zram *zram = dev_to_zram(dev); in io_stat_show() local
1059 down_read(&zram->init_lock); in io_stat_show()
1062 (u64)atomic64_read(&zram->stats.failed_reads), in io_stat_show()
1063 (u64)atomic64_read(&zram->stats.failed_writes), in io_stat_show()
1064 (u64)atomic64_read(&zram->stats.invalid_io), in io_stat_show()
1065 (u64)atomic64_read(&zram->stats.notify_free)); in io_stat_show()
1066 up_read(&zram->init_lock); in io_stat_show()
1074 struct zram *zram = dev_to_zram(dev); in mm_stat_show() local
1082 down_read(&zram->init_lock); in mm_stat_show()
1083 if (init_done(zram)) { in mm_stat_show()
1084 mem_used = zs_get_total_pages(zram->mem_pool); in mm_stat_show()
1085 zs_pool_stats(zram->mem_pool, &pool_stats); in mm_stat_show()
1088 orig_size = atomic64_read(&zram->stats.pages_stored); in mm_stat_show()
1089 max_used = atomic_long_read(&zram->stats.max_used_pages); in mm_stat_show()
1094 (u64)atomic64_read(&zram->stats.compr_data_size), in mm_stat_show()
1096 zram->limit_pages << PAGE_SHIFT, in mm_stat_show()
1098 (u64)atomic64_read(&zram->stats.same_pages), in mm_stat_show()
1100 (u64)atomic64_read(&zram->stats.huge_pages)); in mm_stat_show()
1101 up_read(&zram->init_lock); in mm_stat_show()
1111 struct zram *zram = dev_to_zram(dev); in bd_stat_show() local
1114 down_read(&zram->init_lock); in bd_stat_show()
1117 FOUR_K((u64)atomic64_read(&zram->stats.bd_count)), in bd_stat_show()
1118 FOUR_K((u64)atomic64_read(&zram->stats.bd_reads)), in bd_stat_show()
1119 FOUR_K((u64)atomic64_read(&zram->stats.bd_writes))); in bd_stat_show()
1120 up_read(&zram->init_lock); in bd_stat_show()
1130 struct zram *zram = dev_to_zram(dev); in debug_stat_show() local
1133 down_read(&zram->init_lock); in debug_stat_show()
1137 (u64)atomic64_read(&zram->stats.writestall), in debug_stat_show()
1138 (u64)atomic64_read(&zram->stats.miss_free)); in debug_stat_show()
1139 up_read(&zram->init_lock); in debug_stat_show()
1151 static void zram_meta_free(struct zram *zram, u64 disksize) in zram_meta_free() argument
1158 zram_free_page(zram, index); in zram_meta_free()
1160 zs_destroy_pool(zram->mem_pool); in zram_meta_free()
1161 vfree(zram->table); in zram_meta_free()
1164 static bool zram_meta_alloc(struct zram *zram, u64 disksize) in zram_meta_alloc() argument
1169 zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); in zram_meta_alloc()
1170 if (!zram->table) in zram_meta_alloc()
1173 zram->mem_pool = zs_create_pool(zram->disk->disk_name); in zram_meta_alloc()
1174 if (!zram->mem_pool) { in zram_meta_alloc()
1175 vfree(zram->table); in zram_meta_alloc()
1180 huge_class_size = zs_huge_class_size(zram->mem_pool); in zram_meta_alloc()
1189 static void zram_free_page(struct zram *zram, size_t index) in zram_free_page() argument
1194 zram->table[index].ac_time = 0; in zram_free_page()
1196 if (zram_test_flag(zram, index, ZRAM_IDLE)) in zram_free_page()
1197 zram_clear_flag(zram, index, ZRAM_IDLE); in zram_free_page()
1199 if (zram_test_flag(zram, index, ZRAM_HUGE)) { in zram_free_page()
1200 zram_clear_flag(zram, index, ZRAM_HUGE); in zram_free_page()
1201 atomic64_dec(&zram->stats.huge_pages); in zram_free_page()
1204 if (zram_test_flag(zram, index, ZRAM_WB)) { in zram_free_page()
1205 zram_clear_flag(zram, index, ZRAM_WB); in zram_free_page()
1206 free_block_bdev(zram, zram_get_element(zram, index)); in zram_free_page()
1214 if (zram_test_flag(zram, index, ZRAM_SAME)) { in zram_free_page()
1215 zram_clear_flag(zram, index, ZRAM_SAME); in zram_free_page()
1216 atomic64_dec(&zram->stats.same_pages); in zram_free_page()
1220 handle = zram_get_handle(zram, index); in zram_free_page()
1224 zs_free(zram->mem_pool, handle); in zram_free_page()
1226 atomic64_sub(zram_get_obj_size(zram, index), in zram_free_page()
1227 &zram->stats.compr_data_size); in zram_free_page()
1229 atomic64_dec(&zram->stats.pages_stored); in zram_free_page()
1230 zram_set_handle(zram, index, 0); in zram_free_page()
1231 zram_set_obj_size(zram, index, 0); in zram_free_page()
1232 WARN_ON_ONCE(zram->table[index].flags & in zram_free_page()
1236 static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index, in __zram_bvec_read() argument
1245 zram_slot_lock(zram, index); in __zram_bvec_read()
1246 if (zram_test_flag(zram, index, ZRAM_WB)) { in __zram_bvec_read()
1249 zram_slot_unlock(zram, index); in __zram_bvec_read()
1254 return read_from_bdev(zram, &bvec, in __zram_bvec_read()
1255 zram_get_element(zram, index), in __zram_bvec_read()
1259 handle = zram_get_handle(zram, index); in __zram_bvec_read()
1260 if (!handle || zram_test_flag(zram, index, ZRAM_SAME)) { in __zram_bvec_read()
1264 value = handle ? zram_get_element(zram, index) : 0; in __zram_bvec_read()
1268 zram_slot_unlock(zram, index); in __zram_bvec_read()
1272 size = zram_get_obj_size(zram, index); in __zram_bvec_read()
1275 zstrm = zcomp_stream_get(zram->comp); in __zram_bvec_read()
1277 src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO); in __zram_bvec_read()
1287 zcomp_stream_put(zram->comp); in __zram_bvec_read()
1289 zs_unmap_object(zram->mem_pool, handle); in __zram_bvec_read()
1290 zram_slot_unlock(zram, index); in __zram_bvec_read()
1299 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, in zram_bvec_read() argument
1313 ret = __zram_bvec_read(zram, page, index, bio, is_partial_io(bvec)); in zram_bvec_read()
1332 static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, in __zram_bvec_write() argument
1350 atomic64_inc(&zram->stats.same_pages); in __zram_bvec_write()
1356 zstrm = zcomp_stream_get(zram->comp); in __zram_bvec_write()
1362 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1364 zs_free(zram->mem_pool, handle); in __zram_bvec_write()
1384 handle = zs_malloc(zram->mem_pool, comp_len, in __zram_bvec_write()
1391 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1392 atomic64_inc(&zram->stats.writestall); in __zram_bvec_write()
1393 handle = zs_malloc(zram->mem_pool, comp_len, in __zram_bvec_write()
1401 alloced_pages = zs_get_total_pages(zram->mem_pool); in __zram_bvec_write()
1402 update_used_max(zram, alloced_pages); in __zram_bvec_write()
1404 if (zram->limit_pages && alloced_pages > zram->limit_pages) { in __zram_bvec_write()
1405 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1406 zs_free(zram->mem_pool, handle); in __zram_bvec_write()
1410 dst = zs_map_object(zram->mem_pool, handle, ZS_MM_WO); in __zram_bvec_write()
1419 zcomp_stream_put(zram->comp); in __zram_bvec_write()
1420 zs_unmap_object(zram->mem_pool, handle); in __zram_bvec_write()
1421 atomic64_add(comp_len, &zram->stats.compr_data_size); in __zram_bvec_write()
1427 zram_slot_lock(zram, index); in __zram_bvec_write()
1428 zram_free_page(zram, index); in __zram_bvec_write()
1431 zram_set_flag(zram, index, ZRAM_HUGE); in __zram_bvec_write()
1432 atomic64_inc(&zram->stats.huge_pages); in __zram_bvec_write()
1436 zram_set_flag(zram, index, flags); in __zram_bvec_write()
1437 zram_set_element(zram, index, element); in __zram_bvec_write()
1439 zram_set_handle(zram, index, handle); in __zram_bvec_write()
1440 zram_set_obj_size(zram, index, comp_len); in __zram_bvec_write()
1442 zram_slot_unlock(zram, index); in __zram_bvec_write()
1445 atomic64_inc(&zram->stats.pages_stored); in __zram_bvec_write()
1449 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, in zram_bvec_write() argument
1468 ret = __zram_bvec_read(zram, page, index, bio, true); in zram_bvec_write()
1483 ret = __zram_bvec_write(zram, &vec, index, bio); in zram_bvec_write()
1495 static void zram_bio_discard(struct zram *zram, u32 index, in zram_bio_discard() argument
1519 zram_slot_lock(zram, index); in zram_bio_discard()
1520 zram_free_page(zram, index); in zram_bio_discard()
1521 zram_slot_unlock(zram, index); in zram_bio_discard()
1522 atomic64_inc(&zram->stats.notify_free); in zram_bio_discard()
1533 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, in zram_bvec_rw() argument
1539 atomic64_inc(&zram->stats.num_reads); in zram_bvec_rw()
1540 ret = zram_bvec_read(zram, bvec, index, offset, bio); in zram_bvec_rw()
1543 atomic64_inc(&zram->stats.num_writes); in zram_bvec_rw()
1544 ret = zram_bvec_write(zram, bvec, index, offset, bio); in zram_bvec_rw()
1547 zram_slot_lock(zram, index); in zram_bvec_rw()
1548 zram_accessed(zram, index); in zram_bvec_rw()
1549 zram_slot_unlock(zram, index); in zram_bvec_rw()
1553 atomic64_inc(&zram->stats.failed_reads); in zram_bvec_rw()
1555 atomic64_inc(&zram->stats.failed_writes); in zram_bvec_rw()
1561 static void __zram_make_request(struct zram *zram, struct bio *bio) in __zram_make_request() argument
1576 zram_bio_discard(zram, index, offset, bio); in __zram_make_request()
1591 if (zram_bvec_rw(zram, &bv, index, offset, in __zram_make_request()
1612 struct zram *zram = bio->bi_disk->private_data; in zram_submit_bio() local
1614 if (!valid_io_request(zram, bio->bi_iter.bi_sector, in zram_submit_bio()
1616 atomic64_inc(&zram->stats.invalid_io); in zram_submit_bio()
1620 __zram_make_request(zram, bio); in zram_submit_bio()
1631 struct zram *zram; in zram_slot_free_notify() local
1633 zram = bdev->bd_disk->private_data; in zram_slot_free_notify()
1635 atomic64_inc(&zram->stats.notify_free); in zram_slot_free_notify()
1636 if (!zram_slot_trylock(zram, index)) { in zram_slot_free_notify()
1637 atomic64_inc(&zram->stats.miss_free); in zram_slot_free_notify()
1641 zram_free_page(zram, index); in zram_slot_free_notify()
1642 zram_slot_unlock(zram, index); in zram_slot_free_notify()
1650 struct zram *zram; in zram_rw_page() local
1656 zram = bdev->bd_disk->private_data; in zram_rw_page()
1658 if (!valid_io_request(zram, sector, PAGE_SIZE)) { in zram_rw_page()
1659 atomic64_inc(&zram->stats.invalid_io); in zram_rw_page()
1672 ret = zram_bvec_rw(zram, &bv, index, offset, op, NULL); in zram_rw_page()
1699 static void zram_reset_device(struct zram *zram) in zram_reset_device() argument
1704 down_write(&zram->init_lock); in zram_reset_device()
1706 zram->limit_pages = 0; in zram_reset_device()
1708 if (!init_done(zram)) { in zram_reset_device()
1709 up_write(&zram->init_lock); in zram_reset_device()
1713 comp = zram->comp; in zram_reset_device()
1714 disksize = zram->disksize; in zram_reset_device()
1715 zram->disksize = 0; in zram_reset_device()
1717 set_capacity(zram->disk, 0); in zram_reset_device()
1718 part_stat_set_all(&zram->disk->part0, 0); in zram_reset_device()
1720 up_write(&zram->init_lock); in zram_reset_device()
1722 zram_meta_free(zram, disksize); in zram_reset_device()
1723 memset(&zram->stats, 0, sizeof(zram->stats)); in zram_reset_device()
1725 reset_bdev(zram); in zram_reset_device()
1733 struct zram *zram = dev_to_zram(dev); in disksize_store() local
1740 down_write(&zram->init_lock); in disksize_store()
1741 if (init_done(zram)) { in disksize_store()
1748 if (!zram_meta_alloc(zram, disksize)) { in disksize_store()
1753 comp = zcomp_create(zram->compressor); in disksize_store()
1756 zram->compressor); in disksize_store()
1761 zram->comp = comp; in disksize_store()
1762 zram->disksize = disksize; in disksize_store()
1763 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); in disksize_store()
1765 revalidate_disk_size(zram->disk, true); in disksize_store()
1766 up_write(&zram->init_lock); in disksize_store()
1771 zram_meta_free(zram, disksize); in disksize_store()
1773 up_write(&zram->init_lock); in disksize_store()
1782 struct zram *zram; in reset_store() local
1792 zram = dev_to_zram(dev); in reset_store()
1793 bdev = bdget_disk(zram->disk, 0); in reset_store()
1799 if (bdev->bd_openers || zram->claim) { in reset_store()
1806 zram->claim = true; in reset_store()
1811 zram_reset_device(zram); in reset_store()
1812 revalidate_disk_size(zram->disk, true); in reset_store()
1816 zram->claim = false; in reset_store()
1825 struct zram *zram; in zram_open() local
1829 zram = bdev->bd_disk->private_data; in zram_open()
1831 if (zram->claim) in zram_open()
1908 struct zram *zram; in zram_add() local
1912 zram = kzalloc(sizeof(struct zram), GFP_KERNEL); in zram_add()
1913 if (!zram) in zram_add()
1916 ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL); in zram_add()
1921 init_rwsem(&zram->init_lock); in zram_add()
1923 spin_lock_init(&zram->wb_limit_lock); in zram_add()
1934 zram->disk = alloc_disk(1); in zram_add()
1935 if (!zram->disk) { in zram_add()
1942 zram->disk->major = zram_major; in zram_add()
1943 zram->disk->first_minor = device_id; in zram_add()
1944 zram->disk->fops = &zram_devops; in zram_add()
1945 zram->disk->queue = queue; in zram_add()
1946 zram->disk->private_data = zram; in zram_add()
1947 snprintf(zram->disk->disk_name, 16, "zram%d", device_id); in zram_add()
1950 set_capacity(zram->disk, 0); in zram_add()
1952 blk_queue_flag_set(QUEUE_FLAG_NONROT, zram->disk->queue); in zram_add()
1953 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue); in zram_add()
1959 blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE); in zram_add()
1960 blk_queue_logical_block_size(zram->disk->queue, in zram_add()
1962 blk_queue_io_min(zram->disk->queue, PAGE_SIZE); in zram_add()
1963 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); in zram_add()
1964 zram->disk->queue->limits.discard_granularity = PAGE_SIZE; in zram_add()
1965 blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX); in zram_add()
1966 blk_queue_flag_set(QUEUE_FLAG_DISCARD, zram->disk->queue); in zram_add()
1977 blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX); in zram_add()
1979 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, zram->disk->queue); in zram_add()
1980 device_add_disk(NULL, zram->disk, zram_disk_attr_groups); in zram_add()
1982 strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor)); in zram_add()
1984 zram_debugfs_register(zram); in zram_add()
1985 pr_info("Added device: %s\n", zram->disk->disk_name); in zram_add()
1993 kfree(zram); in zram_add()
1997 static int zram_remove(struct zram *zram) in zram_remove() argument
2001 bdev = bdget_disk(zram->disk, 0); in zram_remove()
2006 if (bdev->bd_openers || zram->claim) { in zram_remove()
2012 zram->claim = true; in zram_remove()
2015 zram_debugfs_unregister(zram); in zram_remove()
2019 zram_reset_device(zram); in zram_remove()
2022 pr_info("Removed device: %s\n", zram->disk->disk_name); in zram_remove()
2024 del_gendisk(zram->disk); in zram_remove()
2025 blk_cleanup_queue(zram->disk->queue); in zram_remove()
2026 put_disk(zram->disk); in zram_remove()
2027 kfree(zram); in zram_remove()
2061 struct zram *zram; in hot_remove_store() local
2073 zram = idr_find(&zram_index_idr, dev_id); in hot_remove_store()
2074 if (zram) { in hot_remove_store()
2075 ret = zram_remove(zram); in hot_remove_store()