53 	int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
54 	void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
55 	int (*decompress_pages)(struct decompress_io_ctx *dic);
204 static int lzo_decompress_pages(struct decompress_io_ctx *dic)  in lzo_decompress_pages()  argument
208 	ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,  in lzo_decompress_pages()
209 						dic->rbuf, &dic->rlen);  in lzo_decompress_pages()
212 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);  in lzo_decompress_pages()
216 	if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {  in lzo_decompress_pages()
219 					F2FS_I_SB(dic->inode)->sb->s_id,  in lzo_decompress_pages()
220 					dic->rlen,  in lzo_decompress_pages()
221 					PAGE_SIZE << dic->log_cluster_size);  in lzo_decompress_pages()
301 static int lz4_decompress_pages(struct decompress_io_ctx *dic)  in lz4_decompress_pages()  argument
305 	ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,  in lz4_decompress_pages()
306 						dic->clen, dic->rlen);  in lz4_decompress_pages()
309 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);  in lz4_decompress_pages()
313 	if (ret != PAGE_SIZE << dic->log_cluster_size) {  in lz4_decompress_pages()
316 					F2FS_I_SB(dic->inode)->sb->s_id, ret,  in lz4_decompress_pages()
317 					PAGE_SIZE << dic->log_cluster_size);  in lz4_decompress_pages()
421 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)  in zstd_init_decompress_ctx()  argument
427 			MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size);  in zstd_init_decompress_ctx()
431 	workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode),  in zstd_init_decompress_ctx()
439 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,  in zstd_init_decompress_ctx()
445 	dic->private = workspace;  in zstd_init_decompress_ctx()
446 	dic->private2 = stream;  in zstd_init_decompress_ctx()
451 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)  in zstd_destroy_decompress_ctx()  argument
453 	kvfree(dic->private);  in zstd_destroy_decompress_ctx()
454 	dic->private = NULL;  in zstd_destroy_decompress_ctx()
455 	dic->private2 = NULL;  in zstd_destroy_decompress_ctx()
458 static int zstd_decompress_pages(struct decompress_io_ctx *dic)  in zstd_decompress_pages()  argument
460 	ZSTD_DStream *stream = dic->private2;  in zstd_decompress_pages()
466 	inbuf.src = dic->cbuf->cdata;  in zstd_decompress_pages()
467 	inbuf.size = dic->clen;  in zstd_decompress_pages()
470 	outbuf.dst = dic->rbuf;  in zstd_decompress_pages()
471 	outbuf.size = dic->rlen;  in zstd_decompress_pages()
476 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,  in zstd_decompress_pages()
481 	if (dic->rlen != outbuf.pos) {  in zstd_decompress_pages()
484 				F2FS_I_SB(dic->inode)->sb->s_id,  in zstd_decompress_pages()
485 				__func__, dic->rlen,  in zstd_decompress_pages()
486 				PAGE_SIZE << dic->log_cluster_size);  in zstd_decompress_pages()
740 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,
742 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,
745 void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task)  in f2fs_decompress_cluster()  argument
747 	struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);  in f2fs_decompress_cluster()
748 	struct f2fs_inode_info *fi = F2FS_I(dic->inode);  in f2fs_decompress_cluster()
754 	trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,  in f2fs_decompress_cluster()
755 				dic->cluster_size, fi->i_compress_algorithm);  in f2fs_decompress_cluster()
757 	if (dic->failed) {  in f2fs_decompress_cluster()
762 	ret = f2fs_prepare_decomp_mem(dic, false);  in f2fs_decompress_cluster()
768 	dic->clen = le32_to_cpu(dic->cbuf->clen);  in f2fs_decompress_cluster()
769 	dic->rlen = PAGE_SIZE << dic->log_cluster_size;  in f2fs_decompress_cluster()
771 	if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {  in f2fs_decompress_cluster()
776 	ret = cops->decompress_pages(dic);  in f2fs_decompress_cluster()
779 		u32 provided = le32_to_cpu(dic->cbuf->chksum);  in f2fs_decompress_cluster()
780 		u32 calculated = f2fs_crc32(sbi, dic->cbuf->cdata, dic->clen);  in f2fs_decompress_cluster()
783 			if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) {  in f2fs_decompress_cluster()
784 				set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT);  in f2fs_decompress_cluster()
787 					KERN_INFO, sbi->sb->s_id, dic->inode->i_ino,  in f2fs_decompress_cluster()
795 	f2fs_release_decomp_mem(dic, bypass_callback, false);  in f2fs_decompress_cluster()
798 	trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,  in f2fs_decompress_cluster()
799 							dic->clen, ret);  in f2fs_decompress_cluster()
800 	f2fs_decompress_end_io(dic, ret, in_task);  in f2fs_decompress_cluster()
812 	struct decompress_io_ctx *dic =  in f2fs_end_read_compressed_page()  local
814 	struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);  in f2fs_end_read_compressed_page()
819 		WRITE_ONCE(dic->failed, true);  in f2fs_end_read_compressed_page()
822 					dic->inode->i_ino, blkaddr);  in f2fs_end_read_compressed_page()
824 	if (atomic_dec_and_test(&dic->remaining_pages))  in f2fs_end_read_compressed_page()
825 		f2fs_decompress_cluster(dic, in_task);  in f2fs_end_read_compressed_page()
1471 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,  in f2fs_prepare_decomp_mem()  argument
1475 		f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm];  in f2fs_prepare_decomp_mem()
1478 	if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc))  in f2fs_prepare_decomp_mem()
1481 	dic->tpages = page_array_alloc(dic->inode, dic->cluster_size);  in f2fs_prepare_decomp_mem()
1482 	if (!dic->tpages)  in f2fs_prepare_decomp_mem()
1485 	for (i = 0; i < dic->cluster_size; i++) {  in f2fs_prepare_decomp_mem()
1486 		if (dic->rpages[i]) {  in f2fs_prepare_decomp_mem()
1487 			dic->tpages[i] = dic->rpages[i];  in f2fs_prepare_decomp_mem()
1491 		dic->tpages[i] = f2fs_compress_alloc_page();  in f2fs_prepare_decomp_mem()
1492 		if (!dic->tpages[i])  in f2fs_prepare_decomp_mem()
1496 	dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size);  in f2fs_prepare_decomp_mem()
1497 	if (!dic->rbuf)  in f2fs_prepare_decomp_mem()
1500 	dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages);  in f2fs_prepare_decomp_mem()
1501 	if (!dic->cbuf)  in f2fs_prepare_decomp_mem()
1505 		int ret = cops->init_decompress_ctx(dic);  in f2fs_prepare_decomp_mem()
1514 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,  in f2fs_release_decomp_mem()  argument
1518 		f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm];  in f2fs_release_decomp_mem()
1520 	if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc))  in f2fs_release_decomp_mem()
1524 		cops->destroy_decompress_ctx(dic);  in f2fs_release_decomp_mem()
1526 	if (dic->cbuf)  in f2fs_release_decomp_mem()
1527 		vm_unmap_ram(dic->cbuf, dic->nr_cpages);  in f2fs_release_decomp_mem()
1529 	if (dic->rbuf)  in f2fs_release_decomp_mem()
1530 		vm_unmap_ram(dic->rbuf, dic->cluster_size);  in f2fs_release_decomp_mem()
1533 static void f2fs_free_dic(struct decompress_io_ctx *dic,
1538 	struct decompress_io_ctx *dic;  in f2fs_alloc_dic()  local
1542 	dic = kmem_cache_zalloc(dic_entry_slab, GFP_NOFS);  in f2fs_alloc_dic()
1543 	if (!dic)  in f2fs_alloc_dic()
1546 	dic->rpages = page_array_alloc(cc->inode, cc->cluster_size);  in f2fs_alloc_dic()
1547 	if (!dic->rpages) {  in f2fs_alloc_dic()
1548 		kmem_cache_free(dic_entry_slab, dic);  in f2fs_alloc_dic()
1552 	dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;  in f2fs_alloc_dic()
1553 	dic->inode = cc->inode;  in f2fs_alloc_dic()
1554 	atomic_set(&dic->remaining_pages, cc->nr_cpages);  in f2fs_alloc_dic()
1555 	dic->cluster_idx = cc->cluster_idx;  in f2fs_alloc_dic()
1556 	dic->cluster_size = cc->cluster_size;  in f2fs_alloc_dic()
1557 	dic->log_cluster_size = cc->log_cluster_size;  in f2fs_alloc_dic()
1558 	dic->nr_cpages = cc->nr_cpages;  in f2fs_alloc_dic()
1559 	refcount_set(&dic->refcnt, 1);  in f2fs_alloc_dic()
1560 	dic->failed = false;  in f2fs_alloc_dic()
1561 	dic->need_verity = f2fs_need_verity(cc->inode, start_idx);  in f2fs_alloc_dic()
1563 	for (i = 0; i < dic->cluster_size; i++)  in f2fs_alloc_dic()
1564 		dic->rpages[i] = cc->rpages[i];  in f2fs_alloc_dic()
1565 	dic->nr_rpages = cc->cluster_size;  in f2fs_alloc_dic()
1567 	dic->cpages = page_array_alloc(dic->inode, dic->nr_cpages);  in f2fs_alloc_dic()
1568 	if (!dic->cpages) {  in f2fs_alloc_dic()
1573 	for (i = 0; i < dic->nr_cpages; i++) {  in f2fs_alloc_dic()
1583 					start_idx + i + 1, dic);  in f2fs_alloc_dic()
1584 		dic->cpages[i] = page;  in f2fs_alloc_dic()
1587 	ret = f2fs_prepare_decomp_mem(dic, true);  in f2fs_alloc_dic()
1591 	return dic;  in f2fs_alloc_dic()
1594 	f2fs_free_dic(dic, true);  in f2fs_alloc_dic()
1598 static void f2fs_free_dic(struct decompress_io_ctx *dic,  in f2fs_free_dic()  argument
1603 	f2fs_release_decomp_mem(dic, bypass_destroy_callback, true);  in f2fs_free_dic()
1605 	if (dic->tpages) {  in f2fs_free_dic()
1606 		for (i = 0; i < dic->cluster_size; i++) {  in f2fs_free_dic()
1607 			if (dic->rpages[i])  in f2fs_free_dic()
1609 			if (!dic->tpages[i])  in f2fs_free_dic()
1611 			f2fs_compress_free_page(dic->tpages[i]);  in f2fs_free_dic()
1613 		page_array_free(dic->inode, dic->tpages, dic->cluster_size);  in f2fs_free_dic()
1616 	if (dic->cpages) {  in f2fs_free_dic()
1617 		for (i = 0; i < dic->nr_cpages; i++) {  in f2fs_free_dic()
1618 			if (!dic->cpages[i])  in f2fs_free_dic()
1620 			f2fs_compress_free_page(dic->cpages[i]);  in f2fs_free_dic()
1622 		page_array_free(dic->inode, dic->cpages, dic->nr_cpages);  in f2fs_free_dic()
1625 	page_array_free(dic->inode, dic->rpages, dic->nr_rpages);  in f2fs_free_dic()
1626 	kmem_cache_free(dic_entry_slab, dic);  in f2fs_free_dic()
1631 	struct decompress_io_ctx *dic =  in f2fs_late_free_dic()  local
1634 	f2fs_free_dic(dic, false);  in f2fs_late_free_dic()
1637 static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task)  in f2fs_put_dic()  argument
1639 	if (refcount_dec_and_test(&dic->refcnt)) {  in f2fs_put_dic()
1641 			f2fs_free_dic(dic, false);  in f2fs_put_dic()
1643 			INIT_WORK(&dic->free_work, f2fs_late_free_dic);  in f2fs_put_dic()
1644 			queue_work(F2FS_I_SB(dic->inode)->post_read_wq,  in f2fs_put_dic()
1645 					&dic->free_work);  in f2fs_put_dic()
1654 static void __f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,  in __f2fs_decompress_end_io()  argument
1659 	for (i = 0; i < dic->cluster_size; i++) {  in __f2fs_decompress_end_io()
1660 		struct page *rpage = dic->rpages[i];  in __f2fs_decompress_end_io()
1676 	f2fs_put_dic(dic, in_task);  in __f2fs_decompress_end_io()
1681 	struct decompress_io_ctx *dic =  in f2fs_verify_cluster()  local
1686 	for (i = 0; i < dic->cluster_size; i++) {  in f2fs_verify_cluster()
1687 		struct page *rpage = dic->rpages[i];  in f2fs_verify_cluster()
1693 	__f2fs_decompress_end_io(dic, false, true);  in f2fs_verify_cluster()
1700 void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,  in f2fs_decompress_end_io()  argument
1703 	if (!failed && dic->need_verity) {  in f2fs_decompress_end_io()
1710 		INIT_WORK(&dic->verity_work, f2fs_verify_cluster);  in f2fs_decompress_end_io()
1711 		fsverity_enqueue_verify_work(&dic->verity_work);  in f2fs_decompress_end_io()
1713 		__f2fs_decompress_end_io(dic, failed, in_task);  in f2fs_decompress_end_io()
1724 	struct decompress_io_ctx *dic =  in f2fs_put_page_dic()  local
1727 	f2fs_put_dic(dic, in_task);  in f2fs_put_page_dic()