25 	int (*prepare_destpages)(struct z_erofs_decompress_req *rq,
27 	int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out);
67 static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,  in z_erofs_lz4_prepare_destpages()  argument
71 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_lz4_prepare_destpages()
76 				EROFS_SB(rq->sb)->lz4.max_distance_pages;  in z_erofs_lz4_prepare_destpages()
82 		struct page *const page = rq->out[i];  in z_erofs_lz4_prepare_destpages()
92 			availables[top++] = rq->out[i - lz4_max_distance_pages];  in z_erofs_lz4_prepare_destpages()
122 		rq->out[i] = victim;  in z_erofs_lz4_prepare_destpages()
127 static void *z_erofs_handle_inplace_io(struct z_erofs_decompress_req *rq,  in z_erofs_handle_inplace_io()  argument
136 	inputsize = rq->inputsize;  in z_erofs_handle_inplace_io()
138 	oend = rq->pageofs_out + rq->outputsize;  in z_erofs_handle_inplace_io()
142 	if (rq->inplace_io) {  in z_erofs_handle_inplace_io()
143 		if (rq->partial_decoding || !support_0padding ||  in z_erofs_handle_inplace_io()
148 			DBG_BUGON(rq->in[i] == NULL);  in z_erofs_handle_inplace_io()
150 				if (rq->out[j] == rq->in[i])  in z_erofs_handle_inplace_io()
161 	src = erofs_vm_map_ram(rq->in, nrpages_in);  in z_erofs_handle_inplace_io()
169 	in = rq->in;  in z_erofs_handle_inplace_io()
178 	total = rq->inputsize;  in z_erofs_handle_inplace_io()
197 static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)  in z_erofs_lz4_decompress()  argument
204 	DBG_BUGON(*rq->in == NULL);  in z_erofs_lz4_decompress()
205 	headpage = kmap_atomic(*rq->in);  in z_erofs_lz4_decompress()
210 	if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) {  in z_erofs_lz4_decompress()
217 		if (inputmargin >= rq->inputsize) {  in z_erofs_lz4_decompress()
223 	rq->inputsize -= inputmargin;  in z_erofs_lz4_decompress()
224 	src = z_erofs_handle_inplace_io(rq, headpage, &inputmargin, &maptype,  in z_erofs_lz4_decompress()
230 	if (rq->partial_decoding || !support_0padding)  in z_erofs_lz4_decompress()
232 				rq->inputsize, rq->outputsize, rq->outputsize);  in z_erofs_lz4_decompress()
235 					  rq->inputsize, rq->outputsize);  in z_erofs_lz4_decompress()
237 	if (ret != rq->outputsize) {  in z_erofs_lz4_decompress()
238 		erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",  in z_erofs_lz4_decompress()
239 			  ret, rq->inputsize, inputmargin, rq->outputsize);  in z_erofs_lz4_decompress()
242 			       16, 1, src + inputmargin, rq->inputsize, true);  in z_erofs_lz4_decompress()
244 			       16, 1, out, rq->outputsize, true);  in z_erofs_lz4_decompress()
247 			memset(out + ret, 0, rq->outputsize - ret);  in z_erofs_lz4_decompress()
254 		vm_unmap_ram(src, PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT);  in z_erofs_lz4_decompress()
302 static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq,  in z_erofs_decompress_generic()  argument
306 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_decompress_generic()
307 	const struct z_erofs_decompressor *alg = decompressors + rq->alg;  in z_erofs_decompress_generic()
313 	if (rq->inputsize <= PAGE_SIZE) {  in z_erofs_decompress_generic()
314 		if (nrpages_out == 1 && !rq->inplace_io) {  in z_erofs_decompress_generic()
315 			DBG_BUGON(!*rq->out);  in z_erofs_decompress_generic()
316 			dst = kmap_atomic(*rq->out);  in z_erofs_decompress_generic()
326 		if (rq->outputsize <= PAGE_SIZE * 7 / 8) {  in z_erofs_decompress_generic()
331 			rq->inplace_io = false;  in z_erofs_decompress_generic()
332 			ret = alg->decompress(rq, dst);  in z_erofs_decompress_generic()
334 				copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,  in z_erofs_decompress_generic()
335 						  rq->outputsize);  in z_erofs_decompress_generic()
343 	ret = alg->prepare_destpages(rq, pagepool);  in z_erofs_decompress_generic()
347 		dst = page_address(*rq->out);  in z_erofs_decompress_generic()
352 	dst = erofs_vm_map_ram(rq->out, nrpages_out);  in z_erofs_decompress_generic()
358 	ret = alg->decompress(rq, dst + rq->pageofs_out);  in z_erofs_decompress_generic()
367 static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq,  in z_erofs_shifted_transform()  argument
371 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_shifted_transform()
372 	const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out;  in z_erofs_shifted_transform()
380 	if (rq->out[0] == *rq->in) {  in z_erofs_shifted_transform()
385 	src = kmap_atomic(*rq->in);  in z_erofs_shifted_transform()
386 	if (rq->out[0]) {  in z_erofs_shifted_transform()
387 		dst = kmap_atomic(rq->out[0]);  in z_erofs_shifted_transform()
388 		memcpy(dst + rq->pageofs_out, src, righthalf);  in z_erofs_shifted_transform()
393 		DBG_BUGON(!rq->out[1]);  in z_erofs_shifted_transform()
394 		if (rq->out[1] == *rq->in) {  in z_erofs_shifted_transform()
395 			memmove(src, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
397 			dst = kmap_atomic(rq->out[1]);  in z_erofs_shifted_transform()
398 			memcpy(dst, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
406 int z_erofs_decompress(struct z_erofs_decompress_req *rq,  in z_erofs_decompress()  argument
409 	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED)  in z_erofs_decompress()
410 		return z_erofs_shifted_transform(rq, pagepool);  in z_erofs_decompress()
411 	return z_erofs_decompress_generic(rq, pagepool);  in z_erofs_decompress()