217 	struct z_erofs_collector clt;  member
231 	.inode = __i, .clt = COLLECTOR_INIT(), \
237 static void preload_compressed_pages(struct z_erofs_collector *clt,  in preload_compressed_pages()  argument
242 	struct z_erofs_pcluster *pcl = clt->pcl;  in preload_compressed_pages()
249 	if (clt->mode < COLLECT_PRIMARY_FOLLOWED)  in preload_compressed_pages()
304 		clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;  in preload_compressed_pages()
366 static bool z_erofs_try_inplace_io(struct z_erofs_collector *clt,  in z_erofs_try_inplace_io()  argument
369 	struct z_erofs_pcluster *const pcl = clt->pcl;  in z_erofs_try_inplace_io()
371 	while (clt->icpage_ptr > pcl->compressed_pages)  in z_erofs_try_inplace_io()
372 		if (!cmpxchg(--clt->icpage_ptr, NULL, page))  in z_erofs_try_inplace_io()
378 static int z_erofs_attach_page(struct z_erofs_collector *clt,  in z_erofs_attach_page()  argument
385 	if (clt->mode >= COLLECT_PRIMARY &&  in z_erofs_attach_page()
387 	    z_erofs_try_inplace_io(clt, page))  in z_erofs_attach_page()
390 	ret = z_erofs_pagevec_enqueue(&clt->vector, page, type,  in z_erofs_attach_page()
392 	clt->cl->vcnt += (unsigned int)ret;  in z_erofs_attach_page()
426 static int z_erofs_lookup_collection(struct z_erofs_collector *clt,  in z_erofs_lookup_collection()  argument
430 	struct z_erofs_pcluster *pcl = clt->pcl;  in z_erofs_lookup_collection()
435 	if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) {  in z_erofs_lookup_collection()
466 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_lookup_collection()
467 		clt->tailpcl = pcl;  in z_erofs_lookup_collection()
468 	clt->mode = try_to_claim_pcluster(pcl, &clt->owned_head);  in z_erofs_lookup_collection()
470 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_lookup_collection()
471 		clt->tailpcl = NULL;  in z_erofs_lookup_collection()
472 	clt->cl = cl;  in z_erofs_lookup_collection()
476 static int z_erofs_register_collection(struct z_erofs_collector *clt,  in z_erofs_register_collection()  argument
503 	pcl->next = clt->owned_head;  in z_erofs_register_collection()
504 	clt->mode = COLLECT_PRIMARY_FOLLOWED;  in z_erofs_register_collection()
523 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_register_collection()
528 	if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_register_collection()
529 		clt->tailpcl = pcl;  in z_erofs_register_collection()
530 	clt->owned_head = &pcl->next;  in z_erofs_register_collection()
531 	clt->pcl = pcl;  in z_erofs_register_collection()
532 	clt->cl = cl;  in z_erofs_register_collection()
541 static int z_erofs_collector_begin(struct z_erofs_collector *clt,  in z_erofs_collector_begin()  argument
548 	DBG_BUGON(clt->cl);  in z_erofs_collector_begin()
551 	DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL);  in z_erofs_collector_begin()
552 	DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);  in z_erofs_collector_begin()
561 		clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);  in z_erofs_collector_begin()
563 		ret = z_erofs_register_collection(clt, inode, map);  in z_erofs_collector_begin()
571 	ret = z_erofs_lookup_collection(clt, inode, map);  in z_erofs_collector_begin()
573 		erofs_workgroup_put(&clt->pcl->obj);  in z_erofs_collector_begin()
578 	z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS,  in z_erofs_collector_begin()
579 				  clt->cl->pagevec, clt->cl->vcnt);  in z_erofs_collector_begin()
582 	clt->icpage_ptr = clt->pcl->compressed_pages + clt->pcl->pclusterpages;  in z_erofs_collector_begin()
616 static bool z_erofs_collector_end(struct z_erofs_collector *clt)  in z_erofs_collector_end()  argument
618 	struct z_erofs_collection *cl = clt->cl;  in z_erofs_collector_end()
623 	z_erofs_pagevec_ctor_exit(&clt->vector, false);  in z_erofs_collector_end()
630 	if (clt->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE)  in z_erofs_collector_end()
633 	clt->cl = NULL;  in z_erofs_collector_end()
657 	struct z_erofs_collector *const clt = &fe->clt;  in z_erofs_do_read_page()  local
678 		if (!clt->cl)  in z_erofs_do_read_page()
686 	if (z_erofs_collector_end(clt))  in z_erofs_do_read_page()
699 	err = z_erofs_collector_begin(clt, inode, map);  in z_erofs_do_read_page()
709 	preload_compressed_pages(clt, MNGD_MAPPING(sbi),  in z_erofs_do_read_page()
719 	tight &= (clt->mode >= COLLECT_PRIMARY_HOOKED &&  in z_erofs_do_read_page()
720 		  clt->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE);  in z_erofs_do_read_page()
735 		tight &= (clt->mode >= COLLECT_PRIMARY_FOLLOWED);  in z_erofs_do_read_page()
738 	err = z_erofs_attach_page(clt, page, page_type,  in z_erofs_do_read_page()
739 				  clt->mode >= COLLECT_PRIMARY_FOLLOWED);  in z_erofs_do_read_page()
746 		err = z_erofs_attach_page(clt, newpage,  in z_erofs_do_read_page()
762 	clt->cl->nr_pages = max_t(pgoff_t, clt->cl->nr_pages, index + 1);  in z_erofs_do_read_page()
1272 	z_erofs_next_pcluster_t owned_head = f->clt.owned_head;  in z_erofs_submit_queue()
1366 	if (f->clt.owned_head == Z_EROFS_PCLUSTER_TAIL)  in z_erofs_runqueue()
1395 	(void)z_erofs_collector_end(&f.clt);  in z_erofs_readpage()
1457 	(void)z_erofs_collector_end(&f.clt);  in z_erofs_readahead()