28 static void bch_data_insert_start(struct closure *cl);
58 static void bch_data_insert_keys(struct closure *cl)  in bch_data_insert_keys()  argument
60 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_keys()
67 					  op->flush_journal ? cl : NULL);  in bch_data_insert_keys()
82 		continue_at(cl, bch_data_insert_start, op->wq);  in bch_data_insert_keys()
87 	closure_return(cl);  in bch_data_insert_keys()
108 static void bch_data_invalidate(struct closure *cl)  in bch_data_invalidate()  argument
110 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_invalidate()
136 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_invalidate()
139 static void bch_data_insert_error(struct closure *cl)  in bch_data_insert_error()  argument
141 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_error()
166 	bch_data_insert_keys(cl);  in bch_data_insert_error()
171 	struct closure *cl = bio->bi_private;  in bch_data_insert_endio()  local
172 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_endio()
179 			set_closure_fn(cl, bch_data_insert_error, op->wq);  in bch_data_insert_endio()
181 			set_closure_fn(cl, NULL, NULL);  in bch_data_insert_endio()
187 static void bch_data_insert_start(struct closure *cl)  in bch_data_insert_start()  argument
189 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_start()
193 		return bch_data_invalidate(cl);  in bch_data_insert_start()
213 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
230 		n->bi_private	= cl;  in bch_data_insert_start()
252 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
272 		return bch_data_invalidate(cl);  in bch_data_insert_start()
282 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
284 			closure_return(cl);  in bch_data_insert_start()
308 void bch_data_insert(struct closure *cl)  in bch_data_insert()  argument
310 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert()
317 	bch_data_insert_start(cl);  in bch_data_insert()
465 	struct closure		cl;  member
488 	struct closure *cl = bio->bi_private;  in bch_cache_read_endio()  local
489 	struct search *s = container_of(cl, struct search, cl);  in bch_cache_read_endio()
561 	n->bi_private	= &s->cl;  in cache_lookup_fn()
578 static void cache_lookup(struct closure *cl)  in cache_lookup()  argument
580 	struct search *s = container_of(cl, struct search, iop.cl);  in cache_lookup()
591 		continue_at(cl, cache_lookup, bcache_wq);  in cache_lookup()
616 	closure_return(cl);  in cache_lookup()
623 	struct closure *cl = bio->bi_private;  in request_endio()  local
626 		struct search *s = container_of(cl, struct search, cl);  in request_endio()
634 	closure_put(cl);  in request_endio()
639 	struct closure *cl = bio->bi_private;  in backing_request_endio()  local
642 		struct search *s = container_of(cl, struct search, cl);  in backing_request_endio()
666 	closure_put(cl);  in backing_request_endio()
697 	bio->bi_private		= &s->cl;  in do_bio_hook()
702 static void search_free(struct closure *cl)  in search_free()  argument
704 	struct search *s = container_of(cl, struct search, cl);  in search_free()
712 	closure_debug_destroy(cl);  in search_free()
723 	closure_init(&s->cl, NULL);  in search_alloc()
751 static void cached_dev_bio_complete(struct closure *cl)  in cached_dev_bio_complete()  argument
753 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_bio_complete()
757 	search_free(cl);  in cached_dev_bio_complete()
762 static void cached_dev_read_error_done(struct closure *cl)  in cached_dev_read_error_done()  argument
764 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_error_done()
772 	cached_dev_bio_complete(cl);  in cached_dev_read_error_done()
775 static void cached_dev_read_error(struct closure *cl)  in cached_dev_read_error()  argument
777 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_error()
797 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_read_error()
800 	continue_at(cl, cached_dev_read_error_done, NULL);  in cached_dev_read_error()
803 static void cached_dev_cache_miss_done(struct closure *cl)  in cached_dev_cache_miss_done()  argument
805 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_cache_miss_done()
814 	cached_dev_bio_complete(cl);  in cached_dev_cache_miss_done()
815 	closure_put(&d->cl);  in cached_dev_cache_miss_done()
818 static void cached_dev_read_done(struct closure *cl)  in cached_dev_read_done()  argument
820 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done()
848 	closure_get(&dc->disk.cl);  in cached_dev_read_done()
854 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_read_done()
857 	continue_at(cl, cached_dev_cache_miss_done, NULL);  in cached_dev_read_done()
860 static void cached_dev_read_done_bh(struct closure *cl)  in cached_dev_read_done_bh()  argument
862 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done_bh()
870 		continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq);  in cached_dev_read_done_bh()
872 		continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq);  in cached_dev_read_done_bh()
874 		continue_at_nobarrier(cl, cached_dev_bio_complete, NULL);  in cached_dev_read_done_bh()
927 	cache_bio->bi_private	= &s->cl;  in cached_dev_cache_miss()
940 	closure_bio_submit(s->iop.c, cache_bio, &s->cl);  in cached_dev_cache_miss()
947 	miss->bi_private	= &s->cl;  in cached_dev_cache_miss()
949 	closure_bio_submit(s->iop.c, miss, &s->cl);  in cached_dev_cache_miss()
955 	struct closure *cl = &s->cl;  in cached_dev_read()  local
957 	closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in cached_dev_read()
958 	continue_at(cl, cached_dev_read_done_bh, NULL);  in cached_dev_read()
963 static void cached_dev_write_complete(struct closure *cl)  in cached_dev_write_complete()  argument
965 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_write_complete()
969 	cached_dev_bio_complete(cl);  in cached_dev_write_complete()
974 	struct closure *cl = &s->cl;  in cached_dev_write()  local
1018 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1039 			flush->bi_private = cl;  in cached_dev_write()
1042 			closure_bio_submit(s->iop.c, flush, cl);  in cached_dev_write()
1048 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1052 	closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_write()
1053 	continue_at(cl, cached_dev_write_complete, NULL);  in cached_dev_write()
1056 static void cached_dev_nodata(struct closure *cl)  in cached_dev_nodata()  argument
1058 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_nodata()
1062 		bch_journal_meta(s->iop.c, cl);  in cached_dev_nodata()
1066 	closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_nodata()
1068 	continue_at(cl, cached_dev_bio_complete, NULL);  in cached_dev_nodata()
1214 			continue_at_nobarrier(&s->cl,  in cached_dev_submit_bio()
1268 static void flash_dev_nodata(struct closure *cl)  in flash_dev_nodata()  argument
1270 	struct search *s = container_of(cl, struct search, cl);  in flash_dev_nodata()
1273 		bch_journal_meta(s->iop.c, cl);  in flash_dev_nodata()
1275 	continue_at(cl, search_free, NULL);  in flash_dev_nodata()
1281 	struct closure *cl;  in flash_dev_submit_bio()  local
1291 	cl = &s->cl;  in flash_dev_submit_bio()
1300 		continue_at_nobarrier(&s->cl,  in flash_dev_submit_bio()
1313 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in flash_dev_submit_bio()
1315 		closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in flash_dev_submit_bio()
1318 	continue_at(cl, search_free, NULL);  in flash_dev_submit_bio()