645 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result);
1416 static bool rbd_img_is_write(struct rbd_img_request *img_req)  in rbd_img_is_write()  argument
1418 	switch (img_req->op_type) {  in rbd_img_is_write()
1644 static void rbd_img_capture_header(struct rbd_img_request *img_req)  in rbd_img_capture_header()  argument
1646 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_capture_header()
1650 	if (rbd_img_is_write(img_req))  in rbd_img_capture_header()
1651 		img_req->snapc = ceph_get_snap_context(rbd_dev->header.snapc);  in rbd_img_capture_header()
1653 		img_req->snap_id = rbd_dev->spec->snap_id;  in rbd_img_capture_header()
1656 		img_request_layered_set(img_req);  in rbd_img_capture_header()
2421 	struct rbd_img_request *img_req = obj_req->img_request;  in count_write_ops()  local
2423 	switch (img_req->op_type) {  in count_write_ops()
2425 		if (!use_object_map(img_req->rbd_dev) ||  in count_write_ops()
2468 static int __rbd_img_fill_request(struct rbd_img_request *img_req)  in __rbd_img_fill_request()  argument
2473 	for_each_obj_request_safe(img_req, obj_req, next_obj_req) {  in __rbd_img_fill_request()
2474 		switch (img_req->op_type) {  in __rbd_img_fill_request()
2493 			rbd_img_obj_request_del(img_req, obj_req);  in __rbd_img_fill_request()
2498 	img_req->state = RBD_IMG_START;  in __rbd_img_fill_request()
2518 	struct rbd_img_request *img_req = arg;  in alloc_object_extent()  local
2525 	rbd_img_obj_request_add(img_req, obj_req);  in alloc_object_extent()
2541 static int rbd_img_fill_request_nocopy(struct rbd_img_request *img_req,  in rbd_img_fill_request_nocopy()  argument
2549 	img_req->data_type = fctx->pos_type;  in rbd_img_fill_request_nocopy()
2557 		ret = ceph_file_to_extents(&img_req->rbd_dev->layout,  in rbd_img_fill_request_nocopy()
2560 					   &img_req->object_extents,  in rbd_img_fill_request_nocopy()
2561 					   alloc_object_extent, img_req,  in rbd_img_fill_request_nocopy()
2567 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request_nocopy()
2583 static int rbd_img_fill_request(struct rbd_img_request *img_req,  in rbd_img_fill_request()  argument
2588 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_fill_request()
2595 		return rbd_img_fill_request_nocopy(img_req, img_extents,  in rbd_img_fill_request()
2598 	img_req->data_type = OBJ_REQUEST_OWN_BVECS;  in rbd_img_fill_request()
2612 					   &img_req->object_extents,  in rbd_img_fill_request()
2613 					   alloc_object_extent, img_req,  in rbd_img_fill_request()
2619 	for_each_obj_request(img_req, obj_req) {  in rbd_img_fill_request()
2636 					   &img_req->object_extents,  in rbd_img_fill_request()
2642 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request()
2645 static int rbd_img_fill_nodata(struct rbd_img_request *img_req,  in rbd_img_fill_nodata()  argument
2655 	return rbd_img_fill_request(img_req, &ex, 1, &fctx);  in rbd_img_fill_nodata()
2695 static int __rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bio()  argument
2708 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bio()
2712 static int rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in rbd_img_fill_from_bio()  argument
2718 	return __rbd_img_fill_from_bio(img_req, &ex, 1, &it);  in rbd_img_fill_from_bio()
2755 static int __rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bvecs()  argument
2768 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bvecs()
2772 static int rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in rbd_img_fill_from_bvecs()  argument
2783 	return __rbd_img_fill_from_bvecs(img_req, img_extents, num_img_extents,  in rbd_img_fill_from_bvecs()
2789 	struct rbd_img_request *img_req =  in rbd_img_handle_request_work()  local
2792 	rbd_img_handle_request(img_req, img_req->work_result);  in rbd_img_handle_request_work()
2795 static void rbd_img_schedule(struct rbd_img_request *img_req, int result)  in rbd_img_schedule()  argument
2797 	INIT_WORK(&img_req->work, rbd_img_handle_request_work);  in rbd_img_schedule()
2798 	img_req->work_result = result;  in rbd_img_schedule()
2799 	queue_work(rbd_wq, &img_req->work);  in rbd_img_schedule()
2840 	struct rbd_img_request *img_req = obj_req->img_request;  in rbd_obj_read_from_parent()  local
2841 	struct rbd_device *parent = img_req->rbd_dev->parent;  in rbd_obj_read_from_parent()
2860 	if (!rbd_img_is_write(img_req)) {  in rbd_obj_read_from_parent()
2861 		switch (img_req->data_type) {  in rbd_obj_read_from_parent()
3433 	struct rbd_img_request *img_req = obj_req->img_request;  in __rbd_obj_handle_request()  local
3434 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_obj_handle_request()
3438 	if (!rbd_img_is_write(img_req))  in __rbd_obj_handle_request()
3447 			 obj_op_name(img_req->op_type), obj_req->ex.oe_objno,  in __rbd_obj_handle_request()
3463 static bool need_exclusive_lock(struct rbd_img_request *img_req)  in need_exclusive_lock()  argument
3465 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in need_exclusive_lock()
3473 	rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));  in need_exclusive_lock()
3478 	return rbd_img_is_write(img_req);  in need_exclusive_lock()
3481 static bool rbd_lock_add_request(struct rbd_img_request *img_req)  in rbd_lock_add_request()  argument
3483 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_add_request()
3489 	rbd_assert(list_empty(&img_req->lock_item));  in rbd_lock_add_request()
3491 		list_add_tail(&img_req->lock_item, &rbd_dev->acquiring_list);  in rbd_lock_add_request()
3493 		list_add_tail(&img_req->lock_item, &rbd_dev->running_list);  in rbd_lock_add_request()
3498 static void rbd_lock_del_request(struct rbd_img_request *img_req)  in rbd_lock_del_request()  argument
3500 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_del_request()
3505 	rbd_assert(!list_empty(&img_req->lock_item));  in rbd_lock_del_request()
3506 	list_del_init(&img_req->lock_item);  in rbd_lock_del_request()
3514 static int rbd_img_exclusive_lock(struct rbd_img_request *img_req)  in rbd_img_exclusive_lock()  argument
3516 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_exclusive_lock()
3518 	if (!need_exclusive_lock(img_req))  in rbd_img_exclusive_lock()
3521 	if (rbd_lock_add_request(img_req))  in rbd_img_exclusive_lock()
3538 static void rbd_img_object_requests(struct rbd_img_request *img_req)  in rbd_img_object_requests()  argument
3542 	rbd_assert(!img_req->pending.result && !img_req->pending.num_pending);  in rbd_img_object_requests()
3544 	for_each_obj_request(img_req, obj_req) {  in rbd_img_object_requests()
3549 				img_req->pending.result = result;  in rbd_img_object_requests()
3553 			img_req->pending.num_pending++;  in rbd_img_object_requests()
3558 static bool rbd_img_advance(struct rbd_img_request *img_req, int *result)  in rbd_img_advance()  argument
3560 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_advance()
3564 	switch (img_req->state) {  in rbd_img_advance()
3568 		ret = rbd_img_exclusive_lock(img_req);  in rbd_img_advance()
3573 		img_req->state = RBD_IMG_EXCLUSIVE_LOCK;  in rbd_img_advance()
3581 		rbd_assert(!need_exclusive_lock(img_req) ||  in rbd_img_advance()
3584 		rbd_img_object_requests(img_req);  in rbd_img_advance()
3585 		if (!img_req->pending.num_pending) {  in rbd_img_advance()
3586 			*result = img_req->pending.result;  in rbd_img_advance()
3587 			img_req->state = RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3590 		img_req->state = __RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3593 		if (!pending_result_dec(&img_req->pending, result))  in rbd_img_advance()
3606 static bool __rbd_img_handle_request(struct rbd_img_request *img_req,  in __rbd_img_handle_request()  argument
3609 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_img_handle_request()
3612 	if (need_exclusive_lock(img_req)) {  in __rbd_img_handle_request()
3614 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3615 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3617 			rbd_lock_del_request(img_req);  in __rbd_img_handle_request()
3618 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3621 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3622 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3623 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3629 		      test_bit(IMG_REQ_CHILD, &img_req->flags) ? "child " : "",  in __rbd_img_handle_request()
3630 		      obj_op_name(img_req->op_type), *result);  in __rbd_img_handle_request()
3635 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result)  in rbd_img_handle_request()  argument
3638 	if (!__rbd_img_handle_request(img_req, &result))  in rbd_img_handle_request()
3641 	if (test_bit(IMG_REQ_CHILD, &img_req->flags)) {  in rbd_img_handle_request()
3642 		struct rbd_obj_request *obj_req = img_req->obj_request;  in rbd_img_handle_request()
3644 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3646 			img_req = obj_req->img_request;  in rbd_img_handle_request()
3650 		struct request *rq = blk_mq_rq_from_pdu(img_req);  in rbd_img_handle_request()
3652 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3871 	struct rbd_img_request *img_req;  in wake_lock_waiters()  local
3885 	list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) {  in wake_lock_waiters()
3886 		mutex_lock(&img_req->state_mutex);  in wake_lock_waiters()
3887 		rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK);  in wake_lock_waiters()
3888 		rbd_img_schedule(img_req, result);  in wake_lock_waiters()
3889 		mutex_unlock(&img_req->state_mutex);  in wake_lock_waiters()
4753 	struct rbd_img_request *img_req = blk_mq_rq_to_pdu(bd->rq);  in rbd_queue_rq()  local
4774 	rbd_img_request_init(img_req, rbd_dev, op_type);  in rbd_queue_rq()
4776 	if (rbd_img_is_write(img_req)) {  in rbd_queue_rq()
4779 				 obj_op_name(img_req->op_type));  in rbd_queue_rq()
4785 	INIT_WORK(&img_req->work, rbd_queue_workfn);  in rbd_queue_rq()
4786 	queue_work(rbd_wq, &img_req->work);  in rbd_queue_rq()