Lines Matching +full:- +full:b
2 * Copyright (C) 2009-2011 Red Hat, Inc.
9 #include <linux/dm-bufio.h>
11 #include <linux/device-mapper.h>
12 #include <linux/dm-io.h>
61 * dm_buffer->list_mode
78 * context), so some clean-not-writing buffers can be held on
164 /*----------------------------------------------------------------*/
166 #define dm_bufio_in_request() (!!current->bio_list)
170 mutex_lock_nested(&c->lock, dm_bufio_in_request()); in dm_bufio_lock()
175 return mutex_trylock(&c->lock); in dm_bufio_trylock()
180 mutex_unlock(&c->lock); in dm_bufio_unlock()
183 /*----------------------------------------------------------------*/
219 /*----------------------------------------------------------------*/
242 static void buffer_record_stack(struct dm_buffer *b) in buffer_record_stack() argument
244 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2); in buffer_record_stack()
248 /*----------------------------------------------------------------
250 *--------------------------------------------------------------*/
253 struct rb_node *n = c->buffer_tree.rb_node; in __find()
254 struct dm_buffer *b; in __find() local
257 b = container_of(n, struct dm_buffer, node); in __find()
259 if (b->block == block) in __find()
260 return b; in __find()
262 n = block < b->block ? n->rb_left : n->rb_right; in __find()
270 struct rb_node *n = c->buffer_tree.rb_node; in __find_next()
271 struct dm_buffer *b; in __find_next() local
275 b = container_of(n, struct dm_buffer, node); in __find_next()
277 if (b->block == block) in __find_next()
278 return b; in __find_next()
280 if (block <= b->block) { in __find_next()
281 n = n->rb_left; in __find_next()
282 best = b; in __find_next()
284 n = n->rb_right; in __find_next()
291 static void __insert(struct dm_bufio_client *c, struct dm_buffer *b) in __insert() argument
293 struct rb_node **new = &c->buffer_tree.rb_node, *parent = NULL; in __insert()
299 if (found->block == b->block) { in __insert()
300 BUG_ON(found != b); in __insert()
305 new = b->block < found->block ? in __insert()
306 &found->node.rb_left : &found->node.rb_right; in __insert()
309 rb_link_node(&b->node, parent, new); in __insert()
310 rb_insert_color(&b->node, &c->buffer_tree); in __insert()
313 static void __remove(struct dm_bufio_client *c, struct dm_buffer *b) in __remove() argument
315 rb_erase(&b->node, &c->buffer_tree); in __remove()
318 /*----------------------------------------------------------------*/
320 static void adjust_total_allocated(struct dm_buffer *b, bool unlink) in adjust_total_allocated() argument
331 data_mode = b->data_mode; in adjust_total_allocated()
332 diff = (long)b->c->block_size; in adjust_total_allocated()
334 diff = -diff; in adjust_total_allocated()
345 b->accessed = 1; in adjust_total_allocated()
348 list_add(&b->global_list, &global_queue); in adjust_total_allocated()
353 list_del(&b->global_list); in adjust_total_allocated()
354 global_num--; in adjust_total_allocated()
361 * Change the number of clients and recalculate per-client limit.
404 if (unlikely(c->slab_cache != NULL)) { in alloc_buffer_data()
406 return kmem_cache_alloc(c->slab_cache, gfp_mask); in alloc_buffer_data()
409 if (c->block_size <= KMALLOC_MAX_SIZE && in alloc_buffer_data()
413 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in alloc_buffer_data()
423 * Consequently, we must set per-process flag PF_MEMALLOC_NOIO so that in alloc_buffer_data()
429 void *ptr = __vmalloc(c->block_size, gfp_mask); in alloc_buffer_data()
435 return __vmalloc(c->block_size, gfp_mask); in alloc_buffer_data()
446 kmem_cache_free(c->slab_cache, data); in free_buffer_data()
451 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in free_buffer_data()
470 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask); in alloc_buffer() local
472 if (!b) in alloc_buffer()
475 b->c = c; in alloc_buffer()
477 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode); in alloc_buffer()
478 if (!b->data) { in alloc_buffer()
479 kmem_cache_free(c->slab_buffer, b); in alloc_buffer()
484 b->stack_len = 0; in alloc_buffer()
486 return b; in alloc_buffer()
492 static void free_buffer(struct dm_buffer *b) in free_buffer() argument
494 struct dm_bufio_client *c = b->c; in free_buffer()
496 free_buffer_data(c, b->data, b->data_mode); in free_buffer()
497 kmem_cache_free(c->slab_buffer, b); in free_buffer()
503 static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty) in __link_buffer() argument
505 struct dm_bufio_client *c = b->c; in __link_buffer()
507 c->n_buffers[dirty]++; in __link_buffer()
508 b->block = block; in __link_buffer()
509 b->list_mode = dirty; in __link_buffer()
510 list_add(&b->lru_list, &c->lru[dirty]); in __link_buffer()
511 __insert(b->c, b); in __link_buffer()
512 b->last_accessed = jiffies; in __link_buffer()
514 adjust_total_allocated(b, false); in __link_buffer()
520 static void __unlink_buffer(struct dm_buffer *b) in __unlink_buffer() argument
522 struct dm_bufio_client *c = b->c; in __unlink_buffer()
524 BUG_ON(!c->n_buffers[b->list_mode]); in __unlink_buffer()
526 c->n_buffers[b->list_mode]--; in __unlink_buffer()
527 __remove(b->c, b); in __unlink_buffer()
528 list_del(&b->lru_list); in __unlink_buffer()
530 adjust_total_allocated(b, true); in __unlink_buffer()
536 static void __relink_lru(struct dm_buffer *b, int dirty) in __relink_lru() argument
538 struct dm_bufio_client *c = b->c; in __relink_lru()
540 b->accessed = 1; in __relink_lru()
542 BUG_ON(!c->n_buffers[b->list_mode]); in __relink_lru()
544 c->n_buffers[b->list_mode]--; in __relink_lru()
545 c->n_buffers[dirty]++; in __relink_lru()
546 b->list_mode = dirty; in __relink_lru()
547 list_move(&b->lru_list, &c->lru[dirty]); in __relink_lru()
548 b->last_accessed = jiffies; in __relink_lru()
551 /*----------------------------------------------------------------
556 * memory-consumption per buffer, so it is not viable);
558 * the memory must be direct-mapped, not vmalloced;
564 * rejects the bio because it is too large, use dm-io layer to do the I/O.
565 * The dm-io layer splits the I/O into multiple requests, avoiding the above
567 *--------------------------------------------------------------*/
570 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
575 struct dm_buffer *b = context; in dmio_complete() local
577 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); in dmio_complete()
580 static void use_dmio(struct dm_buffer *b, int rw, sector_t sector, in use_dmio() argument
588 .notify.context = b, in use_dmio()
589 .client = b->c->dm_io, in use_dmio()
592 .bdev = b->c->bdev, in use_dmio()
597 if (b->data_mode != DATA_MODE_VMALLOC) { in use_dmio()
599 io_req.mem.ptr.addr = (char *)b->data + offset; in use_dmio()
602 io_req.mem.ptr.vma = (char *)b->data + offset; in use_dmio()
607 b->end_io(b, errno_to_blk_status(r)); in use_dmio()
612 struct dm_buffer *b = bio->bi_private; in bio_complete() local
613 blk_status_t status = bio->bi_status; in bio_complete()
615 b->end_io(b, status); in bio_complete()
618 static void use_bio(struct dm_buffer *b, int rw, sector_t sector, in use_bio() argument
625 vec_size = b->c->block_size >> PAGE_SHIFT; in use_bio()
626 if (unlikely(b->c->sectors_per_block_bits < PAGE_SHIFT - SECTOR_SHIFT)) in use_bio()
632 use_dmio(b, rw, sector, n_sectors, offset); in use_bio()
636 bio->bi_iter.bi_sector = sector; in use_bio()
637 bio_set_dev(bio, b->c->bdev); in use_bio()
639 bio->bi_end_io = bio_complete; in use_bio()
640 bio->bi_private = b; in use_bio()
642 ptr = (char *)b->data + offset; in use_bio()
646 unsigned this_step = min((unsigned)(PAGE_SIZE - offset_in_page(ptr)), len); in use_bio()
653 len -= this_step; in use_bio()
664 if (likely(c->sectors_per_block_bits >= 0)) in block_to_sector()
665 sector = block << c->sectors_per_block_bits; in block_to_sector()
667 sector = block * (c->block_size >> SECTOR_SHIFT); in block_to_sector()
668 sector += c->start; in block_to_sector()
673 static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buffer *, blk_status_t)) in submit_io() argument
679 b->end_io = end_io; in submit_io()
681 sector = block_to_sector(b->c, b->block); in submit_io()
684 n_sectors = b->c->block_size >> SECTOR_SHIFT; in submit_io()
687 if (b->c->write_callback) in submit_io()
688 b->c->write_callback(b); in submit_io()
689 offset = b->write_start; in submit_io()
690 end = b->write_end; in submit_io()
691 offset &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
692 end += DM_BUFIO_WRITE_ALIGN - 1; in submit_io()
693 end &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
694 if (unlikely(end > b->c->block_size)) in submit_io()
695 end = b->c->block_size; in submit_io()
698 n_sectors = (end - offset) >> SECTOR_SHIFT; in submit_io()
701 if (b->data_mode != DATA_MODE_VMALLOC) in submit_io()
702 use_bio(b, rw, sector, n_sectors, offset); in submit_io()
704 use_dmio(b, rw, sector, n_sectors, offset); in submit_io()
707 /*----------------------------------------------------------------
709 *--------------------------------------------------------------*/
717 static void write_endio(struct dm_buffer *b, blk_status_t status) in write_endio() argument
719 b->write_error = status; in write_endio()
721 struct dm_bufio_client *c = b->c; in write_endio()
723 (void)cmpxchg(&c->async_write_error, 0, in write_endio()
727 BUG_ON(!test_bit(B_WRITING, &b->state)); in write_endio()
730 clear_bit(B_WRITING, &b->state); in write_endio()
733 wake_up_bit(&b->state, B_WRITING); in write_endio()
739 * - If the buffer is not dirty, exit.
740 * - If there some previous write going on, wait for it to finish (we can't
742 * - Submit our write and don't wait on it. We set B_WRITING indicating
745 static void __write_dirty_buffer(struct dm_buffer *b, in __write_dirty_buffer() argument
748 if (!test_bit(B_DIRTY, &b->state)) in __write_dirty_buffer()
751 clear_bit(B_DIRTY, &b->state); in __write_dirty_buffer()
752 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __write_dirty_buffer()
754 b->write_start = b->dirty_start; in __write_dirty_buffer()
755 b->write_end = b->dirty_end; in __write_dirty_buffer()
758 submit_io(b, REQ_OP_WRITE, write_endio); in __write_dirty_buffer()
760 list_add_tail(&b->write_list, write_list); in __write_dirty_buffer()
768 struct dm_buffer *b = in __flush_write_list() local
769 list_entry(write_list->next, struct dm_buffer, write_list); in __flush_write_list()
770 list_del(&b->write_list); in __flush_write_list()
771 submit_io(b, REQ_OP_WRITE, write_endio); in __flush_write_list()
782 static void __make_buffer_clean(struct dm_buffer *b) in __make_buffer_clean() argument
784 BUG_ON(b->hold_count); in __make_buffer_clean()
786 if (!b->state) /* fast case */ in __make_buffer_clean()
789 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
790 __write_dirty_buffer(b, NULL); in __make_buffer_clean()
791 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
800 struct dm_buffer *b; in __get_unclaimed_buffer() local
802 list_for_each_entry_reverse(b, &c->lru[LIST_CLEAN], lru_list) { in __get_unclaimed_buffer()
803 BUG_ON(test_bit(B_WRITING, &b->state)); in __get_unclaimed_buffer()
804 BUG_ON(test_bit(B_DIRTY, &b->state)); in __get_unclaimed_buffer()
806 if (!b->hold_count) { in __get_unclaimed_buffer()
807 __make_buffer_clean(b); in __get_unclaimed_buffer()
808 __unlink_buffer(b); in __get_unclaimed_buffer()
809 return b; in __get_unclaimed_buffer()
814 list_for_each_entry_reverse(b, &c->lru[LIST_DIRTY], lru_list) { in __get_unclaimed_buffer()
815 BUG_ON(test_bit(B_READING, &b->state)); in __get_unclaimed_buffer()
817 if (!b->hold_count) { in __get_unclaimed_buffer()
818 __make_buffer_clean(b); in __get_unclaimed_buffer()
819 __unlink_buffer(b); in __get_unclaimed_buffer()
820 return b; in __get_unclaimed_buffer()
832 * This function is entered with c->lock held, drops it and regains it
839 add_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
845 remove_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
865 struct dm_buffer *b; in __alloc_buffer_wait_no_callback() local
869 * dm-bufio is resistant to allocation failures (it just keeps in __alloc_buffer_wait_no_callback()
883 b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); in __alloc_buffer_wait_no_callback()
884 if (b) in __alloc_buffer_wait_no_callback()
885 return b; in __alloc_buffer_wait_no_callback()
893 b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); in __alloc_buffer_wait_no_callback()
895 if (b) in __alloc_buffer_wait_no_callback()
896 return b; in __alloc_buffer_wait_no_callback()
900 if (!list_empty(&c->reserved_buffers)) { in __alloc_buffer_wait_no_callback()
901 b = list_entry(c->reserved_buffers.next, in __alloc_buffer_wait_no_callback()
903 list_del(&b->lru_list); in __alloc_buffer_wait_no_callback()
904 c->need_reserved_buffers++; in __alloc_buffer_wait_no_callback()
906 return b; in __alloc_buffer_wait_no_callback()
909 b = __get_unclaimed_buffer(c); in __alloc_buffer_wait_no_callback()
910 if (b) in __alloc_buffer_wait_no_callback()
911 return b; in __alloc_buffer_wait_no_callback()
919 struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf); in __alloc_buffer_wait() local
921 if (!b) in __alloc_buffer_wait()
924 if (c->alloc_callback) in __alloc_buffer_wait()
925 c->alloc_callback(b); in __alloc_buffer_wait()
927 return b; in __alloc_buffer_wait()
933 static void __free_buffer_wake(struct dm_buffer *b) in __free_buffer_wake() argument
935 struct dm_bufio_client *c = b->c; in __free_buffer_wake()
937 if (!c->need_reserved_buffers) in __free_buffer_wake()
938 free_buffer(b); in __free_buffer_wake()
940 list_add(&b->lru_list, &c->reserved_buffers); in __free_buffer_wake()
941 c->need_reserved_buffers--; in __free_buffer_wake()
944 wake_up(&c->free_buffer_wait); in __free_buffer_wake()
950 struct dm_buffer *b, *tmp; in __write_dirty_buffers_async() local
952 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) { in __write_dirty_buffers_async()
953 BUG_ON(test_bit(B_READING, &b->state)); in __write_dirty_buffers_async()
955 if (!test_bit(B_DIRTY, &b->state) && in __write_dirty_buffers_async()
956 !test_bit(B_WRITING, &b->state)) { in __write_dirty_buffers_async()
957 __relink_lru(b, LIST_CLEAN); in __write_dirty_buffers_async()
961 if (no_wait && test_bit(B_WRITING, &b->state)) in __write_dirty_buffers_async()
964 __write_dirty_buffer(b, write_list); in __write_dirty_buffers_async()
977 if (c->n_buffers[LIST_DIRTY] > c->n_buffers[LIST_CLEAN] * DM_BUFIO_WRITEBACK_RATIO) in __check_watermark()
981 /*----------------------------------------------------------------
983 *--------------------------------------------------------------*/
989 struct dm_buffer *b, *new_b = NULL; in __bufio_new() local
993 b = __find(c, block); in __bufio_new()
994 if (b) in __bufio_new()
1008 b = __find(c, block); in __bufio_new()
1009 if (b) { in __bufio_new()
1016 b = new_b; in __bufio_new()
1017 b->hold_count = 1; in __bufio_new()
1018 b->read_error = 0; in __bufio_new()
1019 b->write_error = 0; in __bufio_new()
1020 __link_buffer(b, block, LIST_CLEAN); in __bufio_new()
1023 b->state = 0; in __bufio_new()
1024 return b; in __bufio_new()
1027 b->state = 1 << B_READING; in __bufio_new()
1030 return b; in __bufio_new()
1042 if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state))) in __bufio_new()
1045 b->hold_count++; in __bufio_new()
1046 __relink_lru(b, test_bit(B_DIRTY, &b->state) || in __bufio_new()
1047 test_bit(B_WRITING, &b->state)); in __bufio_new()
1048 return b; in __bufio_new()
1055 static void read_endio(struct dm_buffer *b, blk_status_t status) in read_endio() argument
1057 b->read_error = status; in read_endio()
1059 BUG_ON(!test_bit(B_READING, &b->state)); in read_endio()
1062 clear_bit(B_READING, &b->state); in read_endio()
1065 wake_up_bit(&b->state, B_READING); in read_endio()
1078 struct dm_buffer *b; in new_read() local
1083 b = __bufio_new(c, block, nf, &need_submit, &write_list); in new_read()
1085 if (b && b->hold_count == 1) in new_read()
1086 buffer_record_stack(b); in new_read()
1092 if (!b) in new_read()
1096 submit_io(b, REQ_OP_READ, read_endio); in new_read()
1098 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in new_read()
1100 if (b->read_error) { in new_read()
1101 int error = blk_status_to_errno(b->read_error); in new_read()
1103 dm_bufio_release(b); in new_read()
1108 *bp = b; in new_read()
1110 return b->data; in new_read()
1150 for (; n_blocks--; block++) { in dm_bufio_prefetch()
1152 struct dm_buffer *b; in dm_bufio_prefetch() local
1153 b = __bufio_new(c, block, NF_PREFETCH, &need_submit, in dm_bufio_prefetch()
1162 if (unlikely(b != NULL)) { in dm_bufio_prefetch()
1166 submit_io(b, REQ_OP_READ, read_endio); in dm_bufio_prefetch()
1167 dm_bufio_release(b); in dm_bufio_prefetch()
1184 void dm_bufio_release(struct dm_buffer *b) in dm_bufio_release() argument
1186 struct dm_bufio_client *c = b->c; in dm_bufio_release()
1190 BUG_ON(!b->hold_count); in dm_bufio_release()
1192 b->hold_count--; in dm_bufio_release()
1193 if (!b->hold_count) { in dm_bufio_release()
1194 wake_up(&c->free_buffer_wait); in dm_bufio_release()
1201 if ((b->read_error || b->write_error) && in dm_bufio_release()
1202 !test_bit(B_READING, &b->state) && in dm_bufio_release()
1203 !test_bit(B_WRITING, &b->state) && in dm_bufio_release()
1204 !test_bit(B_DIRTY, &b->state)) { in dm_bufio_release()
1205 __unlink_buffer(b); in dm_bufio_release()
1206 __free_buffer_wake(b); in dm_bufio_release()
1214 void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b, in dm_bufio_mark_partial_buffer_dirty() argument
1217 struct dm_bufio_client *c = b->c; in dm_bufio_mark_partial_buffer_dirty()
1220 BUG_ON(end > b->c->block_size); in dm_bufio_mark_partial_buffer_dirty()
1224 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_mark_partial_buffer_dirty()
1226 if (!test_and_set_bit(B_DIRTY, &b->state)) { in dm_bufio_mark_partial_buffer_dirty()
1227 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
1228 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
1229 __relink_lru(b, LIST_DIRTY); in dm_bufio_mark_partial_buffer_dirty()
1231 if (start < b->dirty_start) in dm_bufio_mark_partial_buffer_dirty()
1232 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
1233 if (end > b->dirty_end) in dm_bufio_mark_partial_buffer_dirty()
1234 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
1241 void dm_bufio_mark_buffer_dirty(struct dm_buffer *b) in dm_bufio_mark_buffer_dirty() argument
1243 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size); in dm_bufio_mark_buffer_dirty()
1271 struct dm_buffer *b, *tmp; in dm_bufio_write_dirty_buffers() local
1282 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) { in dm_bufio_write_dirty_buffers()
1285 if (buffers_processed < c->n_buffers[LIST_DIRTY]) in dm_bufio_write_dirty_buffers()
1288 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_write_dirty_buffers()
1290 if (test_bit(B_WRITING, &b->state)) { in dm_bufio_write_dirty_buffers()
1291 if (buffers_processed < c->n_buffers[LIST_DIRTY]) { in dm_bufio_write_dirty_buffers()
1293 b->hold_count++; in dm_bufio_write_dirty_buffers()
1295 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_write_dirty_buffers()
1298 b->hold_count--; in dm_bufio_write_dirty_buffers()
1300 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_write_dirty_buffers()
1304 if (!test_bit(B_DIRTY, &b->state) && in dm_bufio_write_dirty_buffers()
1305 !test_bit(B_WRITING, &b->state)) in dm_bufio_write_dirty_buffers()
1306 __relink_lru(b, LIST_CLEAN); in dm_bufio_write_dirty_buffers()
1327 wake_up(&c->free_buffer_wait); in dm_bufio_write_dirty_buffers()
1330 a = xchg(&c->async_write_error, 0); in dm_bufio_write_dirty_buffers()
1340 * Use dm-io to send an empty barrier to flush the device.
1349 .client = c->dm_io, in dm_bufio_issue_flush()
1352 .bdev = c->bdev, in dm_bufio_issue_flush()
1364 * Use dm-io to send a discard request to flush the device.
1373 .client = c->dm_io, in dm_bufio_issue_discard()
1376 .bdev = c->bdev, in dm_bufio_issue_discard()
1399 void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block) in dm_bufio_release_move() argument
1401 struct dm_bufio_client *c = b->c; in dm_bufio_release_move()
1411 if (new->hold_count) { in dm_bufio_release_move()
1425 BUG_ON(!b->hold_count); in dm_bufio_release_move()
1426 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_release_move()
1428 __write_dirty_buffer(b, NULL); in dm_bufio_release_move()
1429 if (b->hold_count == 1) { in dm_bufio_release_move()
1430 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_release_move()
1432 set_bit(B_DIRTY, &b->state); in dm_bufio_release_move()
1433 b->dirty_start = 0; in dm_bufio_release_move()
1434 b->dirty_end = c->block_size; in dm_bufio_release_move()
1435 __unlink_buffer(b); in dm_bufio_release_move()
1436 __link_buffer(b, new_block, LIST_DIRTY); in dm_bufio_release_move()
1439 wait_on_bit_lock_io(&b->state, B_WRITING, in dm_bufio_release_move()
1448 old_block = b->block; in dm_bufio_release_move()
1449 __unlink_buffer(b); in dm_bufio_release_move()
1450 __link_buffer(b, new_block, b->list_mode); in dm_bufio_release_move()
1451 submit_io(b, REQ_OP_WRITE, write_endio); in dm_bufio_release_move()
1452 wait_on_bit_io(&b->state, B_WRITING, in dm_bufio_release_move()
1454 __unlink_buffer(b); in dm_bufio_release_move()
1455 __link_buffer(b, old_block, b->list_mode); in dm_bufio_release_move()
1459 dm_bufio_release(b); in dm_bufio_release_move()
1463 static void forget_buffer_locked(struct dm_buffer *b) in forget_buffer_locked() argument
1465 if (likely(!b->hold_count) && likely(!b->state)) { in forget_buffer_locked()
1466 __unlink_buffer(b); in forget_buffer_locked()
1467 __free_buffer_wake(b); in forget_buffer_locked()
1479 struct dm_buffer *b; in dm_bufio_forget() local
1483 b = __find(c, block); in dm_bufio_forget()
1484 if (b) in dm_bufio_forget()
1485 forget_buffer_locked(b); in dm_bufio_forget()
1493 struct dm_buffer *b; in dm_bufio_forget_buffers() local
1499 b = __find_next(c, block); in dm_bufio_forget_buffers()
1500 if (b) { in dm_bufio_forget_buffers()
1501 block = b->block + 1; in dm_bufio_forget_buffers()
1502 forget_buffer_locked(b); in dm_bufio_forget_buffers()
1507 if (!b) in dm_bufio_forget_buffers()
1516 c->minimum_buffers = n; in dm_bufio_set_minimum_buffers()
1522 return c->block_size; in dm_bufio_get_block_size()
1528 sector_t s = i_size_read(c->bdev->bd_inode) >> SECTOR_SHIFT; in dm_bufio_get_device_size()
1529 if (s >= c->start) in dm_bufio_get_device_size()
1530 s -= c->start; in dm_bufio_get_device_size()
1533 if (likely(c->sectors_per_block_bits >= 0)) in dm_bufio_get_device_size()
1534 s >>= c->sectors_per_block_bits; in dm_bufio_get_device_size()
1536 sector_div(s, c->block_size >> SECTOR_SHIFT); in dm_bufio_get_device_size()
1543 return c->dm_io; in dm_bufio_get_dm_io_client()
1547 sector_t dm_bufio_get_block_number(struct dm_buffer *b) in dm_bufio_get_block_number() argument
1549 return b->block; in dm_bufio_get_block_number()
1553 void *dm_bufio_get_block_data(struct dm_buffer *b) in dm_bufio_get_block_data() argument
1555 return b->data; in dm_bufio_get_block_data()
1559 void *dm_bufio_get_aux_data(struct dm_buffer *b) in dm_bufio_get_aux_data() argument
1561 return b + 1; in dm_bufio_get_aux_data()
1565 struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b) in dm_bufio_get_client() argument
1567 return b->c; in dm_bufio_get_client()
1573 struct dm_buffer *b; in drop_buffers() local
1580 * An optimization so that the buffers are not written one-by-one. in drop_buffers()
1586 while ((b = __get_unclaimed_buffer(c))) in drop_buffers()
1587 __free_buffer_wake(b); in drop_buffers()
1590 list_for_each_entry(b, &c->lru[i], lru_list) { in drop_buffers()
1594 (unsigned long long)b->block, b->hold_count, i); in drop_buffers()
1596 stack_trace_print(b->stack_entries, b->stack_len, 1); in drop_buffers()
1598 b->hold_count = 0; in drop_buffers()
1603 while ((b = __get_unclaimed_buffer(c))) in drop_buffers()
1604 __free_buffer_wake(b); in drop_buffers()
1608 BUG_ON(!list_empty(&c->lru[i])); in drop_buffers()
1621 static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp) in __try_evict_buffer() argument
1624 if (test_bit(B_READING, &b->state) || in __try_evict_buffer()
1625 test_bit(B_WRITING, &b->state) || in __try_evict_buffer()
1626 test_bit(B_DIRTY, &b->state)) in __try_evict_buffer()
1630 if (b->hold_count) in __try_evict_buffer()
1633 __make_buffer_clean(b); in __try_evict_buffer()
1634 __unlink_buffer(b); in __try_evict_buffer()
1635 __free_buffer_wake(b); in __try_evict_buffer()
1643 if (likely(c->sectors_per_block_bits >= 0)) in get_retain_buffers()
1644 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; in get_retain_buffers()
1646 retain_bytes /= c->block_size; in get_retain_buffers()
1653 struct dm_buffer *b, *tmp; in __scan() local
1655 unsigned long count = c->n_buffers[LIST_CLEAN] + in __scan()
1656 c->n_buffers[LIST_DIRTY]; in __scan()
1660 list_for_each_entry_safe_reverse(b, tmp, &c->lru[l], lru_list) { in __scan()
1661 if (count - freed <= retain_target) in __scan()
1662 atomic_long_set(&c->need_shrink, 0); in __scan()
1663 if (!atomic_long_read(&c->need_shrink)) in __scan()
1665 if (__try_evict_buffer(b, GFP_KERNEL)) { in __scan()
1666 atomic_long_dec(&c->need_shrink); in __scan()
1688 atomic_long_add(sc->nr_to_scan, &c->need_shrink); in dm_bufio_shrink_scan()
1689 queue_work(dm_bufio_wq, &c->shrink_work); in dm_bufio_shrink_scan()
1691 return sc->nr_to_scan; in dm_bufio_shrink_scan()
1697 unsigned long count = READ_ONCE(c->n_buffers[LIST_CLEAN]) + in dm_bufio_shrink_count()
1698 READ_ONCE(c->n_buffers[LIST_DIRTY]); in dm_bufio_shrink_count()
1700 unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink); in dm_bufio_shrink_count()
1705 count -= retain_target; in dm_bufio_shrink_count()
1710 count -= queued_for_cleanup; in dm_bufio_shrink_count()
1728 if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) { in dm_bufio_client_create()
1729 DMERR("%s: block size not specified or is not multiple of 512b", __func__); in dm_bufio_client_create()
1730 r = -EINVAL; in dm_bufio_client_create()
1736 r = -ENOMEM; in dm_bufio_client_create()
1739 c->buffer_tree = RB_ROOT; in dm_bufio_client_create()
1741 c->bdev = bdev; in dm_bufio_client_create()
1742 c->block_size = block_size; in dm_bufio_client_create()
1744 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT; in dm_bufio_client_create()
1746 c->sectors_per_block_bits = -1; in dm_bufio_client_create()
1748 c->alloc_callback = alloc_callback; in dm_bufio_client_create()
1749 c->write_callback = write_callback; in dm_bufio_client_create()
1752 INIT_LIST_HEAD(&c->lru[i]); in dm_bufio_client_create()
1753 c->n_buffers[i] = 0; in dm_bufio_client_create()
1756 mutex_init(&c->lock); in dm_bufio_client_create()
1757 INIT_LIST_HEAD(&c->reserved_buffers); in dm_bufio_client_create()
1758 c->need_reserved_buffers = reserved_buffers; in dm_bufio_client_create()
1762 init_waitqueue_head(&c->free_buffer_wait); in dm_bufio_client_create()
1763 c->async_write_error = 0; in dm_bufio_client_create()
1765 c->dm_io = dm_io_client_create(); in dm_bufio_client_create()
1766 if (IS_ERR(c->dm_io)) { in dm_bufio_client_create()
1767 r = PTR_ERR(c->dm_io); in dm_bufio_client_create()
1774 snprintf(slab_name, sizeof slab_name, "dm_bufio_cache-%u", block_size); in dm_bufio_client_create()
1775 c->slab_cache = kmem_cache_create(slab_name, block_size, align, in dm_bufio_client_create()
1777 if (!c->slab_cache) { in dm_bufio_client_create()
1778 r = -ENOMEM; in dm_bufio_client_create()
1783 snprintf(slab_name, sizeof slab_name, "dm_bufio_buffer-%u", aux_size); in dm_bufio_client_create()
1786 c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size, in dm_bufio_client_create()
1788 if (!c->slab_buffer) { in dm_bufio_client_create()
1789 r = -ENOMEM; in dm_bufio_client_create()
1793 while (c->need_reserved_buffers) { in dm_bufio_client_create()
1794 struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL); in dm_bufio_client_create() local
1796 if (!b) { in dm_bufio_client_create()
1797 r = -ENOMEM; in dm_bufio_client_create()
1800 __free_buffer_wake(b); in dm_bufio_client_create()
1803 INIT_WORK(&c->shrink_work, shrink_work); in dm_bufio_client_create()
1804 atomic_long_set(&c->need_shrink, 0); in dm_bufio_client_create()
1806 c->shrinker.count_objects = dm_bufio_shrink_count; in dm_bufio_client_create()
1807 c->shrinker.scan_objects = dm_bufio_shrink_scan; in dm_bufio_client_create()
1808 c->shrinker.seeks = 1; in dm_bufio_client_create()
1809 c->shrinker.batch = 0; in dm_bufio_client_create()
1810 r = register_shrinker(&c->shrinker); in dm_bufio_client_create()
1816 list_add(&c->client_list, &dm_bufio_all_clients); in dm_bufio_client_create()
1823 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_create()
1824 struct dm_buffer *b = list_entry(c->reserved_buffers.next, in dm_bufio_client_create() local
1826 list_del(&b->lru_list); in dm_bufio_client_create()
1827 free_buffer(b); in dm_bufio_client_create()
1829 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_create()
1830 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_create()
1831 dm_io_client_destroy(c->dm_io); in dm_bufio_client_create()
1833 mutex_destroy(&c->lock); in dm_bufio_client_create()
1850 unregister_shrinker(&c->shrinker); in dm_bufio_client_destroy()
1851 flush_work(&c->shrink_work); in dm_bufio_client_destroy()
1855 list_del(&c->client_list); in dm_bufio_client_destroy()
1856 dm_bufio_client_count--; in dm_bufio_client_destroy()
1861 BUG_ON(!RB_EMPTY_ROOT(&c->buffer_tree)); in dm_bufio_client_destroy()
1862 BUG_ON(c->need_reserved_buffers); in dm_bufio_client_destroy()
1864 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_destroy()
1865 struct dm_buffer *b = list_entry(c->reserved_buffers.next, in dm_bufio_client_destroy() local
1867 list_del(&b->lru_list); in dm_bufio_client_destroy()
1868 free_buffer(b); in dm_bufio_client_destroy()
1872 if (c->n_buffers[i]) in dm_bufio_client_destroy()
1873 DMERR("leaked buffer count %d: %ld", i, c->n_buffers[i]); in dm_bufio_client_destroy()
1876 BUG_ON(c->n_buffers[i]); in dm_bufio_client_destroy()
1878 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_destroy()
1879 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_destroy()
1880 dm_io_client_destroy(c->dm_io); in dm_bufio_client_destroy()
1881 mutex_destroy(&c->lock); in dm_bufio_client_destroy()
1888 c->start = start; in dm_bufio_set_sector_offset()
1902 static bool older_than(struct dm_buffer *b, unsigned long age_hz) in older_than() argument
1904 return time_after_eq(jiffies, b->last_accessed + age_hz); in older_than()
1909 struct dm_buffer *b, *tmp; in __evict_old_buffers() local
1923 count = c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY]; in __evict_old_buffers()
1924 list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_CLEAN], lru_list) { in __evict_old_buffers()
1928 if (!older_than(b, age_hz)) in __evict_old_buffers()
1931 if (__try_evict_buffer(b, 0)) in __evict_old_buffers()
1932 count--; in __evict_old_buffers()
1944 struct dm_buffer *b; in do_global_cleanup() local
1946 unsigned long threshold = dm_bufio_cache_size - in do_global_cleanup()
1961 if (!loops--) in do_global_cleanup()
1965 b = list_entry(global_queue.prev, struct dm_buffer, global_list); in do_global_cleanup()
1967 if (b->accessed) { in do_global_cleanup()
1968 b->accessed = 0; in do_global_cleanup()
1969 list_move(&b->global_list, &global_queue); in do_global_cleanup()
1976 current_client = b->c; in do_global_cleanup()
1993 if (unlikely(!__try_evict_buffer(b, GFP_KERNEL))) { in do_global_cleanup()
1995 list_move(&b->global_list, &global_queue); in do_global_cleanup()
2031 /*----------------------------------------------------------------
2033 *--------------------------------------------------------------*/
2048 mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(), in dm_bufio_init()
2067 return -ENOMEM; in dm_bufio_init()
2142 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");