Lines Matching refs:pblk
29 struct pblk *pblk = line_ws->pblk; in pblk_line_mark_bb() local
30 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_mark_bb()
39 line = pblk_ppa_to_line(pblk, *ppa); in pblk_line_mark_bb()
42 pblk_err(pblk, "failed to mark bb, line:%d, pos:%d\n", in pblk_line_mark_bb()
47 mempool_free(line_ws, &pblk->gen_ws_pool); in pblk_line_mark_bb()
50 static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line, in pblk_mark_bb() argument
53 struct nvm_tgt_dev *dev = pblk->dev; in pblk_mark_bb()
58 pblk_debug(pblk, "erase failed: line:%d, pos:%d\n", line->id, pos); in pblk_mark_bb()
59 atomic_long_inc(&pblk->erase_failed); in pblk_mark_bb()
63 pblk_err(pblk, "attempted to erase bb: line:%d, pos:%d\n", in pblk_mark_bb()
75 pblk_gen_run_ws(pblk, NULL, ppa, pblk_line_mark_bb, in pblk_mark_bb()
76 GFP_ATOMIC, pblk->bb_wq); in pblk_mark_bb()
79 static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd) in __pblk_end_io_erase() argument
81 struct nvm_tgt_dev *dev = pblk->dev; in __pblk_end_io_erase()
87 line = pblk_ppa_to_line(pblk, rqd->ppa_addr); in __pblk_end_io_erase()
94 trace_pblk_chunk_reset(pblk_disk_name(pblk), in __pblk_end_io_erase()
98 pblk_mark_bb(pblk, line, rqd->ppa_addr); in __pblk_end_io_erase()
100 trace_pblk_chunk_reset(pblk_disk_name(pblk), in __pblk_end_io_erase()
106 trace_pblk_chunk_state(pblk_disk_name(pblk), &rqd->ppa_addr, in __pblk_end_io_erase()
109 atomic_dec(&pblk->inflight_io); in __pblk_end_io_erase()
115 struct pblk *pblk = rqd->private; in pblk_end_io_erase() local
117 __pblk_end_io_erase(pblk, rqd); in pblk_end_io_erase()
118 mempool_free(rqd, &pblk->e_rq_pool); in pblk_end_io_erase()
126 struct nvm_chk_meta *pblk_get_chunk_meta(struct pblk *pblk) in pblk_get_chunk_meta() argument
128 struct nvm_tgt_dev *dev = pblk->dev; in pblk_get_chunk_meta()
151 struct nvm_chk_meta *pblk_chunk_get_off(struct pblk *pblk, in pblk_chunk_get_off() argument
155 struct nvm_tgt_dev *dev = pblk->dev; in pblk_chunk_get_off()
164 void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line, in __pblk_map_invalidate() argument
167 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_map_invalidate()
185 move_list = pblk_line_gc_list(pblk, line); in __pblk_map_invalidate()
204 void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa) in pblk_map_invalidate() argument
215 line = pblk_ppa_to_line(pblk, ppa); in pblk_map_invalidate()
216 paddr = pblk_dev_ppa_to_line_addr(pblk, ppa); in pblk_map_invalidate()
218 __pblk_map_invalidate(pblk, line, paddr); in pblk_map_invalidate()
221 static void pblk_invalidate_range(struct pblk *pblk, sector_t slba, in pblk_invalidate_range() argument
226 spin_lock(&pblk->trans_lock); in pblk_invalidate_range()
230 ppa = pblk_trans_map_get(pblk, lba); in pblk_invalidate_range()
233 pblk_map_invalidate(pblk, ppa); in pblk_invalidate_range()
236 pblk_trans_map_set(pblk, lba, ppa); in pblk_invalidate_range()
238 spin_unlock(&pblk->trans_lock); in pblk_invalidate_range()
241 int pblk_alloc_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd) in pblk_alloc_rqd_meta() argument
243 struct nvm_tgt_dev *dev = pblk->dev; in pblk_alloc_rqd_meta()
253 rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size(pblk); in pblk_alloc_rqd_meta()
254 rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size(pblk); in pblk_alloc_rqd_meta()
259 void pblk_free_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd) in pblk_free_rqd_meta() argument
261 struct nvm_tgt_dev *dev = pblk->dev; in pblk_free_rqd_meta()
269 struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type) in pblk_alloc_rqd() argument
278 pool = &pblk->w_rq_pool; in pblk_alloc_rqd()
282 pool = &pblk->r_rq_pool; in pblk_alloc_rqd()
286 pool = &pblk->e_rq_pool; in pblk_alloc_rqd()
297 void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type) in pblk_free_rqd() argument
306 pool = &pblk->w_rq_pool; in pblk_free_rqd()
309 pool = &pblk->r_rq_pool; in pblk_free_rqd()
312 pool = &pblk->e_rq_pool; in pblk_free_rqd()
315 pblk_err(pblk, "trying to free unknown rqd type\n"); in pblk_free_rqd()
319 pblk_free_rqd_meta(pblk, rqd); in pblk_free_rqd()
323 void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off, in pblk_bio_free_pages() argument
335 mempool_free(page++, &pblk->page_bio_pool); in pblk_bio_free_pages()
339 int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags, in pblk_bio_add_pages() argument
342 struct request_queue *q = pblk->dev->q; in pblk_bio_add_pages()
347 page = mempool_alloc(&pblk->page_bio_pool, flags); in pblk_bio_add_pages()
351 pblk_err(pblk, "could not add page to bio\n"); in pblk_bio_add_pages()
352 mempool_free(page, &pblk->page_bio_pool); in pblk_bio_add_pages()
359 pblk_bio_free_pages(pblk, bio, (bio->bi_vcnt - i), i); in pblk_bio_add_pages()
363 void pblk_write_kick(struct pblk *pblk) in pblk_write_kick() argument
365 wake_up_process(pblk->writer_ts); in pblk_write_kick()
366 mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(1000)); in pblk_write_kick()
371 struct pblk *pblk = from_timer(pblk, t, wtimer); in pblk_write_timer_fn() local
374 pblk_write_kick(pblk); in pblk_write_timer_fn()
377 void pblk_write_should_kick(struct pblk *pblk) in pblk_write_should_kick() argument
379 unsigned int secs_avail = pblk_rb_read_count(&pblk->rwb); in pblk_write_should_kick()
381 if (secs_avail >= pblk->min_write_pgs_data) in pblk_write_should_kick()
382 pblk_write_kick(pblk); in pblk_write_should_kick()
385 static void pblk_wait_for_meta(struct pblk *pblk) in pblk_wait_for_meta() argument
388 if (!atomic_read(&pblk->inflight_io)) in pblk_wait_for_meta()
395 static void pblk_flush_writer(struct pblk *pblk) in pblk_flush_writer() argument
397 pblk_rb_flush(&pblk->rwb); in pblk_flush_writer()
399 if (!pblk_rb_sync_count(&pblk->rwb)) in pblk_flush_writer()
402 pblk_write_kick(pblk); in pblk_flush_writer()
407 struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line) in pblk_line_gc_list() argument
409 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_gc_list()
410 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_gc_list()
412 int packed_meta = (le32_to_cpu(*line->vsc) / pblk->min_write_pgs_data) in pblk_line_gc_list()
413 * (pblk->min_write_pgs - pblk->min_write_pgs_data); in pblk_line_gc_list()
422 pblk_rl_werr_line_in(&pblk->rl); in pblk_line_gc_list()
451 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_gc_list()
456 pblk_err(pblk, "corrupted vsc for line %d, vsc:%d (%d/%d/%d)\n", in pblk_line_gc_list()
465 void pblk_discard(struct pblk *pblk, struct bio *bio) in pblk_discard() argument
470 pblk_invalidate_range(pblk, slba, nr_secs); in pblk_discard()
473 void pblk_log_write_err(struct pblk *pblk, struct nvm_rq *rqd) in pblk_log_write_err() argument
475 atomic_long_inc(&pblk->write_failed); in pblk_log_write_err()
477 pblk_print_failed_rqd(pblk, rqd, rqd->error); in pblk_log_write_err()
481 void pblk_log_read_err(struct pblk *pblk, struct nvm_rq *rqd) in pblk_log_read_err() argument
485 atomic_long_inc(&pblk->read_empty); in pblk_log_read_err()
491 atomic_long_inc(&pblk->read_high_ecc); in pblk_log_read_err()
495 atomic_long_inc(&pblk->read_failed); in pblk_log_read_err()
498 pblk_err(pblk, "unknown read error:%d\n", rqd->error); in pblk_log_read_err()
501 pblk_print_failed_rqd(pblk, rqd, rqd->error); in pblk_log_read_err()
505 void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write) in pblk_set_sec_per_write() argument
507 pblk->sec_per_write = sec_per_write; in pblk_set_sec_per_write()
510 int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd, void *buf) in pblk_submit_io() argument
512 struct nvm_tgt_dev *dev = pblk->dev; in pblk_submit_io()
514 atomic_inc(&pblk->inflight_io); in pblk_submit_io()
517 if (pblk_check_io(pblk, rqd)) in pblk_submit_io()
524 void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd) in pblk_check_chunk_state_update() argument
532 struct nvm_chk_meta *chunk = pblk_dev_ppa_to_chunk(pblk, *ppa); in pblk_check_chunk_state_update()
533 u64 caddr = pblk_dev_ppa_to_chunk_addr(pblk, *ppa); in pblk_check_chunk_state_update()
536 trace_pblk_chunk_state(pblk_disk_name(pblk), in pblk_check_chunk_state_update()
539 trace_pblk_chunk_state(pblk_disk_name(pblk), in pblk_check_chunk_state_update()
544 int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd, void *buf) in pblk_submit_io_sync() argument
546 struct nvm_tgt_dev *dev = pblk->dev; in pblk_submit_io_sync()
549 atomic_inc(&pblk->inflight_io); in pblk_submit_io_sync()
552 if (pblk_check_io(pblk, rqd)) in pblk_submit_io_sync()
560 pblk_check_chunk_state_update(pblk, rqd); in pblk_submit_io_sync()
565 static int pblk_submit_io_sync_sem(struct pblk *pblk, struct nvm_rq *rqd, in pblk_submit_io_sync_sem() argument
571 pblk_down_chunk(pblk, ppa_list[0]); in pblk_submit_io_sync_sem()
572 ret = pblk_submit_io_sync(pblk, rqd, buf); in pblk_submit_io_sync_sem()
573 pblk_up_chunk(pblk, ppa_list[0]); in pblk_submit_io_sync_sem()
578 int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail, in pblk_calc_secs() argument
581 int max = pblk->sec_per_write; in pblk_calc_secs()
582 int min = pblk->min_write_pgs; in pblk_calc_secs()
585 if (skip_meta && pblk->min_write_pgs_data != pblk->min_write_pgs) in pblk_calc_secs()
586 min = max = pblk->min_write_pgs_data; in pblk_calc_secs()
598 void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_dealloc_page() argument
605 pblk->lm.sec_per_line, line->cur_sec); in pblk_dealloc_page()
613 u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in __pblk_alloc_page() argument
621 if (line->cur_sec + nr_secs > pblk->lm.sec_per_line) { in __pblk_alloc_page()
623 nr_secs = pblk->lm.sec_per_line - line->cur_sec; in __pblk_alloc_page()
627 pblk->lm.sec_per_line, line->cur_sec); in __pblk_alloc_page()
634 u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_alloc_page() argument
642 addr = __pblk_alloc_page(pblk, line, nr_secs); in pblk_alloc_page()
650 u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line) in pblk_lookup_page() argument
656 pblk->lm.sec_per_line, line->cur_sec); in pblk_lookup_page()
662 u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line) in pblk_line_smeta_start() argument
664 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_smeta_start()
666 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_smeta_start()
677 int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line) in pblk_line_smeta_read() argument
679 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_smeta_read()
682 u64 paddr = pblk_line_smeta_start(pblk, line); in pblk_line_smeta_read()
687 ret = pblk_alloc_rqd_meta(pblk, &rqd); in pblk_line_smeta_read()
697 ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_smeta_read()
699 ret = pblk_submit_io_sync(pblk, &rqd, line->smeta); in pblk_line_smeta_read()
701 pblk_err(pblk, "smeta I/O submission failed: %d\n", ret); in pblk_line_smeta_read()
705 atomic_dec(&pblk->inflight_io); in pblk_line_smeta_read()
708 pblk_log_read_err(pblk, &rqd); in pblk_line_smeta_read()
713 pblk_free_rqd_meta(pblk, &rqd); in pblk_line_smeta_read()
717 static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line, in pblk_line_smeta_write() argument
720 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_smeta_write()
723 __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); in pblk_line_smeta_write()
729 ret = pblk_alloc_rqd_meta(pblk, &rqd); in pblk_line_smeta_write()
739 struct pblk_sec_meta *meta = pblk_get_meta(pblk, in pblk_line_smeta_write()
742 ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_smeta_write()
746 ret = pblk_submit_io_sync_sem(pblk, &rqd, line->smeta); in pblk_line_smeta_write()
748 pblk_err(pblk, "smeta I/O submission failed: %d\n", ret); in pblk_line_smeta_write()
752 atomic_dec(&pblk->inflight_io); in pblk_line_smeta_write()
755 pblk_log_write_err(pblk, &rqd); in pblk_line_smeta_write()
760 pblk_free_rqd_meta(pblk, &rqd); in pblk_line_smeta_write()
764 int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line, in pblk_line_emeta_read() argument
767 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_emeta_read()
769 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_emeta_read()
775 int min = pblk->min_write_pgs; in pblk_line_emeta_read()
787 ppa_list_buf = meta_list + pblk_dma_meta_size(pblk); in pblk_line_emeta_read()
788 dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk); in pblk_line_emeta_read()
793 rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); in pblk_line_emeta_read()
805 struct ppa_addr ppa = addr_to_gen_ppa(pblk, paddr, line_id); in pblk_line_emeta_read()
808 if (pblk_io_aligned(pblk, rq_ppas)) in pblk_line_emeta_read()
813 if (pblk_boundary_paddr_checks(pblk, paddr)) { in pblk_line_emeta_read()
818 ppa = addr_to_gen_ppa(pblk, paddr, line_id); in pblk_line_emeta_read()
822 if (pblk_boundary_paddr_checks(pblk, paddr + min)) { in pblk_line_emeta_read()
828 ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line_id); in pblk_line_emeta_read()
831 ret = pblk_submit_io_sync(pblk, &rqd, emeta_buf); in pblk_line_emeta_read()
833 pblk_err(pblk, "emeta I/O submission failed: %d\n", ret); in pblk_line_emeta_read()
837 atomic_dec(&pblk->inflight_io); in pblk_line_emeta_read()
840 pblk_log_read_err(pblk, &rqd); in pblk_line_emeta_read()
855 static void pblk_setup_e_rq(struct pblk *pblk, struct nvm_rq *rqd, in pblk_setup_e_rq() argument
865 static int pblk_blk_erase_sync(struct pblk *pblk, struct ppa_addr ppa) in pblk_blk_erase_sync() argument
870 trace_pblk_chunk_reset(pblk_disk_name(pblk), &ppa, in pblk_blk_erase_sync()
873 pblk_setup_e_rq(pblk, &rqd, ppa); in pblk_blk_erase_sync()
878 ret = pblk_submit_io_sync(pblk, &rqd, NULL); in pblk_blk_erase_sync()
879 rqd.private = pblk; in pblk_blk_erase_sync()
880 __pblk_end_io_erase(pblk, &rqd); in pblk_blk_erase_sync()
885 int pblk_line_erase(struct pblk *pblk, struct pblk_line *line) in pblk_line_erase() argument
887 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_erase()
901 ppa = pblk->luns[bit].bppa; /* set ch and lun */ in pblk_line_erase()
908 ret = pblk_blk_erase_sync(pblk, ppa); in pblk_line_erase()
910 pblk_err(pblk, "failed to erase line %d\n", line->id); in pblk_line_erase()
951 static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_metadata() argument
954 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_init_metadata()
956 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_init_metadata()
957 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_init_metadata()
972 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_init_metadata()
979 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_init_metadata()
991 guid_copy((guid_t *)&smeta_buf->header.uuid, &pblk->instance_uuid); in pblk_line_init_metadata()
1012 pblk_calc_meta_header_crc(pblk, &smeta_buf->header)); in pblk_line_init_metadata()
1013 smeta_buf->crc = cpu_to_le32(pblk_calc_smeta_crc(pblk, smeta_buf)); in pblk_line_init_metadata()
1022 pblk_calc_meta_header_crc(pblk, &emeta_buf->header)); in pblk_line_init_metadata()
1034 static int pblk_line_alloc_bitmaps(struct pblk *pblk, struct pblk_line *line) in pblk_line_alloc_bitmaps() argument
1036 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_alloc_bitmaps()
1037 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_alloc_bitmaps()
1059 static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_bb() argument
1062 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_init_bb()
1064 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_init_bb()
1065 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_init_bb()
1090 if (init && pblk_line_smeta_write(pblk, line, off)) { in pblk_line_init_bb()
1091 pblk_debug(pblk, "line smeta I/O failed. Retry\n"); in pblk_line_init_bb()
1120 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_init_bb()
1125 pblk_err(pblk, "unexpected line %d is bad\n", line->id); in pblk_line_init_bb()
1133 static int pblk_prepare_new_line(struct pblk *pblk, struct pblk_line *line) in pblk_prepare_new_line() argument
1135 struct pblk_line_meta *lm = &pblk->lm; in pblk_prepare_new_line()
1136 struct nvm_tgt_dev *dev = pblk->dev; in pblk_prepare_new_line()
1142 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_prepare_new_line()
1157 static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line) in pblk_line_prepare() argument
1159 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_prepare()
1172 blk_to_erase = pblk_prepare_new_line(pblk, line); in pblk_line_prepare()
1174 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_prepare()
1193 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_prepare()
1209 int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_alloc() argument
1211 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_recov_alloc()
1218 ret = pblk_line_prepare(pblk, line); in pblk_line_recov_alloc()
1226 ret = pblk_line_alloc_bitmaps(pblk, line); in pblk_line_recov_alloc()
1230 if (!pblk_line_init_bb(pblk, line, 0)) { in pblk_line_recov_alloc()
1235 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_recov_alloc()
1246 void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_close() argument
1248 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_recov_close()
1268 struct pblk *pblk = line->pblk; in pblk_line_free() local
1269 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_free()
1277 struct pblk_line *pblk_line_get(struct pblk *pblk) in pblk_line_get() argument
1279 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_get()
1280 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_get()
1288 pblk_err(pblk, "no free lines\n"); in pblk_line_get()
1300 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_get()
1306 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_get()
1310 ret = pblk_line_prepare(pblk, line); in pblk_line_get()
1320 pblk_err(pblk, "failed to prepare line %d\n", line->id); in pblk_line_get()
1330 static struct pblk_line *pblk_line_retry(struct pblk *pblk, in pblk_line_retry() argument
1333 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_retry()
1338 retry_line = pblk_line_get(pblk); in pblk_line_retry()
1356 pblk_rl_free_lines_dec(&pblk->rl, line, false); in pblk_line_retry()
1358 if (pblk_line_erase(pblk, retry_line)) in pblk_line_retry()
1364 static void pblk_set_space_limit(struct pblk *pblk) in pblk_set_space_limit() argument
1366 struct pblk_rl *rl = &pblk->rl; in pblk_set_space_limit()
1371 struct pblk_line *pblk_line_get_first_data(struct pblk *pblk) in pblk_line_get_first_data() argument
1373 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_get_first_data()
1377 line = pblk_line_get(pblk); in pblk_line_get_first_data()
1387 pblk_line_setup_metadata(line, l_mg, &pblk->lm); in pblk_line_get_first_data()
1390 l_mg->data_next = pblk_line_get(pblk); in pblk_line_get_first_data()
1396 pblk_set_space_limit(pblk); in pblk_line_get_first_data()
1405 if (pblk_line_alloc_bitmaps(pblk, line)) in pblk_line_get_first_data()
1408 if (pblk_line_erase(pblk, line)) { in pblk_line_get_first_data()
1409 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1415 if (!pblk_line_init_metadata(pblk, line, NULL)) { in pblk_line_get_first_data()
1416 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1423 if (!pblk_line_init_bb(pblk, line, 1)) { in pblk_line_get_first_data()
1424 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1431 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_get_first_data()
1436 void pblk_ppa_to_line_put(struct pblk *pblk, struct ppa_addr ppa) in pblk_ppa_to_line_put() argument
1440 line = pblk_ppa_to_line(pblk, ppa); in pblk_ppa_to_line_put()
1444 void pblk_rq_to_line_put(struct pblk *pblk, struct nvm_rq *rqd) in pblk_rq_to_line_put() argument
1450 pblk_ppa_to_line_put(pblk, ppa_list[i]); in pblk_rq_to_line_put()
1453 static void pblk_stop_writes(struct pblk *pblk, struct pblk_line *line) in pblk_stop_writes() argument
1455 lockdep_assert_held(&pblk->l_mg.free_lock); in pblk_stop_writes()
1457 pblk_set_space_limit(pblk); in pblk_stop_writes()
1458 pblk->state = PBLK_STATE_STOPPING; in pblk_stop_writes()
1459 trace_pblk_state(pblk_disk_name(pblk), pblk->state); in pblk_stop_writes()
1462 static void pblk_line_close_meta_sync(struct pblk *pblk) in pblk_line_close_meta_sync() argument
1464 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_close_meta_sync()
1465 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_close_meta_sync()
1484 ret = pblk_submit_meta_io(pblk, line); in pblk_line_close_meta_sync()
1486 pblk_err(pblk, "sync meta line %d failed (%d)\n", in pblk_line_close_meta_sync()
1493 pblk_wait_for_meta(pblk); in pblk_line_close_meta_sync()
1494 flush_workqueue(pblk->close_wq); in pblk_line_close_meta_sync()
1497 void __pblk_pipeline_flush(struct pblk *pblk) in __pblk_pipeline_flush() argument
1499 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_pipeline_flush()
1503 if (pblk->state == PBLK_STATE_RECOVERING || in __pblk_pipeline_flush()
1504 pblk->state == PBLK_STATE_STOPPED) { in __pblk_pipeline_flush()
1508 pblk->state = PBLK_STATE_RECOVERING; in __pblk_pipeline_flush()
1509 trace_pblk_state(pblk_disk_name(pblk), pblk->state); in __pblk_pipeline_flush()
1512 pblk_flush_writer(pblk); in __pblk_pipeline_flush()
1513 pblk_wait_for_meta(pblk); in __pblk_pipeline_flush()
1515 ret = pblk_recov_pad(pblk); in __pblk_pipeline_flush()
1517 pblk_err(pblk, "could not close data on teardown(%d)\n", ret); in __pblk_pipeline_flush()
1521 flush_workqueue(pblk->bb_wq); in __pblk_pipeline_flush()
1522 pblk_line_close_meta_sync(pblk); in __pblk_pipeline_flush()
1525 void __pblk_pipeline_stop(struct pblk *pblk) in __pblk_pipeline_stop() argument
1527 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_pipeline_stop()
1530 pblk->state = PBLK_STATE_STOPPED; in __pblk_pipeline_stop()
1531 trace_pblk_state(pblk_disk_name(pblk), pblk->state); in __pblk_pipeline_stop()
1537 void pblk_pipeline_stop(struct pblk *pblk) in pblk_pipeline_stop() argument
1539 __pblk_pipeline_flush(pblk); in pblk_pipeline_stop()
1540 __pblk_pipeline_stop(pblk); in pblk_pipeline_stop()
1543 struct pblk_line *pblk_line_replace_data(struct pblk *pblk) in pblk_line_replace_data() argument
1545 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_replace_data()
1557 pblk_line_setup_metadata(new, l_mg, &pblk->lm); in pblk_line_replace_data()
1565 if (pblk_line_erase(pblk, new)) in pblk_line_replace_data()
1573 if (pblk_line_alloc_bitmaps(pblk, new)) in pblk_line_replace_data()
1577 if (!pblk_line_init_metadata(pblk, new, cur)) { in pblk_line_replace_data()
1578 new = pblk_line_retry(pblk, new); in pblk_line_replace_data()
1585 if (!pblk_line_init_bb(pblk, new, 1)) { in pblk_line_replace_data()
1586 new = pblk_line_retry(pblk, new); in pblk_line_replace_data()
1593 pblk_rl_free_lines_dec(&pblk->rl, new, true); in pblk_line_replace_data()
1597 l_mg->data_next = pblk_line_get(pblk); in pblk_line_replace_data()
1603 pblk_stop_writes(pblk, new); in pblk_line_replace_data()
1615 static void __pblk_line_put(struct pblk *pblk, struct pblk_line *line) in __pblk_line_put() argument
1617 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_line_put()
1618 struct pblk_gc *gc = &pblk->gc; in __pblk_line_put()
1624 pblk_err(pblk, "line %d had errors during GC\n", line->id); in __pblk_line_put()
1625 pblk_put_line_back(pblk, line); in __pblk_line_put()
1631 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in __pblk_line_put()
1637 pblk_rl_werr_line_out(&pblk->rl); in __pblk_line_put()
1649 pblk_rl_free_lines_inc(&pblk->rl, line); in __pblk_line_put()
1656 struct pblk *pblk = line_put_ws->pblk; in pblk_line_put_ws() local
1659 __pblk_line_put(pblk, line); in pblk_line_put_ws()
1660 mempool_free(line_put_ws, &pblk->gen_ws_pool); in pblk_line_put_ws()
1666 struct pblk *pblk = line->pblk; in pblk_line_put() local
1668 __pblk_line_put(pblk, line); in pblk_line_put()
1674 struct pblk *pblk = line->pblk; in pblk_line_put_wq() local
1677 line_put_ws = mempool_alloc(&pblk->gen_ws_pool, GFP_ATOMIC); in pblk_line_put_wq()
1681 line_put_ws->pblk = pblk; in pblk_line_put_wq()
1686 queue_work(pblk->r_end_wq, &line_put_ws->ws); in pblk_line_put_wq()
1689 int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr ppa) in pblk_blk_erase_async() argument
1694 rqd = pblk_alloc_rqd(pblk, PBLK_ERASE); in pblk_blk_erase_async()
1696 pblk_setup_e_rq(pblk, rqd, ppa); in pblk_blk_erase_async()
1699 rqd->private = pblk; in pblk_blk_erase_async()
1701 trace_pblk_chunk_reset(pblk_disk_name(pblk), in pblk_blk_erase_async()
1707 err = pblk_submit_io(pblk, rqd, NULL); in pblk_blk_erase_async()
1709 struct nvm_tgt_dev *dev = pblk->dev; in pblk_blk_erase_async()
1712 pblk_err(pblk, "could not async erase line:%d,blk:%d\n", in pblk_blk_erase_async()
1720 struct pblk_line *pblk_line_get_data(struct pblk *pblk) in pblk_line_get_data() argument
1722 return pblk->l_mg.data_line; in pblk_line_get_data()
1726 struct pblk_line *pblk_line_get_erase(struct pblk *pblk) in pblk_line_get_erase() argument
1728 return pblk->l_mg.data_next; in pblk_line_get_erase()
1736 static void pblk_line_should_sync_meta(struct pblk *pblk) in pblk_line_should_sync_meta() argument
1738 if (pblk_rl_is_limit(&pblk->rl)) in pblk_line_should_sync_meta()
1739 pblk_line_close_meta_sync(pblk); in pblk_line_should_sync_meta()
1742 void pblk_line_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_close() argument
1744 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_close()
1746 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_close()
1747 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_close()
1764 move_list = pblk_line_gc_list(pblk, line); in pblk_line_close()
1773 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_line_close()
1784 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_close()
1788 void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) in pblk_line_close_meta() argument
1790 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_close_meta()
1791 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_close_meta()
1797 memcpy(emeta_to_vsc(pblk, emeta_buf), l_mg->vsc_list, lm->vsc_list_len); in pblk_line_close_meta()
1800 wa->user = cpu_to_le64(atomic64_read(&pblk->user_wa)); in pblk_line_close_meta()
1801 wa->pad = cpu_to_le64(atomic64_read(&pblk->pad_wa)); in pblk_line_close_meta()
1802 wa->gc = cpu_to_le64(atomic64_read(&pblk->gc_wa)); in pblk_line_close_meta()
1807 &pblk->instance_uuid); in pblk_line_close_meta()
1813 pblk_calc_meta_header_crc(pblk, &emeta_buf->header)); in pblk_line_close_meta()
1817 emeta_buf->crc = cpu_to_le32(pblk_calc_emeta_crc(pblk, emeta_buf)); in pblk_line_close_meta()
1832 pblk_line_should_sync_meta(pblk); in pblk_line_close_meta()
1835 static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line) in pblk_save_lba_list() argument
1837 struct pblk_line_meta *lm = &pblk->lm; in pblk_save_lba_list()
1843 memcpy(w_err_gc->lba_list, emeta_to_lbas(pblk, emeta->buf), in pblk_save_lba_list()
1851 struct pblk *pblk = line_ws->pblk; in pblk_line_close_ws() local
1859 pblk_save_lba_list(pblk, line); in pblk_line_close_ws()
1861 pblk_line_close(pblk, line); in pblk_line_close_ws()
1862 mempool_free(line_ws, &pblk->gen_ws_pool); in pblk_line_close_ws()
1865 void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, in pblk_gen_run_ws() argument
1871 line_ws = mempool_alloc(&pblk->gen_ws_pool, gfp_mask); in pblk_gen_run_ws()
1873 line_ws->pblk = pblk; in pblk_gen_run_ws()
1881 static void __pblk_down_chunk(struct pblk *pblk, int pos) in __pblk_down_chunk() argument
1883 struct pblk_lun *rlun = &pblk->luns[pos]; in __pblk_down_chunk()
1893 pblk_err(pblk, "taking lun semaphore timed out: err %d\n", in __pblk_down_chunk()
1897 void pblk_down_chunk(struct pblk *pblk, struct ppa_addr ppa) in pblk_down_chunk() argument
1899 struct nvm_tgt_dev *dev = pblk->dev; in pblk_down_chunk()
1903 __pblk_down_chunk(pblk, pos); in pblk_down_chunk()
1906 void pblk_down_rq(struct pblk *pblk, struct ppa_addr ppa, in pblk_down_rq() argument
1909 struct nvm_tgt_dev *dev = pblk->dev; in pblk_down_rq()
1919 __pblk_down_chunk(pblk, pos); in pblk_down_rq()
1922 void pblk_up_chunk(struct pblk *pblk, struct ppa_addr ppa) in pblk_up_chunk() argument
1924 struct nvm_tgt_dev *dev = pblk->dev; in pblk_up_chunk()
1929 rlun = &pblk->luns[pos]; in pblk_up_chunk()
1933 void pblk_up_rq(struct pblk *pblk, unsigned long *lun_bitmap) in pblk_up_rq() argument
1935 struct nvm_tgt_dev *dev = pblk->dev; in pblk_up_rq()
1942 rlun = &pblk->luns[bit]; in pblk_up_rq()
1947 void pblk_update_map(struct pblk *pblk, sector_t lba, struct ppa_addr ppa) in pblk_update_map() argument
1952 if (!(lba < pblk->capacity)) { in pblk_update_map()
1957 spin_lock(&pblk->trans_lock); in pblk_update_map()
1958 ppa_l2p = pblk_trans_map_get(pblk, lba); in pblk_update_map()
1961 pblk_map_invalidate(pblk, ppa_l2p); in pblk_update_map()
1963 pblk_trans_map_set(pblk, lba, ppa); in pblk_update_map()
1964 spin_unlock(&pblk->trans_lock); in pblk_update_map()
1967 void pblk_update_map_cache(struct pblk *pblk, sector_t lba, struct ppa_addr ppa) in pblk_update_map_cache() argument
1973 BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa))); in pblk_update_map_cache()
1976 pblk_update_map(pblk, lba, ppa); in pblk_update_map_cache()
1979 int pblk_update_map_gc(struct pblk *pblk, sector_t lba, struct ppa_addr ppa_new, in pblk_update_map_gc() argument
1988 BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa_new))); in pblk_update_map_gc()
1992 if (!(lba < pblk->capacity)) { in pblk_update_map_gc()
1997 spin_lock(&pblk->trans_lock); in pblk_update_map_gc()
1998 ppa_l2p = pblk_trans_map_get(pblk, lba); in pblk_update_map_gc()
1999 ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, gc_line->id); in pblk_update_map_gc()
2011 pblk_trans_map_set(pblk, lba, ppa_new); in pblk_update_map_gc()
2013 spin_unlock(&pblk->trans_lock); in pblk_update_map_gc()
2017 void pblk_update_map_dev(struct pblk *pblk, sector_t lba, in pblk_update_map_dev() argument
2028 atomic64_inc(&pblk->pad_wa); in pblk_update_map_dev()
2030 atomic_long_inc(&pblk->padded_wb); in pblk_update_map_dev()
2033 pblk_map_invalidate(pblk, ppa_mapped); in pblk_update_map_dev()
2038 if (!(lba < pblk->capacity)) { in pblk_update_map_dev()
2043 spin_lock(&pblk->trans_lock); in pblk_update_map_dev()
2044 ppa_l2p = pblk_trans_map_get(pblk, lba); in pblk_update_map_dev()
2051 pblk_map_invalidate(pblk, ppa_mapped); in pblk_update_map_dev()
2059 pblk_trans_map_set(pblk, lba, ppa_mapped); in pblk_update_map_dev()
2061 spin_unlock(&pblk->trans_lock); in pblk_update_map_dev()
2064 int pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas, in pblk_lookup_l2p_seq() argument
2069 spin_lock(&pblk->trans_lock); in pblk_lookup_l2p_seq()
2073 ppa = ppas[i] = pblk_trans_map_get(pblk, blba + i); in pblk_lookup_l2p_seq()
2077 struct pblk_line *line = pblk_ppa_to_line(pblk, ppa); in pblk_lookup_l2p_seq()
2090 spin_unlock(&pblk->trans_lock); in pblk_lookup_l2p_seq()
2094 void pblk_lookup_l2p_rand(struct pblk *pblk, struct ppa_addr *ppas, in pblk_lookup_l2p_rand() argument
2100 spin_lock(&pblk->trans_lock); in pblk_lookup_l2p_rand()
2105 if (!(lba < pblk->capacity)) { in pblk_lookup_l2p_rand()
2109 ppas[i] = pblk_trans_map_get(pblk, lba); in pblk_lookup_l2p_rand()
2112 spin_unlock(&pblk->trans_lock); in pblk_lookup_l2p_rand()
2115 void *pblk_get_meta_for_writes(struct pblk *pblk, struct nvm_rq *rqd) in pblk_get_meta_for_writes() argument
2119 if (pblk_is_oob_meta_supported(pblk)) { in pblk_get_meta_for_writes()
2133 void pblk_get_packed_meta(struct pblk *pblk, struct nvm_rq *rqd) in pblk_get_packed_meta() argument
2139 if (pblk_is_oob_meta_supported(pblk)) in pblk_get_packed_meta()
2145 memcpy(pblk_get_meta(pblk, meta_list, i), in pblk_get_packed_meta()