59 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd);
119 scsi_set_blocked(struct scsi_cmnd *cmd, int reason)  in scsi_set_blocked()  argument
121 	struct Scsi_Host *host = cmd->device->host;  in scsi_set_blocked()
122 	struct scsi_device *device = cmd->device;  in scsi_set_blocked()
154 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)  in scsi_mq_requeue_cmd()  argument
156 	if (cmd->request->rq_flags & RQF_DONTPREP) {  in scsi_mq_requeue_cmd()
157 		cmd->request->rq_flags &= ~RQF_DONTPREP;  in scsi_mq_requeue_cmd()
158 		scsi_mq_uninit_cmd(cmd);  in scsi_mq_requeue_cmd()
162 	blk_mq_requeue_request(cmd->request, true);  in scsi_mq_requeue_cmd()
177 static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)  in __scsi_queue_insert()  argument
179 	struct scsi_device *device = cmd->device;  in __scsi_queue_insert()
181 	SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,  in __scsi_queue_insert()
182 		"Inserting command %p into mlqueue\n", cmd));  in __scsi_queue_insert()
184 	scsi_set_blocked(cmd, reason);  in __scsi_queue_insert()
191 		scsi_device_unbusy(device, cmd);  in __scsi_queue_insert()
199 	cmd->result = 0;  in __scsi_queue_insert()
201 	blk_mq_requeue_request(cmd->request, true);  in __scsi_queue_insert()
216 void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)  in scsi_queue_insert()  argument
218 	__scsi_queue_insert(cmd, reason, true);  in scsi_queue_insert()
240 int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,  in __scsi_execute()  argument
262 	rq->cmd_len = COMMAND_SIZE(cmd[0]);  in __scsi_execute()
263 	memcpy(rq->cmd, cmd, rq->cmd_len);  in __scsi_execute()
306 static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)  in scsi_dec_host_busy()  argument
311 	__clear_bit(SCMD_STATE_INFLIGHT, &cmd->state);  in scsi_dec_host_busy()
321 void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)  in scsi_device_unbusy()  argument
326 	scsi_dec_host_busy(shost, cmd);  in scsi_device_unbusy()
509 static void scsi_uninit_cmd(struct scsi_cmnd *cmd)  in scsi_uninit_cmd()  argument
511 	if (!blk_rq_is_passthrough(cmd->request)) {  in scsi_uninit_cmd()
512 		struct scsi_driver *drv = scsi_cmd_to_driver(cmd);  in scsi_uninit_cmd()
515 			drv->uninit_command(cmd);  in scsi_uninit_cmd()
519 void scsi_free_sgtables(struct scsi_cmnd *cmd)  in scsi_free_sgtables()  argument
521 	if (cmd->sdb.table.nents)  in scsi_free_sgtables()
522 		sg_free_table_chained(&cmd->sdb.table,  in scsi_free_sgtables()
524 	if (scsi_prot_sg_count(cmd))  in scsi_free_sgtables()
525 		sg_free_table_chained(&cmd->prot_sdb->table,  in scsi_free_sgtables()
530 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)  in scsi_mq_uninit_cmd()  argument
532 	scsi_free_sgtables(cmd);  in scsi_mq_uninit_cmd()
533 	scsi_uninit_cmd(cmd);  in scsi_mq_uninit_cmd()
566 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);  in scsi_end_request()  local
567 	struct scsi_device *sdev = cmd->device;  in scsi_end_request()
577 		WARN_ON_ONCE(!(cmd->flags & SCMD_INITIALIZED));  in scsi_end_request()
578 		cmd->flags &= ~SCMD_INITIALIZED;  in scsi_end_request()
587 	destroy_rcu_head(&cmd->rcu);  in scsi_end_request()
596 	scsi_mq_uninit_cmd(cmd);  in scsi_end_request()
620 static blk_status_t scsi_result_to_blk_status(struct scsi_cmnd *cmd, int result)  in scsi_result_to_blk_status()  argument
635 		set_host_byte(cmd, DID_OK);  in scsi_result_to_blk_status()
638 		set_host_byte(cmd, DID_OK);  in scsi_result_to_blk_status()
641 		set_host_byte(cmd, DID_OK);  in scsi_result_to_blk_status()
644 		set_host_byte(cmd, DID_OK);  in scsi_result_to_blk_status()
652 static void scsi_io_completion_reprep(struct scsi_cmnd *cmd,  in scsi_io_completion_reprep()  argument
656 	scsi_mq_requeue_cmd(cmd);  in scsi_io_completion_reprep()
659 static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd)  in scsi_cmd_runtime_exceeced()  argument
661 	struct request *req = cmd->request;  in scsi_cmd_runtime_exceeced()
664 	if (cmd->allowed == SCSI_CMD_RETRIES_NO_LIMIT)  in scsi_cmd_runtime_exceeced()
667 	wait_for = (cmd->allowed + 1) * req->timeout;  in scsi_cmd_runtime_exceeced()
668 	if (time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {  in scsi_cmd_runtime_exceeced()
669 		scmd_printk(KERN_ERR, cmd, "timing out command, waited %lus\n",  in scsi_cmd_runtime_exceeced()
677 static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)  in scsi_io_completion_action()  argument
679 	struct request_queue *q = cmd->device->request_queue;  in scsi_io_completion_action()
680 	struct request *req = cmd->request;  in scsi_io_completion_action()
689 	sense_valid = scsi_command_normalize_sense(cmd, &sshdr);  in scsi_io_completion_action()
693 	blk_stat = scsi_result_to_blk_status(cmd, result);  in scsi_io_completion_action()
704 			if (cmd->device->removable) {  in scsi_io_completion_action()
708 				cmd->device->changed = 1;  in scsi_io_completion_action()
728 			if ((cmd->device->use_10_for_rw &&  in scsi_io_completion_action()
730 			    (cmd->cmnd[0] == READ_10 ||  in scsi_io_completion_action()
731 			     cmd->cmnd[0] == WRITE_10)) {  in scsi_io_completion_action()
733 				cmd->device->use_10_for_rw = 0;  in scsi_io_completion_action()
798 	if (action != ACTION_FAIL && scsi_cmd_runtime_exceeced(cmd))  in scsi_io_completion_action()
819 				scsi_print_result(cmd, NULL, FAILED);  in scsi_io_completion_action()
821 					scsi_print_sense(cmd);  in scsi_io_completion_action()
822 				scsi_print_command(cmd);  in scsi_io_completion_action()
829 		scsi_io_completion_reprep(cmd, q);  in scsi_io_completion_action()
833 		__scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, false);  in scsi_io_completion_action()
837 		__scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, false);  in scsi_io_completion_action()
847 static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result,  in scsi_io_completion_nz_result()  argument
852 	struct request *req = cmd->request;  in scsi_io_completion_nz_result()
855 	sense_valid = scsi_command_normalize_sense(cmd, &sshdr);  in scsi_io_completion_nz_result()
865 				min(8 + cmd->sense_buffer[7],  in scsi_io_completion_nz_result()
869 			*blk_statp = scsi_result_to_blk_status(cmd, result);  in scsi_io_completion_nz_result()
876 		*blk_statp = scsi_result_to_blk_status(cmd, result);  in scsi_io_completion_nz_result()
896 			scsi_print_sense(cmd);  in scsi_io_completion_nz_result()
937 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)  in scsi_io_completion()  argument
939 	int result = cmd->result;  in scsi_io_completion()
940 	struct request_queue *q = cmd->device->request_queue;  in scsi_io_completion()
941 	struct request *req = cmd->request;  in scsi_io_completion()
945 		result = scsi_io_completion_nz_result(cmd, result, &blk_stat);  in scsi_io_completion()
951 		scsi_req(req)->result = cmd->result;  in scsi_io_completion()
958 	SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd,  in scsi_io_completion()
972 	if (unlikely(blk_stat && scsi_noretry_cmd(cmd))) {  in scsi_io_completion()
984 		scsi_io_completion_reprep(cmd, q);  in scsi_io_completion()
986 		scsi_io_completion_action(cmd, result);  in scsi_io_completion()
1006 blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd)  in scsi_alloc_sgtables()  argument
1008 	struct scsi_device *sdev = cmd->device;  in scsi_alloc_sgtables()
1009 	struct request *rq = cmd->request;  in scsi_alloc_sgtables()
1029 	if (unlikely(sg_alloc_table_chained(&cmd->sdb.table, nr_segs,  in scsi_alloc_sgtables()
1030 			cmd->sdb.table.sgl, SCSI_INLINE_SG_CNT)))  in scsi_alloc_sgtables()
1037 	count = __blk_rq_map_sg(rq->q, rq, cmd->sdb.table.sgl, &last_sg);  in scsi_alloc_sgtables()
1044 		cmd->extra_len += pad_len;  in scsi_alloc_sgtables()
1053 		cmd->extra_len += sdev->dma_drain_len;  in scsi_alloc_sgtables()
1057 	BUG_ON(count > cmd->sdb.table.nents);  in scsi_alloc_sgtables()
1058 	cmd->sdb.table.nents = count;  in scsi_alloc_sgtables()
1059 	cmd->sdb.length = blk_rq_payload_bytes(rq);  in scsi_alloc_sgtables()
1062 		struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;  in scsi_alloc_sgtables()
1089 		cmd->prot_sdb = prot_sdb;  in scsi_alloc_sgtables()
1090 		cmd->prot_sdb->table.nents = count;  in scsi_alloc_sgtables()
1095 	scsi_free_sgtables(cmd);  in scsi_alloc_sgtables()
1113 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);  in scsi_initialize_rq()  local
1115 	scsi_req_init(&cmd->req);  in scsi_initialize_rq()
1116 	init_rcu_head(&cmd->rcu);  in scsi_initialize_rq()
1117 	cmd->jiffies_at_alloc = jiffies;  in scsi_initialize_rq()
1118 	cmd->retries = 0;  in scsi_initialize_rq()
1134 void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)  in scsi_init_command()  argument
1136 	void *buf = cmd->sense_buffer;  in scsi_init_command()
1137 	void *prot = cmd->prot_sdb;  in scsi_init_command()
1138 	struct request *rq = blk_mq_rq_from_pdu(cmd);  in scsi_init_command()
1139 	unsigned int flags = cmd->flags & SCMD_PRESERVED_FLAGS;  in scsi_init_command()
1149 	jiffies_at_alloc = cmd->jiffies_at_alloc;  in scsi_init_command()
1150 	retries = cmd->retries;  in scsi_init_command()
1151 	in_flight = test_bit(SCMD_STATE_INFLIGHT, &cmd->state);  in scsi_init_command()
1157 	to_clear = sizeof(*cmd) - sizeof(cmd->req);  in scsi_init_command()
1160 	memset((char *)cmd + sizeof(cmd->req), 0, to_clear);  in scsi_init_command()
1162 	cmd->device = dev;  in scsi_init_command()
1163 	cmd->sense_buffer = buf;  in scsi_init_command()
1164 	cmd->prot_sdb = prot;  in scsi_init_command()
1165 	cmd->flags = flags;  in scsi_init_command()
1166 	INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);  in scsi_init_command()
1167 	cmd->jiffies_at_alloc = jiffies_at_alloc;  in scsi_init_command()
1168 	cmd->retries = retries;  in scsi_init_command()
1170 		__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);  in scsi_init_command()
1177 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);  in scsi_setup_scsi_cmnd()  local
1186 		blk_status_t ret = scsi_alloc_sgtables(cmd);  in scsi_setup_scsi_cmnd()
1192 		memset(&cmd->sdb, 0, sizeof(cmd->sdb));  in scsi_setup_scsi_cmnd()
1195 	cmd->cmd_len = scsi_req(req)->cmd_len;  in scsi_setup_scsi_cmnd()
1196 	cmd->cmnd = scsi_req(req)->cmd;  in scsi_setup_scsi_cmnd()
1197 	cmd->transfersize = blk_rq_bytes(req);  in scsi_setup_scsi_cmnd()
1198 	cmd->allowed = scsi_req(req)->retries;  in scsi_setup_scsi_cmnd()
1347 				   struct scsi_cmnd *cmd)  in scsi_host_queue_ready()  argument
1378 	__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);  in scsi_host_queue_ready()
1388 	scsi_dec_host_busy(shost, cmd);  in scsi_host_queue_ready()
1428 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);  in scsi_softirq_done()  local
1431 	INIT_LIST_HEAD(&cmd->eh_entry);  in scsi_softirq_done()
1433 	atomic_inc(&cmd->device->iodone_cnt);  in scsi_softirq_done()
1434 	if (cmd->result)  in scsi_softirq_done()
1435 		atomic_inc(&cmd->device->ioerr_cnt);  in scsi_softirq_done()
1437 	disposition = scsi_decide_disposition(cmd);  in scsi_softirq_done()
1438 	if (disposition != SUCCESS && scsi_cmd_runtime_exceeced(cmd))  in scsi_softirq_done()
1441 	scsi_log_completion(cmd, disposition);  in scsi_softirq_done()
1445 		scsi_finish_command(cmd);  in scsi_softirq_done()
1448 		scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);  in scsi_softirq_done()
1451 		scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);  in scsi_softirq_done()
1454 		scsi_eh_scmd_add(cmd);  in scsi_softirq_done()
1466 static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)  in scsi_dispatch_cmd()  argument
1468 	struct Scsi_Host *host = cmd->device->host;  in scsi_dispatch_cmd()
1471 	atomic_inc(&cmd->device->iorequest_cnt);  in scsi_dispatch_cmd()
1474 	if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {  in scsi_dispatch_cmd()
1478 		cmd->result = DID_NO_CONNECT << 16;  in scsi_dispatch_cmd()
1483 	if (unlikely(scsi_device_blocked(cmd->device))) {  in scsi_dispatch_cmd()
1491 		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,  in scsi_dispatch_cmd()
1497 	if (cmd->device->lun_in_cdb)  in scsi_dispatch_cmd()
1498 		cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) |  in scsi_dispatch_cmd()
1499 			       (cmd->device->lun << 5 & 0xe0);  in scsi_dispatch_cmd()
1501 	scsi_log_send(cmd);  in scsi_dispatch_cmd()
1507 	if (cmd->cmd_len > cmd->device->host->max_cmd_len) {  in scsi_dispatch_cmd()
1508 		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,  in scsi_dispatch_cmd()
1511 			       cmd->cmd_len, cmd->device->host->max_cmd_len));  in scsi_dispatch_cmd()
1512 		cmd->result = (DID_ABORT << 16);  in scsi_dispatch_cmd()
1517 		cmd->result = (DID_NO_CONNECT << 16);  in scsi_dispatch_cmd()
1522 	trace_scsi_dispatch_cmd_start(cmd);  in scsi_dispatch_cmd()
1523 	rtn = host->hostt->queuecommand(host, cmd);  in scsi_dispatch_cmd()
1525 		trace_scsi_dispatch_cmd_error(cmd, rtn);  in scsi_dispatch_cmd()
1530 		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,  in scsi_dispatch_cmd()
1536 	cmd->scsi_done(cmd);  in scsi_dispatch_cmd()
1549 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);  in scsi_prepare_cmd()  local
1554 	scsi_init_command(sdev, cmd);  in scsi_prepare_cmd()
1556 	cmd->request = req;  in scsi_prepare_cmd()
1557 	cmd->tag = req->tag;  in scsi_prepare_cmd()
1558 	cmd->prot_op = SCSI_PROT_NORMAL;  in scsi_prepare_cmd()
1560 		cmd->sc_data_direction = rq_dma_dir(req);  in scsi_prepare_cmd()
1562 		cmd->sc_data_direction = DMA_NONE;  in scsi_prepare_cmd()
1564 	sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;  in scsi_prepare_cmd()
1565 	cmd->sdb.table.sgl = sg;  in scsi_prepare_cmd()
1568 		memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer));  in scsi_prepare_cmd()
1570 		cmd->prot_sdb->table.sgl =  in scsi_prepare_cmd()
1571 			(struct scatterlist *)(cmd->prot_sdb + 1);  in scsi_prepare_cmd()
1588 	cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd;  in scsi_prepare_cmd()
1589 	memset(cmd->cmnd, 0, BLK_MAX_CDB);  in scsi_prepare_cmd()
1590 	return scsi_cmd_to_driver(cmd)->init_command(cmd);  in scsi_prepare_cmd()
1593 static void scsi_mq_done(struct scsi_cmnd *cmd)  in scsi_mq_done()  argument
1595 	if (unlikely(blk_should_fake_timeout(cmd->request->q)))  in scsi_mq_done()
1597 	if (unlikely(test_and_set_bit(SCMD_STATE_COMPLETE, &cmd->state)))  in scsi_mq_done()
1599 	trace_scsi_dispatch_cmd_done(cmd);  in scsi_mq_done()
1600 	blk_mq_complete_request(cmd->request);  in scsi_mq_done()
1647 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);  in scsi_queue_rq()  local
1664 	if (!scsi_host_queue_ready(q, shost, sdev, cmd))  in scsi_queue_rq()
1673 		clear_bit(SCMD_STATE_COMPLETE, &cmd->state);  in scsi_queue_rq()
1676 	cmd->flags &= SCMD_PRESERVED_FLAGS;  in scsi_queue_rq()
1678 		cmd->flags |= SCMD_TAGGED;  in scsi_queue_rq()
1680 		cmd->flags |= SCMD_LAST;  in scsi_queue_rq()
1682 	scsi_set_resid(cmd, 0);  in scsi_queue_rq()
1683 	memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);  in scsi_queue_rq()
1684 	cmd->scsi_done = scsi_mq_done;  in scsi_queue_rq()
1687 	reason = scsi_dispatch_cmd(cmd);  in scsi_queue_rq()
1689 		scsi_set_blocked(cmd, reason);  in scsi_queue_rq()
1697 	scsi_dec_host_busy(shost, cmd);  in scsi_queue_rq()
1722 			scsi_mq_uninit_cmd(cmd);  in scsi_queue_rq()
1742 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);  in scsi_mq_init_request()  local
1747 		cmd->flags |= SCMD_UNCHECKED_ISA_DMA;  in scsi_mq_init_request()
1748 	cmd->sense_buffer = scsi_alloc_sense_buffer(unchecked_isa_dma,  in scsi_mq_init_request()
1750 	if (!cmd->sense_buffer)  in scsi_mq_init_request()
1752 	cmd->req.sense = cmd->sense_buffer;  in scsi_mq_init_request()
1755 		sg = (void *)cmd + sizeof(struct scsi_cmnd) +  in scsi_mq_init_request()
1757 		cmd->prot_sdb = (void *)sg + scsi_mq_inline_sgl_size(shost);  in scsi_mq_init_request()
1761 		ret = shost->hostt->init_cmd_priv(shost, cmd);  in scsi_mq_init_request()
1764 					       cmd->sense_buffer);  in scsi_mq_init_request()
1774 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);  in scsi_mq_exit_request()  local
1777 		shost->hostt->exit_cmd_priv(shost, cmd);  in scsi_mq_exit_request()
1778 	scsi_free_sense_buffer(cmd->flags & SCMD_UNCHECKED_ISA_DMA,  in scsi_mq_exit_request()
1779 			       cmd->sense_buffer);  in scsi_mq_exit_request()
2007 	unsigned char cmd[10];  in scsi_mode_select()  local
2011 	memset(cmd, 0, sizeof(cmd));  in scsi_mode_select()
2012 	cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);  in scsi_mode_select()
2031 		cmd[0] = MODE_SELECT_10;  in scsi_mode_select()
2032 		cmd[7] = len >> 8;  in scsi_mode_select()
2033 		cmd[8] = len;  in scsi_mode_select()
2049 		cmd[0] = MODE_SELECT;  in scsi_mode_select()
2050 		cmd[4] = len;  in scsi_mode_select()
2053 	ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,  in scsi_mode_select()
2080 	unsigned char cmd[12];  in scsi_mode_sense()  local
2087 	memset(&cmd[0], 0, 12);  in scsi_mode_sense()
2090 	cmd[1] = dbd & 0x18;	/* allows DBD and LLBA bits */  in scsi_mode_sense()
2091 	cmd[2] = modepage;  in scsi_mode_sense()
2104 		cmd[0] = MODE_SENSE_10;  in scsi_mode_sense()
2105 		cmd[8] = len;  in scsi_mode_sense()
2111 		cmd[0] = MODE_SENSE;  in scsi_mode_sense()
2112 		cmd[4] = len;  in scsi_mode_sense()
2118 	result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,  in scsi_mode_sense()
2193 	char cmd[] = {  in scsi_test_unit_ready()  local
2200 		result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,  in scsi_test_unit_ready()