134 void iscsi_prep_data_out_pdu(struct iscsi_task *task, struct iscsi_r2t_info *r2t,  in iscsi_prep_data_out_pdu()  argument
137 	struct iscsi_conn *conn = task->conn;  in iscsi_prep_data_out_pdu()
140 	task->hdr_len = sizeof(struct iscsi_data);  in iscsi_prep_data_out_pdu()
147 	hdr->lun = task->lun;  in iscsi_prep_data_out_pdu()
148 	hdr->itt = task->hdr_itt;  in iscsi_prep_data_out_pdu()
164 static int iscsi_add_hdr(struct iscsi_task *task, unsigned len)  in iscsi_add_hdr()  argument
166 	unsigned exp_len = task->hdr_len + len;  in iscsi_add_hdr()
168 	if (exp_len > task->hdr_max) {  in iscsi_add_hdr()
174 	task->hdr_len = exp_len;  in iscsi_add_hdr()
181 static int iscsi_prep_ecdb_ahs(struct iscsi_task *task)  in iscsi_prep_ecdb_ahs()  argument
183 	struct scsi_cmnd *cmd = task->sc;  in iscsi_prep_ecdb_ahs()
189 	ecdb_ahdr = iscsi_next_hdr(task);  in iscsi_prep_ecdb_ahs()
197 	rc = iscsi_add_hdr(task, sizeof(ecdb_ahdr->ahslength) +  in iscsi_prep_ecdb_ahs()
210 	ISCSI_DBG_SESSION(task->conn->session,  in iscsi_prep_ecdb_ahs()
214 		          task->hdr_len);  in iscsi_prep_ecdb_ahs()
231 static int iscsi_check_tmf_restrictions(struct iscsi_task *task, int opcode)  in iscsi_check_tmf_restrictions()  argument
233 	struct iscsi_session *session = task->conn->session;  in iscsi_check_tmf_restrictions()
249 		if (hdr_lun != task->sc->device->lun)  in iscsi_check_tmf_restrictions()
259 					     opcode, task->itt, task->hdr_itt);  in iscsi_check_tmf_restrictions()
269 					     opcode, task->itt, task->hdr_itt);  in iscsi_check_tmf_restrictions()
281 		    task->hdr_itt == tmf->rtt) {  in iscsi_check_tmf_restrictions()
285 					  "progress\n", task->itt,  in iscsi_check_tmf_restrictions()
286 					  task->hdr_itt);  in iscsi_check_tmf_restrictions()
302 static int iscsi_prep_scsi_cmd_pdu(struct iscsi_task *task)  in iscsi_prep_scsi_cmd_pdu()  argument
304 	struct iscsi_conn *conn = task->conn;  in iscsi_prep_scsi_cmd_pdu()
306 	struct scsi_cmnd *sc = task->sc;  in iscsi_prep_scsi_cmd_pdu()
312 	rc = iscsi_check_tmf_restrictions(task, ISCSI_OP_SCSI_CMD);  in iscsi_prep_scsi_cmd_pdu()
317 		rc = conn->session->tt->alloc_pdu(task, ISCSI_OP_SCSI_CMD);  in iscsi_prep_scsi_cmd_pdu()
321 	hdr = (struct iscsi_scsi_req *)task->hdr;  in iscsi_prep_scsi_cmd_pdu()
326 		hdr->itt = task->hdr_itt = itt;  in iscsi_prep_scsi_cmd_pdu()
328 		hdr->itt = task->hdr_itt = build_itt(task->itt,  in iscsi_prep_scsi_cmd_pdu()
329 						     task->conn->session->age);  in iscsi_prep_scsi_cmd_pdu()
330 	task->hdr_len = 0;  in iscsi_prep_scsi_cmd_pdu()
331 	rc = iscsi_add_hdr(task, sizeof(*hdr));  in iscsi_prep_scsi_cmd_pdu()
337 	task->lun = hdr->lun;  in iscsi_prep_scsi_cmd_pdu()
343 		rc = iscsi_prep_ecdb_ahs(task);  in iscsi_prep_scsi_cmd_pdu()
350 	task->imm_count = 0;  in iscsi_prep_scsi_cmd_pdu()
352 		task->protected = true;  in iscsi_prep_scsi_cmd_pdu()
357 		struct iscsi_r2t_info *r2t = &task->unsol_r2t;  in iscsi_prep_scsi_cmd_pdu()
378 				task->imm_count = min(session->first_burst,  in iscsi_prep_scsi_cmd_pdu()
381 				task->imm_count = min(transfer_length,  in iscsi_prep_scsi_cmd_pdu()
383 			hton24(hdr->dlength, task->imm_count);  in iscsi_prep_scsi_cmd_pdu()
390 					       task->imm_count;  in iscsi_prep_scsi_cmd_pdu()
391 			r2t->data_offset = task->imm_count;  in iscsi_prep_scsi_cmd_pdu()
396 		if (!task->unsol_r2t.data_length)  in iscsi_prep_scsi_cmd_pdu()
408 	hdrlength = task->hdr_len - sizeof(*hdr);  in iscsi_prep_scsi_cmd_pdu()
415 	hdr->cmdsn = task->cmdsn = cpu_to_be32(session->cmdsn);  in iscsi_prep_scsi_cmd_pdu()
417 	if (session->tt->init_task && session->tt->init_task(task))  in iscsi_prep_scsi_cmd_pdu()
420 	task->state = ISCSI_TASK_RUNNING;  in iscsi_prep_scsi_cmd_pdu()
428 			  task->itt, transfer_length,  in iscsi_prep_scsi_cmd_pdu()
442 static void iscsi_free_task(struct iscsi_task *task)  in iscsi_free_task()  argument
444 	struct iscsi_conn *conn = task->conn;  in iscsi_free_task()
446 	struct scsi_cmnd *sc = task->sc;  in iscsi_free_task()
447 	int oldstate = task->state;  in iscsi_free_task()
450 			  task->itt, task->state, task->sc);  in iscsi_free_task()
452 	session->tt->cleanup_task(task);  in iscsi_free_task()
453 	task->state = ISCSI_TASK_FREE;  in iscsi_free_task()
454 	task->sc = NULL;  in iscsi_free_task()
458 	if (conn->login_task == task)  in iscsi_free_task()
461 	kfifo_in(&session->cmdpool.queue, (void*)&task, sizeof(void*));  in iscsi_free_task()
475 void __iscsi_get_task(struct iscsi_task *task)  in __iscsi_get_task()  argument
477 	refcount_inc(&task->refcount);  in __iscsi_get_task()
481 void __iscsi_put_task(struct iscsi_task *task)  in __iscsi_put_task()  argument
483 	if (refcount_dec_and_test(&task->refcount))  in __iscsi_put_task()
484 		iscsi_free_task(task);  in __iscsi_put_task()
488 void iscsi_put_task(struct iscsi_task *task)  in iscsi_put_task()  argument
490 	struct iscsi_session *session = task->conn->session;  in iscsi_put_task()
494 	__iscsi_put_task(task);  in iscsi_put_task()
506 static void iscsi_complete_task(struct iscsi_task *task, int state)  in iscsi_complete_task()  argument
508 	struct iscsi_conn *conn = task->conn;  in iscsi_complete_task()
512 			  task->itt, task->state, task->sc);  in iscsi_complete_task()
513 	if (task->state == ISCSI_TASK_COMPLETED ||  in iscsi_complete_task()
514 	    task->state == ISCSI_TASK_ABRT_TMF ||  in iscsi_complete_task()
515 	    task->state == ISCSI_TASK_ABRT_SESS_RECOV ||  in iscsi_complete_task()
516 	    task->state == ISCSI_TASK_REQUEUE_SCSIQ)  in iscsi_complete_task()
518 	WARN_ON_ONCE(task->state == ISCSI_TASK_FREE);  in iscsi_complete_task()
519 	task->state = state;  in iscsi_complete_task()
522 	if (!list_empty(&task->running)) {  in iscsi_complete_task()
524 		list_del_init(&task->running);  in iscsi_complete_task()
528 	if (conn->task == task)  in iscsi_complete_task()
529 		conn->task = NULL;  in iscsi_complete_task()
531 	if (READ_ONCE(conn->ping_task) == task)  in iscsi_complete_task()
535 	__iscsi_put_task(task);  in iscsi_complete_task()
549 void iscsi_complete_scsi_task(struct iscsi_task *task,  in iscsi_complete_scsi_task()  argument
552 	struct iscsi_conn *conn = task->conn;  in iscsi_complete_scsi_task()
554 	ISCSI_DBG_SESSION(conn->session, "[itt 0x%x]\n", task->itt);  in iscsi_complete_scsi_task()
558 	iscsi_complete_task(task, ISCSI_TASK_COMPLETED);  in iscsi_complete_scsi_task()
568 static void fail_scsi_task(struct iscsi_task *task, int err)  in fail_scsi_task()  argument
570 	struct iscsi_conn *conn = task->conn;  in fail_scsi_task()
579 	sc = task->sc;  in fail_scsi_task()
583 	if (task->state == ISCSI_TASK_PENDING) {  in fail_scsi_task()
601 	iscsi_complete_task(task, state);  in fail_scsi_task()
606 				struct iscsi_task *task)  in iscsi_prep_mgmt_task()  argument
609 	struct iscsi_hdr *hdr = task->hdr;  in iscsi_prep_mgmt_task()
638 	if (session->tt->init_task && session->tt->init_task(task))  in iscsi_prep_mgmt_task()
644 	task->state = ISCSI_TASK_RUNNING;  in iscsi_prep_mgmt_task()
647 			  hdr->itt, task->data_count);  in iscsi_prep_mgmt_task()
658 	struct iscsi_task *task;  in __iscsi_conn_send_pdu()  local
682 		task = conn->login_task;  in __iscsi_conn_send_pdu()
696 				 (void*)&task, sizeof(void*)))  in __iscsi_conn_send_pdu()
704 	refcount_set(&task->refcount, 1);  in __iscsi_conn_send_pdu()
705 	task->conn = conn;  in __iscsi_conn_send_pdu()
706 	task->sc = NULL;  in __iscsi_conn_send_pdu()
707 	INIT_LIST_HEAD(&task->running);  in __iscsi_conn_send_pdu()
708 	task->state = ISCSI_TASK_PENDING;  in __iscsi_conn_send_pdu()
711 		memcpy(task->data, data, data_size);  in __iscsi_conn_send_pdu()
712 		task->data_count = data_size;  in __iscsi_conn_send_pdu()
714 		task->data_count = 0;  in __iscsi_conn_send_pdu()
717 		if (conn->session->tt->alloc_pdu(task, hdr->opcode)) {  in __iscsi_conn_send_pdu()
724 	itt = task->hdr->itt;  in __iscsi_conn_send_pdu()
725 	task->hdr_len = sizeof(struct iscsi_hdr);  in __iscsi_conn_send_pdu()
726 	memcpy(task->hdr, hdr, sizeof(struct iscsi_hdr));  in __iscsi_conn_send_pdu()
730 			task->hdr->itt = itt;  in __iscsi_conn_send_pdu()
732 			task->hdr->itt = build_itt(task->itt,  in __iscsi_conn_send_pdu()
733 						   task->conn->session->age);  in __iscsi_conn_send_pdu()
737 		WRITE_ONCE(conn->ping_task, task);  in __iscsi_conn_send_pdu()
740 		if (iscsi_prep_mgmt_task(conn, task))  in __iscsi_conn_send_pdu()
743 		if (session->tt->xmit_task(task))  in __iscsi_conn_send_pdu()
747 		list_add_tail(&task->running, &conn->mgmtqueue);  in __iscsi_conn_send_pdu()
752 	return task;  in __iscsi_conn_send_pdu()
757 	__iscsi_put_task(task);  in __iscsi_conn_send_pdu()
789 			       struct iscsi_task *task, char *data,  in iscsi_scsi_cmd_rsp()  argument
794 	struct scsi_cmnd *sc = task->sc;  in iscsi_scsi_cmd_rsp()
801 	if (task->protected) {  in iscsi_scsi_cmd_rsp()
812 		ascq = session->tt->check_protection(task, &sector);  in iscsi_scsi_cmd_rsp()
872 			  sc, sc->result, task->itt);  in iscsi_scsi_cmd_rsp()
874 	iscsi_complete_task(task, ISCSI_TASK_COMPLETED);  in iscsi_scsi_cmd_rsp()
888 		  struct iscsi_task *task)  in iscsi_data_in_rsp()  argument
891 	struct scsi_cmnd *sc = task->sc;  in iscsi_data_in_rsp()
913 			  sc, sc->result, task->itt);  in iscsi_data_in_rsp()
915 	iscsi_complete_task(task, ISCSI_TASK_COMPLETED);  in iscsi_data_in_rsp()
941 	struct iscsi_task *task;  in iscsi_send_nopout()  local
960 	task = __iscsi_conn_send_pdu(conn, (struct iscsi_hdr *)&hdr, NULL, 0);  in iscsi_send_nopout()
961 	if (!task) {  in iscsi_send_nopout()
984 static int iscsi_nop_out_rsp(struct iscsi_task *task,  in iscsi_nop_out_rsp()  argument
987 	struct iscsi_conn *conn = task->conn;  in iscsi_nop_out_rsp()
990 	if (READ_ONCE(conn->ping_task) != task) {  in iscsi_nop_out_rsp()
1000 	iscsi_complete_task(task, ISCSI_TASK_COMPLETED);  in iscsi_nop_out_rsp()
1060 			struct iscsi_task *task;  in iscsi_handle_reject()  local
1065 			task = iscsi_itt_to_task(conn, rejected_pdu.itt);  in iscsi_handle_reject()
1066 			if (!task) {  in iscsi_handle_reject()
1072 				rc = iscsi_nop_out_rsp(task,  in iscsi_handle_reject()
1132 	struct iscsi_task *task;  in __iscsi_complete_pdu()  local
1186 		task = iscsi_itt_to_ctask(conn, hdr->itt);  in __iscsi_complete_pdu()
1187 		if (!task)  in __iscsi_complete_pdu()
1189 		task->last_xfer = jiffies;  in __iscsi_complete_pdu()
1201 		task = iscsi_itt_to_task(conn, hdr->itt);  in __iscsi_complete_pdu()
1202 		if (!task)  in __iscsi_complete_pdu()
1211 		iscsi_scsi_cmd_rsp(conn, hdr, task, data, datalen);  in __iscsi_complete_pdu()
1214 		iscsi_data_in_rsp(conn, hdr, task);  in __iscsi_complete_pdu()
1240 		iscsi_complete_task(task, ISCSI_TASK_COMPLETED);  in __iscsi_complete_pdu()
1250 		rc = iscsi_nop_out_rsp(task, (struct iscsi_nopin*)hdr,  in __iscsi_complete_pdu()
1263 	iscsi_complete_task(task, ISCSI_TASK_COMPLETED);  in __iscsi_complete_pdu()
1323 	struct iscsi_task *task;  in iscsi_itt_to_ctask()  local
1328 	task = iscsi_itt_to_task(conn, itt);  in iscsi_itt_to_ctask()
1329 	if (!task || !task->sc)  in iscsi_itt_to_ctask()
1332 	if (task->sc->SCp.phase != conn->session->age) {  in iscsi_itt_to_ctask()
1335 				  task->sc->SCp.phase, conn->session->age);  in iscsi_itt_to_ctask()
1339 	return task;  in iscsi_itt_to_ctask()
1418 	struct iscsi_task *task = conn->task;  in iscsi_xmit_task()  local
1425 	if (conn->task == NULL) {  in iscsi_xmit_task()
1429 	__iscsi_get_task(task);  in iscsi_xmit_task()
1432 	rc = conn->session->tt->xmit_task(task);  in iscsi_xmit_task()
1436 		task->last_xfer = jiffies;  in iscsi_xmit_task()
1437 		conn->task = NULL;  in iscsi_xmit_task()
1441 	__iscsi_put_task(task);  in iscsi_xmit_task()
1454 void iscsi_requeue_task(struct iscsi_task *task)  in iscsi_requeue_task()  argument
1456 	struct iscsi_conn *conn = task->conn;  in iscsi_requeue_task()
1463 	if (list_empty(&task->running))  in iscsi_requeue_task()
1464 		list_add_tail(&task->running, &conn->requeue);  in iscsi_requeue_task()
1481 	struct iscsi_task *task;  in iscsi_data_xmit()  local
1491 	if (conn->task) {  in iscsi_data_xmit()
1505 		conn->task = list_entry(conn->mgmtqueue.next,  in iscsi_data_xmit()
1507 		list_del_init(&conn->task->running);  in iscsi_data_xmit()
1509 		if (iscsi_prep_mgmt_task(conn, conn->task)) {  in iscsi_data_xmit()
1512 			__iscsi_put_task(conn->task);  in iscsi_data_xmit()
1514 			conn->task = NULL;  in iscsi_data_xmit()
1526 		conn->task = list_entry(conn->cmdqueue.next, struct iscsi_task,  in iscsi_data_xmit()
1528 		list_del_init(&conn->task->running);  in iscsi_data_xmit()
1531 			fail_scsi_task(conn->task, DID_IMM_RETRY);  in iscsi_data_xmit()
1535 		rc = iscsi_prep_scsi_cmd_pdu(conn->task);  in iscsi_data_xmit()
1538 				fail_scsi_task(conn->task, DID_IMM_RETRY);  in iscsi_data_xmit()
1540 				fail_scsi_task(conn->task, DID_ABORT);  in iscsi_data_xmit()
1564 		task = list_entry(conn->requeue.next, struct iscsi_task,  in iscsi_data_xmit()
1566 		if (iscsi_check_tmf_restrictions(task, ISCSI_OP_SCSI_DATA_OUT))  in iscsi_data_xmit()
1569 		conn->task = task;  in iscsi_data_xmit()
1570 		list_del_init(&conn->task->running);  in iscsi_data_xmit()
1571 		conn->task->state = ISCSI_TASK_RUNNING;  in iscsi_data_xmit()
1605 	struct iscsi_task *task;  in iscsi_alloc_task()  local
1608 			 (void *) &task, sizeof(void *)))  in iscsi_alloc_task()
1612 	sc->SCp.ptr = (char *) task;  in iscsi_alloc_task()
1614 	refcount_set(&task->refcount, 1);  in iscsi_alloc_task()
1615 	task->state = ISCSI_TASK_PENDING;  in iscsi_alloc_task()
1616 	task->conn = conn;  in iscsi_alloc_task()
1617 	task->sc = sc;  in iscsi_alloc_task()
1618 	task->have_checked_conn = false;  in iscsi_alloc_task()
1619 	task->last_timeout = jiffies;  in iscsi_alloc_task()
1620 	task->last_xfer = jiffies;  in iscsi_alloc_task()
1621 	task->protected = false;  in iscsi_alloc_task()
1622 	INIT_LIST_HEAD(&task->running);  in iscsi_alloc_task()
1623 	return task;  in iscsi_alloc_task()
1646 	struct iscsi_task *task = NULL;  in iscsi_queuecommand()  local
1723 	task = iscsi_alloc_task(conn, sc);  in iscsi_queuecommand()
1724 	if (!task) {  in iscsi_queuecommand()
1730 		reason = iscsi_prep_scsi_cmd_pdu(task);  in iscsi_queuecommand()
1740 		if (session->tt->xmit_task(task)) {  in iscsi_queuecommand()
1747 		list_add_tail(&task->running, &conn->cmdqueue);  in iscsi_queuecommand()
1758 	iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);  in iscsi_queuecommand()
1768 	iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);  in iscsi_queuecommand()
1810 	struct iscsi_task *task;  in iscsi_exec_task_mgmt_fn()  local
1812 	task = __iscsi_conn_send_pdu(conn, (struct iscsi_hdr *)hdr,  in iscsi_exec_task_mgmt_fn()
1814 	if (!task) {  in iscsi_exec_task_mgmt_fn()
1859 	struct iscsi_task *task;  in fail_scsi_tasks()  local
1863 		task = conn->session->cmds[i];  in fail_scsi_tasks()
1864 		if (!task->sc || task->state == ISCSI_TASK_FREE)  in fail_scsi_tasks()
1867 		if (lun != -1 && lun != task->sc->device->lun)  in fail_scsi_tasks()
1872 				  task->sc, task->itt, task->state);  in fail_scsi_tasks()
1873 		fail_scsi_task(task, error);  in fail_scsi_tasks()
1940 	struct iscsi_task *task = NULL, *running_task;  in iscsi_eh_cmd_timed_out()  local
1952 	task = (struct iscsi_task *)sc->SCp.ptr;  in iscsi_eh_cmd_timed_out()
1953 	if (!task) {  in iscsi_eh_cmd_timed_out()
1998 	if (time_after(task->last_xfer, task->last_timeout)) {  in iscsi_eh_cmd_timed_out()
2002 			     "%lu\n.", task->last_xfer, task->last_timeout);  in iscsi_eh_cmd_timed_out()
2003 		task->have_checked_conn = false;  in iscsi_eh_cmd_timed_out()
2021 		if (!running_task->sc || running_task == task ||  in iscsi_eh_cmd_timed_out()
2030 			       task->sc->jiffies_at_alloc))  in iscsi_eh_cmd_timed_out()
2033 		if (time_after(running_task->last_xfer, task->last_timeout)) {  in iscsi_eh_cmd_timed_out()
2049 				     task->last_xfer, running_task->last_xfer,  in iscsi_eh_cmd_timed_out()
2050 				     task->last_timeout);  in iscsi_eh_cmd_timed_out()
2057 	if (task->have_checked_conn)  in iscsi_eh_cmd_timed_out()
2065 		task->have_checked_conn = true;  in iscsi_eh_cmd_timed_out()
2072 	task->have_checked_conn = true;  in iscsi_eh_cmd_timed_out()
2076 	if (task)  in iscsi_eh_cmd_timed_out()
2077 		task->last_timeout = jiffies;  in iscsi_eh_cmd_timed_out()
2174 static void iscsi_prep_abort_task_pdu(struct iscsi_task *task,  in iscsi_prep_abort_task_pdu()  argument
2181 	hdr->lun = task->lun;  in iscsi_prep_abort_task_pdu()
2182 	hdr->rtt = task->hdr_itt;  in iscsi_prep_abort_task_pdu()
2183 	hdr->refcmdsn = task->cmdsn;  in iscsi_prep_abort_task_pdu()
2191 	struct iscsi_task *task;  in iscsi_eh_abort()  local
2231 	task = (struct iscsi_task *)sc->SCp.ptr;  in iscsi_eh_abort()
2233 		     sc, task->itt);  in iscsi_eh_abort()
2236 	if (!task->sc) {  in iscsi_eh_abort()
2241 	if (task->state == ISCSI_TASK_PENDING) {  in iscsi_eh_abort()
2242 		fail_scsi_task(task, DID_ABORT);  in iscsi_eh_abort()
2252 	iscsi_prep_abort_task_pdu(task, hdr);  in iscsi_eh_abort()
2271 		fail_scsi_task(task, DID_ABORT);  in iscsi_eh_abort()
2300 		     sc, task->itt);  in iscsi_eh_abort()
2308 		     task ? task->itt : 0);  in iscsi_eh_abort()
2866 		struct iscsi_task *task = session->cmds[cmd_i];  in iscsi_session_setup()  local
2869 			task->dd_data = &task[1];  in iscsi_session_setup()
2870 		task->itt = cmd_i;  in iscsi_session_setup()
2871 		task->state = ISCSI_TASK_FREE;  in iscsi_session_setup()
2872 		INIT_LIST_HEAD(&task->running);  in iscsi_session_setup()
3117 	struct iscsi_task *task;  in fail_mgmt_tasks()  local
3121 		task = conn->session->cmds[i];  in fail_mgmt_tasks()
3122 		if (task->sc)  in fail_mgmt_tasks()
3125 		if (task->state == ISCSI_TASK_FREE)  in fail_mgmt_tasks()
3130 				  task->itt, task->state);  in fail_mgmt_tasks()
3132 		if (task->state == ISCSI_TASK_PENDING)  in fail_mgmt_tasks()
3135 		iscsi_complete_task(task, state);  in fail_mgmt_tasks()