52 	struct iscsi_cmd *cmd,  in iscsit_add_r2t_to_list()  argument
60 	lockdep_assert_held(&cmd->r2t_lock);  in iscsit_add_r2t_to_list()
72 	r2t->r2t_sn = (!r2t_sn) ? cmd->r2t_sn++ : r2t_sn;  in iscsit_add_r2t_to_list()
75 	list_add_tail(&r2t->r2t_list, &cmd->cmd_r2t_list);  in iscsit_add_r2t_to_list()
76 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_add_r2t_to_list()
78 	iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);  in iscsit_add_r2t_to_list()
80 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_add_r2t_to_list()
85 	struct iscsi_cmd *cmd,  in iscsit_get_r2t_for_eos()  argument
91 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_for_eos()
92 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {  in iscsit_get_r2t_for_eos()
95 			spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_for_eos()
99 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_for_eos()
106 struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *cmd)  in iscsit_get_r2t_from_list()  argument
110 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_from_list()
111 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {  in iscsit_get_r2t_from_list()
113 			spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_from_list()
117 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_r2t_from_list()
120 			" 0x%08x.\n", cmd->init_task_tag);  in iscsit_get_r2t_from_list()
124 void iscsit_free_r2t(struct iscsi_r2t *r2t, struct iscsi_cmd *cmd)  in iscsit_free_r2t()  argument
126 	lockdep_assert_held(&cmd->r2t_lock);  in iscsit_free_r2t()
132 void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)  in iscsit_free_r2ts_from_list()  argument
136 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_free_r2ts_from_list()
137 	list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list)  in iscsit_free_r2ts_from_list()
138 		iscsit_free_r2t(r2t, cmd);  in iscsit_free_r2ts_from_list()
139 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_free_r2ts_from_list()
174 	struct iscsi_cmd *cmd;  in iscsit_allocate_cmd()  local
185 	cmd = (struct iscsi_cmd *)(se_sess->sess_cmd_map + (tag * size));  in iscsit_allocate_cmd()
186 	memset(cmd, 0, size);  in iscsit_allocate_cmd()
188 	cmd->se_cmd.map_tag = tag;  in iscsit_allocate_cmd()
189 	cmd->se_cmd.map_cpu = cpu;  in iscsit_allocate_cmd()
190 	cmd->conn = conn;  in iscsit_allocate_cmd()
191 	cmd->data_direction = DMA_NONE;  in iscsit_allocate_cmd()
192 	INIT_LIST_HEAD(&cmd->i_conn_node);  in iscsit_allocate_cmd()
193 	INIT_LIST_HEAD(&cmd->datain_list);  in iscsit_allocate_cmd()
194 	INIT_LIST_HEAD(&cmd->cmd_r2t_list);  in iscsit_allocate_cmd()
195 	spin_lock_init(&cmd->datain_lock);  in iscsit_allocate_cmd()
196 	spin_lock_init(&cmd->dataout_timeout_lock);  in iscsit_allocate_cmd()
197 	spin_lock_init(&cmd->istate_lock);  in iscsit_allocate_cmd()
198 	spin_lock_init(&cmd->error_lock);  in iscsit_allocate_cmd()
199 	spin_lock_init(&cmd->r2t_lock);  in iscsit_allocate_cmd()
200 	timer_setup(&cmd->dataout_timer, iscsit_handle_dataout_timeout, 0);  in iscsit_allocate_cmd()
202 	return cmd;  in iscsit_allocate_cmd()
207 	struct iscsi_cmd *cmd,  in iscsit_get_seq_holder_for_datain()  argument
212 	for (i = 0; i < cmd->seq_count; i++)  in iscsit_get_seq_holder_for_datain()
213 		if (cmd->seq_list[i].seq_send_order == seq_send_order)  in iscsit_get_seq_holder_for_datain()
214 			return &cmd->seq_list[i];  in iscsit_get_seq_holder_for_datain()
219 struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *cmd)  in iscsit_get_seq_holder_for_r2t()  argument
223 	if (!cmd->seq_list) {  in iscsit_get_seq_holder_for_r2t()
228 	for (i = 0; i < cmd->seq_count; i++) {  in iscsit_get_seq_holder_for_r2t()
229 		if (cmd->seq_list[i].type != SEQTYPE_NORMAL)  in iscsit_get_seq_holder_for_r2t()
231 		if (cmd->seq_list[i].seq_send_order == cmd->seq_send_order) {  in iscsit_get_seq_holder_for_r2t()
232 			cmd->seq_send_order++;  in iscsit_get_seq_holder_for_r2t()
233 			return &cmd->seq_list[i];  in iscsit_get_seq_holder_for_r2t()
241 	struct iscsi_cmd *cmd,  in iscsit_get_holder_for_r2tsn()  argument
246 	spin_lock_bh(&cmd->r2t_lock);  in iscsit_get_holder_for_r2tsn()
247 	list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {  in iscsit_get_holder_for_r2tsn()
249 			spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_holder_for_r2tsn()
253 	spin_unlock_bh(&cmd->r2t_lock);  in iscsit_get_holder_for_r2tsn()
302 int iscsit_sequence_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,  in iscsit_sequence_cmd()  argument
314 		ret = iscsit_execute_cmd(cmd, 0);  in iscsit_sequence_cmd()
323 		ret = iscsit_handle_ooo_cmdsn(conn->sess, cmd, be32_to_cpu(cmdsn));  in iscsit_sequence_cmd()
334 		cmd->i_state = ISTATE_REMOVE;  in iscsit_sequence_cmd()
335 		iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);  in iscsit_sequence_cmd()
347 		iscsit_reject_cmd(cmd, reason, buf);  in iscsit_sequence_cmd()
353 int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)  in iscsit_check_unsolicited_dataout()  argument
355 	struct iscsi_conn *conn = cmd->conn;  in iscsit_check_unsolicited_dataout()
356 	struct se_cmd *se_cmd = &cmd->se_cmd;  in iscsit_check_unsolicited_dataout()
368 	if ((cmd->first_burst_len + payload_length) >  in iscsit_check_unsolicited_dataout()
372 			(cmd->first_burst_len + payload_length),  in iscsit_check_unsolicited_dataout()
382 	if (((cmd->first_burst_len + payload_length) != cmd->se_cmd.data_length) &&  in iscsit_check_unsolicited_dataout()
383 	    ((cmd->first_burst_len + payload_length) !=  in iscsit_check_unsolicited_dataout()
388 			(cmd->first_burst_len + payload_length),  in iscsit_check_unsolicited_dataout()
389 			conn->sess->sess_ops->FirstBurstLength, cmd->se_cmd.data_length);  in iscsit_check_unsolicited_dataout()
401 	struct iscsi_cmd *cmd;  in iscsit_find_cmd_from_itt()  local
404 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {  in iscsit_find_cmd_from_itt()
405 		if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_from_itt()
407 			return cmd;  in iscsit_find_cmd_from_itt()
423 	struct iscsi_cmd *cmd;  in iscsit_find_cmd_from_itt_or_dump()  local
426 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {  in iscsit_find_cmd_from_itt_or_dump()
427 		if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT)  in iscsit_find_cmd_from_itt_or_dump()
429 		if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_from_itt_or_dump()
431 			return cmd;  in iscsit_find_cmd_from_itt_or_dump()
449 	struct iscsi_cmd *cmd = NULL;  in iscsit_find_cmd_from_ttt()  local
452 	list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {  in iscsit_find_cmd_from_ttt()
453 		if (cmd->targ_xfer_tag == targ_xfer_tag) {  in iscsit_find_cmd_from_ttt()
455 			return cmd;  in iscsit_find_cmd_from_ttt()
471 	struct iscsi_cmd *cmd = NULL;  in iscsit_find_cmd_for_recovery()  local
480 		list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {  in iscsit_find_cmd_for_recovery()
481 			if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_for_recovery()
486 				*cmd_ptr = cmd;  in iscsit_find_cmd_for_recovery()
500 		list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {  in iscsit_find_cmd_for_recovery()
501 			if (cmd->init_task_tag == init_task_tag) {  in iscsit_find_cmd_for_recovery()
506 				*cmd_ptr = cmd;  in iscsit_find_cmd_for_recovery()
518 	struct iscsi_cmd *cmd,  in iscsit_add_cmd_to_immediate_queue()  argument
531 	qr->cmd = cmd;  in iscsit_add_cmd_to_immediate_queue()
536 	atomic_inc(&cmd->immed_queue_count);  in iscsit_add_cmd_to_immediate_queue()
557 	if (qr->cmd)  in iscsit_get_cmd_from_immediate_queue()
558 		atomic_dec(&qr->cmd->immed_queue_count);  in iscsit_get_cmd_from_immediate_queue()
565 	struct iscsi_cmd *cmd,  in iscsit_remove_cmd_from_immediate_queue()  argument
571 	if (!atomic_read(&cmd->immed_queue_count)) {  in iscsit_remove_cmd_from_immediate_queue()
577 		if (qr->cmd != cmd)  in iscsit_remove_cmd_from_immediate_queue()
580 		atomic_dec(&qr->cmd->immed_queue_count);  in iscsit_remove_cmd_from_immediate_queue()
586 	if (atomic_read(&cmd->immed_queue_count)) {  in iscsit_remove_cmd_from_immediate_queue()
588 			cmd->init_task_tag,  in iscsit_remove_cmd_from_immediate_queue()
589 			atomic_read(&cmd->immed_queue_count));  in iscsit_remove_cmd_from_immediate_queue()
594 	struct iscsi_cmd *cmd,  in iscsit_add_cmd_to_response_queue()  argument
607 	qr->cmd = cmd;  in iscsit_add_cmd_to_response_queue()
612 	atomic_inc(&cmd->response_queue_count);  in iscsit_add_cmd_to_response_queue()
633 	if (qr->cmd)  in iscsit_get_cmd_from_response_queue()
634 		atomic_dec(&qr->cmd->response_queue_count);  in iscsit_get_cmd_from_response_queue()
641 	struct iscsi_cmd *cmd,  in iscsit_remove_cmd_from_response_queue()  argument
647 	if (!atomic_read(&cmd->response_queue_count)) {  in iscsit_remove_cmd_from_response_queue()
654 		if (qr->cmd != cmd)  in iscsit_remove_cmd_from_response_queue()
657 		atomic_dec(&qr->cmd->response_queue_count);  in iscsit_remove_cmd_from_response_queue()
663 	if (atomic_read(&cmd->response_queue_count)) {  in iscsit_remove_cmd_from_response_queue()
665 			cmd->init_task_tag,  in iscsit_remove_cmd_from_response_queue()
666 			atomic_read(&cmd->response_queue_count));  in iscsit_remove_cmd_from_response_queue()
695 		if (qr->cmd)  in iscsit_free_queue_reqs_for_conn()
696 			atomic_dec(&qr->cmd->immed_queue_count);  in iscsit_free_queue_reqs_for_conn()
706 		if (qr->cmd)  in iscsit_free_queue_reqs_for_conn()
707 			atomic_dec(&qr->cmd->response_queue_count);  in iscsit_free_queue_reqs_for_conn()
714 void iscsit_release_cmd(struct iscsi_cmd *cmd)  in iscsit_release_cmd()  argument
717 	struct se_cmd *se_cmd = &cmd->se_cmd;  in iscsit_release_cmd()
719 	WARN_ON(!list_empty(&cmd->i_conn_node));  in iscsit_release_cmd()
721 	if (cmd->conn)  in iscsit_release_cmd()
722 		sess = cmd->conn->sess;  in iscsit_release_cmd()
724 		sess = cmd->sess;  in iscsit_release_cmd()
728 	kfree(cmd->buf_ptr);  in iscsit_release_cmd()
729 	kfree(cmd->pdu_list);  in iscsit_release_cmd()
730 	kfree(cmd->seq_list);  in iscsit_release_cmd()
731 	kfree(cmd->tmr_req);  in iscsit_release_cmd()
732 	kfree(cmd->overflow_buf);  in iscsit_release_cmd()
733 	kfree(cmd->iov_data);  in iscsit_release_cmd()
734 	kfree(cmd->text_in_ptr);  in iscsit_release_cmd()
740 void __iscsit_free_cmd(struct iscsi_cmd *cmd, bool check_queues)  in __iscsit_free_cmd()  argument
742 	struct iscsi_conn *conn = cmd->conn;  in __iscsit_free_cmd()
744 	WARN_ON(!list_empty(&cmd->i_conn_node));  in __iscsit_free_cmd()
746 	if (cmd->data_direction == DMA_TO_DEVICE) {  in __iscsit_free_cmd()
747 		iscsit_stop_dataout_timer(cmd);  in __iscsit_free_cmd()
748 		iscsit_free_r2ts_from_list(cmd);  in __iscsit_free_cmd()
750 	if (cmd->data_direction == DMA_FROM_DEVICE)  in __iscsit_free_cmd()
751 		iscsit_free_all_datain_reqs(cmd);  in __iscsit_free_cmd()
754 		iscsit_remove_cmd_from_immediate_queue(cmd, conn);  in __iscsit_free_cmd()
755 		iscsit_remove_cmd_from_response_queue(cmd, conn);  in __iscsit_free_cmd()
759 		conn->conn_transport->iscsit_unmap_cmd(conn, cmd);  in __iscsit_free_cmd()
762 void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)  in iscsit_free_cmd()  argument
764 	struct se_cmd *se_cmd = cmd->se_cmd.se_tfo ? &cmd->se_cmd : NULL;  in iscsit_free_cmd()
767 	WARN_ON(!list_empty(&cmd->i_conn_node));  in iscsit_free_cmd()
769 	__iscsit_free_cmd(cmd, shutdown);  in iscsit_free_cmd()
773 			__iscsit_free_cmd(cmd, shutdown);  in iscsit_free_cmd()
777 		iscsit_release_cmd(cmd);  in iscsit_free_cmd()
889 	struct iscsi_cmd *cmd;  in iscsit_add_nopin()  local
891 	cmd = iscsit_allocate_cmd(conn, TASK_RUNNING);  in iscsit_add_nopin()
892 	if (!cmd)  in iscsit_add_nopin()
895 	cmd->iscsi_opcode = ISCSI_OP_NOOP_IN;  in iscsit_add_nopin()
898 	cmd->init_task_tag = RESERVED_ITT;  in iscsit_add_nopin()
899 	cmd->targ_xfer_tag = (want_response) ?  in iscsit_add_nopin()
902 	list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);  in iscsit_add_nopin()
907 	iscsit_add_cmd_to_immediate_queue(cmd, conn, state);  in iscsit_add_nopin()
1060 	struct iscsi_cmd *cmd,  in iscsit_send_tx_data()  argument
1069 	tx_size = cmd->tx_size;  in iscsit_send_tx_data()
1072 		iov = &cmd->iov_data[0];  in iscsit_send_tx_data()
1073 		iov_count = cmd->iov_data_count;  in iscsit_send_tx_data()
1075 		iov = &cmd->iov_misc[0];  in iscsit_send_tx_data()
1076 		iov_count = cmd->iov_misc_count;  in iscsit_send_tx_data()
1087 	cmd->tx_size = 0;  in iscsit_send_tx_data()
1093 	struct iscsi_cmd *cmd,  in iscsit_fe_sendpage_sg()  argument
1096 	struct scatterlist *sg = cmd->first_data_sg;  in iscsit_fe_sendpage_sg()
1099 	u32 offset = cmd->first_data_sg_off;  in iscsit_fe_sendpage_sg()
1107 	iov.iov_base = cmd->pdu;  in iscsit_fe_sendpage_sg()
1119 	data_len = cmd->tx_size - tx_hdr_size - cmd->padding;  in iscsit_fe_sendpage_sg()
1126 		if (cmd->padding)  in iscsit_fe_sendpage_sg()
1127 			iov_off = (cmd->iov_data_count - 2);  in iscsit_fe_sendpage_sg()
1129 			iov_off = (cmd->iov_data_count - 1);  in iscsit_fe_sendpage_sg()
1131 		iov_off = (cmd->iov_data_count - 1);  in iscsit_fe_sendpage_sg()
1160 	if (cmd->padding) {  in iscsit_fe_sendpage_sg()
1161 		struct kvec *iov_p = &cmd->iov_data[iov_off++];  in iscsit_fe_sendpage_sg()
1163 		tx_sent = tx_data(conn, iov_p, 1, cmd->padding);  in iscsit_fe_sendpage_sg()
1164 		if (cmd->padding != tx_sent) {  in iscsit_fe_sendpage_sg()
1175 		struct kvec *iov_d = &cmd->iov_data[iov_off];  in iscsit_fe_sendpage_sg()