30 static void qedi_process_logout_resp(struct qedi_ctx *qedi,  in qedi_process_logout_resp()  argument
58 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,  in qedi_process_logout_resp()
68 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,  in qedi_process_logout_resp()
81 static void qedi_process_text_resp(struct qedi_ctx *qedi,  in qedi_process_text_resp()  argument
95 	task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id);  in qedi_process_text_resp()
122 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,  in qedi_process_text_resp()
132 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,  in qedi_process_text_resp()
153 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_tmf_resp_work()  local
162 	rval = qedi_cleanup_all_io(qedi, qedi_conn, qedi_cmd->task, true);  in qedi_tmf_resp_work()
175 static void qedi_process_tmf_resp(struct qedi_ctx *qedi,  in qedi_process_tmf_resp()  argument
193 		QEDI_ERR(&qedi->dbg_ctx,  in qedi_process_tmf_resp()
235 		queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work);  in qedi_process_tmf_resp()
246 static void qedi_process_login_resp(struct qedi_ctx *qedi,  in qedi_process_login_resp()  argument
262 	task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id);  in qedi_process_login_resp()
302 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,  in qedi_process_login_resp()
308 static void qedi_get_rq_bdq_buf(struct qedi_ctx *qedi,  in qedi_get_rq_bdq_buf()  argument
314 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_get_rq_bdq_buf()
316 		  len, qedi->bdq_prod_idx,  in qedi_get_rq_bdq_buf()
317 		  (qedi->bdq_prod_idx % qedi->rq_num_entries));  in qedi_get_rq_bdq_buf()
322 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_get_rq_bdq_buf()
328 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_get_rq_bdq_buf()
331 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_get_rq_bdq_buf()
337 			memcpy(ptr, (void *)qedi->bdq[idx].buf_addr, len);  in qedi_get_rq_bdq_buf()
347 static void qedi_put_rq_bdq_buf(struct qedi_ctx *qedi,  in qedi_put_rq_bdq_buf()  argument
357 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_put_rq_bdq_buf()
363 	pbl = (struct scsi_bd *)qedi->bdq_pbl;  in qedi_put_rq_bdq_buf()
364 	pbl += (qedi->bdq_prod_idx % qedi->rq_num_entries);  in qedi_put_rq_bdq_buf()
365 	pbl->address.hi = cpu_to_le32(QEDI_U64_HI(qedi->bdq[idx].buf_dma));  in qedi_put_rq_bdq_buf()
366 	pbl->address.lo = cpu_to_le32(QEDI_U64_LO(qedi->bdq[idx].buf_dma));  in qedi_put_rq_bdq_buf()
367 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_put_rq_bdq_buf()
376 	qedi->bdq_prod_idx += count;  in qedi_put_rq_bdq_buf()
378 	writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);  in qedi_put_rq_bdq_buf()
379 	readw(qedi->bdq_primary_prod);  in qedi_put_rq_bdq_buf()
381 	writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);  in qedi_put_rq_bdq_buf()
382 	readw(qedi->bdq_secondary_prod);  in qedi_put_rq_bdq_buf()
385 static void qedi_unsol_pdu_adjust_bdq(struct qedi_ctx *qedi,  in qedi_unsol_pdu_adjust_bdq()  argument
390 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_unsol_pdu_adjust_bdq()
393 	qedi_get_rq_bdq_buf(qedi, cqe, bdq_data, pdu_len);  in qedi_unsol_pdu_adjust_bdq()
394 	qedi_put_rq_bdq_buf(qedi, cqe, (num_bdqs + 1));  in qedi_unsol_pdu_adjust_bdq()
397 static int qedi_process_nopin_mesg(struct qedi_ctx *qedi,  in qedi_process_nopin_mesg()  argument
429 		spin_lock_irqsave(&qedi->hba_lock, flags);  in qedi_process_nopin_mesg()
430 		qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,  in qedi_process_nopin_mesg()
434 		spin_unlock_irqrestore(&qedi->hba_lock, flags);  in qedi_process_nopin_mesg()
447 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,  in qedi_process_nopin_mesg()
468 static void qedi_process_async_mesg(struct qedi_ctx *qedi,  in qedi_process_async_mesg()  argument
491 		spin_lock_irqsave(&qedi->hba_lock, flags);  in qedi_process_async_mesg()
492 		qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,  in qedi_process_async_mesg()
494 		spin_unlock_irqrestore(&qedi->hba_lock, flags);  in qedi_process_async_mesg()
522 static void qedi_process_reject_mesg(struct qedi_ctx *qedi,  in qedi_process_reject_mesg()  argument
542 		spin_lock_irqsave(&qedi->hba_lock, flags);  in qedi_process_reject_mesg()
543 		qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited,  in qedi_process_reject_mesg()
545 		spin_unlock_irqrestore(&qedi->hba_lock, flags);  in qedi_process_reject_mesg()
564 static void qedi_scsi_completion(struct qedi_ctx *qedi,  in qedi_scsi_completion()  argument
580 	qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];  in qedi_scsi_completion()
591 		QEDI_WARN(&qedi->dbg_ctx, "sc_cmd is NULL!\n");  in qedi_scsi_completion()
596 		QEDI_WARN(&qedi->dbg_ctx,  in qedi_scsi_completion()
602 		QEDI_WARN(&qedi->dbg_ctx,  in qedi_scsi_completion()
609 		QEDI_WARN(&qedi->dbg_ctx,  in qedi_scsi_completion()
638 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,  in qedi_scsi_completion()
656 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,  in qedi_scsi_completion()
661 		qedi_trace_io(qedi, task, cmd->task_id, QEDI_IO_TRACE_RSP);  in qedi_scsi_completion()
669 static void qedi_mtask_completion(struct qedi_ctx *qedi,  in qedi_mtask_completion()  argument
683 		qedi_scsi_completion(qedi, cqe, task, iscsi_conn);  in qedi_mtask_completion()
686 		qedi_process_login_resp(qedi, cqe, task, conn);  in qedi_mtask_completion()
689 		qedi_process_tmf_resp(qedi, cqe, task, conn);  in qedi_mtask_completion()
692 		qedi_process_text_resp(qedi, cqe, task, conn);  in qedi_mtask_completion()
695 		qedi_process_logout_resp(qedi, cqe, task, conn);  in qedi_mtask_completion()
698 		qedi_process_nopin_mesg(qedi, cqe, task, conn, que_idx);  in qedi_mtask_completion()
701 		QEDI_ERR(&qedi->dbg_ctx, "unknown opcode\n");  in qedi_mtask_completion()
705 static void qedi_process_nopin_local_cmpl(struct qedi_ctx *qedi,  in qedi_process_nopin_local_cmpl()  argument
714 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UNSOL,  in qedi_process_nopin_local_cmpl()
725 static void qedi_process_cmd_cleanup_resp(struct qedi_ctx *qedi,  in qedi_process_cmd_cleanup_resp()  argument
743 	qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];  in qedi_process_cmd_cleanup_resp()
745 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,  in qedi_process_cmd_cleanup_resp()
758 				QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_process_cmd_cleanup_resp()
775 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_process_cmd_cleanup_resp()
790 				QEDI_NOTICE(&qedi->dbg_ctx,  in qedi_process_cmd_cleanup_resp()
799 			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_process_cmd_cleanup_resp()
819 		qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt);  in qedi_process_cmd_cleanup_resp()
822 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_process_cmd_cleanup_resp()
829 			QEDI_NOTICE(&qedi->dbg_ctx,  in qedi_process_cmd_cleanup_resp()
837 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID,  in qedi_process_cmd_cleanup_resp()
842 		qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt);  in qedi_process_cmd_cleanup_resp()
845 		QEDI_ERR(&qedi->dbg_ctx,  in qedi_process_cmd_cleanup_resp()
853 	struct qedi_ctx *qedi = work->qedi;  in qedi_fp_process_cqes()  local
871 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,  in qedi_fp_process_cqes()
876 		QEDI_WARN(&qedi->dbg_ctx, "Invalid CqE type\n");  in qedi_fp_process_cqes()
881 	q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];  in qedi_fp_process_cqes()
883 		QEDI_WARN(&qedi->dbg_ctx,  in qedi_fp_process_cqes()
904 			QEDI_WARN(&qedi->dbg_ctx, "task is NULL\n");  in qedi_fp_process_cqes()
912 			qedi_process_nopin_local_cmpl(qedi, &cqe->cqe_solicited,  in qedi_fp_process_cqes()
918 			qedi_mtask_completion(qedi, cqe, task, q_conn, que_idx);  in qedi_fp_process_cqes()
924 			qedi_process_nopin_mesg(qedi, cqe, task, q_conn,  in qedi_fp_process_cqes()
928 			qedi_process_async_mesg(qedi, cqe, task, q_conn,  in qedi_fp_process_cqes()
932 			qedi_process_reject_mesg(qedi, cqe, task, q_conn,  in qedi_fp_process_cqes()
938 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "Dummy CqE\n");  in qedi_fp_process_cqes()
941 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "CleanUp CqE\n");  in qedi_fp_process_cqes()
942 		qedi_process_cmd_cleanup_resp(qedi, &cqe->cqe_solicited, task,  in qedi_fp_process_cqes()
946 		QEDI_ERR(&qedi->dbg_ctx, "Error cqe.\n");  in qedi_fp_process_cqes()
973 	QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_MP_REQ,  in qedi_ring_doorbell()
1004 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_send_iscsi_login()  local
1018 	tid = qedi_get_task_idx(qedi);  in qedi_send_iscsi_login()
1023 	     (struct e4_iscsi_task_context *)qedi_get_task_mem(&qedi->tasks,  in qedi_send_iscsi_login()
1044 	qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);  in qedi_send_iscsi_login()
1108 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_send_iscsi_logout()  local
1119 	tid = qedi_get_task_idx(qedi);  in qedi_send_iscsi_logout()
1124 	     (struct e4_iscsi_task_context *)qedi_get_task_mem(&qedi->tasks,  in qedi_send_iscsi_logout()
1138 	qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);  in qedi_send_iscsi_logout()
1172 int qedi_cleanup_all_io(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn,  in qedi_cleanup_all_io()  argument
1198 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_cleanup_all_io()
1216 				QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_cleanup_all_io()
1232 		QEDI_WARN(&qedi->dbg_ctx,  in qedi_cleanup_all_io()
1242 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_cleanup_all_io()
1251 							  &qedi->flags)),  in qedi_cleanup_all_io()
1254 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_cleanup_all_io()
1263 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_cleanup_all_io()
1269 	iscsi_host_for_each_session(qedi->shost,  in qedi_cleanup_all_io()
1271 	qedi_ops->common->drain(qedi->cdev);  in qedi_cleanup_all_io()
1278 							&qedi->flags),  in qedi_cleanup_all_io()
1280 		iscsi_host_for_each_session(qedi->shost,  in qedi_cleanup_all_io()
1285 	iscsi_host_for_each_session(qedi->shost,  in qedi_cleanup_all_io()
1291 void qedi_clearsq(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn,  in qedi_clearsq()  argument
1302 		QEDI_WARN(&qedi->dbg_ctx,  in qedi_clearsq()
1308 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,  in qedi_clearsq()
1312 	qedi_ops->clear_sq(qedi->cdev, qedi_ep->handle);  in qedi_clearsq()
1314 	rval = qedi_cleanup_all_io(qedi, qedi_conn, task, true);  in qedi_clearsq()
1316 		QEDI_ERR(&qedi->dbg_ctx,  in qedi_clearsq()
1323 static int qedi_wait_for_cleanup_request(struct qedi_ctx *qedi,  in qedi_wait_for_cleanup_request()  argument
1342 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_wait_for_cleanup_request()
1356 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_tmf_work()  local
1372 		QEDI_ERR(&qedi->dbg_ctx, "Task already completed\n");  in qedi_tmf_work()
1377 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,  in qedi_tmf_work()
1383 		QEDI_ERR(&qedi->dbg_ctx, "DONT SEND CLEANUP/ABORT %d\n",  in qedi_tmf_work()
1390 		QEDI_ERR(&qedi->dbg_ctx, "Memory allocation failed\n");  in qedi_tmf_work()
1400 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_tmf_work()
1411 	rval = qedi_wait_for_cleanup_request(qedi, qedi_conn, ctask, qedi_cmd,  in qedi_tmf_work()
1414 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,  in qedi_tmf_work()
1420 	tid = qedi_get_task_idx(qedi);  in qedi_tmf_work()
1422 		QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n",  in qedi_tmf_work()
1459 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_send_iscsi_tmf()  local
1478 	tid = qedi_get_task_idx(qedi);  in qedi_send_iscsi_tmf()
1483 	     (struct e4_iscsi_task_context *)qedi_get_task_mem(&qedi->tasks,  in qedi_send_iscsi_tmf()
1493 	qedi_update_itt_map(qedi, tid, mtask->itt, qedi_cmd);  in qedi_send_iscsi_tmf()
1505 			QEDI_ERR(&qedi->dbg_ctx,  in qedi_send_iscsi_tmf()
1552 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_iscsi_abort_work()  local
1565 		queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work);  in qedi_iscsi_abort_work()
1573 		tid = qedi_get_task_idx(qedi);  in qedi_iscsi_abort_work()
1575 			QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n",  in qedi_iscsi_abort_work()
1584 		QEDI_ERR(&qedi->dbg_ctx, "Invalid tmf, cid=0x%x\n",  in qedi_iscsi_abort_work()
1600 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_send_iscsi_text()  local
1616 	tid = qedi_get_task_idx(qedi);  in qedi_send_iscsi_text()
1621 	     (struct e4_iscsi_task_context *)qedi_get_task_mem(&qedi->tasks,  in qedi_send_iscsi_text()
1636 	qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);  in qedi_send_iscsi_text()
1699 	struct qedi_ctx *qedi = qedi_conn->qedi;  in qedi_send_iscsi_nopout()  local
1715 	tid = qedi_get_task_idx(qedi);  in qedi_send_iscsi_nopout()
1720 	     (struct e4_iscsi_task_context *)qedi_get_task_mem(&qedi->tasks,  in qedi_send_iscsi_nopout()
1742 	qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd);  in qedi_send_iscsi_nopout()
1824 		QEDI_INFO(&cmd->conn->qedi->dbg_ctx, QEDI_LOG_IO,  in qedi_split_bd()
1835 static int qedi_map_scsi_sg(struct qedi_ctx *qedi, struct qedi_cmd *cmd)  in qedi_map_scsi_sg()  argument
1850 	sg_count = dma_map_sg(&qedi->pdev->dev, scsi_sglist(sc),  in qedi_map_scsi_sg()
1866 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,  in qedi_map_scsi_sg()
1902 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "sg[%d] size=0x%x",  in qedi_map_scsi_sg()
1918 		QEDI_ERR(&qedi->dbg_ctx,  in qedi_map_scsi_sg()
1922 		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "byte_count = %d\n",  in qedi_map_scsi_sg()
1936 		bd_count  = qedi_map_scsi_sg(cmd->conn->qedi, cmd);  in qedi_iscsi_map_sg_list()
1970 void qedi_trace_io(struct qedi_ctx *qedi, struct iscsi_task *task,  in qedi_trace_io()  argument
1979 	spin_lock_irqsave(&qedi->io_trace_lock, flags);  in qedi_trace_io()
1981 	io_log = &qedi->io_trace_buf[qedi->io_trace_idx];  in qedi_trace_io()
1993 	io_log->fast_sgs = qedi->fast_sgls;  in qedi_trace_io()
1994 	io_log->cached_sgs = qedi->cached_sgls;  in qedi_trace_io()
1995 	io_log->slow_sgs = qedi->slow_sgls;  in qedi_trace_io()
1996 	io_log->cached_sge = qedi->use_cached_sge;  in qedi_trace_io()
1997 	io_log->slow_sge = qedi->use_slow_sge;  in qedi_trace_io()
1998 	io_log->fast_sge = qedi->use_fast_sge;  in qedi_trace_io()
2005 		io_log->req_cpu = smp_processor_id() % qedi->num_queues;  in qedi_trace_io()
2009 		io_log->req_cpu = smp_processor_id() % qedi->num_queues;  in qedi_trace_io()
2010 		io_log->intr_cpu = qedi->intr_cpu;  in qedi_trace_io()
2014 	qedi->io_trace_idx++;  in qedi_trace_io()
2015 	if (qedi->io_trace_idx == QEDI_IO_TRACE_SIZE)  in qedi_trace_io()
2016 		qedi->io_trace_idx = 0;  in qedi_trace_io()
2018 	qedi->use_cached_sge = false;  in qedi_trace_io()
2019 	qedi->use_slow_sge = false;  in qedi_trace_io()
2020 	qedi->use_fast_sge = false;  in qedi_trace_io()
2022 	spin_unlock_irqrestore(&qedi->io_trace_lock, flags);  in qedi_trace_io()
2030 	struct qedi_ctx *qedi = iscsi_host_priv(shost);  in qedi_iscsi_send_ioreq()  local
2060 	tid = qedi_get_task_idx(qedi);  in qedi_iscsi_send_ioreq()
2065 	     (struct e4_iscsi_task_context *)qedi_get_task_mem(&qedi->tasks,  in qedi_iscsi_send_ioreq()
2078 	cq_idx = smp_processor_id() % qedi->num_queues;  in qedi_iscsi_send_ioreq()
2097 	qedi_update_itt_map(qedi, tid, task->itt, cmd);  in qedi_iscsi_send_ioreq()
2152 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,  in qedi_iscsi_send_ioreq()
2195 	QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_SCSI_TM,  in qedi_iscsi_cleanup_task()