43 	struct hl_device *hdev = cs->ctx->hdev;  in hl_int_hw_queue_update_ci()  local
47 	if (hdev->disabled)  in hl_int_hw_queue_update_ci()
50 	q = &hdev->kernel_queues[0];  in hl_int_hw_queue_update_ci()
51 	for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {  in hl_int_hw_queue_update_ci()
73 static void ext_and_hw_queue_submit_bd(struct hl_device *hdev,  in ext_and_hw_queue_submit_bd()  argument
85 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in ext_and_hw_queue_submit_bd()
106 static int ext_queue_sanity_checks(struct hl_device *hdev,  in ext_queue_sanity_checks()  argument
111 			&hdev->completion_queue[q->cq_id].free_slots_cnt;  in ext_queue_sanity_checks()
118 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in ext_queue_sanity_checks()
132 			dev_dbg(hdev->dev, "No space for %d on CQ %d\n",  in ext_queue_sanity_checks()
155 static int int_queue_sanity_checks(struct hl_device *hdev,  in int_queue_sanity_checks()  argument
162 		dev_err(hdev->dev,  in int_queue_sanity_checks()
172 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in int_queue_sanity_checks()
190 static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,  in hw_queue_sanity_checks()  argument
199 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in hw_queue_sanity_checks()
218 int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,  in hl_hw_queue_send_cb_no_cmpl()  argument
221 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_send_cb_no_cmpl()
233 		hdev->asic_funcs->hw_queues_lock(hdev);  in hl_hw_queue_send_cb_no_cmpl()
235 	if (hdev->disabled) {  in hl_hw_queue_send_cb_no_cmpl()
246 		rc = ext_queue_sanity_checks(hdev, q, 1, false);  in hl_hw_queue_send_cb_no_cmpl()
251 	ext_and_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);  in hl_hw_queue_send_cb_no_cmpl()
255 		hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_hw_queue_send_cb_no_cmpl()
270 	struct hl_device *hdev = job->cs->ctx->hdev;  in ext_queue_schedule_job()  local
271 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in ext_queue_schedule_job()
303 	cq = &hdev->completion_queue[q->cq_id];  in ext_queue_schedule_job()
306 	hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,  in ext_queue_schedule_job()
316 	ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in ext_queue_schedule_job()
329 	struct hl_device *hdev = job->cs->ctx->hdev;  in int_queue_schedule_job()  local
330 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in int_queue_schedule_job()
343 	hdev->asic_funcs->pqe_write(hdev, pi, &bd);  in int_queue_schedule_job()
345 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in int_queue_schedule_job()
358 	struct hl_device *hdev = job->cs->ctx->hdev;  in hw_queue_schedule_job()  local
359 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in hw_queue_schedule_job()
369 	offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1);  in hw_queue_schedule_job()
388 	ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in hw_queue_schedule_job()
400 	struct hl_device *hdev = ctx->hdev;  in init_signal_wait_cs()  local
413 	hw_queue = &hdev->kernel_queues[q_idx];  in init_signal_wait_cs()
421 		dev_dbg(hdev->dev,  in init_signal_wait_cs()
425 		hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,  in init_signal_wait_cs()
444 			dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",  in init_signal_wait_cs()
458 		dev_dbg(hdev->dev,  in init_signal_wait_cs()
463 		hdev->asic_funcs->gen_wait_cb(hdev, job->patched_cb,  in init_signal_wait_cs()
488 	struct hl_device *hdev = ctx->hdev;  in hl_hw_queue_schedule_cs()  local
494 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_hw_queue_schedule_cs()
496 	if (hl_device_disabled_or_in_reset(hdev)) {  in hl_hw_queue_schedule_cs()
498 		dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
504 	max_queues = hdev->asic_prop.max_queues;  in hl_hw_queue_schedule_cs()
506 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
511 				rc = ext_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
515 				rc = int_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
519 				rc = hw_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
523 				dev_err(hdev->dev, "Queue type %d is invalid\n",  in hl_hw_queue_schedule_cs()
542 	spin_lock(&hdev->hw_queues_mirror_lock);  in hl_hw_queue_schedule_cs()
543 	list_add_tail(&cs->mirror_node, &hdev->hw_queues_mirror_list);  in hl_hw_queue_schedule_cs()
546 	if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&  in hl_hw_queue_schedule_cs()
547 			(list_first_entry(&hdev->hw_queues_mirror_list,  in hl_hw_queue_schedule_cs()
550 		schedule_delayed_work(&cs->work_tdr, hdev->timeout_jiffies);  in hl_hw_queue_schedule_cs()
551 		spin_unlock(&hdev->hw_queues_mirror_lock);  in hl_hw_queue_schedule_cs()
553 		spin_unlock(&hdev->hw_queues_mirror_lock);  in hl_hw_queue_schedule_cs()
556 	if (!hdev->cs_active_cnt++) {  in hl_hw_queue_schedule_cs()
559 		ts = &hdev->idle_busy_ts_arr[hdev->idle_busy_ts_idx];  in hl_hw_queue_schedule_cs()
584 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
589 				&hdev->completion_queue[i].free_slots_cnt;  in hl_hw_queue_schedule_cs()
596 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_hw_queue_schedule_cs()
607 void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)  in hl_hw_queue_inc_ci_kernel()  argument
609 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_inc_ci_kernel()
614 static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in ext_and_cpu_queue_init()  argument
621 		p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,  in ext_and_cpu_queue_init()
625 		p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,  in ext_and_cpu_queue_init()
638 		dev_err(hdev->dev,  in ext_and_cpu_queue_init()
653 		hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,  in ext_and_cpu_queue_init()
657 		hdev->asic_funcs->asic_dma_free_coherent(hdev,  in ext_and_cpu_queue_init()
665 static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in int_queue_init()  argument
669 	p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,  in int_queue_init()
672 		dev_err(hdev->dev,  in int_queue_init()
685 static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in cpu_queue_init()  argument
687 	return ext_and_cpu_queue_init(hdev, q, true);  in cpu_queue_init()
690 static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in ext_queue_init()  argument
692 	return ext_and_cpu_queue_init(hdev, q, false);  in ext_queue_init()
695 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in hw_queue_init()  argument
699 	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev,  in hw_queue_init()
715 static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx)  in sync_stream_queue_init()  argument
717 	struct hl_hw_queue *hw_queue = &hdev->kernel_queues[q_idx];  in sync_stream_queue_init()
718 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in sync_stream_queue_init()
720 	int sob, queue_idx = hdev->sync_stream_queue_idx++;  in sync_stream_queue_init()
731 		hw_sob->hdev = hdev;  in sync_stream_queue_init()
738 static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx)  in sync_stream_queue_reset()  argument
740 	struct hl_hw_queue *hw_queue = &hdev->kernel_queues[q_idx];  in sync_stream_queue_reset()
761 static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in queue_init()  argument
770 		rc = ext_queue_init(hdev, q);  in queue_init()
773 		rc = int_queue_init(hdev, q);  in queue_init()
776 		rc = cpu_queue_init(hdev, q);  in queue_init()
779 		rc = hw_queue_init(hdev, q);  in queue_init()
785 		dev_crit(hdev->dev, "wrong queue type %d during init\n",  in queue_init()
792 		sync_stream_queue_init(hdev, q->hw_queue_id);  in queue_init()
810 static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)  in queue_fini()  argument
839 		hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,  in queue_fini()
843 		hdev->asic_funcs->asic_dma_free_coherent(hdev,  in queue_fini()
849 int hl_hw_queues_create(struct hl_device *hdev)  in hl_hw_queues_create()  argument
851 	struct asic_fixed_properties *asic = &hdev->asic_prop;  in hl_hw_queues_create()
855 	hdev->kernel_queues = kcalloc(asic->max_queues,  in hl_hw_queues_create()
856 				sizeof(*hdev->kernel_queues), GFP_KERNEL);  in hl_hw_queues_create()
858 	if (!hdev->kernel_queues) {  in hl_hw_queues_create()
859 		dev_err(hdev->dev, "Not enough memory for H/W queues\n");  in hl_hw_queues_create()
864 	for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;  in hl_hw_queues_create()
870 		rc = queue_init(hdev, q, i);  in hl_hw_queues_create()
872 			dev_err(hdev->dev,  in hl_hw_queues_create()
881 	for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)  in hl_hw_queues_create()
882 		queue_fini(hdev, q);  in hl_hw_queues_create()
884 	kfree(hdev->kernel_queues);  in hl_hw_queues_create()
889 void hl_hw_queues_destroy(struct hl_device *hdev)  in hl_hw_queues_destroy()  argument
892 	u32 max_queues = hdev->asic_prop.max_queues;  in hl_hw_queues_destroy()
895 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)  in hl_hw_queues_destroy()
896 		queue_fini(hdev, q);  in hl_hw_queues_destroy()
898 	kfree(hdev->kernel_queues);  in hl_hw_queues_destroy()
901 void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)  in hl_hw_queue_reset()  argument
904 	u32 max_queues = hdev->asic_prop.max_queues;  in hl_hw_queue_reset()
907 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {  in hl_hw_queue_reset()
915 			sync_stream_queue_reset(hdev, q->hw_queue_id);  in hl_hw_queue_reset()