19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)  in hl_device_disabled_or_in_reset()  argument
21 	if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))  in hl_device_disabled_or_in_reset()
27 enum hl_device_status hl_device_status(struct hl_device *hdev)  in hl_device_status()  argument
31 	if (hdev->disabled)  in hl_device_status()
33 	else if (atomic_read(&hdev->in_reset))  in hl_device_status()
44 	struct hl_device *hdev;  in hpriv_release()  local
48 	hdev = hpriv->hdev;  in hpriv_release()
56 	mutex_lock(&hdev->fpriv_list_lock);  in hpriv_release()
58 	hdev->compute_ctx = NULL;  in hpriv_release()
59 	mutex_unlock(&hdev->fpriv_list_lock);  in hpriv_release()
86 	hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);  in hl_device_release()
87 	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);  in hl_device_release()
99 	struct hl_device *hdev;  in hl_device_release_ctrl()  local
103 	hdev = hpriv->hdev;  in hl_device_release_ctrl()
105 	mutex_lock(&hdev->fpriv_list_lock);  in hl_device_release_ctrl()
107 	mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_release_ctrl()
176 static int device_init_cdev(struct hl_device *hdev, struct class *hclass,  in device_init_cdev()  argument
189 	(*dev)->devt = MKDEV(hdev->major, minor);  in device_init_cdev()
192 	dev_set_drvdata(*dev, hdev);  in device_init_cdev()
198 static int device_cdev_sysfs_add(struct hl_device *hdev)  in device_cdev_sysfs_add()  argument
202 	rc = cdev_device_add(&hdev->cdev, hdev->dev);  in device_cdev_sysfs_add()
204 		dev_err(hdev->dev,  in device_cdev_sysfs_add()
209 	rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);  in device_cdev_sysfs_add()
211 		dev_err(hdev->dev,  in device_cdev_sysfs_add()
217 	rc = hl_sysfs_init(hdev);  in device_cdev_sysfs_add()
219 		dev_err(hdev->dev, "failed to initialize sysfs\n");  in device_cdev_sysfs_add()
223 	hdev->cdev_sysfs_created = true;  in device_cdev_sysfs_add()
228 	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);  in device_cdev_sysfs_add()
230 	cdev_device_del(&hdev->cdev, hdev->dev);  in device_cdev_sysfs_add()
234 static void device_cdev_sysfs_del(struct hl_device *hdev)  in device_cdev_sysfs_del()  argument
236 	if (!hdev->cdev_sysfs_created)  in device_cdev_sysfs_del()
239 	hl_sysfs_fini(hdev);  in device_cdev_sysfs_del()
240 	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);  in device_cdev_sysfs_del()
241 	cdev_device_del(&hdev->cdev, hdev->dev);  in device_cdev_sysfs_del()
244 	put_device(hdev->dev);  in device_cdev_sysfs_del()
245 	put_device(hdev->dev_ctrl);  in device_cdev_sysfs_del()
256 static int device_early_init(struct hl_device *hdev)  in device_early_init()  argument
261 	switch (hdev->asic_type) {  in device_early_init()
263 		goya_set_asic_funcs(hdev);  in device_early_init()
264 		strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));  in device_early_init()
267 		gaudi_set_asic_funcs(hdev);  in device_early_init()
268 		sprintf(hdev->asic_name, "GAUDI");  in device_early_init()
271 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",  in device_early_init()
272 			hdev->asic_type);  in device_early_init()
276 	rc = hdev->asic_funcs->early_init(hdev);  in device_early_init()
280 	rc = hl_asid_init(hdev);  in device_early_init()
284 	if (hdev->asic_prop.completion_queues_count) {  in device_early_init()
285 		hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,  in device_early_init()
286 				sizeof(*hdev->cq_wq),  in device_early_init()
288 		if (!hdev->cq_wq) {  in device_early_init()
294 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {  in device_early_init()
296 		hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);  in device_early_init()
297 		if (hdev->cq_wq[i] == NULL) {  in device_early_init()
298 			dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");  in device_early_init()
304 	hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);  in device_early_init()
305 	if (hdev->eq_wq == NULL) {  in device_early_init()
306 		dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");  in device_early_init()
311 	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),  in device_early_init()
313 	if (!hdev->hl_chip_info) {  in device_early_init()
318 	hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE,  in device_early_init()
321 	if (!hdev->idle_busy_ts_arr) {  in device_early_init()
326 	rc = hl_mmu_if_set_funcs(hdev);  in device_early_init()
330 	hl_cb_mgr_init(&hdev->kernel_cb_mgr);  in device_early_init()
332 	mutex_init(&hdev->send_cpu_message_lock);  in device_early_init()
333 	mutex_init(&hdev->debug_lock);  in device_early_init()
334 	mutex_init(&hdev->mmu_cache_lock);  in device_early_init()
335 	INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);  in device_early_init()
336 	spin_lock_init(&hdev->hw_queues_mirror_lock);  in device_early_init()
337 	INIT_LIST_HEAD(&hdev->fpriv_list);  in device_early_init()
338 	mutex_init(&hdev->fpriv_list_lock);  in device_early_init()
339 	atomic_set(&hdev->in_reset, 0);  in device_early_init()
344 	kfree(hdev->idle_busy_ts_arr);  in device_early_init()
346 	kfree(hdev->hl_chip_info);  in device_early_init()
348 	destroy_workqueue(hdev->eq_wq);  in device_early_init()
350 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in device_early_init()
351 		if (hdev->cq_wq[i])  in device_early_init()
352 			destroy_workqueue(hdev->cq_wq[i]);  in device_early_init()
353 	kfree(hdev->cq_wq);  in device_early_init()
355 	hl_asid_fini(hdev);  in device_early_init()
357 	if (hdev->asic_funcs->early_fini)  in device_early_init()
358 		hdev->asic_funcs->early_fini(hdev);  in device_early_init()
369 static void device_early_fini(struct hl_device *hdev)  in device_early_fini()  argument
373 	mutex_destroy(&hdev->mmu_cache_lock);  in device_early_fini()
374 	mutex_destroy(&hdev->debug_lock);  in device_early_fini()
375 	mutex_destroy(&hdev->send_cpu_message_lock);  in device_early_fini()
377 	mutex_destroy(&hdev->fpriv_list_lock);  in device_early_fini()
379 	hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);  in device_early_fini()
381 	kfree(hdev->idle_busy_ts_arr);  in device_early_fini()
382 	kfree(hdev->hl_chip_info);  in device_early_fini()
384 	destroy_workqueue(hdev->eq_wq);  in device_early_fini()
386 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in device_early_fini()
387 		destroy_workqueue(hdev->cq_wq[i]);  in device_early_fini()
388 	kfree(hdev->cq_wq);  in device_early_fini()
390 	hl_asid_fini(hdev);  in device_early_fini()
392 	if (hdev->asic_funcs->early_fini)  in device_early_fini()
393 		hdev->asic_funcs->early_fini(hdev);  in device_early_fini()
398 	struct hl_device *hdev = container_of(work, struct hl_device,  in set_freq_to_low_job()  local
401 	mutex_lock(&hdev->fpriv_list_lock);  in set_freq_to_low_job()
403 	if (!hdev->compute_ctx)  in set_freq_to_low_job()
404 		hl_device_set_frequency(hdev, PLL_LOW);  in set_freq_to_low_job()
406 	mutex_unlock(&hdev->fpriv_list_lock);  in set_freq_to_low_job()
408 	schedule_delayed_work(&hdev->work_freq,  in set_freq_to_low_job()
414 	struct hl_device *hdev = container_of(work, struct hl_device,  in hl_device_heartbeat()  local
417 	if (hl_device_disabled_or_in_reset(hdev))  in hl_device_heartbeat()
420 	if (!hdev->asic_funcs->send_heartbeat(hdev))  in hl_device_heartbeat()
423 	dev_err(hdev->dev, "Device heartbeat failed!\n");  in hl_device_heartbeat()
424 	hl_device_reset(hdev, true, false);  in hl_device_heartbeat()
429 	schedule_delayed_work(&hdev->work_heartbeat,  in hl_device_heartbeat()
441 static int device_late_init(struct hl_device *hdev)  in device_late_init()  argument
445 	if (hdev->asic_funcs->late_init) {  in device_late_init()
446 		rc = hdev->asic_funcs->late_init(hdev);  in device_late_init()
448 			dev_err(hdev->dev,  in device_late_init()
454 	hdev->high_pll = hdev->asic_prop.high_pll;  in device_late_init()
457 	hdev->curr_pll_profile = PLL_LOW;  in device_late_init()
459 	if (hdev->pm_mng_profile == PM_AUTO)  in device_late_init()
460 		hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);  in device_late_init()
462 		hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);  in device_late_init()
464 	INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);  in device_late_init()
465 	schedule_delayed_work(&hdev->work_freq,  in device_late_init()
468 	if (hdev->heartbeat) {  in device_late_init()
469 		INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);  in device_late_init()
470 		schedule_delayed_work(&hdev->work_heartbeat,  in device_late_init()
474 	hdev->late_init_done = true;  in device_late_init()
485 static void device_late_fini(struct hl_device *hdev)  in device_late_fini()  argument
487 	if (!hdev->late_init_done)  in device_late_fini()
490 	cancel_delayed_work_sync(&hdev->work_freq);  in device_late_fini()
491 	if (hdev->heartbeat)  in device_late_fini()
492 		cancel_delayed_work_sync(&hdev->work_heartbeat);  in device_late_fini()
494 	if (hdev->asic_funcs->late_fini)  in device_late_fini()
495 		hdev->asic_funcs->late_fini(hdev);  in device_late_fini()
497 	hdev->late_init_done = false;  in device_late_fini()
500 uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms)  in hl_device_utilization()  argument
504 	u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx;  in hl_device_utilization()
510 	ts = &hdev->idle_busy_ts_arr[last_index];  in hl_device_utilization()
521 		ts = &hdev->idle_busy_ts_arr[last_index];  in hl_device_utilization()
585 		ts = &hdev->idle_busy_ts_arr[last_index];  in hl_device_utilization()
609 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)  in hl_device_set_frequency()  argument
611 	if ((hdev->pm_mng_profile == PM_MANUAL) ||  in hl_device_set_frequency()
612 			(hdev->curr_pll_profile == freq))  in hl_device_set_frequency()
615 	dev_dbg(hdev->dev, "Changing device frequency to %s\n",  in hl_device_set_frequency()
618 	hdev->asic_funcs->set_pll_profile(hdev, freq);  in hl_device_set_frequency()
620 	hdev->curr_pll_profile = freq;  in hl_device_set_frequency()
625 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)  in hl_device_set_debug_mode()  argument
629 	mutex_lock(&hdev->debug_lock);  in hl_device_set_debug_mode()
632 		if (!hdev->in_debug) {  in hl_device_set_debug_mode()
633 			dev_err(hdev->dev,  in hl_device_set_debug_mode()
639 		if (!hdev->hard_reset_pending)  in hl_device_set_debug_mode()
640 			hdev->asic_funcs->halt_coresight(hdev);  in hl_device_set_debug_mode()
642 		hdev->in_debug = 0;  in hl_device_set_debug_mode()
644 		if (!hdev->hard_reset_pending)  in hl_device_set_debug_mode()
645 			hdev->asic_funcs->set_clock_gating(hdev);  in hl_device_set_debug_mode()
650 	if (hdev->in_debug) {  in hl_device_set_debug_mode()
651 		dev_err(hdev->dev,  in hl_device_set_debug_mode()
657 	hdev->asic_funcs->disable_clock_gating(hdev);  in hl_device_set_debug_mode()
658 	hdev->in_debug = 1;  in hl_device_set_debug_mode()
661 	mutex_unlock(&hdev->debug_lock);  in hl_device_set_debug_mode()
675 int hl_device_suspend(struct hl_device *hdev)  in hl_device_suspend()  argument
679 	pci_save_state(hdev->pdev);  in hl_device_suspend()
682 	rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_suspend()
684 		dev_err(hdev->dev, "Can't suspend while in reset\n");  in hl_device_suspend()
689 	hdev->disabled = true;  in hl_device_suspend()
695 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_device_suspend()
696 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_device_suspend()
699 	mutex_lock(&hdev->send_cpu_message_lock);  in hl_device_suspend()
700 	mutex_unlock(&hdev->send_cpu_message_lock);  in hl_device_suspend()
702 	rc = hdev->asic_funcs->suspend(hdev);  in hl_device_suspend()
704 		dev_err(hdev->dev,  in hl_device_suspend()
708 	pci_disable_device(hdev->pdev);  in hl_device_suspend()
709 	pci_set_power_state(hdev->pdev, PCI_D3hot);  in hl_device_suspend()
723 int hl_device_resume(struct hl_device *hdev)  in hl_device_resume()  argument
727 	pci_set_power_state(hdev->pdev, PCI_D0);  in hl_device_resume()
728 	pci_restore_state(hdev->pdev);  in hl_device_resume()
729 	rc = pci_enable_device_mem(hdev->pdev);  in hl_device_resume()
731 		dev_err(hdev->dev,  in hl_device_resume()
736 	pci_set_master(hdev->pdev);  in hl_device_resume()
738 	rc = hdev->asic_funcs->resume(hdev);  in hl_device_resume()
740 		dev_err(hdev->dev, "Failed to resume device after suspend\n");  in hl_device_resume()
745 	hdev->disabled = false;  in hl_device_resume()
746 	atomic_set(&hdev->in_reset, 0);  in hl_device_resume()
748 	rc = hl_device_reset(hdev, true, false);  in hl_device_resume()
750 		dev_err(hdev->dev, "Failed to reset device during resume\n");  in hl_device_resume()
757 	pci_clear_master(hdev->pdev);  in hl_device_resume()
758 	pci_disable_device(hdev->pdev);  in hl_device_resume()
763 static int device_kill_open_processes(struct hl_device *hdev)  in device_kill_open_processes()  argument
769 	if (hdev->pldm)  in device_kill_open_processes()
777 	if (!list_empty(&hdev->fpriv_list))  in device_kill_open_processes()
780 	mutex_lock(&hdev->fpriv_list_lock);  in device_kill_open_processes()
785 	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {  in device_kill_open_processes()
788 			dev_info(hdev->dev, "Killing user process pid=%d\n",  in device_kill_open_processes()
797 	mutex_unlock(&hdev->fpriv_list_lock);  in device_kill_open_processes()
807 	while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {  in device_kill_open_processes()
808 		dev_info(hdev->dev,  in device_kill_open_processes()
816 	return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;  in device_kill_open_processes()
823 	struct hl_device *hdev = device_reset_work->hdev;  in device_hard_reset_pending()  local
825 	hl_device_reset(hdev, true, true);  in device_hard_reset_pending()
848 int hl_device_reset(struct hl_device *hdev, bool hard_reset,  in hl_device_reset()  argument
853 	if (!hdev->init_done) {  in hl_device_reset()
854 		dev_err(hdev->dev,  in hl_device_reset()
859 	if ((!hard_reset) && (!hdev->supports_soft_reset)) {  in hl_device_reset()
860 		dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");  in hl_device_reset()
871 		rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_reset()
885 			if (hl_fw_send_pci_access_msg(hdev,  in hl_device_reset()
887 				dev_warn(hdev->dev,  in hl_device_reset()
892 		hdev->disabled = true;  in hl_device_reset()
897 		hdev->asic_funcs->hw_queues_lock(hdev);  in hl_device_reset()
898 		hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_device_reset()
901 		mutex_lock(&hdev->fpriv_list_lock);  in hl_device_reset()
902 		mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_reset()
904 		dev_err(hdev->dev, "Going to RESET device!\n");  in hl_device_reset()
911 		hdev->hard_reset_pending = true;  in hl_device_reset()
927 		device_reset_work->hdev = hdev;  in hl_device_reset()
934 		device_late_fini(hdev);  in hl_device_reset()
940 		mutex_lock(&hdev->send_cpu_message_lock);  in hl_device_reset()
941 		mutex_unlock(&hdev->send_cpu_message_lock);  in hl_device_reset()
949 	hdev->asic_funcs->halt_engines(hdev, hard_reset);  in hl_device_reset()
952 	hl_cs_rollback_all(hdev);  in hl_device_reset()
959 		rc = device_kill_open_processes(hdev);  in hl_device_reset()
961 			dev_crit(hdev->dev,  in hl_device_reset()
969 		flush_workqueue(hdev->eq_wq);  in hl_device_reset()
973 	hdev->asic_funcs->hw_fini(hdev, hard_reset);  in hl_device_reset()
977 		if (hl_ctx_put(hdev->kernel_ctx) == 1)  in hl_device_reset()
978 			hdev->kernel_ctx = NULL;  in hl_device_reset()
979 		hl_vm_fini(hdev);  in hl_device_reset()
980 		hl_mmu_fini(hdev);  in hl_device_reset()
981 		hl_eq_reset(hdev, &hdev->event_queue);  in hl_device_reset()
985 	hl_hw_queue_reset(hdev, hard_reset);  in hl_device_reset()
986 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in hl_device_reset()
987 		hl_cq_reset(hdev, &hdev->completion_queue[i]);  in hl_device_reset()
989 	hdev->idle_busy_ts_idx = 0;  in hl_device_reset()
990 	hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0);  in hl_device_reset()
991 	hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0);  in hl_device_reset()
993 	if (hdev->cs_active_cnt)  in hl_device_reset()
994 		dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n",  in hl_device_reset()
995 			hdev->cs_active_cnt);  in hl_device_reset()
997 	mutex_lock(&hdev->fpriv_list_lock);  in hl_device_reset()
1000 	if (hdev->compute_ctx) {  in hl_device_reset()
1001 		atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);  in hl_device_reset()
1002 		hdev->compute_ctx->thread_ctx_switch_wait_token = 0;  in hl_device_reset()
1005 	mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_reset()
1010 		hdev->device_cpu_disabled = false;  in hl_device_reset()
1011 		hdev->hard_reset_pending = false;  in hl_device_reset()
1013 		if (hdev->kernel_ctx) {  in hl_device_reset()
1014 			dev_crit(hdev->dev,  in hl_device_reset()
1020 		rc = hl_mmu_init(hdev);  in hl_device_reset()
1022 			dev_err(hdev->dev,  in hl_device_reset()
1028 		hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),  in hl_device_reset()
1030 		if (!hdev->kernel_ctx) {  in hl_device_reset()
1032 			hl_mmu_fini(hdev);  in hl_device_reset()
1036 		hdev->compute_ctx = NULL;  in hl_device_reset()
1038 		rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);  in hl_device_reset()
1040 			dev_err(hdev->dev,  in hl_device_reset()
1042 			kfree(hdev->kernel_ctx);  in hl_device_reset()
1043 			hdev->kernel_ctx = NULL;  in hl_device_reset()
1044 			hl_mmu_fini(hdev);  in hl_device_reset()
1053 	hdev->disabled = false;  in hl_device_reset()
1055 	rc = hdev->asic_funcs->hw_init(hdev);  in hl_device_reset()
1057 		dev_err(hdev->dev,  in hl_device_reset()
1063 	rc = hdev->asic_funcs->test_queues(hdev);  in hl_device_reset()
1065 		dev_err(hdev->dev,  in hl_device_reset()
1071 		rc = device_late_init(hdev);  in hl_device_reset()
1073 			dev_err(hdev->dev,  in hl_device_reset()
1078 		rc = hl_vm_init(hdev);  in hl_device_reset()
1080 			dev_err(hdev->dev,  in hl_device_reset()
1085 		hl_set_max_power(hdev);  in hl_device_reset()
1087 		rc = hdev->asic_funcs->soft_reset_late_init(hdev);  in hl_device_reset()
1089 			dev_err(hdev->dev,  in hl_device_reset()
1095 	atomic_set(&hdev->in_reset, 0);  in hl_device_reset()
1098 		hdev->hard_reset_cnt++;  in hl_device_reset()
1100 		hdev->soft_reset_cnt++;  in hl_device_reset()
1102 	dev_warn(hdev->dev, "Successfully finished resetting the device\n");  in hl_device_reset()
1107 	hdev->disabled = true;  in hl_device_reset()
1110 		dev_err(hdev->dev,  in hl_device_reset()
1112 		hdev->hard_reset_cnt++;  in hl_device_reset()
1114 		dev_err(hdev->dev,  in hl_device_reset()
1116 		hdev->soft_reset_cnt++;  in hl_device_reset()
1121 	atomic_set(&hdev->in_reset, 0);  in hl_device_reset()
1135 int hl_device_init(struct hl_device *hdev, struct class *hclass)  in hl_device_init()  argument
1141 	name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);  in hl_device_init()
1148 	rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,  in hl_device_init()
1149 				&hdev->cdev, &hdev->dev);  in hl_device_init()
1156 	name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);  in hl_device_init()
1163 	rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,  in hl_device_init()
1164 				name, &hdev->cdev_ctrl, &hdev->dev_ctrl);  in hl_device_init()
1172 	rc = device_early_init(hdev);  in hl_device_init()
1180 	rc = hdev->asic_funcs->sw_init(hdev);  in hl_device_init()
1189 	rc = hl_hw_queues_create(hdev);  in hl_device_init()
1191 		dev_err(hdev->dev, "failed to initialize kernel queues\n");  in hl_device_init()
1195 	cq_cnt = hdev->asic_prop.completion_queues_count;  in hl_device_init()
1203 		hdev->completion_queue = kcalloc(cq_cnt,  in hl_device_init()
1204 				sizeof(*hdev->completion_queue),  in hl_device_init()
1207 		if (!hdev->completion_queue) {  in hl_device_init()
1208 			dev_err(hdev->dev,  in hl_device_init()
1216 		rc = hl_cq_init(hdev, &hdev->completion_queue[i],  in hl_device_init()
1217 				hdev->asic_funcs->get_queue_id_for_cq(hdev, i));  in hl_device_init()
1219 			dev_err(hdev->dev,  in hl_device_init()
1223 		hdev->completion_queue[i].cq_idx = i;  in hl_device_init()
1231 	rc = hl_eq_init(hdev, &hdev->event_queue);  in hl_device_init()
1233 		dev_err(hdev->dev, "failed to initialize event queue\n");  in hl_device_init()
1238 	rc = hl_mmu_init(hdev);  in hl_device_init()
1240 		dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");  in hl_device_init()
1245 	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);  in hl_device_init()
1246 	if (!hdev->kernel_ctx) {  in hl_device_init()
1251 	hdev->compute_ctx = NULL;  in hl_device_init()
1253 	rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);  in hl_device_init()
1255 		dev_err(hdev->dev, "failed to initialize kernel context\n");  in hl_device_init()
1256 		kfree(hdev->kernel_ctx);  in hl_device_init()
1260 	rc = hl_cb_pool_init(hdev);  in hl_device_init()
1262 		dev_err(hdev->dev, "failed to initialize CB pool\n");  in hl_device_init()
1266 	hl_debugfs_add_device(hdev);  in hl_device_init()
1268 	if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {  in hl_device_init()
1269 		dev_info(hdev->dev,  in hl_device_init()
1271 		hdev->asic_funcs->halt_engines(hdev, true);  in hl_device_init()
1272 		hdev->asic_funcs->hw_fini(hdev, true);  in hl_device_init()
1285 	hdev->disabled = false;  in hl_device_init()
1287 	rc = hdev->asic_funcs->hw_init(hdev);  in hl_device_init()
1289 		dev_err(hdev->dev, "failed to initialize the H/W\n");  in hl_device_init()
1295 	rc = hdev->asic_funcs->test_queues(hdev);  in hl_device_init()
1297 		dev_err(hdev->dev, "Failed to detect if device is alive\n");  in hl_device_init()
1302 	rc = device_late_init(hdev);  in hl_device_init()
1304 		dev_err(hdev->dev, "Failed late initialization\n");  in hl_device_init()
1309 	dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",  in hl_device_init()
1310 		hdev->asic_name,  in hl_device_init()
1311 		hdev->asic_prop.dram_size / 1024 / 1024 / 1024);  in hl_device_init()
1313 	rc = hl_vm_init(hdev);  in hl_device_init()
1315 		dev_err(hdev->dev, "Failed to initialize memory module\n");  in hl_device_init()
1326 	rc = device_cdev_sysfs_add(hdev);  in hl_device_init()
1328 		dev_err(hdev->dev,  in hl_device_init()
1337 	hl_set_max_power(hdev);  in hl_device_init()
1345 	rc = hl_hwmon_init(hdev);  in hl_device_init()
1347 		dev_err(hdev->dev, "Failed to initialize hwmon\n");  in hl_device_init()
1352 	dev_notice(hdev->dev,  in hl_device_init()
1355 	hdev->init_done = true;  in hl_device_init()
1360 	if (hl_ctx_put(hdev->kernel_ctx) != 1)  in hl_device_init()
1361 		dev_err(hdev->dev,  in hl_device_init()
1364 	hl_mmu_fini(hdev);  in hl_device_init()
1366 	hl_eq_fini(hdev, &hdev->event_queue);  in hl_device_init()
1369 		hl_cq_fini(hdev, &hdev->completion_queue[i]);  in hl_device_init()
1370 	kfree(hdev->completion_queue);  in hl_device_init()
1372 	hl_hw_queues_destroy(hdev);  in hl_device_init()
1374 	hdev->asic_funcs->sw_fini(hdev);  in hl_device_init()
1376 	device_early_fini(hdev);  in hl_device_init()
1378 	put_device(hdev->dev_ctrl);  in hl_device_init()
1380 	put_device(hdev->dev);  in hl_device_init()
1382 	hdev->disabled = true;  in hl_device_init()
1384 		device_cdev_sysfs_add(hdev);  in hl_device_init()
1385 	if (hdev->pdev)  in hl_device_init()
1386 		dev_err(&hdev->pdev->dev,  in hl_device_init()
1388 			hdev->id / 2);  in hl_device_init()
1391 			hdev->id / 2);  in hl_device_init()
1403 void hl_device_fini(struct hl_device *hdev)  in hl_device_fini()  argument
1408 	dev_info(hdev->dev, "Removing device\n");  in hl_device_fini()
1420 	rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_fini()
1423 		rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);  in hl_device_fini()
1437 	hl_fw_send_pci_access_msg(hdev,	CPUCP_PACKET_DISABLE_PCI_ACCESS);  in hl_device_fini()
1440 	hdev->disabled = true;  in hl_device_fini()
1445 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_device_fini()
1446 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_device_fini()
1449 	mutex_lock(&hdev->fpriv_list_lock);  in hl_device_fini()
1450 	mutex_unlock(&hdev->fpriv_list_lock);  in hl_device_fini()
1452 	hdev->hard_reset_pending = true;  in hl_device_fini()
1454 	hl_hwmon_fini(hdev);  in hl_device_fini()
1456 	device_late_fini(hdev);  in hl_device_fini()
1458 	hl_debugfs_remove_device(hdev);  in hl_device_fini()
1465 	hdev->asic_funcs->halt_engines(hdev, true);  in hl_device_fini()
1468 	hl_cs_rollback_all(hdev);  in hl_device_fini()
1474 	rc = device_kill_open_processes(hdev);  in hl_device_fini()
1476 		dev_crit(hdev->dev, "Failed to kill all open processes\n");  in hl_device_fini()
1478 	hl_cb_pool_fini(hdev);  in hl_device_fini()
1481 	hdev->asic_funcs->hw_fini(hdev, true);  in hl_device_fini()
1484 	if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))  in hl_device_fini()
1485 		dev_err(hdev->dev, "kernel ctx is still alive\n");  in hl_device_fini()
1487 	hl_vm_fini(hdev);  in hl_device_fini()
1489 	hl_mmu_fini(hdev);  in hl_device_fini()
1491 	hl_eq_fini(hdev, &hdev->event_queue);  in hl_device_fini()
1493 	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)  in hl_device_fini()
1494 		hl_cq_fini(hdev, &hdev->completion_queue[i]);  in hl_device_fini()
1495 	kfree(hdev->completion_queue);  in hl_device_fini()
1497 	hl_hw_queues_destroy(hdev);  in hl_device_fini()
1500 	hdev->asic_funcs->sw_fini(hdev);  in hl_device_fini()
1502 	device_early_fini(hdev);  in hl_device_fini()
1505 	device_cdev_sysfs_del(hdev);  in hl_device_fini()
1523 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)  in hl_rreg()  argument
1525 	return readl(hdev->rmmio + reg);  in hl_rreg()
1538 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)  in hl_wreg()  argument
1540 	writel(val, hdev->rmmio + reg);  in hl_wreg()