500 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_set_hbm_bar_base()  local
505 	if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr))  in gaudi_set_hbm_bar_base()
516 	if (gaudi) {  in gaudi_set_hbm_bar_base()
517 		old_addr = gaudi->hbm_bar_cur_addr;  in gaudi_set_hbm_bar_base()
518 		gaudi->hbm_bar_cur_addr = addr;  in gaudi_set_hbm_bar_base()
797 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_late_init()  local
800 	rc = gaudi->cpucp_info_get(hdev);  in gaudi_late_init()
915 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_free_internal_qmans_pq_mem()  local
920 		q = &gaudi->internal_qmans[i];  in gaudi_free_internal_qmans_pq_mem()
931 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_alloc_internal_qmans_pq_mem()  local
939 		q = &gaudi->internal_qmans[i];  in gaudi_alloc_internal_qmans_pq_mem()
977 	struct gaudi_device *gaudi;  in gaudi_sw_init()  local
982 	gaudi = kzalloc(sizeof(*gaudi), GFP_KERNEL);  in gaudi_sw_init()
983 	if (!gaudi)  in gaudi_sw_init()
996 			gaudi->events[event_id++] =  in gaudi_sw_init()
1001 	gaudi->cpucp_info_get = gaudi_cpucp_info_get;  in gaudi_sw_init()
1003 	gaudi->max_freq_value = GAUDI_MAX_CLK_FREQ;  in gaudi_sw_init()
1005 	hdev->asic_specific = gaudi;  in gaudi_sw_init()
1042 	spin_lock_init(&gaudi->hw_queues_lock);  in gaudi_sw_init()
1043 	mutex_init(&gaudi->clk_gate_mutex);  in gaudi_sw_init()
1062 	kfree(gaudi);  in gaudi_sw_init()
1068 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_sw_fini()  local
1083 	mutex_destroy(&gaudi->clk_gate_mutex);  in gaudi_sw_fini()
1085 	kfree(gaudi);  in gaudi_sw_fini()
1175 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_enable_msi()  local
1178 	if (gaudi->hw_cap_initialized & HW_CAP_MSI)  in gaudi_enable_msi()
1189 		gaudi->multi_msi_mode = false;  in gaudi_enable_msi()
1192 		gaudi->multi_msi_mode = true;  in gaudi_enable_msi()
1199 	gaudi->hw_cap_initialized |= HW_CAP_MSI;  in gaudi_enable_msi()
1210 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_sync_irqs()  local
1213 	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))  in gaudi_sync_irqs()
1217 	if (gaudi->multi_msi_mode) {  in gaudi_sync_irqs()
1231 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_msi()  local
1234 	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))  in gaudi_disable_msi()
1239 	if (gaudi->multi_msi_mode) {  in gaudi_disable_msi()
1254 	gaudi->hw_cap_initialized &= ~HW_CAP_MSI;  in gaudi_disable_msi()
1259 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_scrambler_sram()  local
1261 	if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER)  in gaudi_init_scrambler_sram()
1318 	gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER;  in gaudi_init_scrambler_sram()
1323 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_scrambler_hbm()  local
1325 	if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER)  in gaudi_init_scrambler_hbm()
1382 	gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER;  in gaudi_init_scrambler_hbm()
1965 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_pci_dma_qmans()  local
1969 	if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)  in gaudi_init_pci_dma_qmans()
2001 	gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA;  in gaudi_init_pci_dma_qmans()
2087 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_hbm_dma_qmans()  local
2092 	if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)  in gaudi_init_hbm_dma_qmans()
2105 			q = &gaudi->internal_qmans[internal_q_index];  in gaudi_init_hbm_dma_qmans()
2119 	gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA;  in gaudi_init_hbm_dma_qmans()
2205 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_mme_qmans()  local
2211 	if (gaudi->hw_cap_initialized & HW_CAP_MME)  in gaudi_init_mme_qmans()
2223 		q = &gaudi->internal_qmans[internal_q_index];  in gaudi_init_mme_qmans()
2239 	gaudi->hw_cap_initialized |= HW_CAP_MME;  in gaudi_init_mme_qmans()
2326 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_tpc_qmans()  local
2334 	if (gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)  in gaudi_init_tpc_qmans()
2344 			q = &gaudi->internal_qmans[internal_q_index];  in gaudi_init_tpc_qmans()
2364 		gaudi->hw_cap_initialized |=  in gaudi_init_tpc_qmans()
2371 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_pci_dma_qmans()  local
2373 	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))  in gaudi_disable_pci_dma_qmans()
2383 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_hbm_dma_qmans()  local
2385 	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))  in gaudi_disable_hbm_dma_qmans()
2397 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_mme_qmans()  local
2399 	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))  in gaudi_disable_mme_qmans()
2408 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_tpc_qmans()  local
2412 	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi_disable_tpc_qmans()
2423 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_pci_dma_qmans()  local
2425 	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))  in gaudi_stop_pci_dma_qmans()
2436 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_hbm_dma_qmans()  local
2438 	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))  in gaudi_stop_hbm_dma_qmans()
2452 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_mme_qmans()  local
2454 	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))  in gaudi_stop_mme_qmans()
2464 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_stop_tpc_qmans()  local
2466 	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi_stop_tpc_qmans()
2481 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_pci_dma_stall()  local
2483 	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))  in gaudi_pci_dma_stall()
2493 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hbm_dma_stall()  local
2495 	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))  in gaudi_hbm_dma_stall()
2507 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mme_stall()  local
2509 	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))  in gaudi_mme_stall()
2533 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_tpc_stall()  local
2535 	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi_tpc_stall()
2550 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_set_clock_gating()  local
2603 	gaudi->hw_cap_initialized |= HW_CAP_CLK_GATE;  in gaudi_set_clock_gating()
2608 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_disable_clock_gating()  local
2612 	if (!(gaudi->hw_cap_initialized & HW_CAP_CLK_GATE))  in gaudi_disable_clock_gating()
2634 	gaudi->hw_cap_initialized &= ~(HW_CAP_CLK_GATE);  in gaudi_disable_clock_gating()
2698 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_init()  local
2705 	if (gaudi->hw_cap_initialized & HW_CAP_MMU)  in gaudi_mmu_init()
2738 	gaudi->mmu_cache_inv_pi = 1;  in gaudi_mmu_init()
2740 	gaudi->hw_cap_initialized |= HW_CAP_MMU;  in gaudi_mmu_init()
2806 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_cpu()  local
2812 	if (gaudi->hw_cap_initialized & HW_CAP_CPU)  in gaudi_init_cpu()
2831 	gaudi->hw_cap_initialized |= HW_CAP_CPU;  in gaudi_init_cpu()
2838 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_init_cpu_queues()  local
2848 	if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)  in gaudi_init_cpu_queues()
2873 	if (gaudi->multi_msi_mode)  in gaudi_init_cpu_queues()
2895 	gaudi->hw_cap_initialized |= HW_CAP_CPU_Q;  in gaudi_init_cpu_queues()
3020 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hw_fini()  local
3086 	gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |  in gaudi_hw_fini()
3095 	memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat));  in gaudi_hw_fini()
3132 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_ring_doorbell()  local
3195 		if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)  in gaudi_ring_doorbell()
3418 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_get_int_queue_base()  local
3427 	q = &gaudi->internal_qmans[queue_id];  in gaudi_get_int_queue_base()
3437 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_send_cpu_message()  local
3439 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) {  in gaudi_send_cpu_message()
3531 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_test_cpu_queue()  local
3537 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_test_cpu_queue()
4285 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_cs_parser()  local
4290 	if (gaudi->hw_cap_initialized & HW_CAP_MMU)  in gaudi_cs_parser()
4301 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_add_end_of_cb_packets()  local
4324 	if (!gaudi->multi_msi_mode)  in gaudi_add_end_of_cb_packets()
4521 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_clear_pgt_range()  local
4525 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))  in gaudi_mmu_clear_pgt_range()
4539 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_read32()  local
4545 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_read32()
4586 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_write32()  local
4592 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_write32()
4633 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_read64()  local
4639 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_read64()
4684 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_debugfs_write64()  local
4690 		if ((gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) &&  in gaudi_debugfs_write64()
4733 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_read_pte()  local
4739 			(addr - gaudi->hbm_bar_cur_addr));  in gaudi_read_pte()
4744 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_write_pte()  local
4750 			(addr - gaudi->hbm_bar_cur_addr));  in gaudi_write_pte()
4762 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_prepare()  local
4764 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))  in gaudi_mmu_prepare()
4772 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_mmu_prepare()
4923 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_mmu_prepare()
5200 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_print_mmu_error_info()  local
5204 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))  in gaudi_print_mmu_error_info()
5254 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_extract_ecc_info()  local
5271 		mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_extract_ecc_info()
5316 		mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_extract_ecc_info()
5524 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_soft_reset_late_init()  local
5529 	return hl_fw_unmask_irq_arr(hdev, gaudi->events, sizeof(gaudi->events));  in gaudi_soft_reset_late_init()
5634 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_tpc_read_interrupts()  local
5643 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_tpc_read_interrupts()
5666 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_tpc_read_interrupts()
5719 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_handle_eqe()  local
5732 	gaudi->events_stat[event_type]++;  in gaudi_handle_eqe()
5733 	gaudi->events_stat_aggregate[event_type]++;  in gaudi_handle_eqe()
5930 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_get_events_stat()  local
5933 		*size = (u32) sizeof(gaudi->events_stat_aggregate);  in gaudi_get_events_stat()
5934 		return gaudi->events_stat_aggregate;  in gaudi_get_events_stat()
5937 	*size = (u32) sizeof(gaudi->events_stat);  in gaudi_get_events_stat()
5938 	return gaudi->events_stat;  in gaudi_get_events_stat()
5944 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_invalidate_cache()  local
5948 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||  in gaudi_mmu_invalidate_cache()
5961 	WREG32(mmSTLB_CACHE_INV, gaudi->mmu_cache_inv_pi++);  in gaudi_mmu_invalidate_cache()
5988 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_mmu_invalidate_cache_range()  local
5994 	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||  in gaudi_mmu_invalidate_cache_range()
6073 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_send_heartbeat()  local
6075 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_send_heartbeat()
6083 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_cpucp_info_get()  local
6087 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_cpucp_info_get()
6113 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_is_device_idle()  local
6121 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_is_device_idle()
6213 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_is_device_idle()
6219 	__acquires(&gaudi->hw_queues_lock)  in gaudi_hw_queues_lock()
6221 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hw_queues_lock()  local
6223 	spin_lock(&gaudi->hw_queues_lock);  in gaudi_hw_queues_lock()
6227 	__releases(&gaudi->hw_queues_lock)  in gaudi_hw_queues_unlock()
6229 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_hw_queues_unlock()  local
6231 	spin_unlock(&gaudi->hw_queues_lock);  in gaudi_hw_queues_unlock()
6242 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_get_eeprom_data()  local
6244 	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi_get_eeprom_data()
6257 	struct gaudi_device *gaudi = hdev->asic_specific;  in gaudi_run_tpc_kernel()  local
6269 	mutex_lock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()
6313 		mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()
6338 		mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()
6351 	mutex_unlock(&gaudi->clk_gate_mutex);  in gaudi_run_tpc_kernel()