Lines Matching refs:hdev
355 static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
356 static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
357 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
358 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
360 int goya_get_fixed_properties(struct hl_device *hdev) in goya_get_fixed_properties() argument
362 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_get_fixed_properties()
407 if (hdev->pldm) in goya_get_fixed_properties()
471 static int goya_pci_bars_map(struct hl_device *hdev) in goya_pci_bars_map() argument
477 rc = hl_pci_bars_map(hdev, name, is_wc); in goya_pci_bars_map()
481 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_pci_bars_map()
487 static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr) in goya_set_ddr_bar_base() argument
489 struct goya_device *goya = hdev->asic_specific; in goya_set_ddr_bar_base()
501 rc = hl_pci_set_inbound_region(hdev, 1, &pci_region); in goya_set_ddr_bar_base()
521 static int goya_init_iatu(struct hl_device *hdev) in goya_init_iatu() argument
531 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); in goya_init_iatu()
539 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); in goya_init_iatu()
543 hdev->asic_funcs->set_dma_mask_from_fw(hdev); in goya_init_iatu()
548 rc = hl_pci_set_outbound_region(hdev, &outbound_region); in goya_init_iatu()
565 static int goya_early_init(struct hl_device *hdev) in goya_early_init() argument
567 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_early_init()
568 struct pci_dev *pdev = hdev->pdev; in goya_early_init()
572 rc = goya_get_fixed_properties(hdev); in goya_early_init()
574 dev_err(hdev->dev, "Failed to get fixed properties\n"); in goya_early_init()
580 dev_err(hdev->dev, in goya_early_init()
591 dev_err(hdev->dev, in goya_early_init()
603 rc = hl_pci_init(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, in goya_early_init()
610 dev_info(hdev->dev, "firmware-level security is disabled\n"); in goya_early_init()
612 if (!hdev->pldm) { in goya_early_init()
615 dev_warn(hdev->dev, in goya_early_init()
622 kfree(hdev->asic_prop.hw_queues_props); in goya_early_init()
634 static int goya_early_fini(struct hl_device *hdev) in goya_early_fini() argument
636 kfree(hdev->asic_prop.hw_queues_props); in goya_early_fini()
637 hl_pci_fini(hdev); in goya_early_fini()
642 static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid) in goya_mmu_prepare_reg() argument
649 static void goya_qman0_set_security(struct hl_device *hdev, bool secure) in goya_qman0_set_security() argument
651 struct goya_device *goya = hdev->asic_specific; in goya_qman0_set_security()
670 static void goya_fetch_psoc_frequency(struct hl_device *hdev) in goya_fetch_psoc_frequency() argument
672 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_fetch_psoc_frequency()
694 dev_warn(hdev->dev, in goya_fetch_psoc_frequency()
705 int goya_late_init(struct hl_device *hdev) in goya_late_init() argument
707 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_late_init()
710 goya_fetch_psoc_frequency(hdev); in goya_late_init()
712 rc = goya_mmu_clear_pgt_range(hdev); in goya_late_init()
714 dev_err(hdev->dev, in goya_late_init()
719 rc = goya_mmu_set_dram_default_page(hdev); in goya_late_init()
721 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); in goya_late_init()
725 rc = goya_mmu_add_mappings_for_device_cpu(hdev); in goya_late_init()
729 rc = goya_init_cpu_queues(hdev); in goya_late_init()
733 rc = goya_test_cpu_queue(hdev); in goya_late_init()
737 rc = goya_cpucp_info_get(hdev); in goya_late_init()
739 dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc); in goya_late_init()
749 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS); in goya_late_init()
751 dev_err(hdev->dev, in goya_late_init()
769 void goya_late_fini(struct hl_device *hdev) in goya_late_fini() argument
774 if (!hdev->hl_chip_info->info) in goya_late_fini()
777 channel_info_arr = hdev->hl_chip_info->info; in goya_late_fini()
787 hdev->hl_chip_info->info = NULL; in goya_late_fini()
796 static int goya_sw_init(struct hl_device *hdev) in goya_sw_init() argument
813 hdev->asic_specific = goya; in goya_sw_init()
816 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), in goya_sw_init()
817 &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0); in goya_sw_init()
818 if (!hdev->dma_pool) { in goya_sw_init()
819 dev_err(hdev->dev, "failed to create DMA pool\n"); in goya_sw_init()
824 hdev->cpu_accessible_dma_mem = in goya_sw_init()
825 hdev->asic_funcs->asic_dma_alloc_coherent(hdev, in goya_sw_init()
827 &hdev->cpu_accessible_dma_address, in goya_sw_init()
830 if (!hdev->cpu_accessible_dma_mem) { in goya_sw_init()
835 dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n", in goya_sw_init()
836 &hdev->cpu_accessible_dma_address); in goya_sw_init()
838 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in goya_sw_init()
839 if (!hdev->cpu_accessible_dma_pool) { in goya_sw_init()
840 dev_err(hdev->dev, in goya_sw_init()
846 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, in goya_sw_init()
847 (uintptr_t) hdev->cpu_accessible_dma_mem, in goya_sw_init()
850 dev_err(hdev->dev, in goya_sw_init()
857 hdev->supports_coresight = true; in goya_sw_init()
858 hdev->supports_soft_reset = true; in goya_sw_init()
863 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_init()
865 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_init()
867 hdev->cpu_accessible_dma_mem, in goya_sw_init()
868 hdev->cpu_accessible_dma_address); in goya_sw_init()
870 dma_pool_destroy(hdev->dma_pool); in goya_sw_init()
883 static int goya_sw_fini(struct hl_device *hdev) in goya_sw_fini() argument
885 struct goya_device *goya = hdev->asic_specific; in goya_sw_fini()
887 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_fini()
889 hdev->asic_funcs->asic_dma_free_coherent(hdev, in goya_sw_fini()
891 hdev->cpu_accessible_dma_mem, in goya_sw_fini()
892 hdev->cpu_accessible_dma_address); in goya_sw_fini()
894 dma_pool_destroy(hdev->dma_pool); in goya_sw_fini()
901 static void goya_init_dma_qman(struct hl_device *hdev, int dma_id, in goya_init_dma_qman() argument
904 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qman()
946 if (hdev->stop_on_err) in goya_init_dma_qman()
953 static void goya_init_dma_ch(struct hl_device *hdev, int dma_id) in goya_init_dma_ch() argument
987 void goya_init_dma_qmans(struct hl_device *hdev) in goya_init_dma_qmans() argument
989 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qmans()
996 q = &hdev->kernel_queues[0]; in goya_init_dma_qmans()
1000 goya_init_dma_qman(hdev, i, q->bus_address); in goya_init_dma_qmans()
1001 goya_init_dma_ch(hdev, i); in goya_init_dma_qmans()
1013 static void goya_disable_external_queues(struct hl_device *hdev) in goya_disable_external_queues() argument
1015 struct goya_device *goya = hdev->asic_specific; in goya_disable_external_queues()
1027 static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg, in goya_stop_queue() argument
1040 hdev, in goya_stop_queue()
1053 hdev, in goya_stop_queue()
1061 dev_err(hdev->dev, in goya_stop_queue()
1077 static int goya_stop_external_queues(struct hl_device *hdev) in goya_stop_external_queues() argument
1081 struct goya_device *goya = hdev->asic_specific; in goya_stop_external_queues()
1086 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1092 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n"); in goya_stop_external_queues()
1096 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1102 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n"); in goya_stop_external_queues()
1106 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1112 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n"); in goya_stop_external_queues()
1116 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1122 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n"); in goya_stop_external_queues()
1126 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1132 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n"); in goya_stop_external_queues()
1147 int goya_init_cpu_queues(struct hl_device *hdev) in goya_init_cpu_queues() argument
1149 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu_queues()
1152 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_init_cpu_queues()
1155 if (!hdev->cpu_queues_enable) in goya_init_cpu_queues()
1161 eq = &hdev->event_queue; in goya_init_cpu_queues()
1189 hdev, in goya_init_cpu_queues()
1197 dev_err(hdev->dev, in goya_init_cpu_queues()
1206 static void goya_set_pll_refclk(struct hl_device *hdev) in goya_set_pll_refclk() argument
1244 static void goya_disable_clk_rlx(struct hl_device *hdev) in goya_disable_clk_rlx() argument
1250 static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id) in _goya_tpc_mbist_workaround() argument
1268 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n", in _goya_tpc_mbist_workaround()
1288 hdev, in _goya_tpc_mbist_workaround()
1296 dev_err(hdev->dev, in _goya_tpc_mbist_workaround()
1315 static void goya_tpc_mbist_workaround(struct hl_device *hdev) in goya_tpc_mbist_workaround() argument
1317 struct goya_device *goya = hdev->asic_specific; in goya_tpc_mbist_workaround()
1320 if (hdev->pldm) in goya_tpc_mbist_workaround()
1329 _goya_tpc_mbist_workaround(hdev, i); in goya_tpc_mbist_workaround()
1342 static void goya_init_golden_registers(struct hl_device *hdev) in goya_init_golden_registers() argument
1344 struct goya_device *goya = hdev->asic_specific; in goya_init_golden_registers()
1637 static void goya_init_mme_qman(struct hl_device *hdev) in goya_init_mme_qman() argument
1654 qman_base_addr = hdev->asic_prop.sram_base_address + in goya_init_mme_qman()
1687 static void goya_init_mme_cmdq(struct hl_device *hdev) in goya_init_mme_cmdq() argument
1723 void goya_init_mme_qmans(struct hl_device *hdev) in goya_init_mme_qmans() argument
1725 struct goya_device *goya = hdev->asic_specific; in goya_init_mme_qmans()
1737 goya_init_mme_qman(hdev); in goya_init_mme_qmans()
1738 goya_init_mme_cmdq(hdev); in goya_init_mme_qmans()
1743 static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id) in goya_init_tpc_qman() argument
1761 qman_base_addr = hdev->asic_prop.sram_base_address + base_off; in goya_init_tpc_qman()
1793 static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id) in goya_init_tpc_cmdq() argument
1830 void goya_init_tpc_qmans(struct hl_device *hdev) in goya_init_tpc_qmans() argument
1832 struct goya_device *goya = hdev->asic_specific; in goya_init_tpc_qmans()
1851 goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0); in goya_init_tpc_qmans()
1852 goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1); in goya_init_tpc_qmans()
1853 goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2); in goya_init_tpc_qmans()
1854 goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3); in goya_init_tpc_qmans()
1855 goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4); in goya_init_tpc_qmans()
1856 goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5); in goya_init_tpc_qmans()
1857 goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6); in goya_init_tpc_qmans()
1858 goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7); in goya_init_tpc_qmans()
1861 goya_init_tpc_cmdq(hdev, i); in goya_init_tpc_qmans()
1872 static void goya_disable_internal_queues(struct hl_device *hdev) in goya_disable_internal_queues() argument
1874 struct goya_device *goya = hdev->asic_specific; in goya_disable_internal_queues()
1919 static int goya_stop_internal_queues(struct hl_device *hdev) in goya_stop_internal_queues() argument
1921 struct goya_device *goya = hdev->asic_specific; in goya_stop_internal_queues()
1933 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1939 dev_err(hdev->dev, "failed to stop MME QMAN\n"); in goya_stop_internal_queues()
1943 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1949 dev_err(hdev->dev, "failed to stop MME CMDQ\n"); in goya_stop_internal_queues()
1957 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1963 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n"); in goya_stop_internal_queues()
1967 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1973 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n"); in goya_stop_internal_queues()
1977 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1983 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n"); in goya_stop_internal_queues()
1987 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
1993 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n"); in goya_stop_internal_queues()
1997 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2003 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n"); in goya_stop_internal_queues()
2007 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2013 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n"); in goya_stop_internal_queues()
2017 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2023 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n"); in goya_stop_internal_queues()
2027 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2033 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n"); in goya_stop_internal_queues()
2037 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2043 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n"); in goya_stop_internal_queues()
2047 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2053 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n"); in goya_stop_internal_queues()
2057 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2063 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n"); in goya_stop_internal_queues()
2067 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2073 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n"); in goya_stop_internal_queues()
2077 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2083 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n"); in goya_stop_internal_queues()
2087 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2093 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n"); in goya_stop_internal_queues()
2097 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2103 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n"); in goya_stop_internal_queues()
2107 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2113 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n"); in goya_stop_internal_queues()
2120 static void goya_dma_stall(struct hl_device *hdev) in goya_dma_stall() argument
2122 struct goya_device *goya = hdev->asic_specific; in goya_dma_stall()
2134 static void goya_tpc_stall(struct hl_device *hdev) in goya_tpc_stall() argument
2136 struct goya_device *goya = hdev->asic_specific; in goya_tpc_stall()
2151 static void goya_mme_stall(struct hl_device *hdev) in goya_mme_stall() argument
2153 struct goya_device *goya = hdev->asic_specific; in goya_mme_stall()
2161 static int goya_enable_msix(struct hl_device *hdev) in goya_enable_msix() argument
2163 struct goya_device *goya = hdev->asic_specific; in goya_enable_msix()
2164 int cq_cnt = hdev->asic_prop.completion_queues_count; in goya_enable_msix()
2170 rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES, in goya_enable_msix()
2173 dev_err(hdev->dev, in goya_enable_msix()
2180 irq = pci_irq_vector(hdev->pdev, i); in goya_enable_msix()
2182 &hdev->completion_queue[i]); in goya_enable_msix()
2184 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2189 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_enable_msix()
2193 &hdev->event_queue); in goya_enable_msix()
2195 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2204 free_irq(pci_irq_vector(hdev->pdev, i), in goya_enable_msix()
2205 &hdev->completion_queue[i]); in goya_enable_msix()
2207 pci_free_irq_vectors(hdev->pdev); in goya_enable_msix()
2211 static void goya_sync_irqs(struct hl_device *hdev) in goya_sync_irqs() argument
2213 struct goya_device *goya = hdev->asic_specific; in goya_sync_irqs()
2220 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in goya_sync_irqs()
2221 synchronize_irq(pci_irq_vector(hdev->pdev, i)); in goya_sync_irqs()
2223 synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX)); in goya_sync_irqs()
2226 static void goya_disable_msix(struct hl_device *hdev) in goya_disable_msix() argument
2228 struct goya_device *goya = hdev->asic_specific; in goya_disable_msix()
2234 goya_sync_irqs(hdev); in goya_disable_msix()
2236 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_disable_msix()
2237 free_irq(irq, &hdev->event_queue); in goya_disable_msix()
2239 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in goya_disable_msix()
2240 irq = pci_irq_vector(hdev->pdev, i); in goya_disable_msix()
2241 free_irq(irq, &hdev->completion_queue[i]); in goya_disable_msix()
2244 pci_free_irq_vectors(hdev->pdev); in goya_disable_msix()
2249 static void goya_enable_timestamp(struct hl_device *hdev) in goya_enable_timestamp() argument
2262 static void goya_disable_timestamp(struct hl_device *hdev) in goya_disable_timestamp() argument
2268 static void goya_halt_engines(struct hl_device *hdev, bool hard_reset) in goya_halt_engines() argument
2272 dev_info(hdev->dev, in goya_halt_engines()
2275 if (hdev->pldm) in goya_halt_engines()
2280 goya_stop_external_queues(hdev); in goya_halt_engines()
2281 goya_stop_internal_queues(hdev); in goya_halt_engines()
2285 goya_dma_stall(hdev); in goya_halt_engines()
2286 goya_tpc_stall(hdev); in goya_halt_engines()
2287 goya_mme_stall(hdev); in goya_halt_engines()
2291 goya_disable_external_queues(hdev); in goya_halt_engines()
2292 goya_disable_internal_queues(hdev); in goya_halt_engines()
2294 goya_disable_timestamp(hdev); in goya_halt_engines()
2297 goya_disable_msix(hdev); in goya_halt_engines()
2298 goya_mmu_remove_device_cpu_mappings(hdev); in goya_halt_engines()
2300 goya_sync_irqs(hdev); in goya_halt_engines()
2312 static int goya_load_firmware_to_device(struct hl_device *hdev) in goya_load_firmware_to_device() argument
2316 dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET; in goya_load_firmware_to_device()
2318 return hl_fw_load_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst); in goya_load_firmware_to_device()
2329 static int goya_load_boot_fit_to_device(struct hl_device *hdev) in goya_load_boot_fit_to_device() argument
2333 dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET; in goya_load_boot_fit_to_device()
2335 return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst); in goya_load_boot_fit_to_device()
2342 static void goya_read_device_fw_version(struct hl_device *hdev, in goya_read_device_fw_version() argument
2352 dest = hdev->asic_prop.uboot_ver; in goya_read_device_fw_version()
2357 dest = hdev->asic_prop.preboot_ver; in goya_read_device_fw_version()
2361 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc); in goya_read_device_fw_version()
2368 memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off, in goya_read_device_fw_version()
2371 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n", in goya_read_device_fw_version()
2377 static int goya_init_cpu(struct hl_device *hdev) in goya_init_cpu() argument
2379 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu()
2382 if (!hdev->cpu_enable) in goya_init_cpu()
2392 if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { in goya_init_cpu()
2393 dev_err(hdev->dev, in goya_init_cpu()
2398 rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS, in goya_init_cpu()
2412 static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, in goya_mmu_update_asid_hop0_addr() argument
2418 if (hdev->pldm) in goya_mmu_update_asid_hop0_addr()
2428 hdev, in goya_mmu_update_asid_hop0_addr()
2436 dev_err(hdev->dev, in goya_mmu_update_asid_hop0_addr()
2444 int goya_mmu_init(struct hl_device *hdev) in goya_mmu_init() argument
2446 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_init()
2447 struct goya_device *goya = hdev->asic_specific; in goya_mmu_init()
2451 if (!hdev->mmu_enable) in goya_mmu_init()
2457 hdev->dram_supports_virtual_memory = true; in goya_mmu_init()
2458 hdev->dram_default_page_mapping = true; in goya_mmu_init()
2464 rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr); in goya_mmu_init()
2466 dev_err(hdev->dev, in goya_mmu_init()
2483 hdev->asic_funcs->mmu_invalidate_cache(hdev, true, in goya_mmu_init()
2503 static int goya_hw_init(struct hl_device *hdev) in goya_hw_init() argument
2505 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_hw_init()
2508 dev_info(hdev->dev, "Starting initialization of H/W\n"); in goya_hw_init()
2521 rc = goya_init_cpu(hdev); in goya_hw_init()
2523 dev_err(hdev->dev, "failed to initialize CPU\n"); in goya_hw_init()
2527 goya_tpc_mbist_workaround(hdev); in goya_hw_init()
2529 goya_init_golden_registers(hdev); in goya_hw_init()
2535 if (goya_set_ddr_bar_base(hdev, (MMU_PAGE_TABLES_ADDR & in goya_hw_init()
2537 dev_err(hdev->dev, in goya_hw_init()
2542 rc = goya_mmu_init(hdev); in goya_hw_init()
2546 goya_init_security(hdev); in goya_hw_init()
2548 goya_init_dma_qmans(hdev); in goya_hw_init()
2550 goya_init_mme_qmans(hdev); in goya_hw_init()
2552 goya_init_tpc_qmans(hdev); in goya_hw_init()
2554 goya_enable_timestamp(hdev); in goya_hw_init()
2557 rc = goya_enable_msix(hdev); in goya_hw_init()
2567 goya_disable_internal_queues(hdev); in goya_hw_init()
2568 goya_disable_external_queues(hdev); in goya_hw_init()
2580 static void goya_hw_fini(struct hl_device *hdev, bool hard_reset) in goya_hw_fini() argument
2582 struct goya_device *goya = hdev->asic_specific; in goya_hw_fini()
2585 if (hdev->pldm) { in goya_hw_fini()
2603 goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE); in goya_hw_fini()
2604 goya_disable_clk_rlx(hdev); in goya_hw_fini()
2605 goya_set_pll_refclk(hdev); in goya_hw_fini()
2608 dev_info(hdev->dev, in goya_hw_fini()
2613 dev_info(hdev->dev, in goya_hw_fini()
2627 dev_err(hdev->dev, in goya_hw_fini()
2654 int goya_suspend(struct hl_device *hdev) in goya_suspend() argument
2658 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS); in goya_suspend()
2660 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); in goya_suspend()
2665 int goya_resume(struct hl_device *hdev) in goya_resume() argument
2667 return goya_init_iatu(hdev); in goya_resume()
2670 static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma, in goya_cb_mmap() argument
2678 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, in goya_cb_mmap()
2681 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in goya_cb_mmap()
2686 void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) in goya_ring_doorbell() argument
2753 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", in goya_ring_doorbell()
2768 void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) in goya_pqe_write() argument
2774 static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size, in goya_dma_alloc_coherent() argument
2777 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, in goya_dma_alloc_coherent()
2787 static void goya_dma_free_coherent(struct hl_device *hdev, size_t size, in goya_dma_free_coherent() argument
2793 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); in goya_dma_free_coherent()
2796 void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, in goya_get_int_queue_base() argument
2802 *dma_handle = hdev->asic_prop.sram_base_address; in goya_get_int_queue_base()
2804 base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; in goya_get_int_queue_base()
2844 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); in goya_get_int_queue_base()
2854 static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) in goya_send_job_on_qman0() argument
2863 if (hdev->pldm) in goya_send_job_on_qman0()
2868 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) { in goya_send_job_on_qman0()
2869 dev_err_ratelimited(hdev->dev, in goya_send_job_on_qman0()
2874 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_send_job_on_qman0()
2877 dev_err(hdev->dev, in goya_send_job_on_qman0()
2882 goya_qman0_set_security(hdev, true); in goya_send_job_on_qman0()
2896 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0, in goya_send_job_on_qman0()
2899 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); in goya_send_job_on_qman0()
2903 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, in goya_send_job_on_qman0()
2907 hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0); in goya_send_job_on_qman0()
2910 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); in goya_send_job_on_qman0()
2915 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_send_job_on_qman0()
2918 goya_qman0_set_security(hdev, false); in goya_send_job_on_qman0()
2923 int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, in goya_send_cpu_message() argument
2926 struct goya_device *goya = hdev->asic_specific; in goya_send_cpu_message()
2937 return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len, in goya_send_cpu_message()
2941 int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id) in goya_test_queue() argument
2952 fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, in goya_test_queue()
2955 dev_err(hdev->dev, in goya_test_queue()
2963 fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, in goya_test_queue()
2967 dev_err(hdev->dev, in goya_test_queue()
2981 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, in goya_test_queue()
2985 dev_err(hdev->dev, in goya_test_queue()
2991 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val), in goya_test_queue()
2994 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); in goya_test_queue()
2997 dev_err(hdev->dev, in goya_test_queue()
3004 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt, in goya_test_queue()
3007 hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr, in goya_test_queue()
3012 int goya_test_cpu_queue(struct hl_device *hdev) in goya_test_cpu_queue() argument
3014 struct goya_device *goya = hdev->asic_specific; in goya_test_cpu_queue()
3023 return hl_fw_test_cpu_queue(hdev); in goya_test_cpu_queue()
3026 int goya_test_queues(struct hl_device *hdev) in goya_test_queues() argument
3031 rc = goya_test_queue(hdev, i); in goya_test_queues()
3039 static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size, in goya_dma_pool_zalloc() argument
3047 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in goya_dma_pool_zalloc()
3056 static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr, in goya_dma_pool_free() argument
3062 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); in goya_dma_pool_free()
3065 void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, in goya_cpu_accessible_dma_pool_alloc() argument
3070 vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); in goya_cpu_accessible_dma_pool_alloc()
3071 *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + in goya_cpu_accessible_dma_pool_alloc()
3077 void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, in goya_cpu_accessible_dma_pool_free() argument
3080 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); in goya_cpu_accessible_dma_pool_free()
3083 static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl, in goya_dma_map_sg() argument
3089 if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir)) in goya_dma_map_sg()
3099 static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl, in goya_dma_unmap_sg() argument
3109 dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir); in goya_dma_unmap_sg()
3112 u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt) in goya_get_dma_desc_list_size() argument
3153 static int goya_pin_memory_before_cs(struct hl_device *hdev, in goya_pin_memory_before_cs() argument
3161 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3169 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3176 rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl, in goya_pin_memory_before_cs()
3179 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in goya_pin_memory_before_cs()
3188 goya_get_dma_desc_list_size(hdev, userptr->sgt); in goya_pin_memory_before_cs()
3194 hl_unpin_host_memory(hdev, userptr); in goya_pin_memory_before_cs()
3200 static int goya_validate_dma_pkt_host(struct hl_device *hdev, in goya_validate_dma_pkt_host() argument
3223 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n"); in goya_validate_dma_pkt_host()
3233 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3241 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n"); in goya_validate_dma_pkt_host()
3250 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3256 dev_err(hdev->dev, "DMA direction is undefined\n"); in goya_validate_dma_pkt_host()
3263 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_host()
3264 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_host()
3266 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3275 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_host()
3276 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_host()
3278 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3291 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3296 rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt, in goya_validate_dma_pkt_host()
3303 static int goya_validate_dma_pkt_no_host(struct hl_device *hdev, in goya_validate_dma_pkt_no_host() argument
3316 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n"); in goya_validate_dma_pkt_no_host()
3320 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n"); in goya_validate_dma_pkt_no_host()
3327 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_no_host()
3328 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_no_host()
3329 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3336 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_no_host()
3337 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_no_host()
3338 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3348 static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev, in goya_validate_dma_pkt_no_mmu() argument
3356 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_no_mmu()
3357 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3359 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3361 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_no_mmu()
3372 dev_err(hdev->dev, in goya_validate_dma_pkt_no_mmu()
3378 rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt); in goya_validate_dma_pkt_no_mmu()
3380 rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt); in goya_validate_dma_pkt_no_mmu()
3385 static int goya_validate_dma_pkt_mmu(struct hl_device *hdev, in goya_validate_dma_pkt_mmu() argument
3389 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_mmu()
3390 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3392 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3394 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_mmu()
3404 hdev->asic_prop.pmmu.start_addr, in goya_validate_dma_pkt_mmu()
3405 hdev->asic_prop.pmmu.end_addr)) { in goya_validate_dma_pkt_mmu()
3406 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3412 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3422 static int goya_validate_wreg32(struct hl_device *hdev, in goya_validate_wreg32() argument
3426 struct goya_device *goya = hdev->asic_specific; in goya_validate_wreg32()
3433 dev_dbg(hdev->dev, "WREG32 packet details:\n"); in goya_validate_wreg32()
3434 dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset); in goya_validate_wreg32()
3435 dev_dbg(hdev->dev, "value == 0x%x\n", in goya_validate_wreg32()
3439 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n", in goya_validate_wreg32()
3458 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n", in goya_validate_wreg32()
3466 static int goya_validate_cb(struct hl_device *hdev, in goya_validate_cb() argument
3488 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_validate_cb()
3496 dev_err(hdev->dev, in goya_validate_cb()
3509 rc = goya_validate_wreg32(hdev, in goya_validate_cb()
3515 dev_err(hdev->dev, in goya_validate_cb()
3521 dev_err(hdev->dev, in goya_validate_cb()
3527 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_validate_cb()
3532 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_validate_cb()
3538 rc = goya_validate_dma_pkt_mmu(hdev, parser, in goya_validate_cb()
3541 rc = goya_validate_dma_pkt_no_mmu(hdev, parser, in goya_validate_cb()
3553 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_validate_cb()
3573 static int goya_patch_dma_packet(struct hl_device *hdev, in goya_patch_dma_packet() argument
3620 (hl_userptr_is_pinned(hdev, addr, in goya_patch_dma_packet()
3623 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", in goya_patch_dma_packet()
3689 dev_err(hdev->dev, in goya_patch_dma_packet()
3703 static int goya_patch_cb(struct hl_device *hdev, in goya_patch_cb() argument
3727 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_patch_cb()
3735 dev_err(hdev->dev, in goya_patch_cb()
3743 rc = goya_patch_dma_packet(hdev, parser, in goya_patch_cb()
3753 rc = goya_validate_wreg32(hdev, parser, in goya_patch_cb()
3758 dev_err(hdev->dev, in goya_patch_cb()
3764 dev_err(hdev->dev, in goya_patch_cb()
3770 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_patch_cb()
3775 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_patch_cb()
3788 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_patch_cb()
3801 static int goya_parse_cb_mmu(struct hl_device *hdev, in goya_parse_cb_mmu() argument
3817 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, in goya_parse_cb_mmu()
3822 dev_err(hdev->dev, in goya_parse_cb_mmu()
3829 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3852 rc = goya_validate_cb(hdev, parser, true); in goya_parse_cb_mmu()
3861 dev_err(hdev->dev, "user CB size mismatch\n"); in goya_parse_cb_mmu()
3874 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_mmu()
3880 static int goya_parse_cb_no_mmu(struct hl_device *hdev, in goya_parse_cb_no_mmu() argument
3886 rc = goya_validate_cb(hdev, parser, false); in goya_parse_cb_no_mmu()
3891 rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx, in goya_parse_cb_no_mmu()
3895 dev_err(hdev->dev, in goya_parse_cb_no_mmu()
3901 parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
3911 rc = goya_patch_cb(hdev, parser); in goya_parse_cb_no_mmu()
3923 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, in goya_parse_cb_no_mmu()
3928 hl_userptr_delete_list(hdev, parser->job_userptr_list); in goya_parse_cb_no_mmu()
3932 static int goya_parse_cb_no_ext_queue(struct hl_device *hdev, in goya_parse_cb_no_ext_queue() argument
3935 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in goya_parse_cb_no_ext_queue()
3936 struct goya_device *goya = hdev->asic_specific; in goya_parse_cb_no_ext_queue()
3956 dev_err(hdev->dev, in goya_parse_cb_no_ext_queue()
3963 int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) in goya_cs_parser() argument
3965 struct goya_device *goya = hdev->asic_specific; in goya_cs_parser()
3968 return goya_parse_cb_no_ext_queue(hdev, parser); in goya_cs_parser()
3971 return goya_parse_cb_mmu(hdev, parser); in goya_cs_parser()
3973 return goya_parse_cb_no_mmu(hdev, parser); in goya_cs_parser()
3976 void goya_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address, in goya_add_end_of_cb_packets() argument
4001 void goya_update_eq_ci(struct hl_device *hdev, u32 val) in goya_update_eq_ci() argument
4006 void goya_restore_phase_topology(struct hl_device *hdev) in goya_restore_phase_topology() argument
4011 static void goya_clear_sm_regs(struct hl_device *hdev) in goya_clear_sm_regs() argument
4046 static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val) in goya_debugfs_read32() argument
4048 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_read32()
4058 *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_read32()
4061 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) { in goya_debugfs_read32()
4066 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr); in goya_debugfs_read32()
4068 *val = readl(hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_read32()
4071 ddr_bar_addr = goya_set_ddr_bar_base(hdev, in goya_debugfs_read32()
4102 static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val) in goya_debugfs_write32() argument
4104 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_write32()
4114 writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_write32()
4117 } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) { in goya_debugfs_write32()
4122 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr); in goya_debugfs_write32()
4124 writel(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_write32()
4127 ddr_bar_addr = goya_set_ddr_bar_base(hdev, in goya_debugfs_write32()
4143 static int goya_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val) in goya_debugfs_read64() argument
4145 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_read64()
4158 *val = readq(hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_read64()
4162 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) { in goya_debugfs_read64()
4167 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr); in goya_debugfs_read64()
4169 *val = readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_read64()
4172 ddr_bar_addr = goya_set_ddr_bar_base(hdev, in goya_debugfs_read64()
4188 static int goya_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val) in goya_debugfs_write64() argument
4190 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_debugfs_write64()
4201 writeq(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_debugfs_write64()
4205 DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) { in goya_debugfs_write64()
4210 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr); in goya_debugfs_write64()
4212 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_debugfs_write64()
4215 ddr_bar_addr = goya_set_ddr_bar_base(hdev, in goya_debugfs_write64()
4231 static u64 goya_read_pte(struct hl_device *hdev, u64 addr) in goya_read_pte() argument
4233 struct goya_device *goya = hdev->asic_specific; in goya_read_pte()
4235 if (hdev->hard_reset_pending) in goya_read_pte()
4238 return readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_read_pte()
4242 static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val) in goya_write_pte() argument
4244 struct goya_device *goya = hdev->asic_specific; in goya_write_pte()
4246 if (hdev->hard_reset_pending) in goya_write_pte()
4249 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_write_pte()
4446 static void goya_print_razwi_info(struct hl_device *hdev) in goya_print_razwi_info() argument
4449 dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n"); in goya_print_razwi_info()
4454 dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n"); in goya_print_razwi_info()
4459 dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n"); in goya_print_razwi_info()
4464 dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n"); in goya_print_razwi_info()
4469 static void goya_print_mmu_error_info(struct hl_device *hdev) in goya_print_mmu_error_info() argument
4471 struct goya_device *goya = hdev->asic_specific; in goya_print_mmu_error_info()
4484 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", in goya_print_mmu_error_info()
4491 static void goya_print_irq_info(struct hl_device *hdev, u16 event_type, in goya_print_irq_info() argument
4497 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in goya_print_irq_info()
4501 goya_print_razwi_info(hdev); in goya_print_irq_info()
4502 goya_print_mmu_error_info(hdev); in goya_print_irq_info()
4506 static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr, in goya_unmask_irq_arr() argument
4524 dev_err(hdev->dev, "too many elements in IRQ array\n"); in goya_unmask_irq_arr()
4546 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt, in goya_unmask_irq_arr()
4550 dev_err(hdev->dev, "failed to unmask IRQ array\n"); in goya_unmask_irq_arr()
4557 static int goya_soft_reset_late_init(struct hl_device *hdev) in goya_soft_reset_late_init() argument
4563 return goya_unmask_irq_arr(hdev, goya_all_events, in goya_soft_reset_late_init()
4567 static int goya_unmask_irq(struct hl_device *hdev, u16 event_type) in goya_unmask_irq() argument
4579 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), in goya_unmask_irq()
4583 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type); in goya_unmask_irq()
4588 static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type) in goya_print_clk_change_info() argument
4592 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4593 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4597 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4598 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4602 hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4603 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4607 hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4608 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4613 dev_err(hdev->dev, "Received invalid clock change event %d\n", in goya_print_clk_change_info()
4619 void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) in goya_handle_eqe() argument
4624 struct goya_device *goya = hdev->asic_specific; in goya_handle_eqe()
4627 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in goya_handle_eqe()
4659 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4660 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4661 hl_device_reset(hdev, true, false); in goya_handle_eqe()
4691 goya_print_irq_info(hdev, event_type, true); in goya_handle_eqe()
4692 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4705 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4706 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4713 goya_print_clk_change_info(hdev, event_type); in goya_handle_eqe()
4714 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4718 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", in goya_handle_eqe()
4724 void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) in goya_get_events_stat() argument
4726 struct goya_device *goya = hdev->asic_specific; in goya_get_events_stat()
4737 static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, in goya_memset_device_memory() argument
4749 cb = hl_cb_kernel_create(hdev, cb_size, false); in goya_memset_device_memory()
4779 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); in goya_memset_device_memory()
4781 dev_err(hdev->dev, "Failed to allocate a new job\n"); in goya_memset_device_memory()
4794 hl_debugfs_add_job(hdev, job); in goya_memset_device_memory()
4796 rc = goya_send_job_on_qman0(hdev, job); in goya_memset_device_memory()
4798 hl_debugfs_remove_job(hdev, job); in goya_memset_device_memory()
4804 hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); in goya_memset_device_memory()
4809 int goya_context_switch(struct hl_device *hdev, u32 asid) in goya_context_switch() argument
4811 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_context_switch()
4813 u32 size = hdev->pldm ? 0x10000 : prop->sram_size; in goya_context_switch()
4819 rc = goya_memset_device_memory(hdev, addr, size, val, false); in goya_context_switch()
4821 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n"); in goya_context_switch()
4838 goya_mmu_prepare(hdev, asid); in goya_context_switch()
4840 goya_clear_sm_regs(hdev); in goya_context_switch()
4845 static int goya_mmu_clear_pgt_range(struct hl_device *hdev) in goya_mmu_clear_pgt_range() argument
4847 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_clear_pgt_range()
4848 struct goya_device *goya = hdev->asic_specific; in goya_mmu_clear_pgt_range()
4856 return goya_memset_device_memory(hdev, addr, size, 0, true); in goya_mmu_clear_pgt_range()
4859 static int goya_mmu_set_dram_default_page(struct hl_device *hdev) in goya_mmu_set_dram_default_page() argument
4861 struct goya_device *goya = hdev->asic_specific; in goya_mmu_set_dram_default_page()
4862 u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; in goya_mmu_set_dram_default_page()
4869 return goya_memset_device_memory(hdev, addr, size, val, true); in goya_mmu_set_dram_default_page()
4872 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev) in goya_mmu_add_mappings_for_device_cpu() argument
4874 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_add_mappings_for_device_cpu()
4875 struct goya_device *goya = hdev->asic_specific; in goya_mmu_add_mappings_for_device_cpu()
4883 rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off, in goya_mmu_add_mappings_for_device_cpu()
4887 dev_err(hdev->dev, "Map failed for address 0x%llx\n", in goya_mmu_add_mappings_for_device_cpu()
4893 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_add_mappings_for_device_cpu()
4894 rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, in goya_mmu_add_mappings_for_device_cpu()
4895 hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB, true); in goya_mmu_add_mappings_for_device_cpu()
4898 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4905 rc = hl_mmu_map(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4907 hdev->cpu_accessible_dma_address + cpu_off, in goya_mmu_add_mappings_for_device_cpu()
4910 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4918 goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID); in goya_mmu_add_mappings_for_device_cpu()
4919 goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID); in goya_mmu_add_mappings_for_device_cpu()
4932 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4935 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4940 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4943 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4950 void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev) in goya_mmu_remove_device_cpu_mappings() argument
4952 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_remove_device_cpu_mappings()
4953 struct goya_device *goya = hdev->asic_specific; in goya_mmu_remove_device_cpu_mappings()
4965 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_remove_device_cpu_mappings()
4966 if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR, in goya_mmu_remove_device_cpu_mappings()
4968 dev_warn(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4972 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4976 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4982 if (hl_mmu_unmap(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4985 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4992 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) in goya_mmu_prepare() argument
4994 struct goya_device *goya = hdev->asic_specific; in goya_mmu_prepare()
5007 goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid); in goya_mmu_prepare()
5010 static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, in goya_mmu_invalidate_cache() argument
5013 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache()
5018 hdev->hard_reset_pending) in goya_mmu_invalidate_cache()
5025 if (hdev->pldm) in goya_mmu_invalidate_cache()
5030 mutex_lock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache()
5036 hdev, in goya_mmu_invalidate_cache()
5043 mutex_unlock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache()
5046 dev_err_ratelimited(hdev->dev, in goya_mmu_invalidate_cache()
5048 hl_device_reset(hdev, true, false); in goya_mmu_invalidate_cache()
5054 static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, in goya_mmu_invalidate_cache_range() argument
5057 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache_range()
5062 hdev->hard_reset_pending) in goya_mmu_invalidate_cache_range()
5069 if (hdev->pldm) in goya_mmu_invalidate_cache_range()
5074 mutex_lock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache_range()
5091 hdev, in goya_mmu_invalidate_cache_range()
5098 mutex_unlock(&hdev->mmu_cache_lock); in goya_mmu_invalidate_cache_range()
5101 dev_err_ratelimited(hdev->dev, in goya_mmu_invalidate_cache_range()
5103 hl_device_reset(hdev, true, false); in goya_mmu_invalidate_cache_range()
5109 int goya_send_heartbeat(struct hl_device *hdev) in goya_send_heartbeat() argument
5111 struct goya_device *goya = hdev->asic_specific; in goya_send_heartbeat()
5116 return hl_fw_send_heartbeat(hdev); in goya_send_heartbeat()
5119 int goya_cpucp_info_get(struct hl_device *hdev) in goya_cpucp_info_get() argument
5121 struct goya_device *goya = hdev->asic_specific; in goya_cpucp_info_get()
5122 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_cpucp_info_get()
5129 rc = hl_fw_cpucp_info_get(hdev); in goya_cpucp_info_get()
5137 dev_err(hdev->dev, in goya_cpucp_info_get()
5154 static void goya_set_clock_gating(struct hl_device *hdev) in goya_set_clock_gating() argument
5159 static void goya_disable_clock_gating(struct hl_device *hdev) in goya_disable_clock_gating() argument
5164 static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask, in goya_is_device_idle() argument
5244 static void goya_hw_queues_lock(struct hl_device *hdev) in goya_hw_queues_lock() argument
5247 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_lock()
5252 static void goya_hw_queues_unlock(struct hl_device *hdev) in goya_hw_queues_unlock() argument
5255 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_unlock()
5260 static u32 goya_get_pci_id(struct hl_device *hdev) in goya_get_pci_id() argument
5262 return hdev->pdev->device; in goya_get_pci_id()
5265 static int goya_get_eeprom_data(struct hl_device *hdev, void *data, in goya_get_eeprom_data() argument
5268 struct goya_device *goya = hdev->asic_specific; in goya_get_eeprom_data()
5273 return hl_fw_get_eeprom_data(hdev, data, max_size); in goya_get_eeprom_data()
5276 static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev) in goya_get_hw_state() argument
5286 u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) in goya_get_queue_id_for_cq() argument
5291 static u32 goya_get_signal_cb_size(struct hl_device *hdev) in goya_get_signal_cb_size() argument
5296 static u32 goya_get_wait_cb_size(struct hl_device *hdev) in goya_get_wait_cb_size() argument
5301 static void goya_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id) in goya_gen_signal_cb() argument
5306 static void goya_gen_wait_cb(struct hl_device *hdev, void *data, u16 sob_id, in goya_gen_wait_cb() argument
5312 static void goya_reset_sob(struct hl_device *hdev, void *data) in goya_reset_sob() argument
5317 static void goya_set_dma_mask_from_fw(struct hl_device *hdev) in goya_set_dma_mask_from_fw() argument
5321 dev_dbg(hdev->dev, "Working in 64-bit DMA mode\n"); in goya_set_dma_mask_from_fw()
5322 hdev->power9_64bit_dma_enable = 1; in goya_set_dma_mask_from_fw()
5323 hdev->dma_mask = 64; in goya_set_dma_mask_from_fw()
5325 dev_dbg(hdev->dev, "Working in 48-bit DMA mode\n"); in goya_set_dma_mask_from_fw()
5326 hdev->power9_64bit_dma_enable = 0; in goya_set_dma_mask_from_fw()
5327 hdev->dma_mask = 48; in goya_set_dma_mask_from_fw()
5331 u64 goya_get_device_time(struct hl_device *hdev) in goya_get_device_time() argument
5419 void goya_set_asic_funcs(struct hl_device *hdev) in goya_set_asic_funcs() argument
5421 hdev->asic_funcs = &goya_funcs; in goya_set_asic_funcs()