Lines Matching refs:idxd
43 const char *idxd_get_dev_name(struct idxd_device *idxd) in idxd_get_dev_name() argument
45 return idxd_name[idxd->type]; in idxd_get_dev_name()
48 static int idxd_setup_interrupts(struct idxd_device *idxd) in idxd_setup_interrupts() argument
50 struct pci_dev *pdev = idxd->pdev; in idxd_setup_interrupts()
63 idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) * in idxd_setup_interrupts()
65 if (!idxd->msix_entries) { in idxd_setup_interrupts()
71 idxd->msix_entries[i].entry = i; in idxd_setup_interrupts()
73 rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt); in idxd_setup_interrupts()
84 idxd->irq_entries = devm_kcalloc(dev, msixcnt, in idxd_setup_interrupts()
87 if (!idxd->irq_entries) { in idxd_setup_interrupts()
93 idxd->irq_entries[i].id = i; in idxd_setup_interrupts()
94 idxd->irq_entries[i].idxd = idxd; in idxd_setup_interrupts()
97 msix = &idxd->msix_entries[0]; in idxd_setup_interrupts()
98 irq_entry = &idxd->irq_entries[0]; in idxd_setup_interrupts()
111 idxd->num_wq_irqs = msixcnt - 1; in idxd_setup_interrupts()
114 msix = &idxd->msix_entries[i]; in idxd_setup_interrupts()
115 irq_entry = &idxd->irq_entries[i]; in idxd_setup_interrupts()
117 init_llist_head(&idxd->irq_entries[i].pending_llist); in idxd_setup_interrupts()
118 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); in idxd_setup_interrupts()
132 idxd_unmask_error_interrupts(idxd); in idxd_setup_interrupts()
138 idxd_mask_error_interrupts(idxd); in idxd_setup_interrupts()
144 static int idxd_setup_internals(struct idxd_device *idxd) in idxd_setup_internals() argument
146 struct device *dev = &idxd->pdev->dev; in idxd_setup_internals()
149 init_waitqueue_head(&idxd->cmd_waitq); in idxd_setup_internals()
150 idxd->groups = devm_kcalloc(dev, idxd->max_groups, in idxd_setup_internals()
152 if (!idxd->groups) in idxd_setup_internals()
155 for (i = 0; i < idxd->max_groups; i++) { in idxd_setup_internals()
156 idxd->groups[i].idxd = idxd; in idxd_setup_internals()
157 idxd->groups[i].id = i; in idxd_setup_internals()
158 idxd->groups[i].tc_a = -1; in idxd_setup_internals()
159 idxd->groups[i].tc_b = -1; in idxd_setup_internals()
162 idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq), in idxd_setup_internals()
164 if (!idxd->wqs) in idxd_setup_internals()
167 idxd->engines = devm_kcalloc(dev, idxd->max_engines, in idxd_setup_internals()
169 if (!idxd->engines) in idxd_setup_internals()
172 for (i = 0; i < idxd->max_wqs; i++) { in idxd_setup_internals()
173 struct idxd_wq *wq = &idxd->wqs[i]; in idxd_setup_internals()
176 wq->idxd = idxd; in idxd_setup_internals()
179 wq->max_xfer_bytes = idxd->max_xfer_bytes; in idxd_setup_internals()
180 wq->max_batch_size = idxd->max_batch_size; in idxd_setup_internals()
181 wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL); in idxd_setup_internals()
186 for (i = 0; i < idxd->max_engines; i++) { in idxd_setup_internals()
187 idxd->engines[i].idxd = idxd; in idxd_setup_internals()
188 idxd->engines[i].id = i; in idxd_setup_internals()
191 idxd->wq = create_workqueue(dev_name(dev)); in idxd_setup_internals()
192 if (!idxd->wq) in idxd_setup_internals()
198 static void idxd_read_table_offsets(struct idxd_device *idxd) in idxd_read_table_offsets() argument
201 struct device *dev = &idxd->pdev->dev; in idxd_read_table_offsets()
203 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); in idxd_read_table_offsets()
204 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET in idxd_read_table_offsets()
206 idxd->grpcfg_offset = offsets.grpcfg * 0x100; in idxd_read_table_offsets()
207 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); in idxd_read_table_offsets()
208 idxd->wqcfg_offset = offsets.wqcfg * 0x100; in idxd_read_table_offsets()
210 idxd->wqcfg_offset); in idxd_read_table_offsets()
211 idxd->msix_perm_offset = offsets.msix_perm * 0x100; in idxd_read_table_offsets()
213 idxd->msix_perm_offset); in idxd_read_table_offsets()
214 idxd->perfmon_offset = offsets.perfmon * 0x100; in idxd_read_table_offsets()
215 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); in idxd_read_table_offsets()
218 static void idxd_read_caps(struct idxd_device *idxd) in idxd_read_caps() argument
220 struct device *dev = &idxd->pdev->dev; in idxd_read_caps()
224 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); in idxd_read_caps()
225 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); in idxd_read_caps()
226 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; in idxd_read_caps()
227 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); in idxd_read_caps()
228 idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; in idxd_read_caps()
229 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); in idxd_read_caps()
230 if (idxd->hw.gen_cap.config_en) in idxd_read_caps()
231 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); in idxd_read_caps()
234 idxd->hw.group_cap.bits = in idxd_read_caps()
235 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); in idxd_read_caps()
236 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); in idxd_read_caps()
237 idxd->max_groups = idxd->hw.group_cap.num_groups; in idxd_read_caps()
238 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); in idxd_read_caps()
239 idxd->max_tokens = idxd->hw.group_cap.total_tokens; in idxd_read_caps()
240 dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens); in idxd_read_caps()
241 idxd->nr_tokens = idxd->max_tokens; in idxd_read_caps()
244 idxd->hw.engine_cap.bits = in idxd_read_caps()
245 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); in idxd_read_caps()
246 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); in idxd_read_caps()
247 idxd->max_engines = idxd->hw.engine_cap.num_engines; in idxd_read_caps()
248 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); in idxd_read_caps()
251 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); in idxd_read_caps()
252 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); in idxd_read_caps()
253 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; in idxd_read_caps()
254 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); in idxd_read_caps()
255 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; in idxd_read_caps()
256 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); in idxd_read_caps()
257 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); in idxd_read_caps()
258 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); in idxd_read_caps()
262 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + in idxd_read_caps()
264 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); in idxd_read_caps()
272 struct idxd_device *idxd; in idxd_alloc() local
274 idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL); in idxd_alloc()
275 if (!idxd) in idxd_alloc()
278 idxd->pdev = pdev; in idxd_alloc()
279 idxd->reg_base = iomap[IDXD_MMIO_BAR]; in idxd_alloc()
280 spin_lock_init(&idxd->dev_lock); in idxd_alloc()
282 return idxd; in idxd_alloc()
285 static int idxd_probe(struct idxd_device *idxd) in idxd_probe() argument
287 struct pci_dev *pdev = idxd->pdev; in idxd_probe()
292 rc = idxd_device_init_reset(idxd); in idxd_probe()
298 idxd_read_caps(idxd); in idxd_probe()
299 idxd_read_table_offsets(idxd); in idxd_probe()
301 rc = idxd_setup_internals(idxd); in idxd_probe()
305 rc = idxd_setup_interrupts(idxd); in idxd_probe()
312 idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL); in idxd_probe()
314 if (idxd->id < 0) { in idxd_probe()
319 idxd->major = idxd_cdev_get_major(idxd); in idxd_probe()
321 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); in idxd_probe()
325 idxd_mask_error_interrupts(idxd); in idxd_probe()
326 idxd_mask_msix_vectors(idxd); in idxd_probe()
335 struct idxd_device *idxd; in idxd_pci_probe() local
367 idxd = idxd_alloc(pdev, iomap); in idxd_pci_probe()
368 if (!idxd) in idxd_pci_probe()
371 idxd_set_type(idxd); in idxd_pci_probe()
375 pci_set_drvdata(pdev, idxd); in idxd_pci_probe()
377 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); in idxd_pci_probe()
378 rc = idxd_probe(idxd); in idxd_pci_probe()
384 rc = idxd_setup_sysfs(idxd); in idxd_pci_probe()
390 idxd->state = IDXD_DEV_CONF_READY; in idxd_pci_probe()
393 idxd->hw.version); in idxd_pci_probe()
426 struct idxd_device *idxd = pci_get_drvdata(pdev); in idxd_shutdown() local
431 rc = idxd_device_disable(idxd); in idxd_shutdown()
436 idxd_mask_msix_vectors(idxd); in idxd_shutdown()
437 idxd_mask_error_interrupts(idxd); in idxd_shutdown()
440 irq_entry = &idxd->irq_entries[i]; in idxd_shutdown()
441 synchronize_irq(idxd->msix_entries[i].vector); in idxd_shutdown()
448 destroy_workqueue(idxd->wq); in idxd_shutdown()
453 struct idxd_device *idxd = pci_get_drvdata(pdev); in idxd_remove() local
456 idxd_cleanup_sysfs(idxd); in idxd_remove()
459 idr_remove(&idxd_idrs[idxd->type], idxd->id); in idxd_remove()