16 static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
20 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)  in idxd_mask_msix_vector()  argument
22 	struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);  in idxd_mask_msix_vector()
27 void idxd_mask_msix_vectors(struct idxd_device *idxd)  in idxd_mask_msix_vectors()  argument
29 	struct pci_dev *pdev = idxd->pdev;  in idxd_mask_msix_vectors()
34 		idxd_mask_msix_vector(idxd, i);  in idxd_mask_msix_vectors()
37 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)  in idxd_unmask_msix_vector()  argument
39 	struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);  in idxd_unmask_msix_vector()
44 void idxd_unmask_error_interrupts(struct idxd_device *idxd)  in idxd_unmask_error_interrupts()  argument
48 	genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);  in idxd_unmask_error_interrupts()
50 	iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);  in idxd_unmask_error_interrupts()
53 void idxd_mask_error_interrupts(struct idxd_device *idxd)  in idxd_mask_error_interrupts()  argument
57 	genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);  in idxd_mask_error_interrupts()
59 	iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);  in idxd_mask_error_interrupts()
74 	struct device *dev = &wq->idxd->pdev->dev;  in alloc_hw_descs()
107 	struct device *dev = &wq->idxd->pdev->dev;  in alloc_descs()
131 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_alloc_resources()  local
132 	struct device *dev = &idxd->pdev->dev;  in idxd_wq_alloc_resources()
187 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_wq_free_resources()
200 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_enable()  local
201 	struct device *dev = &idxd->pdev->dev;  in idxd_wq_enable()
209 	idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);  in idxd_wq_enable()
224 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable()  local
225 	struct device *dev = &idxd->pdev->dev;  in idxd_wq_disable()
236 	idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status);  in idxd_wq_disable()
250 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_drain()  local
251 	struct device *dev = &idxd->pdev->dev;  in idxd_wq_drain()
261 	idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_WQ, operand, NULL);  in idxd_wq_drain()
266 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_reset()  local
267 	struct device *dev = &idxd->pdev->dev;  in idxd_wq_reset()
276 	idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL);  in idxd_wq_reset()
282 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_map_portal()  local
283 	struct pci_dev *pdev = idxd->pdev;  in idxd_wq_map_portal()
300 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_wq_unmap_portal()
307 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable_cleanup()  local
309 	lockdep_assert_held(&idxd->dev_lock);  in idxd_wq_disable_cleanup()
310 	memset(wq->wqcfg, 0, idxd->wqcfg_size);  in idxd_wq_disable_cleanup()
321 static inline bool idxd_is_enabled(struct idxd_device *idxd)  in idxd_is_enabled()  argument
325 	gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);  in idxd_is_enabled()
332 static inline bool idxd_device_is_halted(struct idxd_device *idxd)  in idxd_device_is_halted()  argument
336 	gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);  in idxd_device_is_halted()
346 int idxd_device_init_reset(struct idxd_device *idxd)  in idxd_device_init_reset()  argument
348 	struct device *dev = &idxd->pdev->dev;  in idxd_device_init_reset()
352 	if (idxd_device_is_halted(idxd)) {  in idxd_device_init_reset()
353 		dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");  in idxd_device_init_reset()
360 	spin_lock_irqsave(&idxd->dev_lock, flags);  in idxd_device_init_reset()
361 	iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);  in idxd_device_init_reset()
363 	while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &  in idxd_device_init_reset()
366 	spin_unlock_irqrestore(&idxd->dev_lock, flags);  in idxd_device_init_reset()
370 static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,  in idxd_cmd_exec()  argument
377 	if (idxd_device_is_halted(idxd)) {  in idxd_cmd_exec()
378 		dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");  in idxd_cmd_exec()
389 	spin_lock_irqsave(&idxd->dev_lock, flags);  in idxd_cmd_exec()
390 	wait_event_lock_irq(idxd->cmd_waitq,  in idxd_cmd_exec()
391 			    !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),  in idxd_cmd_exec()
392 			    idxd->dev_lock);  in idxd_cmd_exec()
394 	dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",  in idxd_cmd_exec()
397 	idxd->cmd_status = 0;  in idxd_cmd_exec()
398 	__set_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);  in idxd_cmd_exec()
399 	idxd->cmd_done = &done;  in idxd_cmd_exec()
400 	iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);  in idxd_cmd_exec()
406 	spin_unlock_irqrestore(&idxd->dev_lock, flags);  in idxd_cmd_exec()
408 	spin_lock_irqsave(&idxd->dev_lock, flags);  in idxd_cmd_exec()
410 		*status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);  in idxd_cmd_exec()
411 		idxd->cmd_status = *status & GENMASK(7, 0);  in idxd_cmd_exec()
414 	__clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);  in idxd_cmd_exec()
416 	wake_up(&idxd->cmd_waitq);  in idxd_cmd_exec()
417 	spin_unlock_irqrestore(&idxd->dev_lock, flags);  in idxd_cmd_exec()
420 int idxd_device_enable(struct idxd_device *idxd)  in idxd_device_enable()  argument
422 	struct device *dev = &idxd->pdev->dev;  in idxd_device_enable()
425 	if (idxd_is_enabled(idxd)) {  in idxd_device_enable()
430 	idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_DEVICE, 0, &status);  in idxd_device_enable()
439 	idxd->state = IDXD_DEV_ENABLED;  in idxd_device_enable()
443 void idxd_device_wqs_clear_state(struct idxd_device *idxd)  in idxd_device_wqs_clear_state()  argument
447 	lockdep_assert_held(&idxd->dev_lock);  in idxd_device_wqs_clear_state()
449 	for (i = 0; i < idxd->max_wqs; i++) {  in idxd_device_wqs_clear_state()
450 		struct idxd_wq *wq = &idxd->wqs[i];  in idxd_device_wqs_clear_state()
459 int idxd_device_disable(struct idxd_device *idxd)  in idxd_device_disable()  argument
461 	struct device *dev = &idxd->pdev->dev;  in idxd_device_disable()
465 	if (!idxd_is_enabled(idxd)) {  in idxd_device_disable()
470 	idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_DEVICE, 0, &status);  in idxd_device_disable()
479 	spin_lock_irqsave(&idxd->dev_lock, flags);  in idxd_device_disable()
480 	idxd_device_wqs_clear_state(idxd);  in idxd_device_disable()
481 	idxd->state = IDXD_DEV_CONF_READY;  in idxd_device_disable()
482 	spin_unlock_irqrestore(&idxd->dev_lock, flags);  in idxd_device_disable()
486 void idxd_device_reset(struct idxd_device *idxd)  in idxd_device_reset()  argument
490 	idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);  in idxd_device_reset()
491 	spin_lock_irqsave(&idxd->dev_lock, flags);  in idxd_device_reset()
492 	idxd_device_wqs_clear_state(idxd);  in idxd_device_reset()
493 	idxd->state = IDXD_DEV_CONF_READY;  in idxd_device_reset()
494 	spin_unlock_irqrestore(&idxd->dev_lock, flags);  in idxd_device_reset()
500 	struct idxd_device *idxd = group->idxd;  in idxd_group_config_write()  local
501 	struct device *dev = &idxd->pdev->dev;  in idxd_group_config_write()
509 		grpcfg_offset = idxd->grpcfg_offset +  in idxd_group_config_write()
512 			  idxd->reg_base + grpcfg_offset);  in idxd_group_config_write()
515 			ioread64(idxd->reg_base + grpcfg_offset));  in idxd_group_config_write()
519 	grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;  in idxd_group_config_write()
520 	iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);  in idxd_group_config_write()
522 		grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));  in idxd_group_config_write()
525 	grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;  in idxd_group_config_write()
526 	iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);  in idxd_group_config_write()
529 		ioread32(idxd->reg_base + grpcfg_offset));  in idxd_group_config_write()
532 static int idxd_groups_config_write(struct idxd_device *idxd)  in idxd_groups_config_write()  argument
537 	struct device *dev = &idxd->pdev->dev;  in idxd_groups_config_write()
540 	if (idxd->token_limit) {  in idxd_groups_config_write()
541 		reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);  in idxd_groups_config_write()
542 		reg.token_limit = idxd->token_limit;  in idxd_groups_config_write()
543 		iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);  in idxd_groups_config_write()
547 		ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));  in idxd_groups_config_write()
549 	for (i = 0; i < idxd->max_groups; i++) {  in idxd_groups_config_write()
550 		struct idxd_group *group = &idxd->groups[i];  in idxd_groups_config_write()
560 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_config_write()  local
561 	struct device *dev = &idxd->pdev->dev;  in idxd_wq_config_write()
572 	for (i = 0; i < WQCFG_STRIDES(idxd); i++) {  in idxd_wq_config_write()
573 		wq_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_config_write()
574 		wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset);  in idxd_wq_config_write()
598 	for (i = 0; i < WQCFG_STRIDES(idxd); i++) {  in idxd_wq_config_write()
599 		wq_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_config_write()
600 		iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);  in idxd_wq_config_write()
603 			ioread32(idxd->reg_base + wq_offset));  in idxd_wq_config_write()
609 static int idxd_wqs_config_write(struct idxd_device *idxd)  in idxd_wqs_config_write()  argument
613 	for (i = 0; i < idxd->max_wqs; i++) {  in idxd_wqs_config_write()
614 		struct idxd_wq *wq = &idxd->wqs[i];  in idxd_wqs_config_write()
624 static void idxd_group_flags_setup(struct idxd_device *idxd)  in idxd_group_flags_setup()  argument
629 	for (i = 0; i < idxd->max_groups; i++) {  in idxd_group_flags_setup()
630 		struct idxd_group *group = &idxd->groups[i];  in idxd_group_flags_setup()
646 			group->grpcfg.flags.tokens_allowed = idxd->max_tokens;  in idxd_group_flags_setup()
650 static int idxd_engines_setup(struct idxd_device *idxd)  in idxd_engines_setup()  argument
656 	for (i = 0; i < idxd->max_groups; i++) {  in idxd_engines_setup()
657 		group = &idxd->groups[i];  in idxd_engines_setup()
661 	for (i = 0; i < idxd->max_engines; i++) {  in idxd_engines_setup()
662 		eng = &idxd->engines[i];  in idxd_engines_setup()
678 static int idxd_wqs_setup(struct idxd_device *idxd)  in idxd_wqs_setup()  argument
683 	struct device *dev = &idxd->pdev->dev;  in idxd_wqs_setup()
685 	for (i = 0; i < idxd->max_groups; i++) {  in idxd_wqs_setup()
686 		group = &idxd->groups[i];  in idxd_wqs_setup()
691 	for (i = 0; i < idxd->max_wqs; i++) {  in idxd_wqs_setup()
692 		wq = &idxd->wqs[i];  in idxd_wqs_setup()
715 int idxd_device_config(struct idxd_device *idxd)  in idxd_device_config()  argument
719 	lockdep_assert_held(&idxd->dev_lock);  in idxd_device_config()
720 	rc = idxd_wqs_setup(idxd);  in idxd_device_config()
724 	rc = idxd_engines_setup(idxd);  in idxd_device_config()
728 	idxd_group_flags_setup(idxd);  in idxd_device_config()
730 	rc = idxd_wqs_config_write(idxd);  in idxd_device_config()
734 	rc = idxd_groups_config_write(idxd);  in idxd_device_config()