Lines Matching refs:cptvf
16 void *cptvf; member
29 vq_post_process(cwqe->cptvf, cwqe->qno); in vq_work_handler()
32 static int init_worker_threads(struct cpt_vf *cptvf) in init_worker_threads() argument
34 struct pci_dev *pdev = cptvf->pdev; in init_worker_threads()
42 if (cptvf->nr_queues) { in init_worker_threads()
44 cptvf->nr_queues); in init_worker_threads()
47 for (i = 0; i < cptvf->nr_queues; i++) { in init_worker_threads()
51 cwqe_info->vq_wqe[i].cptvf = cptvf; in init_worker_threads()
54 cptvf->wqe_info = cwqe_info; in init_worker_threads()
59 static void cleanup_worker_threads(struct cpt_vf *cptvf) in cleanup_worker_threads() argument
62 struct pci_dev *pdev = cptvf->pdev; in cleanup_worker_threads()
65 cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; in cleanup_worker_threads()
69 if (cptvf->nr_queues) { in cleanup_worker_threads()
71 cptvf->nr_queues); in cleanup_worker_threads()
74 for (i = 0; i < cptvf->nr_queues; i++) in cleanup_worker_threads()
78 cptvf->wqe_info = NULL; in cleanup_worker_threads()
139 static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) in init_pending_queues() argument
141 struct pci_dev *pdev = cptvf->pdev; in init_pending_queues()
147 ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues); in init_pending_queues()
157 static void cleanup_pending_queues(struct cpt_vf *cptvf) in cleanup_pending_queues() argument
159 struct pci_dev *pdev = cptvf->pdev; in cleanup_pending_queues()
161 if (!cptvf->nr_queues) in cleanup_pending_queues()
165 cptvf->nr_queues); in cleanup_pending_queues()
166 free_pending_queues(&cptvf->pqinfo); in cleanup_pending_queues()
169 static void free_command_queues(struct cpt_vf *cptvf, in free_command_queues() argument
175 struct pci_dev *pdev = cptvf->pdev; in free_command_queues()
179 for (i = 0; i < cptvf->nr_queues; i++) { in free_command_queues()
203 static int alloc_command_queues(struct cpt_vf *cptvf, in alloc_command_queues() argument
210 struct pci_dev *pdev = cptvf->pdev; in alloc_command_queues()
215 cptvf->qsize = min(qlen, cqinfo->qchunksize) * in alloc_command_queues()
221 for (i = 0; i < cptvf->nr_queues; i++) { in alloc_command_queues()
275 free_command_queues(cptvf, cqinfo); in alloc_command_queues()
279 static int init_command_queues(struct cpt_vf *cptvf, u32 qlen) in init_command_queues() argument
281 struct pci_dev *pdev = cptvf->pdev; in init_command_queues()
285 ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE, in init_command_queues()
289 cptvf->nr_queues); in init_command_queues()
296 static void cleanup_command_queues(struct cpt_vf *cptvf) in cleanup_command_queues() argument
298 struct pci_dev *pdev = cptvf->pdev; in cleanup_command_queues()
300 if (!cptvf->nr_queues) in cleanup_command_queues()
304 cptvf->nr_queues); in cleanup_command_queues()
305 free_command_queues(cptvf, &cptvf->cqinfo); in cleanup_command_queues()
308 static void cptvf_sw_cleanup(struct cpt_vf *cptvf) in cptvf_sw_cleanup() argument
310 cleanup_worker_threads(cptvf); in cptvf_sw_cleanup()
311 cleanup_pending_queues(cptvf); in cptvf_sw_cleanup()
312 cleanup_command_queues(cptvf); in cptvf_sw_cleanup()
315 static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues) in cptvf_sw_init() argument
317 struct pci_dev *pdev = cptvf->pdev; in cptvf_sw_init()
324 cptvf->nr_queues = nr_queues; in cptvf_sw_init()
326 ret = init_command_queues(cptvf, qlen); in cptvf_sw_init()
333 ret = init_pending_queues(cptvf, qlen, nr_queues); in cptvf_sw_init()
341 ret = init_worker_threads(cptvf); in cptvf_sw_init()
350 cleanup_worker_threads(cptvf); in cptvf_sw_init()
351 cleanup_pending_queues(cptvf); in cptvf_sw_init()
354 cleanup_command_queues(cptvf); in cptvf_sw_init()
359 static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec) in cptvf_free_irq_affinity() argument
361 irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL); in cptvf_free_irq_affinity()
362 free_cpumask_var(cptvf->affinity_mask[vec]); in cptvf_free_irq_affinity()
365 static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val) in cptvf_write_vq_ctl() argument
369 vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0)); in cptvf_write_vq_ctl()
371 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u); in cptvf_write_vq_ctl()
374 void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val) in cptvf_write_vq_doorbell() argument
378 vqx_dbell.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_doorbell()
381 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0), in cptvf_write_vq_doorbell()
385 static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val) in cptvf_write_vq_inprog() argument
389 vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0)); in cptvf_write_vq_inprog()
391 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u); in cptvf_write_vq_inprog()
394 static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val) in cptvf_write_vq_done_numwait() argument
398 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_done_numwait()
401 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), in cptvf_write_vq_done_numwait()
405 static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time) in cptvf_write_vq_done_timewait() argument
409 vqx_dwait.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_done_timewait()
412 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0), in cptvf_write_vq_done_timewait()
416 static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf) in cptvf_enable_swerr_interrupts() argument
420 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, in cptvf_enable_swerr_interrupts()
424 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), in cptvf_enable_swerr_interrupts()
428 static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf) in cptvf_enable_mbox_interrupts() argument
432 vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base, in cptvf_enable_mbox_interrupts()
436 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0), in cptvf_enable_mbox_interrupts()
440 static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf) in cptvf_enable_done_interrupts() argument
444 vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base, in cptvf_enable_done_interrupts()
448 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0), in cptvf_enable_done_interrupts()
452 static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf) in cptvf_clear_dovf_intr() argument
456 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_dovf_intr()
460 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_dovf_intr()
464 static void cptvf_clear_irde_intr(struct cpt_vf *cptvf) in cptvf_clear_irde_intr() argument
468 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_irde_intr()
472 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_irde_intr()
476 static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf) in cptvf_clear_nwrp_intr() argument
480 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_nwrp_intr()
484 cpt_write_csr64(cptvf->reg_base, in cptvf_clear_nwrp_intr()
488 static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf) in cptvf_clear_mbox_intr() argument
492 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_mbox_intr()
496 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_mbox_intr()
500 static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf) in cptvf_clear_swerr_intr() argument
504 vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base, in cptvf_clear_swerr_intr()
508 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0), in cptvf_clear_swerr_intr()
512 static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf) in cptvf_read_vf_misc_intr_status() argument
514 return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0)); in cptvf_read_vf_misc_intr_status()
519 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; in cptvf_misc_intr_handler() local
520 struct pci_dev *pdev = cptvf->pdev; in cptvf_misc_intr_handler()
523 intr = cptvf_read_vf_misc_intr_status(cptvf); in cptvf_misc_intr_handler()
527 intr, cptvf->vfid); in cptvf_misc_intr_handler()
528 cptvf_handle_mbox_intr(cptvf); in cptvf_misc_intr_handler()
529 cptvf_clear_mbox_intr(cptvf); in cptvf_misc_intr_handler()
531 cptvf_clear_dovf_intr(cptvf); in cptvf_misc_intr_handler()
533 cptvf_write_vq_doorbell(cptvf, 0); in cptvf_misc_intr_handler()
535 intr, cptvf->vfid); in cptvf_misc_intr_handler()
537 cptvf_clear_irde_intr(cptvf); in cptvf_misc_intr_handler()
539 intr, cptvf->vfid); in cptvf_misc_intr_handler()
541 cptvf_clear_nwrp_intr(cptvf); in cptvf_misc_intr_handler()
543 intr, cptvf->vfid); in cptvf_misc_intr_handler()
545 cptvf_clear_swerr_intr(cptvf); in cptvf_misc_intr_handler()
547 intr, cptvf->vfid); in cptvf_misc_intr_handler()
550 cptvf->vfid); in cptvf_misc_intr_handler()
556 static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf, in get_cptvf_vq_wqe() argument
561 if (unlikely(qno >= cptvf->nr_queues)) in get_cptvf_vq_wqe()
563 nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info; in get_cptvf_vq_wqe()
568 static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf) in cptvf_read_vq_done_count() argument
572 vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0)); in cptvf_read_vq_done_count()
576 static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf, in cptvf_write_vq_done_ack() argument
581 vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base, in cptvf_write_vq_done_ack()
584 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0), in cptvf_write_vq_done_ack()
590 struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq; in cptvf_done_intr_handler() local
591 struct pci_dev *pdev = cptvf->pdev; in cptvf_done_intr_handler()
593 u32 intr = cptvf_read_vq_done_count(cptvf); in cptvf_done_intr_handler()
601 cptvf_write_vq_done_ack(cptvf, intr); in cptvf_done_intr_handler()
602 wqe = get_cptvf_vq_wqe(cptvf, 0); in cptvf_done_intr_handler()
605 cptvf->vfid); in cptvf_done_intr_handler()
614 static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec) in cptvf_set_irq_affinity() argument
616 struct pci_dev *pdev = cptvf->pdev; in cptvf_set_irq_affinity()
619 if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec], in cptvf_set_irq_affinity()
622 cptvf->vfid); in cptvf_set_irq_affinity()
626 cpu = cptvf->vfid % num_online_cpus(); in cptvf_set_irq_affinity()
627 cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node), in cptvf_set_irq_affinity()
628 cptvf->affinity_mask[vec]); in cptvf_set_irq_affinity()
630 cptvf->affinity_mask[vec]); in cptvf_set_irq_affinity()
633 static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val) in cptvf_write_vq_saddr() argument
638 cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u); in cptvf_write_vq_saddr()
641 static void cptvf_device_init(struct cpt_vf *cptvf) in cptvf_device_init() argument
646 cptvf_write_vq_ctl(cptvf, 0); in cptvf_device_init()
648 cptvf_write_vq_doorbell(cptvf, 0); in cptvf_device_init()
650 cptvf_write_vq_inprog(cptvf, 0); in cptvf_device_init()
653 base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr); in cptvf_device_init()
654 cptvf_write_vq_saddr(cptvf, base_addr); in cptvf_device_init()
656 cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD); in cptvf_device_init()
657 cptvf_write_vq_done_numwait(cptvf, 1); in cptvf_device_init()
659 cptvf_write_vq_ctl(cptvf, 1); in cptvf_device_init()
661 cptvf->flags |= CPT_FLAG_DEVICE_READY; in cptvf_device_init()
667 struct cpt_vf *cptvf; in cptvf_probe() local
670 cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); in cptvf_probe()
671 if (!cptvf) in cptvf_probe()
674 pci_set_drvdata(pdev, cptvf); in cptvf_probe()
675 cptvf->pdev = pdev; in cptvf_probe()
689 cptvf->flags |= CPT_FLAG_VF_DRIVER; in cptvf_probe()
703 cptvf->reg_base = pcim_iomap(pdev, 0, 0); in cptvf_probe()
704 if (!cptvf->reg_base) { in cptvf_probe()
710 cptvf->node = dev_to_node(&pdev->dev); in cptvf_probe()
721 cptvf); in cptvf_probe()
728 cptvf_enable_mbox_interrupts(cptvf); in cptvf_probe()
729 cptvf_enable_swerr_interrupts(cptvf); in cptvf_probe()
733 err = cptvf_check_pf_ready(cptvf); in cptvf_probe()
740 cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE; in cptvf_probe()
741 err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF); in cptvf_probe()
747 err = cptvf_send_vq_size_msg(cptvf); in cptvf_probe()
754 cptvf_device_init(cptvf); in cptvf_probe()
756 cptvf->vfgrp = 1; in cptvf_probe()
757 err = cptvf_send_vf_to_grp_msg(cptvf); in cptvf_probe()
763 cptvf->priority = 1; in cptvf_probe()
764 err = cptvf_send_vf_priority_msg(cptvf); in cptvf_probe()
772 cptvf); in cptvf_probe()
779 cptvf_enable_done_interrupts(cptvf); in cptvf_probe()
782 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in cptvf_probe()
783 cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in cptvf_probe()
785 err = cptvf_send_vf_up(cptvf); in cptvf_probe()
790 err = cvm_crypto_init(cptvf); in cptvf_probe()
798 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in cptvf_probe()
799 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in cptvf_probe()
801 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); in cptvf_probe()
803 pci_free_irq_vectors(cptvf->pdev); in cptvf_probe()
815 struct cpt_vf *cptvf = pci_get_drvdata(pdev); in cptvf_remove() local
817 if (!cptvf) { in cptvf_remove()
823 if (cptvf_send_vf_down(cptvf)) { in cptvf_remove()
826 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE); in cptvf_remove()
827 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC); in cptvf_remove()
828 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf); in cptvf_remove()
829 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf); in cptvf_remove()
830 pci_free_irq_vectors(cptvf->pdev); in cptvf_remove()
831 cptvf_sw_cleanup(cptvf); in cptvf_remove()