111 	int	(*readfn)(struct vfio_pci_device *vdev, int pos, int count,
113 	int	(*writefn)(struct vfio_pci_device *vdev, int pos, int count,
174 static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_default_config_read()  argument
180 	memcpy(val, vdev->vconfig + pos, count);  in vfio_default_config_read()
186 		struct pci_dev *pdev = vdev->pdev;  in vfio_default_config_read()
200 static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_default_config_write()  argument
217 		memcpy(&virt_val, vdev->vconfig + pos, count);  in vfio_default_config_write()
222 		memcpy(vdev->vconfig + pos, &virt_val, count);  in vfio_default_config_write()
227 		struct pci_dev *pdev = vdev->pdev;  in vfio_default_config_write()
247 static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_direct_config_read()  argument
253 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);  in vfio_direct_config_read()
259 			memcpy(val, vdev->vconfig + pos, count);  in vfio_direct_config_read()
262 			memcpy(val, vdev->vconfig + pos,  in vfio_direct_config_read()
265 			memcpy(val, vdev->vconfig + pos, 1);  in vfio_direct_config_read()
272 static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_raw_config_write()  argument
278 	ret = vfio_user_config_write(vdev->pdev, pos, val, count);  in vfio_raw_config_write()
285 static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_raw_config_read()  argument
291 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);  in vfio_raw_config_read()
299 static int vfio_virt_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_virt_config_write()  argument
303 	memcpy(vdev->vconfig + pos, &val, count);  in vfio_virt_config_write()
307 static int vfio_virt_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_virt_config_read()  argument
311 	memcpy(val, vdev->vconfig + pos, count);  in vfio_virt_config_read()
399 bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev)  in __vfio_pci_memory_enabled()  argument
401 	struct pci_dev *pdev = vdev->pdev;  in __vfio_pci_memory_enabled()
402 	u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]);  in __vfio_pci_memory_enabled()
416 static void vfio_bar_restore(struct vfio_pci_device *vdev)  in vfio_bar_restore()  argument
418 	struct pci_dev *pdev = vdev->pdev;  in vfio_bar_restore()
419 	u32 *rbar = vdev->rbar;  in vfio_bar_restore()
433 	if (vdev->nointx) {  in vfio_bar_restore()
463 static void vfio_bar_fixup(struct vfio_pci_device *vdev)  in vfio_bar_fixup()  argument
465 	struct pci_dev *pdev = vdev->pdev;  in vfio_bar_fixup()
470 	if (!vdev->bardirty)  in vfio_bar_fixup()
473 	vbar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0];  in vfio_bar_fixup()
495 	vbar = (__le32 *)&vdev->vconfig[PCI_ROM_ADDRESS];  in vfio_bar_fixup()
514 	vdev->bardirty = false;  in vfio_bar_fixup()
517 static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_basic_config_read()  argument
522 		vfio_bar_fixup(vdev);  in vfio_basic_config_read()
524 	count = vfio_default_config_read(vdev, pos, count, perm, offset, val);  in vfio_basic_config_read()
527 	if (offset == PCI_COMMAND && vdev->pdev->no_command_memory) {  in vfio_basic_config_read()
528 		u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]);  in vfio_basic_config_read()
539 static bool vfio_need_bar_restore(struct vfio_pci_device *vdev)  in vfio_need_bar_restore()  argument
545 		if (vdev->rbar[i]) {  in vfio_need_bar_restore()
546 			ret = pci_user_read_config_dword(vdev->pdev, pos, &bar);  in vfio_need_bar_restore()
547 			if (ret || vdev->rbar[i] != bar)  in vfio_need_bar_restore()
555 static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_basic_config_write()  argument
559 	struct pci_dev *pdev = vdev->pdev;  in vfio_basic_config_write()
564 	virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND];  in vfio_basic_config_write()
585 			vfio_pci_zap_and_down_write_memory_lock(vdev);  in vfio_basic_config_write()
587 			down_write(&vdev->memory_lock);  in vfio_basic_config_write()
601 		    vfio_need_bar_restore(vdev))  in vfio_basic_config_write()
602 			vfio_bar_restore(vdev);  in vfio_basic_config_write()
605 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_basic_config_write()
608 			up_write(&vdev->memory_lock);  in vfio_basic_config_write()
622 		up_write(&vdev->memory_lock);  in vfio_basic_config_write()
632 		if (virt_intx_disable && !vdev->virq_disabled) {  in vfio_basic_config_write()
633 			vdev->virq_disabled = true;  in vfio_basic_config_write()
634 			vfio_pci_intx_mask(vdev);  in vfio_basic_config_write()
635 		} else if (!virt_intx_disable && vdev->virq_disabled) {  in vfio_basic_config_write()
636 			vdev->virq_disabled = false;  in vfio_basic_config_write()
637 			vfio_pci_intx_unmask(vdev);  in vfio_basic_config_write()
642 		vdev->bardirty = true;  in vfio_basic_config_write()
695 static int vfio_pm_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_pm_config_write()  argument
699 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_pm_config_write()
721 		vfio_pci_set_power_state(vdev, state);  in vfio_pm_config_write()
750 static int vfio_vpd_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_vpd_config_write()  argument
754 	struct pci_dev *pdev = vdev->pdev;  in vfio_vpd_config_write()
755 	__le16 *paddr = (__le16 *)(vdev->vconfig + pos - offset + PCI_VPD_ADDR);  in vfio_vpd_config_write()
756 	__le32 *pdata = (__le32 *)(vdev->vconfig + pos - offset + PCI_VPD_DATA);  in vfio_vpd_config_write()
765 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_vpd_config_write()
832 static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_exp_config_write()  argument
836 	__le16 *ctrl = (__le16 *)(vdev->vconfig + pos -  in vfio_exp_config_write()
840 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_exp_config_write()
856 		ret = pci_user_read_config_dword(vdev->pdev,  in vfio_exp_config_write()
861 			vfio_pci_zap_and_down_write_memory_lock(vdev);  in vfio_exp_config_write()
862 			pci_try_reset_function(vdev->pdev);  in vfio_exp_config_write()
863 			up_write(&vdev->memory_lock);  in vfio_exp_config_write()
883 		readrq = max(readrq, pcie_get_mps(vdev->pdev));  in vfio_exp_config_write()
885 		pcie_set_readrq(vdev->pdev, readrq);  in vfio_exp_config_write()
916 static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_af_config_write()  argument
920 	u8 *ctrl = vdev->vconfig + pos - offset + PCI_AF_CTRL;  in vfio_af_config_write()
922 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_af_config_write()
938 		ret = pci_user_read_config_byte(vdev->pdev,  in vfio_af_config_write()
943 			vfio_pci_zap_and_down_write_memory_lock(vdev);  in vfio_af_config_write()
944 			pci_try_reset_function(vdev->pdev);  in vfio_af_config_write()
945 			up_write(&vdev->memory_lock);  in vfio_af_config_write()
1075 static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos)  in vfio_find_cap_start()  argument
1080 	cap = vdev->pci_config_map[pos];  in vfio_find_cap_start()
1086 	while (pos - 1 >= base && vdev->pci_config_map[pos - 1] == cap)  in vfio_find_cap_start()
1092 static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos,  in vfio_msi_config_read()  argument
1101 		start = vfio_find_cap_start(vdev, pos);  in vfio_msi_config_read()
1103 		flags = (__le16 *)&vdev->vconfig[start];  in vfio_msi_config_read()
1106 		*flags |= cpu_to_le16(vdev->msi_qmax << 1);  in vfio_msi_config_read()
1109 	return vfio_default_config_read(vdev, pos, count, perm, offset, val);  in vfio_msi_config_read()
1112 static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,  in vfio_msi_config_write()  argument
1116 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);  in vfio_msi_config_write()
1126 		start = vfio_find_cap_start(vdev, pos);  in vfio_msi_config_write()
1128 		pflags = (__le16 *)&vdev->vconfig[start + PCI_MSI_FLAGS];  in vfio_msi_config_write()
1133 		if  (!is_msi(vdev))  in vfio_msi_config_write()
1137 		if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->msi_qmax) {  in vfio_msi_config_write()
1139 			flags |= vdev->msi_qmax << 4;  in vfio_msi_config_write()
1144 		ret = pci_user_write_config_word(vdev->pdev,  in vfio_msi_config_write()
1192 static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos)  in vfio_msi_cap_len()  argument
1194 	struct pci_dev *pdev = vdev->pdev;  in vfio_msi_cap_len()
1208 	if (vdev->msi_perm)  in vfio_msi_cap_len()
1211 	vdev->msi_perm = kmalloc(sizeof(struct perm_bits), GFP_KERNEL);  in vfio_msi_cap_len()
1212 	if (!vdev->msi_perm)  in vfio_msi_cap_len()
1215 	ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags);  in vfio_msi_cap_len()
1217 		kfree(vdev->msi_perm);  in vfio_msi_cap_len()
1225 static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos)  in vfio_vc_cap_len()  argument
1227 	struct pci_dev *pdev = vdev->pdev;  in vfio_vc_cap_len()
1266 static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)  in vfio_cap_len()  argument
1268 	struct pci_dev *pdev = vdev->pdev;  in vfio_cap_len()
1276 		return vfio_msi_cap_len(vdev, pos);  in vfio_cap_len()
1287 				vdev->extended_caps = (dword != 0);  in vfio_cap_len()
1303 			vdev->extended_caps = (dword != 0);  in vfio_cap_len()
1341 static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos)  in vfio_ext_cap_len()  argument
1343 	struct pci_dev *pdev = vdev->pdev;  in vfio_ext_cap_len()
1358 		return vfio_vc_cap_len(vdev, epos);  in vfio_ext_cap_len()
1415 static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,  in vfio_fill_vconfig_bytes()  argument
1418 	struct pci_dev *pdev = vdev->pdev;  in vfio_fill_vconfig_bytes()
1430 			__le32 *dwordp = (__le32 *)&vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1439 			__le16 *wordp = (__le16 *)&vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1448 			u8 *byte = &vdev->vconfig[offset];  in vfio_fill_vconfig_bytes()
1462 static int vfio_cap_init(struct vfio_pci_device *vdev)  in vfio_cap_init()  argument
1464 	struct pci_dev *pdev = vdev->pdev;  in vfio_cap_init()
1465 	u8 *map = vdev->pci_config_map;  in vfio_cap_init()
1483 	prev = &vdev->vconfig[PCI_CAPABILITY_LIST];  in vfio_cap_init()
1508 				len = vfio_cap_len(vdev, cap, pos);  in vfio_cap_init()
1534 		ret = vfio_fill_vconfig_bytes(vdev, pos, len);  in vfio_cap_init()
1538 		prev = &vdev->vconfig[pos + PCI_CAP_LIST_NEXT];  in vfio_cap_init()
1545 		__le16 *vstatus = (__le16 *)&vdev->vconfig[PCI_STATUS];  in vfio_cap_init()
1552 static int vfio_ecap_init(struct vfio_pci_device *vdev)  in vfio_ecap_init()  argument
1554 	struct pci_dev *pdev = vdev->pdev;  in vfio_ecap_init()
1555 	u8 *map = vdev->pci_config_map;  in vfio_ecap_init()
1560 	if (!vdev->extended_caps)  in vfio_ecap_init()
1582 				len = vfio_ext_cap_len(vdev, ecap, epos);  in vfio_ecap_init()
1624 		ret = vfio_fill_vconfig_bytes(vdev, epos, len);  in vfio_ecap_init()
1635 			*(__le32 *)&vdev->vconfig[epos] &=  in vfio_ecap_init()
1640 		prev = (__le32 *)&vdev->vconfig[epos];  in vfio_ecap_init()
1645 		*(u32 *)&vdev->vconfig[PCI_CFG_SPACE_SIZE] = 0;  in vfio_ecap_init()
1672 int vfio_config_init(struct vfio_pci_device *vdev)  in vfio_config_init()  argument
1674 	struct pci_dev *pdev = vdev->pdev;  in vfio_config_init()
1694 	vdev->pci_config_map = map;  in vfio_config_init()
1695 	vdev->vconfig = vconfig;  in vfio_config_init()
1701 	ret = vfio_fill_vconfig_bytes(vdev, 0, PCI_STD_HEADER_SIZEOF);  in vfio_config_init()
1705 	vdev->bardirty = true;  in vfio_config_init()
1713 	vdev->rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]);  in vfio_config_init()
1714 	vdev->rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]);  in vfio_config_init()
1715 	vdev->rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]);  in vfio_config_init()
1716 	vdev->rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]);  in vfio_config_init()
1717 	vdev->rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]);  in vfio_config_init()
1718 	vdev->rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]);  in vfio_config_init()
1719 	vdev->rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]);  in vfio_config_init()
1755 	if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->nointx)  in vfio_config_init()
1758 	ret = vfio_cap_init(vdev);  in vfio_config_init()
1762 	ret = vfio_ecap_init(vdev);  in vfio_config_init()
1770 	vdev->pci_config_map = NULL;  in vfio_config_init()
1772 	vdev->vconfig = NULL;  in vfio_config_init()
1776 void vfio_config_free(struct vfio_pci_device *vdev)  in vfio_config_free()  argument
1778 	kfree(vdev->vconfig);  in vfio_config_free()
1779 	vdev->vconfig = NULL;  in vfio_config_free()
1780 	kfree(vdev->pci_config_map);  in vfio_config_free()
1781 	vdev->pci_config_map = NULL;  in vfio_config_free()
1782 	if (vdev->msi_perm) {  in vfio_config_free()
1783 		free_perm_bits(vdev->msi_perm);  in vfio_config_free()
1784 		kfree(vdev->msi_perm);  in vfio_config_free()
1785 		vdev->msi_perm = NULL;  in vfio_config_free()
1793 static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev,  in vfio_pci_cap_remaining_dword()  argument
1796 	u8 cap = vdev->pci_config_map[pos];  in vfio_pci_cap_remaining_dword()
1799 	for (i = 1; (pos + i) % 4 && vdev->pci_config_map[pos + i] == cap; i++)  in vfio_pci_cap_remaining_dword()
1805 static ssize_t vfio_config_do_rw(struct vfio_pci_device *vdev, char __user *buf,  in vfio_config_do_rw()  argument
1808 	struct pci_dev *pdev = vdev->pdev;  in vfio_config_do_rw()
1823 	count = min(count, vfio_pci_cap_remaining_dword(vdev, *ppos));  in vfio_config_do_rw()
1833 	cap_id = vdev->pci_config_map[*ppos];  in vfio_config_do_rw()
1846 			cap_start = vfio_find_cap_start(vdev, *ppos);  in vfio_config_do_rw()
1853 				perm = vdev->msi_perm;  in vfio_config_do_rw()
1856 				cap_start = vfio_find_cap_start(vdev, *ppos);  in vfio_config_do_rw()
1872 		ret = perm->writefn(vdev, *ppos, count, perm, offset, val);  in vfio_config_do_rw()
1875 			ret = perm->readfn(vdev, *ppos, count,  in vfio_config_do_rw()
1888 ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev, char __user *buf,  in vfio_pci_config_rw()  argument
1898 		ret = vfio_config_do_rw(vdev, buf, count, &pos, iswrite);  in vfio_pci_config_rw()