44  * @pa: pfn_array on which to perform the operation
56  *   -EINVAL if pa->pa_nr is not initially zero, or pa->pa_iova_pfn is not NULL
59 static int pfn_array_alloc(struct pfn_array *pa, u64 iova, unsigned int len)  in pfn_array_alloc()  argument
63 	if (pa->pa_nr || pa->pa_iova_pfn)  in pfn_array_alloc()
66 	pa->pa_iova = iova;  in pfn_array_alloc()
68 	pa->pa_nr = ((iova & ~PAGE_MASK) + len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;  in pfn_array_alloc()
69 	if (!pa->pa_nr)  in pfn_array_alloc()
72 	pa->pa_iova_pfn = kcalloc(pa->pa_nr,  in pfn_array_alloc()
73 				  sizeof(*pa->pa_iova_pfn) +  in pfn_array_alloc()
74 				  sizeof(*pa->pa_pfn),  in pfn_array_alloc()
76 	if (unlikely(!pa->pa_iova_pfn)) {  in pfn_array_alloc()
77 		pa->pa_nr = 0;  in pfn_array_alloc()
80 	pa->pa_pfn = pa->pa_iova_pfn + pa->pa_nr;  in pfn_array_alloc()
82 	pa->pa_iova_pfn[0] = pa->pa_iova >> PAGE_SHIFT;  in pfn_array_alloc()
83 	pa->pa_pfn[0] = -1ULL;  in pfn_array_alloc()
84 	for (i = 1; i < pa->pa_nr; i++) {  in pfn_array_alloc()
85 		pa->pa_iova_pfn[i] = pa->pa_iova_pfn[i - 1] + 1;  in pfn_array_alloc()
86 		pa->pa_pfn[i] = -1ULL;  in pfn_array_alloc()
94  * @pa: pfn_array on which to perform the operation
101 static int pfn_array_pin(struct pfn_array *pa, struct device *mdev)  in pfn_array_pin()  argument
105 	ret = vfio_pin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr,  in pfn_array_pin()
106 			     IOMMU_READ | IOMMU_WRITE, pa->pa_pfn);  in pfn_array_pin()
110 	} else if (ret > 0 && ret != pa->pa_nr) {  in pfn_array_pin()
111 		vfio_unpin_pages(mdev, pa->pa_iova_pfn, ret);  in pfn_array_pin()
119 	pa->pa_nr = 0;  in pfn_array_pin()
125 static void pfn_array_unpin_free(struct pfn_array *pa, struct device *mdev)  in pfn_array_unpin_free()  argument
128 	if (pa->pa_nr)  in pfn_array_unpin_free()
129 		vfio_unpin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr);  in pfn_array_unpin_free()
130 	pa->pa_nr = 0;  in pfn_array_unpin_free()
131 	kfree(pa->pa_iova_pfn);  in pfn_array_unpin_free()
134 static bool pfn_array_iova_pinned(struct pfn_array *pa, unsigned long iova)  in pfn_array_iova_pinned()  argument
139 	for (i = 0; i < pa->pa_nr; i++)  in pfn_array_iova_pinned()
140 		if (pa->pa_iova_pfn[i] == iova_pfn)  in pfn_array_iova_pinned()
147 	struct pfn_array *pa,  in pfn_array_idal_create_words()  argument
160 	for (i = 0; i < pa->pa_nr; i++)  in pfn_array_idal_create_words()
161 		idaws[i] = pa->pa_pfn[i] << PAGE_SHIFT;  in pfn_array_idal_create_words()
164 	idaws[0] += pa->pa_iova & (PAGE_SIZE - 1);  in pfn_array_idal_create_words()
197 	struct pfn_array pa = {0};  in copy_from_iova()  local
202 	ret = pfn_array_alloc(&pa, iova, n);  in copy_from_iova()
206 	ret = pfn_array_pin(&pa, mdev);  in copy_from_iova()
208 		pfn_array_unpin_free(&pa, mdev);  in copy_from_iova()
213 	for (i = 0; i < pa.pa_nr; i++) {  in copy_from_iova()
214 		from = pa.pa_pfn[i] << PAGE_SHIFT;  in copy_from_iova()
229 	pfn_array_unpin_free(&pa, mdev);  in copy_from_iova()
512 	struct pfn_array *pa;  in ccwchain_fetch_direct()  local
551 	pa = chain->ch_pa + idx;  in ccwchain_fetch_direct()
552 	ret = pfn_array_alloc(pa, iova, bytes);  in ccwchain_fetch_direct()
567 			pa->pa_iova_pfn[i] = idaws[i] >> PAGE_SHIFT;  in ccwchain_fetch_direct()
577 		ret = pfn_array_pin(pa, cp->mdev);  in ccwchain_fetch_direct()
581 		pa->pa_nr = 0;  in ccwchain_fetch_direct()
588 	pfn_array_idal_create_words(pa, idaws);  in ccwchain_fetch_direct()
593 	pfn_array_unpin_free(pa, cp->mdev);  in ccwchain_fetch_direct()