1296 static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,  in i40iw_copy_user_pgaddrs()  argument
1300 	struct ib_umem *region = iwmr->region;  in i40iw_copy_user_pgaddrs()
1301 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_copy_user_pgaddrs()
1309 	if (iwmr->type == IW_MEMREG_TYPE_QP)  in i40iw_copy_user_pgaddrs()
1312 	rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) {  in i40iw_copy_user_pgaddrs()
1377 			     struct i40iw_mr *iwmr,  in i40iw_setup_pbles()  argument
1380 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_setup_pbles()
1389 		status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);  in i40iw_setup_pbles()
1399 		pbl = iwmr->pgaddrmem;  in i40iw_setup_pbles()
1402 	i40iw_copy_user_pgaddrs(iwmr, pbl, level);  in i40iw_setup_pbles()
1405 		iwmr->pgaddrmem[0] = *pbl;  in i40iw_setup_pbles()
1423 	struct i40iw_mr *iwmr = iwpbl->iwmr;  in i40iw_handle_q_mem()  local
1427 	u64 *arr = iwmr->pgaddrmem;  in i40iw_handle_q_mem()
1434 	pg_size = iwmr->page_size;  in i40iw_handle_q_mem()
1436 	err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);  in i40iw_handle_q_mem()
1449 	if (iwmr->type == IW_MEMREG_TYPE_QP) {  in i40iw_handle_q_mem()
1494 static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr)  in i40iw_hw_alloc_stag()  argument
1497 	struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);  in i40iw_hw_alloc_stag()
1511 	info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;  in i40iw_hw_alloc_stag()
1513 	info->total_len = iwmr->length;  in i40iw_hw_alloc_stag()
1541 	struct i40iw_mr *iwmr;  in i40iw_alloc_mr()  local
1546 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in i40iw_alloc_mr()
1547 	if (!iwmr)  in i40iw_alloc_mr()
1556 	iwmr->stag = stag;  in i40iw_alloc_mr()
1557 	iwmr->ibmr.rkey = stag;  in i40iw_alloc_mr()
1558 	iwmr->ibmr.lkey = stag;  in i40iw_alloc_mr()
1559 	iwmr->ibmr.pd = pd;  in i40iw_alloc_mr()
1560 	iwmr->ibmr.device = pd->device;  in i40iw_alloc_mr()
1561 	iwpbl = &iwmr->iwpbl;  in i40iw_alloc_mr()
1562 	iwpbl->iwmr = iwmr;  in i40iw_alloc_mr()
1563 	iwmr->type = IW_MEMREG_TYPE_MEM;  in i40iw_alloc_mr()
1565 	iwmr->page_cnt = max_num_sg;  in i40iw_alloc_mr()
1567 	status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);  in i40iw_alloc_mr()
1574 	err_code = i40iw_hw_alloc_stag(iwdev, iwmr);  in i40iw_alloc_mr()
1579 	return &iwmr->ibmr;  in i40iw_alloc_mr()
1585 	kfree(iwmr);  in i40iw_alloc_mr()
1596 	struct i40iw_mr *iwmr = to_iwmr(ibmr);  in i40iw_set_page()  local
1597 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_set_page()
1601 	if (unlikely(iwmr->npages == iwmr->page_cnt))  in i40iw_set_page()
1605 	pbl[iwmr->npages++] = cpu_to_le64(addr);  in i40iw_set_page()
1618 	struct i40iw_mr *iwmr = to_iwmr(ibmr);  in i40iw_map_mr_sg()  local
1620 	iwmr->npages = 0;  in i40iw_map_mr_sg()
1657 			  struct i40iw_mr *iwmr,  in i40iw_hwreg_mr()  argument
1660 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_hwreg_mr()
1662 	struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);  in i40iw_hwreg_mr()
1677 	stag_info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;  in i40iw_hwreg_mr()
1678 	stag_info->stag_key = (u8)iwmr->stag;  in i40iw_hwreg_mr()
1679 	stag_info->total_len = iwmr->length;  in i40iw_hwreg_mr()
1683 	stag_info->page_size = iwmr->page_size;  in i40iw_hwreg_mr()
1694 		stag_info->reg_addr_pa = iwmr->pgaddrmem[0];  in i40iw_hwreg_mr()
1732 	struct i40iw_mr *iwmr;  in i40iw_reg_user_mr()  local
1759 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in i40iw_reg_user_mr()
1760 	if (!iwmr) {  in i40iw_reg_user_mr()
1765 	iwpbl = &iwmr->iwpbl;  in i40iw_reg_user_mr()
1766 	iwpbl->iwmr = iwmr;  in i40iw_reg_user_mr()
1767 	iwmr->region = region;  in i40iw_reg_user_mr()
1768 	iwmr->ibmr.pd = pd;  in i40iw_reg_user_mr()
1769 	iwmr->ibmr.device = pd->device;  in i40iw_reg_user_mr()
1771 	iwmr->page_size = PAGE_SIZE;  in i40iw_reg_user_mr()
1773 		iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K | SZ_2M,  in i40iw_reg_user_mr()
1775 	iwmr->length = region->length;  in i40iw_reg_user_mr()
1780 	iwmr->type = req.reg_type;  in i40iw_reg_user_mr()
1781 	iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);  in i40iw_reg_user_mr()
1806 		use_pbles = (iwmr->page_cnt != 1);  in i40iw_reg_user_mr()
1809 		err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);  in i40iw_reg_user_mr()
1814 			ret = i40iw_check_mr_contiguous(palloc, iwmr->page_size);  in i40iw_reg_user_mr()
1828 		iwmr->stag = stag;  in i40iw_reg_user_mr()
1829 		iwmr->ibmr.rkey = stag;  in i40iw_reg_user_mr()
1830 		iwmr->ibmr.lkey = stag;  in i40iw_reg_user_mr()
1832 		err = i40iw_hwreg_mr(iwdev, iwmr, access);  in i40iw_reg_user_mr()
1843 	iwmr->type = req.reg_type;  in i40iw_reg_user_mr()
1846 	return &iwmr->ibmr;  in i40iw_reg_user_mr()
1852 	kfree(iwmr);  in i40iw_reg_user_mr()
1873 	struct i40iw_mr *iwmr;  in i40iw_reg_phys_mr()  local
1879 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in i40iw_reg_phys_mr()
1880 	if (!iwmr)  in i40iw_reg_phys_mr()
1882 	iwmr->ibmr.pd = pd;  in i40iw_reg_phys_mr()
1883 	iwmr->ibmr.device = pd->device;  in i40iw_reg_phys_mr()
1884 	iwpbl = &iwmr->iwpbl;  in i40iw_reg_phys_mr()
1885 	iwpbl->iwmr = iwmr;  in i40iw_reg_phys_mr()
1886 	iwmr->type = IW_MEMREG_TYPE_MEM;  in i40iw_reg_phys_mr()
1894 	iwmr->stag = stag;  in i40iw_reg_phys_mr()
1895 	iwmr->ibmr.rkey = stag;  in i40iw_reg_phys_mr()
1896 	iwmr->ibmr.lkey = stag;  in i40iw_reg_phys_mr()
1897 	iwmr->page_cnt = 1;  in i40iw_reg_phys_mr()
1898 	iwmr->pgaddrmem[0]  = addr;  in i40iw_reg_phys_mr()
1899 	iwmr->length = size;  in i40iw_reg_phys_mr()
1900 	status = i40iw_hwreg_mr(iwdev, iwmr, access);  in i40iw_reg_phys_mr()
1908 	return &iwmr->ibmr;  in i40iw_reg_phys_mr()
1910 	kfree(iwmr);  in i40iw_reg_phys_mr()
1931 static void i40iw_del_memlist(struct i40iw_mr *iwmr,  in i40iw_del_memlist()  argument
1934 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_del_memlist()
1937 	switch (iwmr->type) {  in i40iw_del_memlist()
1967 	struct i40iw_mr *iwmr = to_iwmr(ib_mr);  in i40iw_dereg_mr()  local
1971 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;  in i40iw_dereg_mr()
1977 	ib_umem_release(iwmr->region);  in i40iw_dereg_mr()
1979 	if (iwmr->type != IW_MEMREG_TYPE_MEM) {  in i40iw_dereg_mr()
1981 		if (iwmr->region) {  in i40iw_dereg_mr()
1988 			i40iw_del_memlist(iwmr, ucontext);  in i40iw_dereg_mr()
1990 		if (iwpbl->pbl_allocated && iwmr->type != IW_MEMREG_TYPE_QP)  in i40iw_dereg_mr()
1992 		kfree(iwmr);  in i40iw_dereg_mr()
2019 	i40iw_free_stag(iwdev, iwmr->stag);  in i40iw_dereg_mr()
2022 	kfree(iwmr);  in i40iw_dereg_mr()
2212 			struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);  in i40iw_post_send()  local
2214 			struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;  in i40iw_post_send()
2227 			info.va = (void *)(uintptr_t)iwmr->ibmr.iova;  in i40iw_post_send()
2228 			info.total_len = iwmr->ibmr.length;  in i40iw_post_send()
2234 			if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR)  in i40iw_post_send()