1*4882a593Smuzhiyun /* 2*4882a593Smuzhiyun * Copyright (c) 2016 Oracle. All rights reserved. 3*4882a593Smuzhiyun * 4*4882a593Smuzhiyun * This software is available to you under a choice of one of two 5*4882a593Smuzhiyun * licenses. You may choose to be licensed under the terms of the GNU 6*4882a593Smuzhiyun * General Public License (GPL) Version 2, available from the file 7*4882a593Smuzhiyun * COPYING in the main directory of this source tree, or the 8*4882a593Smuzhiyun * OpenIB.org BSD license below: 9*4882a593Smuzhiyun * 10*4882a593Smuzhiyun * Redistribution and use in source and binary forms, with or 11*4882a593Smuzhiyun * without modification, are permitted provided that the following 12*4882a593Smuzhiyun * conditions are met: 13*4882a593Smuzhiyun * 14*4882a593Smuzhiyun * - Redistributions of source code must retain the above 15*4882a593Smuzhiyun * copyright notice, this list of conditions and the following 16*4882a593Smuzhiyun * disclaimer. 17*4882a593Smuzhiyun * 18*4882a593Smuzhiyun * - Redistributions in binary form must reproduce the above 19*4882a593Smuzhiyun * copyright notice, this list of conditions and the following 20*4882a593Smuzhiyun * disclaimer in the documentation and/or other materials 21*4882a593Smuzhiyun * provided with the distribution. 22*4882a593Smuzhiyun * 23*4882a593Smuzhiyun * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24*4882a593Smuzhiyun * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25*4882a593Smuzhiyun * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26*4882a593Smuzhiyun * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27*4882a593Smuzhiyun * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28*4882a593Smuzhiyun * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29*4882a593Smuzhiyun * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30*4882a593Smuzhiyun * SOFTWARE. 31*4882a593Smuzhiyun */ 32*4882a593Smuzhiyun #ifndef _RDS_IB_MR_H 33*4882a593Smuzhiyun #define _RDS_IB_MR_H 34*4882a593Smuzhiyun 35*4882a593Smuzhiyun #include <linux/kernel.h> 36*4882a593Smuzhiyun 37*4882a593Smuzhiyun #include "rds.h" 38*4882a593Smuzhiyun #include "ib.h" 39*4882a593Smuzhiyun 40*4882a593Smuzhiyun #define RDS_MR_1M_POOL_SIZE (8192 / 2) 41*4882a593Smuzhiyun #define RDS_MR_1M_MSG_SIZE 256 42*4882a593Smuzhiyun #define RDS_MR_8K_MSG_SIZE 2 43*4882a593Smuzhiyun #define RDS_MR_8K_SCALE (256 / (RDS_MR_8K_MSG_SIZE + 1)) 44*4882a593Smuzhiyun #define RDS_MR_8K_POOL_SIZE (RDS_MR_8K_SCALE * (8192 / 2)) 45*4882a593Smuzhiyun 46*4882a593Smuzhiyun enum rds_ib_fr_state { 47*4882a593Smuzhiyun FRMR_IS_FREE, /* mr invalidated & ready for use */ 48*4882a593Smuzhiyun FRMR_IS_INUSE, /* mr is in use or used & can be invalidated */ 49*4882a593Smuzhiyun FRMR_IS_STALE, /* Stale MR and needs to be dropped */ 50*4882a593Smuzhiyun }; 51*4882a593Smuzhiyun 52*4882a593Smuzhiyun struct rds_ib_frmr { 53*4882a593Smuzhiyun struct ib_mr *mr; 54*4882a593Smuzhiyun enum rds_ib_fr_state fr_state; 55*4882a593Smuzhiyun bool fr_inv; 56*4882a593Smuzhiyun wait_queue_head_t fr_inv_done; 57*4882a593Smuzhiyun bool fr_reg; 58*4882a593Smuzhiyun wait_queue_head_t fr_reg_done; 59*4882a593Smuzhiyun struct ib_send_wr fr_wr; 60*4882a593Smuzhiyun unsigned int dma_npages; 61*4882a593Smuzhiyun unsigned int sg_byte_len; 62*4882a593Smuzhiyun }; 63*4882a593Smuzhiyun 64*4882a593Smuzhiyun /* This is stored as mr->r_trans_private. */ 65*4882a593Smuzhiyun struct rds_ib_mr { 66*4882a593Smuzhiyun struct delayed_work work; 67*4882a593Smuzhiyun struct rds_ib_device *device; 68*4882a593Smuzhiyun struct rds_ib_mr_pool *pool; 69*4882a593Smuzhiyun struct rds_ib_connection *ic; 70*4882a593Smuzhiyun 71*4882a593Smuzhiyun struct llist_node llnode; 72*4882a593Smuzhiyun 73*4882a593Smuzhiyun /* unmap_list is for freeing */ 74*4882a593Smuzhiyun struct list_head unmap_list; 75*4882a593Smuzhiyun unsigned int remap_count; 76*4882a593Smuzhiyun 77*4882a593Smuzhiyun struct scatterlist *sg; 78*4882a593Smuzhiyun unsigned int sg_len; 79*4882a593Smuzhiyun int sg_dma_len; 80*4882a593Smuzhiyun 81*4882a593Smuzhiyun u8 odp:1; 82*4882a593Smuzhiyun union { 83*4882a593Smuzhiyun struct rds_ib_frmr frmr; 84*4882a593Smuzhiyun struct ib_mr *mr; 85*4882a593Smuzhiyun } u; 86*4882a593Smuzhiyun }; 87*4882a593Smuzhiyun 88*4882a593Smuzhiyun /* Our own little MR pool */ 89*4882a593Smuzhiyun struct rds_ib_mr_pool { 90*4882a593Smuzhiyun unsigned int pool_type; 91*4882a593Smuzhiyun struct mutex flush_lock; /* serialize fmr invalidate */ 92*4882a593Smuzhiyun struct delayed_work flush_worker; /* flush worker */ 93*4882a593Smuzhiyun 94*4882a593Smuzhiyun atomic_t item_count; /* total # of MRs */ 95*4882a593Smuzhiyun atomic_t dirty_count; /* # dirty of MRs */ 96*4882a593Smuzhiyun 97*4882a593Smuzhiyun struct llist_head drop_list; /* MRs not reached max_maps */ 98*4882a593Smuzhiyun struct llist_head free_list; /* unused MRs */ 99*4882a593Smuzhiyun struct llist_head clean_list; /* unused & unmapped MRs */ 100*4882a593Smuzhiyun wait_queue_head_t flush_wait; 101*4882a593Smuzhiyun spinlock_t clean_lock; /* "clean_list" concurrency */ 102*4882a593Smuzhiyun 103*4882a593Smuzhiyun atomic_t free_pinned; /* memory pinned by free MRs */ 104*4882a593Smuzhiyun unsigned long max_items; 105*4882a593Smuzhiyun unsigned long max_items_soft; 106*4882a593Smuzhiyun unsigned long max_free_pinned; 107*4882a593Smuzhiyun unsigned int max_pages; 108*4882a593Smuzhiyun }; 109*4882a593Smuzhiyun 110*4882a593Smuzhiyun extern struct workqueue_struct *rds_ib_mr_wq; 111*4882a593Smuzhiyun extern bool prefer_frmr; 112*4882a593Smuzhiyun 113*4882a593Smuzhiyun struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_dev, 114*4882a593Smuzhiyun int npages); 115*4882a593Smuzhiyun void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev, 116*4882a593Smuzhiyun struct rds_info_rdma_connection *iinfo); 117*4882a593Smuzhiyun void rds6_ib_get_mr_info(struct rds_ib_device *rds_ibdev, 118*4882a593Smuzhiyun struct rds6_info_rdma_connection *iinfo6); 119*4882a593Smuzhiyun void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *); 120*4882a593Smuzhiyun void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, 121*4882a593Smuzhiyun struct rds_sock *rs, u32 *key_ret, 122*4882a593Smuzhiyun struct rds_connection *conn, u64 start, u64 length, 123*4882a593Smuzhiyun int need_odp); 124*4882a593Smuzhiyun void rds_ib_sync_mr(void *trans_private, int dir); 125*4882a593Smuzhiyun void rds_ib_free_mr(void *trans_private, int invalidate); 126*4882a593Smuzhiyun void rds_ib_flush_mrs(void); 127*4882a593Smuzhiyun int rds_ib_mr_init(void); 128*4882a593Smuzhiyun void rds_ib_mr_exit(void); 129*4882a593Smuzhiyun u32 rds_ib_get_lkey(void *trans_private); 130*4882a593Smuzhiyun 131*4882a593Smuzhiyun void __rds_ib_teardown_mr(struct rds_ib_mr *); 132*4882a593Smuzhiyun void rds_ib_teardown_mr(struct rds_ib_mr *); 133*4882a593Smuzhiyun struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *); 134*4882a593Smuzhiyun int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *, int, struct rds_ib_mr **); 135*4882a593Smuzhiyun struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *); 136*4882a593Smuzhiyun struct rds_ib_mr *rds_ib_reg_frmr(struct rds_ib_device *rds_ibdev, 137*4882a593Smuzhiyun struct rds_ib_connection *ic, 138*4882a593Smuzhiyun struct scatterlist *sg, 139*4882a593Smuzhiyun unsigned long nents, u32 *key); 140*4882a593Smuzhiyun void rds_ib_unreg_frmr(struct list_head *list, unsigned int *nfreed, 141*4882a593Smuzhiyun unsigned long *unpinned, unsigned int goal); 142*4882a593Smuzhiyun void rds_ib_free_frmr_list(struct rds_ib_mr *); 143*4882a593Smuzhiyun #endif 144