63 static void do_remove(struct mmu_rb_handler *handler,
87 			 struct mmu_rb_handler **handler)  in hfi1_mmu_rb_register()  argument
113 	*handler = h;  in hfi1_mmu_rb_register()
117 void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)  in hfi1_mmu_rb_unregister()  argument
125 	mmgrab(handler->mn.mm);  in hfi1_mmu_rb_unregister()
128 	mmu_notifier_unregister(&handler->mn, handler->mn.mm);  in hfi1_mmu_rb_unregister()
131 	 * Make sure the wq delete handler is finished running.  It will not  in hfi1_mmu_rb_unregister()
134 	flush_work(&handler->del_work);  in hfi1_mmu_rb_unregister()
138 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_unregister()
139 	while ((node = rb_first_cached(&handler->root))) {  in hfi1_mmu_rb_unregister()
141 		rb_erase_cached(node, &handler->root);  in hfi1_mmu_rb_unregister()
145 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_unregister()
147 	do_remove(handler, &del_list);  in hfi1_mmu_rb_unregister()
150 	mmdrop(handler->mn.mm);  in hfi1_mmu_rb_unregister()
152 	kfree(handler);  in hfi1_mmu_rb_unregister()
155 int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,  in hfi1_mmu_rb_insert()  argument
164 	if (current->mm != handler->mn.mm)  in hfi1_mmu_rb_insert()
167 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_insert()
168 	node = __mmu_rb_search(handler, mnode->addr, mnode->len);  in hfi1_mmu_rb_insert()
173 	__mmu_int_rb_insert(mnode, &handler->root);  in hfi1_mmu_rb_insert()
174 	list_add(&mnode->list, &handler->lru_list);  in hfi1_mmu_rb_insert()
176 	ret = handler->ops->insert(handler->ops_arg, mnode);  in hfi1_mmu_rb_insert()
178 		__mmu_int_rb_remove(mnode, &handler->root);  in hfi1_mmu_rb_insert()
181 	mnode->handler = handler;  in hfi1_mmu_rb_insert()
183 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_insert()
187 /* Caller must hold handler lock */
188 static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,  in __mmu_rb_search()  argument
195 	if (!handler->ops->filter) {  in __mmu_rb_search()
196 		node = __mmu_int_rb_iter_first(&handler->root, addr,  in __mmu_rb_search()
199 		for (node = __mmu_int_rb_iter_first(&handler->root, addr,  in __mmu_rb_search()
204 			if (handler->ops->filter(node, addr, len))  in __mmu_rb_search()
211 bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,  in hfi1_mmu_rb_remove_unless_exact()  argument
219 	if (current->mm != handler->mn.mm)  in hfi1_mmu_rb_remove_unless_exact()
222 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_remove_unless_exact()
223 	node = __mmu_rb_search(handler, addr, len);  in hfi1_mmu_rb_remove_unless_exact()
227 		__mmu_int_rb_remove(node, &handler->root);  in hfi1_mmu_rb_remove_unless_exact()
232 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_remove_unless_exact()
237 void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)  in hfi1_mmu_rb_evict()  argument
244 	if (current->mm != handler->mn.mm)  in hfi1_mmu_rb_evict()
249 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_evict()
250 	list_for_each_entry_safe_reverse(rbnode, ptr, &handler->lru_list,  in hfi1_mmu_rb_evict()
252 		if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg,  in hfi1_mmu_rb_evict()
254 			__mmu_int_rb_remove(rbnode, &handler->root);  in hfi1_mmu_rb_evict()
261 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_evict()
266 		handler->ops->remove(handler->ops_arg, rbnode);  in hfi1_mmu_rb_evict()
275 void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,  in hfi1_mmu_rb_remove()  argument
280 	if (current->mm != handler->mn.mm)  in hfi1_mmu_rb_remove()
283 	/* Validity of handler and node pointers has been checked by caller. */  in hfi1_mmu_rb_remove()
285 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_remove()
286 	__mmu_int_rb_remove(node, &handler->root);  in hfi1_mmu_rb_remove()
288 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_remove()
290 	handler->ops->remove(handler->ops_arg, node);  in hfi1_mmu_rb_remove()
296 	struct mmu_rb_handler *handler =  in mmu_notifier_range_start()  local
298 	struct rb_root_cached *root = &handler->root;  in mmu_notifier_range_start()
303 	spin_lock_irqsave(&handler->lock, flags);  in mmu_notifier_range_start()
310 		if (handler->ops->invalidate(handler->ops_arg, node)) {  in mmu_notifier_range_start()
313 			list_move(&node->list, &handler->del_list);  in mmu_notifier_range_start()
317 	spin_unlock_irqrestore(&handler->lock, flags);  in mmu_notifier_range_start()
320 		queue_work(handler->wq, &handler->del_work);  in mmu_notifier_range_start()
326  * Call the remove function for the given handler and the list.  This
327  * is expected to be called with a delete list extracted from handler.
328  * The caller should not be holding the handler lock.
330 static void do_remove(struct mmu_rb_handler *handler,  in do_remove()  argument
338 		handler->ops->remove(handler->ops_arg, node);  in do_remove()
349 	struct mmu_rb_handler *handler = container_of(work,  in handle_remove()  local
356 	spin_lock_irqsave(&handler->lock, flags);  in handle_remove()
357 	list_replace_init(&handler->del_list, &del_list);  in handle_remove()
358 	spin_unlock_irqrestore(&handler->lock, flags);  in handle_remove()
360 	do_remove(handler, &del_list);  in handle_remove()