308 	struct mmu_notifier *subscription;  in mn_hlist_release()  local
316 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_release()
324 		if (subscription->ops->release)  in mn_hlist_release()
325 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
329 		subscription = hlist_entry(subscriptions->list.first,  in mn_hlist_release()
337 		hlist_del_init_rcu(&subscription->hlist);  in mn_hlist_release()
375 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_flush_young()  local
379 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_flush_young()
382 		if (subscription->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
383 			young |= subscription->ops->clear_flush_young(  in __mmu_notifier_clear_flush_young()
384 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
395 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_young()  local
399 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_young()
402 		if (subscription->ops->clear_young)  in __mmu_notifier_clear_young()
403 			young |= subscription->ops->clear_young(subscription,  in __mmu_notifier_clear_young()
414 	struct mmu_notifier *subscription;  in __mmu_notifier_test_young()  local
418 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_test_young()
421 		if (subscription->ops->test_young) {  in __mmu_notifier_test_young()
422 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
436 	struct mmu_notifier *subscription;  in __mmu_notifier_change_pte()  local
440 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_change_pte()
443 		if (subscription->ops->change_pte)  in __mmu_notifier_change_pte()
444 			subscription->ops->change_pte(subscription, mm, address,  in __mmu_notifier_change_pte()
485 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_range_start()  local
490 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_range_start()
492 		const struct mmu_notifier_ops *ops = subscription->ops;  in mn_hlist_invalidate_range_start()
499 			_ret = ops->invalidate_range_start(subscription, range);  in mn_hlist_invalidate_range_start()
528 		hlist_for_each_entry_rcu(subscription, &subscriptions->list,  in mn_hlist_invalidate_range_start()
530 			if (!subscription->ops->invalidate_range_end)  in mn_hlist_invalidate_range_start()
533 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_range_start()
562 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_end()  local
566 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_end()
581 		if (!only_end && subscription->ops->invalidate_range)  in mn_hlist_invalidate_end()
582 			subscription->ops->invalidate_range(subscription,  in mn_hlist_invalidate_end()
586 		if (subscription->ops->invalidate_range_end) {  in mn_hlist_invalidate_end()
589 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_end()
616 	struct mmu_notifier *subscription;  in __mmu_notifier_invalidate_range()  local
620 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_invalidate_range()
623 		if (subscription->ops->invalidate_range)  in __mmu_notifier_invalidate_range()
624 			subscription->ops->invalidate_range(subscription, mm,  in __mmu_notifier_invalidate_range()
666 int __mmu_notifier_register(struct mmu_notifier *subscription,  in __mmu_notifier_register()  argument
724 	if (subscription) {  in __mmu_notifier_register()
727 		subscription->mm = mm;  in __mmu_notifier_register()
728 		subscription->users = 1;  in __mmu_notifier_register()
731 		hlist_add_head_rcu(&subscription->hlist,  in __mmu_notifier_register()
767 int mmu_notifier_register(struct mmu_notifier *subscription,  in mmu_notifier_register()  argument
773 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
782 	struct mmu_notifier *subscription;  in find_get_mmu_notifier()  local
785 	hlist_for_each_entry_rcu(subscription,  in find_get_mmu_notifier()
788 		if (subscription->ops != ops)  in find_get_mmu_notifier()
791 		if (likely(subscription->users != UINT_MAX))  in find_get_mmu_notifier()
792 			subscription->users++;  in find_get_mmu_notifier()
794 			subscription = ERR_PTR(-EOVERFLOW);  in find_get_mmu_notifier()
796 		return subscription;  in find_get_mmu_notifier()
822 	struct mmu_notifier *subscription;  in mmu_notifier_get_locked()  local
828 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
829 		if (subscription)  in mmu_notifier_get_locked()
830 			return subscription;  in mmu_notifier_get_locked()
833 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
834 	if (IS_ERR(subscription))  in mmu_notifier_get_locked()
835 		return subscription;  in mmu_notifier_get_locked()
836 	subscription->ops = ops;  in mmu_notifier_get_locked()
837 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
840 	return subscription;  in mmu_notifier_get_locked()
842 	subscription->ops->free_notifier(subscription);  in mmu_notifier_get_locked()
865 void mmu_notifier_unregister(struct mmu_notifier *subscription,  in mmu_notifier_unregister()  argument
870 	if (!hlist_unhashed(&subscription->hlist)) {  in mmu_notifier_unregister()
882 		if (subscription->ops->release)  in mmu_notifier_unregister()
883 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
891 		hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_unregister()
909 	struct mmu_notifier *subscription =  in mmu_notifier_free_rcu()  local
911 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()
913 	subscription->ops->free_notifier(subscription);  in mmu_notifier_free_rcu()
940 void mmu_notifier_put(struct mmu_notifier *subscription)  in mmu_notifier_put()  argument
942 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()
945 	if (WARN_ON(!subscription->users) || --subscription->users)  in mmu_notifier_put()
947 	hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_put()
950 	call_srcu(&srcu, &subscription->rcu, mmu_notifier_free_rcu);  in mmu_notifier_put()