195 		interval_sub->mm->notifier_subscriptions;  in mmu_interval_read_begin()
268 			     struct mm_struct *mm)  in mn_itree_release()  argument
273 		.mm = mm,  in mn_itree_release()
306 			     struct mm_struct *mm)  in mn_hlist_release()  argument
325 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
354 void __mmu_notifier_release(struct mm_struct *mm)  in __mmu_notifier_release()  argument
357 		mm->notifier_subscriptions;  in __mmu_notifier_release()
360 		mn_itree_release(subscriptions, mm);  in __mmu_notifier_release()
363 		mn_hlist_release(subscriptions, mm);  in __mmu_notifier_release()
371 int __mmu_notifier_clear_flush_young(struct mm_struct *mm,  in __mmu_notifier_clear_flush_young()  argument
380 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_clear_flush_young()
384 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
391 int __mmu_notifier_clear_young(struct mm_struct *mm,  in __mmu_notifier_clear_young()  argument
400 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_clear_young()
404 								mm, start, end);  in __mmu_notifier_clear_young()
411 int __mmu_notifier_test_young(struct mm_struct *mm,  in __mmu_notifier_test_young()  argument
419 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_test_young()
422 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
433 void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,  in __mmu_notifier_change_pte()  argument
441 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_change_pte()
444 			subscription->ops->change_pte(subscription, mm, address,  in __mmu_notifier_change_pte()
545 		range->mm->notifier_subscriptions;  in __mmu_notifier_invalidate_range_start()
583 							    range->mm,  in mn_hlist_invalidate_end()
602 		range->mm->notifier_subscriptions;  in __mmu_notifier_invalidate_range_end()
613 void __mmu_notifier_invalidate_range(struct mm_struct *mm,  in __mmu_notifier_invalidate_range()  argument
621 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_invalidate_range()
624 			subscription->ops->invalidate_range(subscription, mm,  in __mmu_notifier_invalidate_range()
632 static inline void mmu_notifier_write_lock(struct mm_struct *mm)  in mmu_notifier_write_lock()  argument
635 		&mm->notifier_subscriptions->hdr.mmu_notifier_lock->rw_sem);  in mmu_notifier_write_lock()
638 static inline void mmu_notifier_write_unlock(struct mm_struct *mm)  in mmu_notifier_write_unlock()  argument
641 		&mm->notifier_subscriptions->hdr.mmu_notifier_lock->rw_sem);  in mmu_notifier_write_unlock()
646 static inline void mmu_notifier_write_lock(struct mm_struct *mm) {}  in mmu_notifier_write_lock()  argument
647 static inline void mmu_notifier_write_unlock(struct mm_struct *mm) {}  in mmu_notifier_write_unlock()  argument
667 			    struct mm_struct *mm)  in __mmu_notifier_register()  argument
672 	mmap_assert_write_locked(mm);  in __mmu_notifier_register()
673 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in __mmu_notifier_register()
682 	if (!mm->notifier_subscriptions) {  in __mmu_notifier_register()
696 	mmu_notifier_write_lock(mm);  in __mmu_notifier_register()
698 	ret = mm_take_all_locks(mm);  in __mmu_notifier_register()
700 		mmu_notifier_write_unlock(mm);  in __mmu_notifier_register()
721 		smp_store_release(&mm->notifier_subscriptions, subscriptions);  in __mmu_notifier_register()
722 	mm->notifier_subscriptions->hdr.valid = true;  in __mmu_notifier_register()
726 		mmgrab(mm);  in __mmu_notifier_register()
727 		subscription->mm = mm;  in __mmu_notifier_register()
730 		spin_lock(&mm->notifier_subscriptions->lock);  in __mmu_notifier_register()
732 				   &mm->notifier_subscriptions->list);  in __mmu_notifier_register()
733 		spin_unlock(&mm->notifier_subscriptions->lock);  in __mmu_notifier_register()
735 		mm->notifier_subscriptions->has_itree = true;  in __mmu_notifier_register()
737 	mm_drop_all_locks(mm);  in __mmu_notifier_register()
738 	mmu_notifier_write_unlock(mm);  in __mmu_notifier_register()
739 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in __mmu_notifier_register()
768 			  struct mm_struct *mm)  in mmu_notifier_register()  argument
772 	mmap_write_lock(mm);  in mmu_notifier_register()
773 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
774 	mmap_write_unlock(mm);  in mmu_notifier_register()
780 find_get_mmu_notifier(struct mm_struct *mm, const struct mmu_notifier_ops *ops)  in find_get_mmu_notifier()  argument
784 	spin_lock(&mm->notifier_subscriptions->lock);  in find_get_mmu_notifier()
786 				 &mm->notifier_subscriptions->list, hlist,  in find_get_mmu_notifier()
787 				 lockdep_is_held(&mm->notifier_subscriptions->lock)) {  in find_get_mmu_notifier()
795 		spin_unlock(&mm->notifier_subscriptions->lock);  in find_get_mmu_notifier()
798 	spin_unlock(&mm->notifier_subscriptions->lock);  in find_get_mmu_notifier()
820 					     struct mm_struct *mm)  in mmu_notifier_get_locked()  argument
825 	mmap_assert_write_locked(mm);  in mmu_notifier_get_locked()
827 	if (mm->notifier_subscriptions) {  in mmu_notifier_get_locked()
828 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
833 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
837 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
848 void __mmu_notifier_subscriptions_destroy(struct mm_struct *mm)  in __mmu_notifier_subscriptions_destroy()  argument
850 	BUG_ON(!hlist_empty(&mm->notifier_subscriptions->list));  in __mmu_notifier_subscriptions_destroy()
851 	kfree(mm->notifier_subscriptions);  in __mmu_notifier_subscriptions_destroy()
852 	mm->notifier_subscriptions = LIST_POISON1; /* debug */  in __mmu_notifier_subscriptions_destroy()
866 			     struct mm_struct *mm)  in mmu_notifier_unregister()  argument
868 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
883 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
886 		spin_lock(&mm->notifier_subscriptions->lock);  in mmu_notifier_unregister()
892 		spin_unlock(&mm->notifier_subscriptions->lock);  in mmu_notifier_unregister()
901 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
903 	mmdrop(mm);  in mmu_notifier_unregister()
911 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()  local
915 	mmdrop(mm);  in mmu_notifier_free_rcu()
942 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()  local
944 	spin_lock(&mm->notifier_subscriptions->lock);  in mmu_notifier_put()
948 	spin_unlock(&mm->notifier_subscriptions->lock);  in mmu_notifier_put()
954 	spin_unlock(&mm->notifier_subscriptions->lock);  in mmu_notifier_put()
959 	struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,  in __mmu_interval_notifier_insert()  argument
963 	interval_sub->mm = mm;  in __mmu_interval_notifier_insert()
977 	if (WARN_ON(atomic_read(&mm->mm_users) <= 0))  in __mmu_interval_notifier_insert()
981 	mmgrab(mm);  in __mmu_interval_notifier_insert()
1041 				 struct mm_struct *mm, unsigned long start,  in mmu_interval_notifier_insert()  argument
1048 	might_lock(&mm->mmap_lock);  in mmu_interval_notifier_insert()
1050 	subscriptions = smp_load_acquire(&mm->notifier_subscriptions);  in mmu_interval_notifier_insert()
1052 		ret = mmu_notifier_register(NULL, mm);  in mmu_interval_notifier_insert()
1055 		subscriptions = mm->notifier_subscriptions;  in mmu_interval_notifier_insert()
1057 	return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions,  in mmu_interval_notifier_insert()
1063 	struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,  in mmu_interval_notifier_insert_locked()  argument
1068 		mm->notifier_subscriptions;  in mmu_interval_notifier_insert_locked()
1071 	mmap_assert_write_locked(mm);  in mmu_interval_notifier_insert_locked()
1074 		ret = __mmu_notifier_register(NULL, mm);  in mmu_interval_notifier_insert_locked()
1077 		subscriptions = mm->notifier_subscriptions;  in mmu_interval_notifier_insert_locked()
1079 	return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions,  in mmu_interval_notifier_insert_locked()
1108 	struct mm_struct *mm = interval_sub->mm;  in mmu_interval_notifier_remove()  local
1110 		mm->notifier_subscriptions;  in mmu_interval_notifier_remove()
1146 	mmdrop(mm);  in mmu_interval_notifier_remove()
1181 bool mmu_notifier_subscriptions_init(struct mm_struct *mm)  in mmu_notifier_subscriptions_init()  argument
1202 	mm->notifier_subscriptions = subscriptions;  in mmu_notifier_subscriptions_init()
1207 void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)  in mmu_notifier_subscriptions_destroy()  argument
1210 			mm->notifier_subscriptions->hdr.mmu_notifier_lock);  in mmu_notifier_subscriptions_destroy()
1211 	kfree(mm->notifier_subscriptions);  in mmu_notifier_subscriptions_destroy()
1212 	mm->notifier_subscriptions = NULL;  in mmu_notifier_subscriptions_destroy()