94 void fsnotify_get_mark(struct fsnotify_mark *mark)  in fsnotify_get_mark()  argument
96 	WARN_ON_ONCE(!refcount_read(&mark->refcnt));  in fsnotify_get_mark()
97 	refcount_inc(&mark->refcnt);  in fsnotify_get_mark()
122 	struct fsnotify_mark *mark;  in __fsnotify_recalc_mask()  local
128 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in __fsnotify_recalc_mask()
129 		if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)  in __fsnotify_recalc_mask()
130 			new_mask |= mark->mask;  in __fsnotify_recalc_mask()
199 static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)  in fsnotify_final_mark_destroy()  argument
201 	struct fsnotify_group *group = mark->group;  in fsnotify_final_mark_destroy()
205 	group->ops->free_mark(mark);  in fsnotify_final_mark_destroy()
227 void fsnotify_put_mark(struct fsnotify_mark *mark)  in fsnotify_put_mark()  argument
229 	struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);  in fsnotify_put_mark()
236 		if (refcount_dec_and_test(&mark->refcnt))  in fsnotify_put_mark()
237 			fsnotify_final_mark_destroy(mark);  in fsnotify_put_mark()
245 	if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))  in fsnotify_put_mark()
248 	hlist_del_init_rcu(&mark->obj_list);  in fsnotify_put_mark()
255 	WRITE_ONCE(mark->connector, NULL);  in fsnotify_put_mark()
274 	list_add(&mark->g_list, &destroy_list);  in fsnotify_put_mark()
288 static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)  in fsnotify_get_mark_safe()  argument
290 	if (!mark)  in fsnotify_get_mark_safe()
293 	if (refcount_inc_not_zero(&mark->refcnt)) {  in fsnotify_get_mark_safe()
294 		spin_lock(&mark->lock);  in fsnotify_get_mark_safe()
295 		if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {  in fsnotify_get_mark_safe()
297 			atomic_inc(&mark->group->user_waits);  in fsnotify_get_mark_safe()
298 			spin_unlock(&mark->lock);  in fsnotify_get_mark_safe()
301 		spin_unlock(&mark->lock);  in fsnotify_get_mark_safe()
302 		fsnotify_put_mark(mark);  in fsnotify_get_mark_safe()
312 static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)  in fsnotify_put_mark_wake()  argument
314 	if (mark) {  in fsnotify_put_mark_wake()
315 		struct fsnotify_group *group = mark->group;  in fsnotify_put_mark_wake()
317 		fsnotify_put_mark(mark);  in fsnotify_put_mark_wake()
375 void fsnotify_detach_mark(struct fsnotify_mark *mark)  in fsnotify_detach_mark()  argument
377 	struct fsnotify_group *group = mark->group;  in fsnotify_detach_mark()
381 		     refcount_read(&mark->refcnt) < 1 +  in fsnotify_detach_mark()
382 			!!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));  in fsnotify_detach_mark()
384 	spin_lock(&mark->lock);  in fsnotify_detach_mark()
386 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {  in fsnotify_detach_mark()
387 		spin_unlock(&mark->lock);  in fsnotify_detach_mark()
390 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;  in fsnotify_detach_mark()
391 	list_del_init(&mark->g_list);  in fsnotify_detach_mark()
392 	spin_unlock(&mark->lock);  in fsnotify_detach_mark()
397 	fsnotify_put_mark(mark);  in fsnotify_detach_mark()
408 void fsnotify_free_mark(struct fsnotify_mark *mark)  in fsnotify_free_mark()  argument
410 	struct fsnotify_group *group = mark->group;  in fsnotify_free_mark()
412 	spin_lock(&mark->lock);  in fsnotify_free_mark()
414 	if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {  in fsnotify_free_mark()
415 		spin_unlock(&mark->lock);  in fsnotify_free_mark()
418 	mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;  in fsnotify_free_mark()
419 	spin_unlock(&mark->lock);  in fsnotify_free_mark()
427 		group->ops->freeing_mark(mark, group);  in fsnotify_free_mark()
430 void fsnotify_destroy_mark(struct fsnotify_mark *mark,  in fsnotify_destroy_mark()  argument
434 	fsnotify_detach_mark(mark);  in fsnotify_destroy_mark()
436 	fsnotify_free_mark(mark);  in fsnotify_destroy_mark()
547 static int fsnotify_add_mark_list(struct fsnotify_mark *mark,  in fsnotify_add_mark_list()  argument
564 	spin_lock(&mark->lock);  in fsnotify_add_mark_list()
567 		spin_unlock(&mark->lock);  in fsnotify_add_mark_list()
597 		hlist_add_head_rcu(&mark->obj_list, &conn->list);  in fsnotify_add_mark_list()
605 		if ((lmark->group == mark->group) &&  in fsnotify_add_mark_list()
612 		cmp = fsnotify_compare_groups(lmark->group, mark->group);  in fsnotify_add_mark_list()
614 			hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);  in fsnotify_add_mark_list()
621 	hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);  in fsnotify_add_mark_list()
628 	WRITE_ONCE(mark->connector, conn);  in fsnotify_add_mark_list()
631 	spin_unlock(&mark->lock);  in fsnotify_add_mark_list()
640 int fsnotify_add_mark_locked(struct fsnotify_mark *mark,  in fsnotify_add_mark_locked()  argument
644 	struct fsnotify_group *group = mark->group;  in fsnotify_add_mark_locked()
655 	spin_lock(&mark->lock);  in fsnotify_add_mark_locked()
656 	mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;  in fsnotify_add_mark_locked()
658 	list_add(&mark->g_list, &group->marks_list);  in fsnotify_add_mark_locked()
660 	fsnotify_get_mark(mark); /* for g_list */  in fsnotify_add_mark_locked()
661 	spin_unlock(&mark->lock);  in fsnotify_add_mark_locked()
663 	ret = fsnotify_add_mark_list(mark, connp, type, allow_dups, fsid);  in fsnotify_add_mark_locked()
667 	if (mark->mask)  in fsnotify_add_mark_locked()
668 		fsnotify_recalc_mask(mark->connector);  in fsnotify_add_mark_locked()
672 	spin_lock(&mark->lock);  in fsnotify_add_mark_locked()
673 	mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |  in fsnotify_add_mark_locked()
675 	list_del_init(&mark->g_list);  in fsnotify_add_mark_locked()
676 	spin_unlock(&mark->lock);  in fsnotify_add_mark_locked()
679 	fsnotify_put_mark(mark);  in fsnotify_add_mark_locked()
683 int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp,  in fsnotify_add_mark()  argument
687 	struct fsnotify_group *group = mark->group;  in fsnotify_add_mark()
690 	ret = fsnotify_add_mark_locked(mark, connp, type, allow_dups, fsid);  in fsnotify_add_mark()
704 	struct fsnotify_mark *mark;  in fsnotify_find_mark()  local
710 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_find_mark()
711 		if (mark->group == group &&  in fsnotify_find_mark()
712 		    (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {  in fsnotify_find_mark()
713 			fsnotify_get_mark(mark);  in fsnotify_find_mark()
715 			return mark;  in fsnotify_find_mark()
727 	struct fsnotify_mark *lmark, *mark;  in fsnotify_clear_marks_by_group()  local
746 	list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {  in fsnotify_clear_marks_by_group()
747 		if ((1U << mark->connector->type) & type_mask)  in fsnotify_clear_marks_by_group()
748 			list_move(&mark->g_list, &to_free);  in fsnotify_clear_marks_by_group()
759 		mark = list_first_entry(head, struct fsnotify_mark, g_list);  in fsnotify_clear_marks_by_group()
760 		fsnotify_get_mark(mark);  in fsnotify_clear_marks_by_group()
761 		fsnotify_detach_mark(mark);  in fsnotify_clear_marks_by_group()
763 		fsnotify_free_mark(mark);  in fsnotify_clear_marks_by_group()
764 		fsnotify_put_mark(mark);  in fsnotify_clear_marks_by_group()
772 	struct fsnotify_mark *mark, *old_mark = NULL;  in fsnotify_destroy_marks()  local
786 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_destroy_marks()
787 		fsnotify_get_mark(mark);  in fsnotify_destroy_marks()
791 		old_mark = mark;  in fsnotify_destroy_marks()
792 		fsnotify_destroy_mark(mark, mark->group);  in fsnotify_destroy_marks()
810 void fsnotify_init_mark(struct fsnotify_mark *mark,  in fsnotify_init_mark()  argument
813 	memset(mark, 0, sizeof(*mark));  in fsnotify_init_mark()
814 	spin_lock_init(&mark->lock);  in fsnotify_init_mark()
815 	refcount_set(&mark->refcnt, 1);  in fsnotify_init_mark()
817 	mark->group = group;  in fsnotify_init_mark()
818 	WRITE_ONCE(mark->connector, NULL);  in fsnotify_init_mark()
828 	struct fsnotify_mark *mark, *next;  in fsnotify_mark_destroy_workfn()  local
838 	list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {  in fsnotify_mark_destroy_workfn()
839 		list_del_init(&mark->g_list);  in fsnotify_mark_destroy_workfn()
840 		fsnotify_final_mark_destroy(mark);  in fsnotify_mark_destroy_workfn()