35 static bool kernfs_active(struct kernfs_node *kn)  in kernfs_active()  argument
38 	return atomic_read(&kn->active) >= 0;  in kernfs_active()
41 static bool kernfs_lockdep(struct kernfs_node *kn)  in kernfs_lockdep()  argument
44 	return kn->flags & KERNFS_LOCKDEP;  in kernfs_lockdep()
50 static int kernfs_name_locked(struct kernfs_node *kn, char *buf, size_t buflen)  in kernfs_name_locked()  argument
52 	if (!kn)  in kernfs_name_locked()
55 	return strlcpy(buf, kn->parent ? kn->name : "/", buflen);  in kernfs_name_locked()
79 	da = kernfs_depth(ra->kn, a);  in kernfs_common_ancestor()
80 	db = kernfs_depth(rb->kn, b);  in kernfs_common_ancestor()
134 	struct kernfs_node *kn, *common;  in kernfs_path_from_node_locked()  local
143 		kn_from = kernfs_root(kn_to)->kn;  in kernfs_path_from_node_locked()
166 		for (kn = kn_to, j = 0; j < i; j++)  in kernfs_path_from_node_locked()
167 			kn = kn->parent;  in kernfs_path_from_node_locked()
170 		len += strlcpy(buf + len, kn->name,  in kernfs_path_from_node_locked()
191 int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen)  in kernfs_name()  argument
197 	ret = kernfs_name_locked(kn, buf, buflen);  in kernfs_name()
237 void pr_cont_kernfs_name(struct kernfs_node *kn)  in pr_cont_kernfs_name()  argument
243 	kernfs_name(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf));  in pr_cont_kernfs_name()
255 void pr_cont_kernfs_path(struct kernfs_node *kn)  in pr_cont_kernfs_path()  argument
262 	sz = kernfs_path_from_node(kn, NULL, kernfs_pr_cont_buf,  in pr_cont_kernfs_path()
287 struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn)  in kernfs_get_parent()  argument
293 	parent = kn->parent;  in kernfs_get_parent()
324 			       const void *ns, const struct kernfs_node *kn)  in kernfs_name_compare()  argument
326 	if (hash < kn->hash)  in kernfs_name_compare()
328 	if (hash > kn->hash)  in kernfs_name_compare()
330 	if (ns < kn->ns)  in kernfs_name_compare()
332 	if (ns > kn->ns)  in kernfs_name_compare()
334 	return strcmp(name, kn->name);  in kernfs_name_compare()
356 static int kernfs_link_sibling(struct kernfs_node *kn)  in kernfs_link_sibling()  argument
358 	struct rb_node **node = &kn->parent->dir.children.rb_node;  in kernfs_link_sibling()
367 		result = kernfs_sd_compare(kn, pos);  in kernfs_link_sibling()
377 	rb_link_node(&kn->rb, parent, node);  in kernfs_link_sibling()
378 	rb_insert_color(&kn->rb, &kn->parent->dir.children);  in kernfs_link_sibling()
381 	if (kernfs_type(kn) == KERNFS_DIR)  in kernfs_link_sibling()
382 		kn->parent->dir.subdirs++;  in kernfs_link_sibling()
398 static bool kernfs_unlink_sibling(struct kernfs_node *kn)  in kernfs_unlink_sibling()  argument
400 	if (RB_EMPTY_NODE(&kn->rb))  in kernfs_unlink_sibling()
403 	if (kernfs_type(kn) == KERNFS_DIR)  in kernfs_unlink_sibling()
404 		kn->parent->dir.subdirs--;  in kernfs_unlink_sibling()
406 	rb_erase(&kn->rb, &kn->parent->dir.children);  in kernfs_unlink_sibling()
407 	RB_CLEAR_NODE(&kn->rb);  in kernfs_unlink_sibling()
421 struct kernfs_node *kernfs_get_active(struct kernfs_node *kn)  in kernfs_get_active()  argument
423 	if (unlikely(!kn))  in kernfs_get_active()
426 	if (!atomic_inc_unless_negative(&kn->active))  in kernfs_get_active()
429 	if (kernfs_lockdep(kn))  in kernfs_get_active()
430 		rwsem_acquire_read(&kn->dep_map, 0, 1, _RET_IP_);  in kernfs_get_active()
431 	return kn;  in kernfs_get_active()
441 void kernfs_put_active(struct kernfs_node *kn)  in kernfs_put_active()  argument
445 	if (unlikely(!kn))  in kernfs_put_active()
448 	if (kernfs_lockdep(kn))  in kernfs_put_active()
449 		rwsem_release(&kn->dep_map, _RET_IP_);  in kernfs_put_active()
450 	v = atomic_dec_return(&kn->active);  in kernfs_put_active()
454 	wake_up_all(&kernfs_root(kn)->deactivate_waitq);  in kernfs_put_active()
465 static void kernfs_drain(struct kernfs_node *kn)  in kernfs_drain()  argument
468 	struct kernfs_root *root = kernfs_root(kn);  in kernfs_drain()
471 	WARN_ON_ONCE(kernfs_active(kn));  in kernfs_drain()
475 	if (kernfs_lockdep(kn)) {  in kernfs_drain()
476 		rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_);  in kernfs_drain()
477 		if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS)  in kernfs_drain()
478 			lock_contended(&kn->dep_map, _RET_IP_);  in kernfs_drain()
483 		   atomic_read(&kn->active) == KN_DEACTIVATED_BIAS);  in kernfs_drain()
485 	if (kernfs_lockdep(kn)) {  in kernfs_drain()
486 		lock_acquired(&kn->dep_map, _RET_IP_);  in kernfs_drain()
487 		rwsem_release(&kn->dep_map, _RET_IP_);  in kernfs_drain()
490 	kernfs_drain_open_files(kn);  in kernfs_drain()
499 void kernfs_get(struct kernfs_node *kn)  in kernfs_get()  argument
501 	if (kn) {  in kernfs_get()
502 		WARN_ON(!atomic_read(&kn->count));  in kernfs_get()
503 		atomic_inc(&kn->count);  in kernfs_get()
514 void kernfs_put(struct kernfs_node *kn)  in kernfs_put()  argument
519 	if (!kn || !atomic_dec_and_test(&kn->count))  in kernfs_put()
521 	root = kernfs_root(kn);  in kernfs_put()
527 	parent = kn->parent;  in kernfs_put()
529 	WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS,  in kernfs_put()
531 		  parent ? parent->name : "", kn->name, atomic_read(&kn->active));  in kernfs_put()
533 	if (kernfs_type(kn) == KERNFS_LINK)  in kernfs_put()
534 		kernfs_put(kn->symlink.target_kn);  in kernfs_put()
536 	kfree_const(kn->name);  in kernfs_put()
538 	if (kn->iattr) {  in kernfs_put()
539 		simple_xattrs_free(&kn->iattr->xattrs);  in kernfs_put()
540 		kmem_cache_free(kernfs_iattrs_cache, kn->iattr);  in kernfs_put()
543 	idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));  in kernfs_put()
545 	kmem_cache_free(kernfs_node_cache, kn);  in kernfs_put()
547 	kn = parent;  in kernfs_put()
548 	if (kn) {  in kernfs_put()
549 		if (atomic_dec_and_test(&kn->count))  in kernfs_put()
561 	struct kernfs_node *kn;  in kernfs_dop_revalidate()  local
570 	kn = kernfs_dentry_node(dentry);  in kernfs_dop_revalidate()
574 	if (!kernfs_active(kn))  in kernfs_dop_revalidate()
578 	if (kernfs_dentry_node(dentry->d_parent) != kn->parent)  in kernfs_dop_revalidate()
582 	if (strcmp(dentry->d_name.name, kn->name) != 0)  in kernfs_dop_revalidate()
586 	if (kn->parent && kernfs_ns_enabled(kn->parent) &&  in kernfs_dop_revalidate()
587 	    kernfs_info(dentry->d_sb)->ns != kn->ns)  in kernfs_dop_revalidate()
627 	struct kernfs_node *kn;  in __kernfs_new_node()  local
635 	kn = kmem_cache_zalloc(kernfs_node_cache, GFP_KERNEL);  in __kernfs_new_node()
636 	if (!kn)  in __kernfs_new_node()
641 	ret = idr_alloc_cyclic(&root->ino_idr, kn, 1, 0, GFP_ATOMIC);  in __kernfs_new_node()
651 	kn->id = (u64)id_highbits << 32 | ret;  in __kernfs_new_node()
653 	atomic_set(&kn->count, 1);  in __kernfs_new_node()
654 	atomic_set(&kn->active, KN_DEACTIVATED_BIAS);  in __kernfs_new_node()
655 	RB_CLEAR_NODE(&kn->rb);  in __kernfs_new_node()
657 	kn->name = name;  in __kernfs_new_node()
658 	kn->mode = mode;  in __kernfs_new_node()
659 	kn->flags = flags;  in __kernfs_new_node()
668 		ret = __kernfs_setattr(kn, &iattr);  in __kernfs_new_node()
674 		ret = security_kernfs_init_security(parent, kn);  in __kernfs_new_node()
679 	return kn;  in __kernfs_new_node()
682 	idr_remove(&root->ino_idr, (u32)kernfs_ino(kn));  in __kernfs_new_node()
684 	kmem_cache_free(kernfs_node_cache, kn);  in __kernfs_new_node()
695 	struct kernfs_node *kn;  in kernfs_new_node()  local
697 	kn = __kernfs_new_node(kernfs_root(parent), parent,  in kernfs_new_node()
699 	if (kn) {  in kernfs_new_node()
701 		kn->parent = parent;  in kernfs_new_node()
703 	return kn;  in kernfs_new_node()
720 	struct kernfs_node *kn;  in kernfs_find_and_get_node_by_id()  local
726 	kn = idr_find(&root->ino_idr, (u32)ino);  in kernfs_find_and_get_node_by_id()
727 	if (!kn)  in kernfs_find_and_get_node_by_id()
732 		if (kernfs_ino(kn) != ino)  in kernfs_find_and_get_node_by_id()
736 		if (unlikely(gen && kernfs_gen(kn) != gen))  in kernfs_find_and_get_node_by_id()
745 	if (unlikely(!(kn->flags & KERNFS_ACTIVATED) ||  in kernfs_find_and_get_node_by_id()
746 		     !atomic_inc_not_zero(&kn->count)))  in kernfs_find_and_get_node_by_id()
750 	return kn;  in kernfs_find_and_get_node_by_id()
768 int kernfs_add_one(struct kernfs_node *kn)  in kernfs_add_one()  argument
770 	struct kernfs_node *parent = kn->parent;  in kernfs_add_one()
779 	if (WARN(has_ns != (bool)kn->ns, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",  in kernfs_add_one()
780 		 has_ns ? "required" : "invalid", parent->name, kn->name))  in kernfs_add_one()
793 	kn->hash = kernfs_name_hash(kn->name, kn->ns);  in kernfs_add_one()
795 	ret = kernfs_link_sibling(kn);  in kernfs_add_one()
815 	if (!(kernfs_root(kn)->flags & KERNFS_ROOT_CREATE_DEACTIVATED))  in kernfs_add_one()
816 		kernfs_activate(kn);  in kernfs_add_one()
851 		struct kernfs_node *kn;  in kernfs_find_ns()  local
854 		kn = rb_to_kn(node);  in kernfs_find_ns()
855 		result = kernfs_name_compare(hash, name, ns, kn);  in kernfs_find_ns()
861 			return kn;  in kernfs_find_ns()
910 	struct kernfs_node *kn;  in kernfs_find_and_get_ns()  local
913 	kn = kernfs_find_ns(parent, name, ns);  in kernfs_find_and_get_ns()
914 	kernfs_get(kn);  in kernfs_find_and_get_ns()
917 	return kn;  in kernfs_find_and_get_ns()
934 	struct kernfs_node *kn;  in kernfs_walk_and_get_ns()  local
937 	kn = kernfs_walk_ns(parent, path, ns);  in kernfs_walk_and_get_ns()
938 	kernfs_get(kn);  in kernfs_walk_and_get_ns()
941 	return kn;  in kernfs_walk_and_get_ns()
957 	struct kernfs_node *kn;  in kernfs_create_root()  local
977 	kn = __kernfs_new_node(root, NULL, "", S_IFDIR | S_IRUGO | S_IXUGO,  in kernfs_create_root()
980 	if (!kn) {  in kernfs_create_root()
986 	kn->priv = priv;  in kernfs_create_root()
987 	kn->dir.root = root;  in kernfs_create_root()
991 	root->kn = kn;  in kernfs_create_root()
995 		kernfs_activate(kn);  in kernfs_create_root()
1009 	kernfs_remove(root->kn);	/* will also free @root */  in kernfs_destroy_root()
1029 	struct kernfs_node *kn;  in kernfs_create_dir_ns()  local
1033 	kn = kernfs_new_node(parent, name, mode | S_IFDIR,  in kernfs_create_dir_ns()
1035 	if (!kn)  in kernfs_create_dir_ns()
1038 	kn->dir.root = parent->dir.root;  in kernfs_create_dir_ns()
1039 	kn->ns = ns;  in kernfs_create_dir_ns()
1040 	kn->priv = priv;  in kernfs_create_dir_ns()
1043 	rc = kernfs_add_one(kn);  in kernfs_create_dir_ns()
1045 		return kn;  in kernfs_create_dir_ns()
1047 	kernfs_put(kn);  in kernfs_create_dir_ns()
1061 	struct kernfs_node *kn;  in kernfs_create_empty_dir()  local
1065 	kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR,  in kernfs_create_empty_dir()
1067 	if (!kn)  in kernfs_create_empty_dir()
1070 	kn->flags |= KERNFS_EMPTY_DIR;  in kernfs_create_empty_dir()
1071 	kn->dir.root = parent->dir.root;  in kernfs_create_empty_dir()
1072 	kn->ns = NULL;  in kernfs_create_empty_dir()
1073 	kn->priv = NULL;  in kernfs_create_empty_dir()
1076 	rc = kernfs_add_one(kn);  in kernfs_create_empty_dir()
1078 		return kn;  in kernfs_create_empty_dir()
1080 	kernfs_put(kn);  in kernfs_create_empty_dir()
1090 	struct kernfs_node *kn;  in kernfs_iop_lookup()  local
1099 	kn = kernfs_find_ns(parent, dentry->d_name.name, ns);  in kernfs_iop_lookup()
1102 	if (!kn || !kernfs_active(kn)) {  in kernfs_iop_lookup()
1108 	inode = kernfs_get_inode(dir->i_sb, kn);  in kernfs_iop_lookup()
1142 	struct kernfs_node *kn  = kernfs_dentry_node(dentry);  in kernfs_iop_rmdir()  local
1143 	struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;  in kernfs_iop_rmdir()
1149 	if (!kernfs_get_active(kn))  in kernfs_iop_rmdir()
1152 	ret = scops->rmdir(kn);  in kernfs_iop_rmdir()
1154 	kernfs_put_active(kn);  in kernfs_iop_rmdir()
1162 	struct kernfs_node *kn = kernfs_dentry_node(old_dentry);  in kernfs_iop_rename()  local
1164 	struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;  in kernfs_iop_rename()
1173 	if (!kernfs_get_active(kn))  in kernfs_iop_rename()
1177 		kernfs_put_active(kn);  in kernfs_iop_rename()
1181 	ret = scops->rename(kn, new_parent, new_dentry->d_name.name);  in kernfs_iop_rename()
1184 	kernfs_put_active(kn);  in kernfs_iop_rename()
1268 void kernfs_activate(struct kernfs_node *kn)  in kernfs_activate()  argument
1275 	while ((pos = kernfs_next_descendant_post(pos, kn))) {  in kernfs_activate()
1289 static void __kernfs_remove(struct kernfs_node *kn)  in __kernfs_remove()  argument
1300 	if (!kn || (kn->parent && RB_EMPTY_NODE(&kn->rb)))  in __kernfs_remove()
1303 	pr_debug("kernfs %s: removing\n", kn->name);  in __kernfs_remove()
1307 	while ((pos = kernfs_next_descendant_post(pos, kn)))  in __kernfs_remove()
1313 		pos = kernfs_leftmost_descendant(kn);  in __kernfs_remove()
1329 		if (kn->flags & KERNFS_ACTIVATED)  in __kernfs_remove()
1332 			WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS);  in __kernfs_remove()
1352 	} while (pos != kn);  in __kernfs_remove()
1361 void kernfs_remove(struct kernfs_node *kn)  in kernfs_remove()  argument
1364 	__kernfs_remove(kn);  in kernfs_remove()
1382 void kernfs_break_active_protection(struct kernfs_node *kn)  in kernfs_break_active_protection()  argument
1388 	kernfs_put_active(kn);  in kernfs_break_active_protection()
1406 void kernfs_unbreak_active_protection(struct kernfs_node *kn)  in kernfs_unbreak_active_protection()  argument
1417 	atomic_inc(&kn->active);  in kernfs_unbreak_active_protection()
1418 	if (kernfs_lockdep(kn))  in kernfs_unbreak_active_protection()
1419 		rwsem_acquire(&kn->dep_map, 0, 1, _RET_IP_);  in kernfs_unbreak_active_protection()
1448 bool kernfs_remove_self(struct kernfs_node *kn)  in kernfs_remove_self()  argument
1453 	kernfs_break_active_protection(kn);  in kernfs_remove_self()
1464 	if (!(kn->flags & KERNFS_SUICIDAL)) {  in kernfs_remove_self()
1465 		kn->flags |= KERNFS_SUICIDAL;  in kernfs_remove_self()
1466 		__kernfs_remove(kn);  in kernfs_remove_self()
1467 		kn->flags |= KERNFS_SUICIDED;  in kernfs_remove_self()
1470 		wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq;  in kernfs_remove_self()
1476 			if ((kn->flags & KERNFS_SUICIDED) &&  in kernfs_remove_self()
1477 			    atomic_read(&kn->active) == KN_DEACTIVATED_BIAS)  in kernfs_remove_self()
1485 		WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb));  in kernfs_remove_self()
1493 	kernfs_unbreak_active_protection(kn);  in kernfs_remove_self()
1511 	struct kernfs_node *kn;  in kernfs_remove_by_name_ns()  local
1521 	kn = kernfs_find_ns(parent, name, ns);  in kernfs_remove_by_name_ns()
1522 	if (kn) {  in kernfs_remove_by_name_ns()
1523 		kernfs_get(kn);  in kernfs_remove_by_name_ns()
1524 		__kernfs_remove(kn);  in kernfs_remove_by_name_ns()
1525 		kernfs_put(kn);  in kernfs_remove_by_name_ns()
1530 	if (kn)  in kernfs_remove_by_name_ns()
1543 int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,  in kernfs_rename_ns()  argument
1551 	if (!kn->parent)  in kernfs_rename_ns()
1557 	if (!kernfs_active(kn) || !kernfs_active(new_parent) ||  in kernfs_rename_ns()
1562 	if ((kn->parent == new_parent) && (kn->ns == new_ns) &&  in kernfs_rename_ns()
1563 	    (strcmp(kn->name, new_name) == 0))  in kernfs_rename_ns()
1571 	if (strcmp(kn->name, new_name) != 0) {  in kernfs_rename_ns()
1583 	kernfs_unlink_sibling(kn);  in kernfs_rename_ns()
1589 	old_parent = kn->parent;  in kernfs_rename_ns()
1590 	kn->parent = new_parent;  in kernfs_rename_ns()
1592 	kn->ns = new_ns;  in kernfs_rename_ns()
1594 		old_name = kn->name;  in kernfs_rename_ns()
1595 		kn->name = new_name;  in kernfs_rename_ns()
1600 	kn->hash = kernfs_name_hash(kn->name, kn->ns);  in kernfs_rename_ns()
1601 	kernfs_link_sibling(kn);  in kernfs_rename_ns()
1613 static inline unsigned char dt_type(struct kernfs_node *kn)  in dt_type()  argument
1615 	return (kn->mode >> 12) & 15;  in dt_type()