258 	struct epitem *epi;  member
341 static inline int ep_is_linked(struct epitem *epi)  in ep_is_linked()  argument
343 	return !list_empty(&epi->rdllink);  in ep_is_linked()
360 	return container_of(p, struct ep_pqueue, pt)->epi;  in ep_item_from_epqueue()
415 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)  in ep_set_busy_poll_napi_id()  argument
426 	sock = sock_from_file(epi->ffd.file, &err);  in ep_set_busy_poll_napi_id()
435 	ep = epi->ep;  in ep_set_busy_poll_napi_id()
458 static inline void ep_set_busy_poll_napi_id(struct epitem *epi)  in ep_set_busy_poll_napi_id()  argument
554 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,  in ep_poll_safewake()  argument
576 	if (epi) {  in ep_poll_safewake()
577 		if ((is_file_epoll(epi->ffd.file))) {  in ep_poll_safewake()
578 			ep_src = epi->ffd.file->private_data;  in ep_poll_safewake()
593 static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,  in ep_poll_safewake()  argument
623 static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)  in ep_unregister_pollwait()  argument
625 	struct list_head *lsthead = &epi->pwqlist;  in ep_unregister_pollwait()
638 static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)  in ep_wakeup_source()  argument
640 	return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));  in ep_wakeup_source()
644 static inline void ep_pm_stay_awake(struct epitem *epi)  in ep_pm_stay_awake()  argument
646 	struct wakeup_source *ws = ep_wakeup_source(epi);  in ep_pm_stay_awake()
652 static inline bool ep_has_wakeup_source(struct epitem *epi)  in ep_has_wakeup_source()  argument
654 	return rcu_access_pointer(epi->ws) ? true : false;  in ep_has_wakeup_source()
658 static inline void ep_pm_stay_awake_rcu(struct epitem *epi)  in ep_pm_stay_awake_rcu()  argument
663 	ws = rcu_dereference(epi->ws);  in ep_pm_stay_awake_rcu()
688 	struct epitem *epi, *nepi;  in ep_scan_ready_list()  local
725 	for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;  in ep_scan_ready_list()
726 	     nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {  in ep_scan_ready_list()
733 		if (!ep_is_linked(epi)) {  in ep_scan_ready_list()
738 			list_add(&epi->rdllink, &ep->rdllist);  in ep_scan_ready_list()
739 			ep_pm_stay_awake(epi);  in ep_scan_ready_list()
770 	struct epitem *epi = container_of(head, struct epitem, rcu);  in epi_rcu_free()  local
771 	kmem_cache_free(epi_cache, epi);  in epi_rcu_free()
778 static int ep_remove(struct eventpoll *ep, struct epitem *epi)  in ep_remove()  argument
780 	struct file *file = epi->ffd.file;  in ep_remove()
787 	ep_unregister_pollwait(ep, epi);  in ep_remove()
791 	list_del_rcu(&epi->fllink);  in ep_remove()
794 	rb_erase_cached(&epi->rbn, &ep->rbr);  in ep_remove()
797 	if (ep_is_linked(epi))  in ep_remove()
798 		list_del_init(&epi->rdllink);  in ep_remove()
801 	wakeup_source_unregister(ep_wakeup_source(epi));  in ep_remove()
809 	call_rcu(&epi->rcu, epi_rcu_free);  in ep_remove()
819 	struct epitem *epi;  in ep_free()  local
839 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
841 		ep_unregister_pollwait(ep, epi);  in ep_free()
855 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_free()
856 		ep_remove(ep, epi);  in ep_free()
888 static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,  in ep_item_poll()  argument
894 	pt->_key = epi->event.events;  in ep_item_poll()
895 	if (!is_file_epoll(epi->ffd.file))  in ep_item_poll()
896 		return vfs_poll(epi->ffd.file, pt) & epi->event.events;  in ep_item_poll()
898 	ep = epi->ffd.file->private_data;  in ep_item_poll()
899 	poll_wait(epi->ffd.file, &ep->poll_wait, pt);  in ep_item_poll()
902 	return ep_scan_ready_list(epi->ffd.file->private_data,  in ep_item_poll()
904 				  locked) & epi->event.events;  in ep_item_poll()
910 	struct epitem *epi, *tmp;  in ep_read_events_proc()  local
917 	list_for_each_entry_safe(epi, tmp, head, rdllink) {  in ep_read_events_proc()
918 		if (ep_item_poll(epi, &pt, depth)) {  in ep_read_events_proc()
926 			__pm_relax(ep_wakeup_source(epi));  in ep_read_events_proc()
927 			list_del_init(&epi->rdllink);  in ep_read_events_proc()
958 		struct epitem *epi = rb_entry(rbp, struct epitem, rbn);  in ep_show_fdinfo()  local
959 		struct inode *inode = file_inode(epi->ffd.file);  in ep_show_fdinfo()
963 			   epi->ffd.fd, epi->event.events,  in ep_show_fdinfo()
964 			   (long long)epi->event.data,  in ep_show_fdinfo()
965 			   (long long)epi->ffd.file->f_pos,  in ep_show_fdinfo()
992 	struct epitem *epi, *next;  in eventpoll_release_file()  local
1008 	list_for_each_entry_safe(epi, next, &file->f_ep_links, fllink) {  in eventpoll_release_file()
1009 		ep = epi->ep;  in eventpoll_release_file()
1011 		ep_remove(ep, epi);  in eventpoll_release_file()
1056 	struct epitem *epi, *epir = NULL;  in ep_find()  local
1061 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_find()
1062 		kcmp = ep_cmp_ffd(&ffd, &epi->ffd);  in ep_find()
1068 			epir = epi;  in ep_find()
1080 	struct epitem *epi;  in ep_find_tfd()  local
1083 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_find_tfd()
1084 		if (epi->ffd.fd == tfd) {  in ep_find_tfd()
1086 				return epi;  in ep_find_tfd()
1101 	struct epitem *epi;  in get_epoll_tfile_raw_ptr()  local
1109 	epi = ep_find_tfd(ep, tfd, toff);  in get_epoll_tfile_raw_ptr()
1110 	if (epi)  in get_epoll_tfile_raw_ptr()
1111 		file_raw = epi->ffd.file;  in get_epoll_tfile_raw_ptr()
1179 static inline bool chain_epi_lockless(struct epitem *epi)  in chain_epi_lockless()  argument
1181 	struct eventpoll *ep = epi->ep;  in chain_epi_lockless()
1184 	if (epi->next != EP_UNACTIVE_PTR)  in chain_epi_lockless()
1188 	if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)  in chain_epi_lockless()
1192 	epi->next = xchg(&ep->ovflist, epi);  in chain_epi_lockless()
1218 	struct epitem *epi = ep_item_from_wait(wait);  in ep_poll_callback()  local
1219 	struct eventpoll *ep = epi->ep;  in ep_poll_callback()
1226 	ep_set_busy_poll_napi_id(epi);  in ep_poll_callback()
1234 	if (!(epi->event.events & ~EP_PRIVATE_BITS))  in ep_poll_callback()
1243 	if (pollflags && !(pollflags & epi->event.events))  in ep_poll_callback()
1253 		if (chain_epi_lockless(epi))  in ep_poll_callback()
1254 			ep_pm_stay_awake_rcu(epi);  in ep_poll_callback()
1255 	} else if (!ep_is_linked(epi)) {  in ep_poll_callback()
1257 		if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist))  in ep_poll_callback()
1258 			ep_pm_stay_awake_rcu(epi);  in ep_poll_callback()
1266 		if ((epi->event.events & EPOLLEXCLUSIVE) &&  in ep_poll_callback()
1270 				if (epi->event.events & EPOLLIN)  in ep_poll_callback()
1274 				if (epi->event.events & EPOLLOUT)  in ep_poll_callback()
1292 		ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE);  in ep_poll_callback()
1294 	if (!(epi->event.events & EPOLLEXCLUSIVE))  in ep_poll_callback()
1323 	struct epitem *epi = ep_item_from_epqueue(pt);  in ep_ptable_queue_proc()  local
1326 	if (epi->nwait >= 0 && (pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL))) {  in ep_ptable_queue_proc()
1329 		pwq->base = epi;  in ep_ptable_queue_proc()
1330 		if (epi->event.events & EPOLLEXCLUSIVE)  in ep_ptable_queue_proc()
1334 		list_add_tail(&pwq->llink, &epi->pwqlist);  in ep_ptable_queue_proc()
1335 		epi->nwait++;  in ep_ptable_queue_proc()
1338 		epi->nwait = -1;  in ep_ptable_queue_proc()
1342 static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)  in ep_rbtree_insert()  argument
1352 		kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);  in ep_rbtree_insert()
1359 	rb_link_node(&epi->rbn, parent, p);  in ep_rbtree_insert()
1360 	rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);  in ep_rbtree_insert()
1404 	struct epitem *epi;  in reverse_path_check_proc()  local
1408 	list_for_each_entry_rcu(epi, &file->f_ep_links, fllink) {  in reverse_path_check_proc()
1409 		child_file = epi->ep->file;  in reverse_path_check_proc()
1460 static int ep_create_wakeup_source(struct epitem *epi)  in ep_create_wakeup_source()  argument
1468 	if (!epi->ep->ws) {  in ep_create_wakeup_source()
1469 		epi->ep->ws = wakeup_source_register(NULL, ws_name);  in ep_create_wakeup_source()
1470 		if (!epi->ep->ws)  in ep_create_wakeup_source()
1474 	take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);  in ep_create_wakeup_source()
1482 	rcu_assign_pointer(epi->ws, ws);  in ep_create_wakeup_source()
1488 static noinline void ep_destroy_wakeup_source(struct epitem *epi)  in ep_destroy_wakeup_source()  argument
1490 	struct wakeup_source *ws = ep_wakeup_source(epi);  in ep_destroy_wakeup_source()
1492 	RCU_INIT_POINTER(epi->ws, NULL);  in ep_destroy_wakeup_source()
1512 	struct epitem *epi;  in ep_insert()  local
1520 	if (!(epi = kmem_cache_alloc(epi_cache, GFP_KERNEL)))  in ep_insert()
1524 	INIT_LIST_HEAD(&epi->rdllink);  in ep_insert()
1525 	INIT_LIST_HEAD(&epi->fllink);  in ep_insert()
1526 	INIT_LIST_HEAD(&epi->pwqlist);  in ep_insert()
1527 	epi->ep = ep;  in ep_insert()
1528 	ep_set_ffd(&epi->ffd, tfile, fd);  in ep_insert()
1529 	epi->event = *event;  in ep_insert()
1530 	epi->nwait = 0;  in ep_insert()
1531 	epi->next = EP_UNACTIVE_PTR;  in ep_insert()
1532 	if (epi->event.events & EPOLLWAKEUP) {  in ep_insert()
1533 		error = ep_create_wakeup_source(epi);  in ep_insert()
1537 		RCU_INIT_POINTER(epi->ws, NULL);  in ep_insert()
1542 	list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);  in ep_insert()
1549 	ep_rbtree_insert(ep, epi);  in ep_insert()
1557 	epq.epi = epi;  in ep_insert()
1567 	revents = ep_item_poll(epi, &epq.pt, 1);  in ep_insert()
1575 	if (epi->nwait < 0)  in ep_insert()
1582 	ep_set_busy_poll_napi_id(epi);  in ep_insert()
1585 	if (revents && !ep_is_linked(epi)) {  in ep_insert()
1586 		list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_insert()
1587 		ep_pm_stay_awake(epi);  in ep_insert()
1607 	ep_unregister_pollwait(ep, epi);  in ep_insert()
1610 	list_del_rcu(&epi->fllink);  in ep_insert()
1613 	rb_erase_cached(&epi->rbn, &ep->rbr);  in ep_insert()
1622 	if (ep_is_linked(epi))  in ep_insert()
1623 		list_del_init(&epi->rdllink);  in ep_insert()
1626 	wakeup_source_unregister(ep_wakeup_source(epi));  in ep_insert()
1629 	kmem_cache_free(epi_cache, epi);  in ep_insert()
1638 static int ep_modify(struct eventpoll *ep, struct epitem *epi,  in ep_modify()  argument
1653 	epi->event.events = event->events; /* need barrier below */  in ep_modify()
1654 	epi->event.data = event->data; /* protected by mtx */  in ep_modify()
1655 	if (epi->event.events & EPOLLWAKEUP) {  in ep_modify()
1656 		if (!ep_has_wakeup_source(epi))  in ep_modify()
1657 			ep_create_wakeup_source(epi);  in ep_modify()
1658 	} else if (ep_has_wakeup_source(epi)) {  in ep_modify()
1659 		ep_destroy_wakeup_source(epi);  in ep_modify()
1688 	if (ep_item_poll(epi, &pt, 1)) {  in ep_modify()
1690 		if (!ep_is_linked(epi)) {  in ep_modify()
1691 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_modify()
1692 			ep_pm_stay_awake(epi);  in ep_modify()
1715 	struct epitem *epi, *tmp;  in ep_send_events_proc()  local
1730 	list_for_each_entry_safe(epi, tmp, head, rdllink) {  in ep_send_events_proc()
1743 		ws = ep_wakeup_source(epi);  in ep_send_events_proc()
1750 		list_del_init(&epi->rdllink);  in ep_send_events_proc()
1758 		revents = ep_item_poll(epi, &pt, 1);  in ep_send_events_proc()
1763 		    __put_user(epi->event.data, &uevent->data)) {  in ep_send_events_proc()
1764 			list_add(&epi->rdllink, head);  in ep_send_events_proc()
1765 			ep_pm_stay_awake(epi);  in ep_send_events_proc()
1772 		if (epi->event.events & EPOLLONESHOT)  in ep_send_events_proc()
1773 			epi->event.events &= EP_PRIVATE_BITS;  in ep_send_events_proc()
1774 		else if (!(epi->event.events & EPOLLET)) {  in ep_send_events_proc()
1786 			list_add_tail(&epi->rdllink, &ep->rdllist);  in ep_send_events_proc()
1787 			ep_pm_stay_awake(epi);  in ep_send_events_proc()
2015 	struct epitem *epi;  in ep_loop_check_proc()  local
2020 		epi = rb_entry(rbp, struct epitem, rbn);  in ep_loop_check_proc()
2021 		if (unlikely(is_file_epoll(epi->ffd.file))) {  in ep_loop_check_proc()
2022 			ep_tovisit = epi->ffd.file->private_data;  in ep_loop_check_proc()
2026 					ep_loop_check_proc, epi->ffd.file,  in ep_loop_check_proc()
2039 			if (list_empty(&epi->ffd.file->f_tfile_llink)) {  in ep_loop_check_proc()
2040 				if (get_file_rcu(epi->ffd.file))  in ep_loop_check_proc()
2041 					list_add(&epi->ffd.file->f_tfile_llink,  in ep_loop_check_proc()
2160 	struct epitem *epi;  in do_epoll_ctl()  local
2266 	epi = ep_find(ep, tf.file, fd);  in do_epoll_ctl()
2271 		if (!epi) {  in do_epoll_ctl()
2278 		if (epi)  in do_epoll_ctl()
2279 			error = ep_remove(ep, epi);  in do_epoll_ctl()
2284 		if (epi) {  in do_epoll_ctl()
2285 			if (!(epi->event.events & EPOLLEXCLUSIVE)) {  in do_epoll_ctl()
2287 				error = ep_modify(ep, epi, epds);  in do_epoll_ctl()