565 svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)  in svc_init_buffer()  argument
570 	if (svc_is_backchannel(rqstp))  in svc_init_buffer()
584 		rqstp->rq_pages[arghi++] = p;  in svc_init_buffer()
594 svc_release_buffer(struct svc_rqst *rqstp)  in svc_release_buffer()  argument
598 	for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)  in svc_release_buffer()
599 		if (rqstp->rq_pages[i])  in svc_release_buffer()
600 			put_page(rqstp->rq_pages[i]);  in svc_release_buffer()
606 	struct svc_rqst	*rqstp;  in svc_rqst_alloc()  local
608 	rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);  in svc_rqst_alloc()
609 	if (!rqstp)  in svc_rqst_alloc()
610 		return rqstp;  in svc_rqst_alloc()
612 	__set_bit(RQ_BUSY, &rqstp->rq_flags);  in svc_rqst_alloc()
613 	spin_lock_init(&rqstp->rq_lock);  in svc_rqst_alloc()
614 	rqstp->rq_server = serv;  in svc_rqst_alloc()
615 	rqstp->rq_pool = pool;  in svc_rqst_alloc()
617 	rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);  in svc_rqst_alloc()
618 	if (!rqstp->rq_argp)  in svc_rqst_alloc()
621 	rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);  in svc_rqst_alloc()
622 	if (!rqstp->rq_resp)  in svc_rqst_alloc()
625 	if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))  in svc_rqst_alloc()
628 	return rqstp;  in svc_rqst_alloc()
630 	svc_rqst_free(rqstp);  in svc_rqst_alloc()
638 	struct svc_rqst	*rqstp;  in svc_prepare_thread()  local
640 	rqstp = svc_rqst_alloc(serv, pool, node);  in svc_prepare_thread()
641 	if (!rqstp)  in svc_prepare_thread()
647 	list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);  in svc_prepare_thread()
649 	return rqstp;  in svc_prepare_thread()
690 		struct svc_rqst *rqstp;  in choose_victim()  local
696 		rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);  in choose_victim()
697 		set_bit(RQ_VICTIM, &rqstp->rq_flags);  in choose_victim()
698 		list_del_rcu(&rqstp->rq_all);  in choose_victim()
699 		task = rqstp->rq_task;  in choose_victim()
710 	struct svc_rqst	*rqstp;  in svc_start_kthreads()  local
721 		rqstp = svc_prepare_thread(serv, chosen_pool, node);  in svc_start_kthreads()
722 		if (IS_ERR(rqstp))  in svc_start_kthreads()
723 			return PTR_ERR(rqstp);  in svc_start_kthreads()
726 		task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp,  in svc_start_kthreads()
730 			svc_exit_thread(rqstp);  in svc_start_kthreads()
734 		rqstp->rq_task = task;  in svc_start_kthreads()
842 svc_rqst_free(struct svc_rqst *rqstp)  in svc_rqst_free()  argument
844 	svc_release_buffer(rqstp);  in svc_rqst_free()
845 	kfree(rqstp->rq_resp);  in svc_rqst_free()
846 	kfree(rqstp->rq_argp);  in svc_rqst_free()
847 	kfree(rqstp->rq_auth_data);  in svc_rqst_free()
848 	kfree_rcu(rqstp, rq_rcu_head);  in svc_rqst_free()
853 svc_exit_thread(struct svc_rqst *rqstp)  in svc_exit_thread()  argument
855 	struct svc_serv	*serv = rqstp->rq_server;  in svc_exit_thread()
856 	struct svc_pool	*pool = rqstp->rq_pool;  in svc_exit_thread()
860 	if (!test_and_set_bit(RQ_VICTIM, &rqstp->rq_flags))  in svc_exit_thread()
861 		list_del_rcu(&rqstp->rq_all);  in svc_exit_thread()
864 	svc_rqst_free(rqstp);  in svc_exit_thread()
1141 void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)  in svc_printk()  argument
1152 	dprintk("svc: %s: %pV", svc_print_addr(rqstp, buf, sizeof(buf)), &vaf);  in svc_printk()
1157 static __printf(2,3) void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...) {}  in svc_printk()  argument
1161 svc_return_autherr(struct svc_rqst *rqstp, __be32 auth_err)  in svc_return_autherr()  argument
1163 	set_bit(RQ_AUTHERR, &rqstp->rq_flags);  in svc_return_autherr()
1169 svc_get_autherr(struct svc_rqst *rqstp, __be32 *statp)  in svc_get_autherr()  argument
1171 	if (test_and_clear_bit(RQ_AUTHERR, &rqstp->rq_flags))  in svc_get_autherr()
1177 svc_generic_dispatch(struct svc_rqst *rqstp, __be32 *statp)  in svc_generic_dispatch()  argument
1179 	struct kvec *argv = &rqstp->rq_arg.head[0];  in svc_generic_dispatch()
1180 	struct kvec *resv = &rqstp->rq_res.head[0];  in svc_generic_dispatch()
1181 	const struct svc_procedure *procp = rqstp->rq_procinfo;  in svc_generic_dispatch()
1188 	    !procp->pc_decode(rqstp, argv->iov_base)) {  in svc_generic_dispatch()
1193 	*statp = procp->pc_func(rqstp);  in svc_generic_dispatch()
1196 	    test_bit(RQ_DROPME, &rqstp->rq_flags))  in svc_generic_dispatch()
1199 	if (test_bit(RQ_AUTHERR, &rqstp->rq_flags))  in svc_generic_dispatch()
1207 	    !procp->pc_encode(rqstp, resv->iov_base + resv->iov_len)) {  in svc_generic_dispatch()
1216 svc_generic_init_request(struct svc_rqst *rqstp,  in svc_generic_init_request()  argument
1223 	if (rqstp->rq_vers >= progp->pg_nvers )  in svc_generic_init_request()
1225 	versp = progp->pg_vers[rqstp->rq_vers];  in svc_generic_init_request()
1240 	if (versp->vs_need_cong_ctrl && rqstp->rq_xprt &&  in svc_generic_init_request()
1241 	    !test_bit(XPT_CONG_CTRL, &rqstp->rq_xprt->xpt_flags))  in svc_generic_init_request()
1244 	if (rqstp->rq_proc >= versp->vs_nproc)  in svc_generic_init_request()
1246 	rqstp->rq_procinfo = procp = &versp->vs_proc[rqstp->rq_proc];  in svc_generic_init_request()
1251 	memset(rqstp->rq_argp, 0, procp->pc_argsize);  in svc_generic_init_request()
1252 	memset(rqstp->rq_resp, 0, procp->pc_ressize);  in svc_generic_init_request()
1255 	versp->vs_count[rqstp->rq_proc]++;  in svc_generic_init_request()
1272 svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)  in svc_process_common()  argument
1276 	struct svc_serv		*serv = rqstp->rq_server;  in svc_process_common()
1290 	set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);  in svc_process_common()
1292 	set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);  in svc_process_common()
1293 	clear_bit(RQ_DROPME, &rqstp->rq_flags);  in svc_process_common()
1295 	svc_putu32(resv, rqstp->rq_xid);  in svc_process_common()
1310 	rqstp->rq_prog = prog = svc_getnl(argv);	/* program number */  in svc_process_common()
1311 	rqstp->rq_vers = svc_getnl(argv);	/* version number */  in svc_process_common()
1312 	rqstp->rq_proc = svc_getnl(argv);	/* procedure number */  in svc_process_common()
1323 	auth_res = svc_authenticate(rqstp, &auth_stat);  in svc_process_common()
1327 		auth_res = progp->pg_authenticate(rqstp);  in svc_process_common()
1330 		trace_svc_authenticate(rqstp, auth_res, auth_stat);  in svc_process_common()
1352 	rpc_stat = progp->pg_init_request(rqstp, progp, &process);  in svc_process_common()
1364 	procp = rqstp->rq_procinfo;  in svc_process_common()
1371 	trace_svc_process(rqstp, progp->pg_name);  in svc_process_common()
1381 		svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);  in svc_process_common()
1385 		if (!svc_generic_dispatch(rqstp, statp))  in svc_process_common()
1389 		auth_stat = svc_get_autherr(rqstp, statp);  in svc_process_common()
1394 		if (!process.dispatch(rqstp, statp))  in svc_process_common()
1404 		procp->pc_release(rqstp);  in svc_process_common()
1410 	if (svc_authorise(rqstp))  in svc_process_common()
1416 		procp->pc_release(rqstp);  in svc_process_common()
1418 	svc_authorise(rqstp);	/* doesn't hurt to call this twice */  in svc_process_common()
1423 	svc_authorise(rqstp);  in svc_process_common()
1425 	if (rqstp->rq_xprt && test_bit(XPT_TEMP, &rqstp->rq_xprt->xpt_flags))  in svc_process_common()
1426 		svc_close_xprt(rqstp->rq_xprt);  in svc_process_common()
1431 	svc_printk(rqstp, "short len %zd, dropping request\n",  in svc_process_common()
1445 		procp->pc_release(rqstp);  in svc_process_common()
1450 	xdr_ressize_check(rqstp, reply_statp);  in svc_process_common()
1463 	svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",  in svc_process_common()
1464 		       rqstp->rq_vers, rqstp->rq_prog, progp->pg_name);  in svc_process_common()
1473 	svc_printk(rqstp, "unknown procedure (%d)\n", rqstp->rq_proc);  in svc_process_common()
1480 	svc_printk(rqstp, "failed to decode args\n");  in svc_process_common()
1493 svc_process(struct svc_rqst *rqstp)  in svc_process()  argument
1495 	struct kvec		*argv = &rqstp->rq_arg.head[0];  in svc_process()
1496 	struct kvec		*resv = &rqstp->rq_res.head[0];  in svc_process()
1497 	struct svc_serv		*serv = rqstp->rq_server;  in svc_process()
1504 	rqstp->rq_next_page = &rqstp->rq_respages[1];  in svc_process()
1505 	resv->iov_base = page_address(rqstp->rq_respages[0]);  in svc_process()
1507 	rqstp->rq_res.pages = rqstp->rq_respages + 1;  in svc_process()
1508 	rqstp->rq_res.len = 0;  in svc_process()
1509 	rqstp->rq_res.page_base = 0;  in svc_process()
1510 	rqstp->rq_res.page_len = 0;  in svc_process()
1511 	rqstp->rq_res.buflen = PAGE_SIZE;  in svc_process()
1512 	rqstp->rq_res.tail[0].iov_base = NULL;  in svc_process()
1513 	rqstp->rq_res.tail[0].iov_len = 0;  in svc_process()
1518 		svc_printk(rqstp, "bad direction %d, dropping request\n", dir);  in svc_process()
1524 	if (likely(svc_process_common(rqstp, argv, resv)))  in svc_process()
1525 		return svc_send(rqstp);  in svc_process()
1528 	svc_drop(rqstp);  in svc_process()
1540 	       struct svc_rqst *rqstp)  in bc_svc_process()  argument
1542 	struct kvec	*argv = &rqstp->rq_arg.head[0];  in bc_svc_process()
1543 	struct kvec	*resv = &rqstp->rq_res.head[0];  in bc_svc_process()
1551 	rqstp->rq_xid = req->rq_xid;  in bc_svc_process()
1552 	rqstp->rq_prot = req->rq_xprt->prot;  in bc_svc_process()
1553 	rqstp->rq_server = serv;  in bc_svc_process()
1554 	rqstp->rq_bc_net = req->rq_xprt->xprt_net;  in bc_svc_process()
1556 	rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);  in bc_svc_process()
1557 	memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);  in bc_svc_process()
1558 	memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));  in bc_svc_process()
1559 	memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));  in bc_svc_process()
1562 	rqstp->rq_arg.len = req->rq_private_buf.len;  in bc_svc_process()
1563 	if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {  in bc_svc_process()
1564 		rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;  in bc_svc_process()
1565 		rqstp->rq_arg.page_len = 0;  in bc_svc_process()
1566 	} else if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len +  in bc_svc_process()
1567 			rqstp->rq_arg.page_len)  in bc_svc_process()
1568 		rqstp->rq_arg.page_len = rqstp->rq_arg.len -  in bc_svc_process()
1569 			rqstp->rq_arg.head[0].iov_len;  in bc_svc_process()
1571 		rqstp->rq_arg.len = rqstp->rq_arg.head[0].iov_len +  in bc_svc_process()
1572 			rqstp->rq_arg.page_len;  in bc_svc_process()
1585 	proc_error = svc_process_common(rqstp, argv, resv);  in bc_svc_process()
1595 	memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));  in bc_svc_process()
1616 u32 svc_max_payload(const struct svc_rqst *rqstp)  in svc_max_payload()  argument
1618 	u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;  in svc_max_payload()
1620 	if (rqstp->rq_server->sv_max_payload < max)  in svc_max_payload()
1621 		max = rqstp->rq_server->sv_max_payload;  in svc_max_payload()
1635 int svc_encode_read_payload(struct svc_rqst *rqstp, unsigned int offset,  in svc_encode_read_payload()  argument
1638 	return rqstp->rq_xprt->xpt_ops->xpo_read_payload(rqstp, offset, length);  in svc_encode_read_payload()
1651 unsigned int svc_fill_write_vector(struct svc_rqst *rqstp, struct page **pages,  in svc_fill_write_vector()  argument
1654 	struct kvec *vec = rqstp->rq_vec;  in svc_fill_write_vector()
1676 	WARN_ON_ONCE(i > ARRAY_SIZE(rqstp->rq_vec));  in svc_fill_write_vector()
1692 char *svc_fill_symlink_pathname(struct svc_rqst *rqstp, struct kvec *first,  in svc_fill_symlink_pathname()  argument