1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4 #include <linux/skmsg.h>
5 #include <linux/filter.h>
6 #include <linux/bpf.h>
7 #include <linux/init.h>
8 #include <linux/wait.h>
9
10 #include <net/inet_common.h>
11 #include <net/tls.h>
12
__tcp_bpf_recvmsg(struct sock * sk,struct sk_psock * psock,struct msghdr * msg,int len,int flags)13 int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
14 struct msghdr *msg, int len, int flags)
15 {
16 struct iov_iter *iter = &msg->msg_iter;
17 int peek = flags & MSG_PEEK;
18 struct sk_msg *msg_rx;
19 int i, copied = 0;
20
21 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
22 struct sk_msg, list);
23
24 while (copied != len) {
25 struct scatterlist *sge;
26
27 if (unlikely(!msg_rx))
28 break;
29
30 i = msg_rx->sg.start;
31 do {
32 struct page *page;
33 int copy;
34
35 sge = sk_msg_elem(msg_rx, i);
36 copy = sge->length;
37 page = sg_page(sge);
38 if (copied + copy > len)
39 copy = len - copied;
40 copy = copy_page_to_iter(page, sge->offset, copy, iter);
41 if (!copy)
42 return copied ? copied : -EFAULT;
43
44 copied += copy;
45 if (likely(!peek)) {
46 sge->offset += copy;
47 sge->length -= copy;
48 if (!msg_rx->skb)
49 sk_mem_uncharge(sk, copy);
50 msg_rx->sg.size -= copy;
51
52 if (!sge->length) {
53 sk_msg_iter_var_next(i);
54 if (!msg_rx->skb)
55 put_page(page);
56 }
57 } else {
58 /* Lets not optimize peek case if copy_page_to_iter
59 * didn't copy the entire length lets just break.
60 */
61 if (copy != sge->length)
62 return copied;
63 sk_msg_iter_var_next(i);
64 }
65
66 if (copied == len)
67 break;
68 } while (i != msg_rx->sg.end);
69
70 if (unlikely(peek)) {
71 if (msg_rx == list_last_entry(&psock->ingress_msg,
72 struct sk_msg, list))
73 break;
74 msg_rx = list_next_entry(msg_rx, list);
75 continue;
76 }
77
78 msg_rx->sg.start = i;
79 if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
80 list_del(&msg_rx->list);
81 if (msg_rx->skb)
82 consume_skb(msg_rx->skb);
83 kfree(msg_rx);
84 }
85 msg_rx = list_first_entry_or_null(&psock->ingress_msg,
86 struct sk_msg, list);
87 }
88
89 return copied;
90 }
91 EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
92
bpf_tcp_ingress(struct sock * sk,struct sk_psock * psock,struct sk_msg * msg,u32 apply_bytes,int flags)93 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
94 struct sk_msg *msg, u32 apply_bytes, int flags)
95 {
96 bool apply = apply_bytes;
97 struct scatterlist *sge;
98 u32 size, copied = 0;
99 struct sk_msg *tmp;
100 int i, ret = 0;
101
102 tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL);
103 if (unlikely(!tmp))
104 return -ENOMEM;
105
106 lock_sock(sk);
107 tmp->sg.start = msg->sg.start;
108 i = msg->sg.start;
109 do {
110 sge = sk_msg_elem(msg, i);
111 size = (apply && apply_bytes < sge->length) ?
112 apply_bytes : sge->length;
113 if (!sk_wmem_schedule(sk, size)) {
114 if (!copied)
115 ret = -ENOMEM;
116 break;
117 }
118
119 sk_mem_charge(sk, size);
120 sk_msg_xfer(tmp, msg, i, size);
121 copied += size;
122 if (sge->length)
123 get_page(sk_msg_page(tmp, i));
124 sk_msg_iter_var_next(i);
125 tmp->sg.end = i;
126 if (apply) {
127 apply_bytes -= size;
128 if (!apply_bytes)
129 break;
130 }
131 } while (i != msg->sg.end);
132
133 if (!ret) {
134 msg->sg.start = i;
135 sk_psock_queue_msg(psock, tmp);
136 sk_psock_data_ready(sk, psock);
137 } else {
138 sk_msg_free(sk, tmp);
139 kfree(tmp);
140 }
141
142 release_sock(sk);
143 return ret;
144 }
145
tcp_bpf_push(struct sock * sk,struct sk_msg * msg,u32 apply_bytes,int flags,bool uncharge)146 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
147 int flags, bool uncharge)
148 {
149 bool apply = apply_bytes;
150 struct scatterlist *sge;
151 struct page *page;
152 int size, ret = 0;
153 u32 off;
154
155 while (1) {
156 bool has_tx_ulp;
157
158 sge = sk_msg_elem(msg, msg->sg.start);
159 size = (apply && apply_bytes < sge->length) ?
160 apply_bytes : sge->length;
161 off = sge->offset;
162 page = sg_page(sge);
163
164 tcp_rate_check_app_limited(sk);
165 retry:
166 has_tx_ulp = tls_sw_has_ctx_tx(sk);
167 if (has_tx_ulp) {
168 flags |= MSG_SENDPAGE_NOPOLICY;
169 ret = kernel_sendpage_locked(sk,
170 page, off, size, flags);
171 } else {
172 ret = do_tcp_sendpages(sk, page, off, size, flags);
173 }
174
175 if (ret <= 0)
176 return ret;
177 if (apply)
178 apply_bytes -= ret;
179 msg->sg.size -= ret;
180 sge->offset += ret;
181 sge->length -= ret;
182 if (uncharge)
183 sk_mem_uncharge(sk, ret);
184 if (ret != size) {
185 size -= ret;
186 off += ret;
187 goto retry;
188 }
189 if (!sge->length) {
190 put_page(page);
191 sk_msg_iter_next(msg, start);
192 sg_init_table(sge, 1);
193 if (msg->sg.start == msg->sg.end)
194 break;
195 }
196 if (apply && !apply_bytes)
197 break;
198 }
199
200 return 0;
201 }
202
tcp_bpf_push_locked(struct sock * sk,struct sk_msg * msg,u32 apply_bytes,int flags,bool uncharge)203 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,
204 u32 apply_bytes, int flags, bool uncharge)
205 {
206 int ret;
207
208 lock_sock(sk);
209 ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge);
210 release_sock(sk);
211 return ret;
212 }
213
tcp_bpf_sendmsg_redir(struct sock * sk,struct sk_msg * msg,u32 bytes,int flags)214 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,
215 u32 bytes, int flags)
216 {
217 bool ingress = sk_msg_to_ingress(msg);
218 struct sk_psock *psock = sk_psock_get(sk);
219 int ret;
220
221 if (unlikely(!psock))
222 return -EPIPE;
223
224 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :
225 tcp_bpf_push_locked(sk, msg, bytes, flags, false);
226 sk_psock_put(sk, psock);
227 return ret;
228 }
229 EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir);
230
231 #ifdef CONFIG_BPF_STREAM_PARSER
tcp_bpf_stream_read(const struct sock * sk)232 static bool tcp_bpf_stream_read(const struct sock *sk)
233 {
234 struct sk_psock *psock;
235 bool empty = true;
236
237 rcu_read_lock();
238 psock = sk_psock(sk);
239 if (likely(psock))
240 empty = list_empty(&psock->ingress_msg);
241 rcu_read_unlock();
242 return !empty;
243 }
244
tcp_bpf_wait_data(struct sock * sk,struct sk_psock * psock,int flags,long timeo,int * err)245 static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,
246 int flags, long timeo, int *err)
247 {
248 DEFINE_WAIT_FUNC(wait, woken_wake_function);
249 int ret = 0;
250
251 if (sk->sk_shutdown & RCV_SHUTDOWN)
252 return 1;
253
254 if (!timeo)
255 return ret;
256
257 add_wait_queue(sk_sleep(sk), &wait);
258 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
259 ret = sk_wait_event(sk, &timeo,
260 !list_empty(&psock->ingress_msg) ||
261 !skb_queue_empty(&sk->sk_receive_queue), &wait);
262 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
263 remove_wait_queue(sk_sleep(sk), &wait);
264 return ret;
265 }
266
tcp_bpf_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int nonblock,int flags,int * addr_len)267 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
268 int nonblock, int flags, int *addr_len)
269 {
270 struct sk_psock *psock;
271 int copied, ret;
272
273 if (unlikely(flags & MSG_ERRQUEUE))
274 return inet_recv_error(sk, msg, len, addr_len);
275
276 psock = sk_psock_get(sk);
277 if (unlikely(!psock))
278 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
279 if (!skb_queue_empty(&sk->sk_receive_queue) &&
280 sk_psock_queue_empty(psock)) {
281 sk_psock_put(sk, psock);
282 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
283 }
284 lock_sock(sk);
285 msg_bytes_ready:
286 copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);
287 if (!copied) {
288 int data, err = 0;
289 long timeo;
290
291 timeo = sock_rcvtimeo(sk, nonblock);
292 data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);
293 if (data) {
294 if (!sk_psock_queue_empty(psock))
295 goto msg_bytes_ready;
296 release_sock(sk);
297 sk_psock_put(sk, psock);
298 return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
299 }
300 if (err) {
301 ret = err;
302 goto out;
303 }
304 copied = -EAGAIN;
305 }
306 ret = copied;
307 out:
308 release_sock(sk);
309 sk_psock_put(sk, psock);
310 return ret;
311 }
312
tcp_bpf_send_verdict(struct sock * sk,struct sk_psock * psock,struct sk_msg * msg,int * copied,int flags)313 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,
314 struct sk_msg *msg, int *copied, int flags)
315 {
316 bool cork = false, enospc = sk_msg_full(msg);
317 struct sock *sk_redir;
318 u32 tosend, origsize, sent, delta = 0;
319 u32 eval = __SK_NONE;
320 int ret;
321
322 more_data:
323 if (psock->eval == __SK_NONE) {
324 /* Track delta in msg size to add/subtract it on SK_DROP from
325 * returned to user copied size. This ensures user doesn't
326 * get a positive return code with msg_cut_data and SK_DROP
327 * verdict.
328 */
329 delta = msg->sg.size;
330 psock->eval = sk_psock_msg_verdict(sk, psock, msg);
331 delta -= msg->sg.size;
332 }
333
334 if (msg->cork_bytes &&
335 msg->cork_bytes > msg->sg.size && !enospc) {
336 psock->cork_bytes = msg->cork_bytes - msg->sg.size;
337 if (!psock->cork) {
338 psock->cork = kzalloc(sizeof(*psock->cork),
339 GFP_ATOMIC | __GFP_NOWARN);
340 if (!psock->cork)
341 return -ENOMEM;
342 }
343 memcpy(psock->cork, msg, sizeof(*msg));
344 return 0;
345 }
346
347 tosend = msg->sg.size;
348 if (psock->apply_bytes && psock->apply_bytes < tosend)
349 tosend = psock->apply_bytes;
350
351 switch (psock->eval) {
352 case __SK_PASS:
353 ret = tcp_bpf_push(sk, msg, tosend, flags, true);
354 if (unlikely(ret)) {
355 *copied -= sk_msg_free(sk, msg);
356 break;
357 }
358 sk_msg_apply_bytes(psock, tosend);
359 break;
360 case __SK_REDIRECT:
361 sk_redir = psock->sk_redir;
362 sk_msg_apply_bytes(psock, tosend);
363 if (!psock->apply_bytes) {
364 /* Clean up before releasing the sock lock. */
365 eval = psock->eval;
366 psock->eval = __SK_NONE;
367 psock->sk_redir = NULL;
368 }
369 if (psock->cork) {
370 cork = true;
371 psock->cork = NULL;
372 }
373 sk_msg_return(sk, msg, tosend);
374 release_sock(sk);
375
376 origsize = msg->sg.size;
377 ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);
378 sent = origsize - msg->sg.size;
379
380 if (eval == __SK_REDIRECT)
381 sock_put(sk_redir);
382
383 lock_sock(sk);
384 if (unlikely(ret < 0)) {
385 int free = sk_msg_free_nocharge(sk, msg);
386
387 if (!cork)
388 *copied -= free;
389 }
390 if (cork) {
391 sk_msg_free(sk, msg);
392 kfree(msg);
393 msg = NULL;
394 ret = 0;
395 }
396 break;
397 case __SK_DROP:
398 default:
399 sk_msg_free_partial(sk, msg, tosend);
400 sk_msg_apply_bytes(psock, tosend);
401 *copied -= (tosend + delta);
402 return -EACCES;
403 }
404
405 if (likely(!ret)) {
406 if (!psock->apply_bytes) {
407 psock->eval = __SK_NONE;
408 if (psock->sk_redir) {
409 sock_put(psock->sk_redir);
410 psock->sk_redir = NULL;
411 }
412 }
413 if (msg &&
414 msg->sg.data[msg->sg.start].page_link &&
415 msg->sg.data[msg->sg.start].length) {
416 if (eval == __SK_REDIRECT)
417 sk_mem_charge(sk, tosend - sent);
418 goto more_data;
419 }
420 }
421 return ret;
422 }
423
tcp_bpf_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)424 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
425 {
426 struct sk_msg tmp, *msg_tx = NULL;
427 int copied = 0, err = 0;
428 struct sk_psock *psock;
429 long timeo;
430 int flags;
431
432 /* Don't let internal do_tcp_sendpages() flags through */
433 flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);
434 flags |= MSG_NO_SHARED_FRAGS;
435
436 psock = sk_psock_get(sk);
437 if (unlikely(!psock))
438 return tcp_sendmsg(sk, msg, size);
439
440 lock_sock(sk);
441 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
442 while (msg_data_left(msg)) {
443 bool enospc = false;
444 u32 copy, osize;
445
446 if (sk->sk_err) {
447 err = -sk->sk_err;
448 goto out_err;
449 }
450
451 copy = msg_data_left(msg);
452 if (!sk_stream_memory_free(sk))
453 goto wait_for_sndbuf;
454 if (psock->cork) {
455 msg_tx = psock->cork;
456 } else {
457 msg_tx = &tmp;
458 sk_msg_init(msg_tx);
459 }
460
461 osize = msg_tx->sg.size;
462 err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1);
463 if (err) {
464 if (err != -ENOSPC)
465 goto wait_for_memory;
466 enospc = true;
467 copy = msg_tx->sg.size - osize;
468 }
469
470 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx,
471 copy);
472 if (err < 0) {
473 sk_msg_trim(sk, msg_tx, osize);
474 goto out_err;
475 }
476
477 copied += copy;
478 if (psock->cork_bytes) {
479 if (size > psock->cork_bytes)
480 psock->cork_bytes = 0;
481 else
482 psock->cork_bytes -= size;
483 if (psock->cork_bytes && !enospc)
484 goto out_err;
485 /* All cork bytes are accounted, rerun the prog. */
486 psock->eval = __SK_NONE;
487 psock->cork_bytes = 0;
488 }
489
490 err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags);
491 if (unlikely(err < 0))
492 goto out_err;
493 continue;
494 wait_for_sndbuf:
495 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
496 wait_for_memory:
497 err = sk_stream_wait_memory(sk, &timeo);
498 if (err) {
499 if (msg_tx && msg_tx != psock->cork)
500 sk_msg_free(sk, msg_tx);
501 goto out_err;
502 }
503 }
504 out_err:
505 if (err < 0)
506 err = sk_stream_error(sk, msg->msg_flags, err);
507 release_sock(sk);
508 sk_psock_put(sk, psock);
509 return copied ? copied : err;
510 }
511
tcp_bpf_sendpage(struct sock * sk,struct page * page,int offset,size_t size,int flags)512 static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset,
513 size_t size, int flags)
514 {
515 struct sk_msg tmp, *msg = NULL;
516 int err = 0, copied = 0;
517 struct sk_psock *psock;
518 bool enospc = false;
519
520 psock = sk_psock_get(sk);
521 if (unlikely(!psock))
522 return tcp_sendpage(sk, page, offset, size, flags);
523
524 lock_sock(sk);
525 if (psock->cork) {
526 msg = psock->cork;
527 } else {
528 msg = &tmp;
529 sk_msg_init(msg);
530 }
531
532 /* Catch case where ring is full and sendpage is stalled. */
533 if (unlikely(sk_msg_full(msg)))
534 goto out_err;
535
536 sk_msg_page_add(msg, page, size, offset);
537 sk_mem_charge(sk, size);
538 copied = size;
539 if (sk_msg_full(msg))
540 enospc = true;
541 if (psock->cork_bytes) {
542 if (size > psock->cork_bytes)
543 psock->cork_bytes = 0;
544 else
545 psock->cork_bytes -= size;
546 if (psock->cork_bytes && !enospc)
547 goto out_err;
548 /* All cork bytes are accounted, rerun the prog. */
549 psock->eval = __SK_NONE;
550 psock->cork_bytes = 0;
551 }
552
553 err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);
554 out_err:
555 release_sock(sk);
556 sk_psock_put(sk, psock);
557 return copied ? copied : err;
558 }
559
560 enum {
561 TCP_BPF_IPV4,
562 TCP_BPF_IPV6,
563 TCP_BPF_NUM_PROTS,
564 };
565
566 enum {
567 TCP_BPF_BASE,
568 TCP_BPF_TX,
569 TCP_BPF_NUM_CFGS,
570 };
571
572 static struct proto *tcpv6_prot_saved __read_mostly;
573 static DEFINE_SPINLOCK(tcpv6_prot_lock);
574 static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS];
575
tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],struct proto * base)576 static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],
577 struct proto *base)
578 {
579 prot[TCP_BPF_BASE] = *base;
580 prot[TCP_BPF_BASE].close = sock_map_close;
581 prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg;
582 prot[TCP_BPF_BASE].stream_memory_read = tcp_bpf_stream_read;
583
584 prot[TCP_BPF_TX] = prot[TCP_BPF_BASE];
585 prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg;
586 prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage;
587 }
588
tcp_bpf_check_v6_needs_rebuild(struct proto * ops)589 static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops)
590 {
591 if (unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) {
592 spin_lock_bh(&tcpv6_prot_lock);
593 if (likely(ops != tcpv6_prot_saved)) {
594 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops);
595 smp_store_release(&tcpv6_prot_saved, ops);
596 }
597 spin_unlock_bh(&tcpv6_prot_lock);
598 }
599 }
600
tcp_bpf_v4_build_proto(void)601 static int __init tcp_bpf_v4_build_proto(void)
602 {
603 tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot);
604 return 0;
605 }
606 late_initcall(tcp_bpf_v4_build_proto);
607
tcp_bpf_assert_proto_ops(struct proto * ops)608 static int tcp_bpf_assert_proto_ops(struct proto *ops)
609 {
610 /* In order to avoid retpoline, we make assumptions when we call
611 * into ops if e.g. a psock is not present. Make sure they are
612 * indeed valid assumptions.
613 */
614 return ops->recvmsg == tcp_recvmsg &&
615 ops->sendmsg == tcp_sendmsg &&
616 ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
617 }
618
tcp_bpf_get_proto(struct sock * sk,struct sk_psock * psock)619 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)
620 {
621 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
622 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
623
624 if (sk->sk_family == AF_INET6) {
625 if (tcp_bpf_assert_proto_ops(psock->sk_proto))
626 return ERR_PTR(-EINVAL);
627
628 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
629 }
630
631 return &tcp_bpf_prots[family][config];
632 }
633
634 /* If a child got cloned from a listening socket that had tcp_bpf
635 * protocol callbacks installed, we need to restore the callbacks to
636 * the default ones because the child does not inherit the psock state
637 * that tcp_bpf callbacks expect.
638 */
tcp_bpf_clone(const struct sock * sk,struct sock * newsk)639 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
640 {
641 int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
642 struct proto *prot = newsk->sk_prot;
643
644 if (prot == &tcp_bpf_prots[family][TCP_BPF_BASE])
645 newsk->sk_prot = sk->sk_prot_creator;
646 }
647 #endif /* CONFIG_BPF_STREAM_PARSER */
648