Lines Matching +full:data +full:- +full:out
1 // SPDX-License-Identifier: GPL-2.0-only
14 #include <linux/error-injection.h>
31 if (t->mode == NO_PREEMPT) in bpf_test_timer_enter()
36 t->time_start = ktime_get_ns(); in bpf_test_timer_enter()
42 t->time_start = 0; in bpf_test_timer_leave()
44 if (t->mode == NO_PREEMPT) in bpf_test_timer_leave()
54 t->i++; in bpf_test_timer_continue()
55 if (t->i >= repeat) { in bpf_test_timer_continue()
57 t->time_spent += ktime_get_ns() - t->time_start; in bpf_test_timer_continue()
58 do_div(t->time_spent, t->i); in bpf_test_timer_continue()
59 *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent; in bpf_test_timer_continue()
66 *err = -EINTR; in bpf_test_timer_continue()
72 t->time_spent += ktime_get_ns() - t->time_start; in bpf_test_timer_continue()
82 t->i = 0; in bpf_test_timer_continue()
100 return -ENOMEM; in bpf_test_run()
130 union bpf_attr __user *uattr, const void *data, in bpf_test_finish() argument
133 void __user *data_out = u64_to_user_ptr(kattr->test.data_out); in bpf_test_finish()
134 int err = -EFAULT; in bpf_test_finish()
140 if (kattr->test.data_size_out && in bpf_test_finish()
141 copy_size > kattr->test.data_size_out) { in bpf_test_finish()
142 copy_size = kattr->test.data_size_out; in bpf_test_finish()
143 err = -ENOSPC; in bpf_test_finish()
146 if (data_out && copy_to_user(data_out, data, copy_size)) in bpf_test_finish()
147 goto out; in bpf_test_finish()
148 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) in bpf_test_finish()
149 goto out; in bpf_test_finish()
150 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_test_finish()
151 goto out; in bpf_test_finish()
152 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) in bpf_test_finish()
153 goto out; in bpf_test_finish()
154 if (err != -ENOSPC) in bpf_test_finish()
156 out: in bpf_test_finish()
166 __diag_ignore(GCC, 8, "-Wmissing-prototypes",
209 return (long)arg->a; in bpf_fentry_test8()
224 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); in bpf_test_init()
225 u32 user_size = kattr->test.data_size_in; in bpf_test_init()
226 void *data; in bpf_test_init() local
228 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) in bpf_test_init()
229 return ERR_PTR(-EINVAL); in bpf_test_init()
232 return ERR_PTR(-EMSGSIZE); in bpf_test_init()
235 data = kzalloc(size + headroom + tailroom, GFP_USER); in bpf_test_init()
236 if (!data) in bpf_test_init()
237 return ERR_PTR(-ENOMEM); in bpf_test_init()
239 if (copy_from_user(data + headroom, data_in, user_size)) { in bpf_test_init()
240 kfree(data); in bpf_test_init()
241 return ERR_PTR(-EFAULT); in bpf_test_init()
244 return data; in bpf_test_init()
253 int b = 2, err = -EFAULT; in bpf_prog_test_run_tracing()
256 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_tracing()
257 return -EINVAL; in bpf_prog_test_run_tracing()
259 switch (prog->expected_attach_type) { in bpf_prog_test_run_tracing()
270 goto out; in bpf_prog_test_run_tracing()
278 goto out; in bpf_prog_test_run_tracing()
282 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) in bpf_prog_test_run_tracing()
283 goto out; in bpf_prog_test_run_tracing()
286 out: in bpf_prog_test_run_tracing()
298 __bpf_prog_test_run_raw_tp(void *data) in __bpf_prog_test_run_raw_tp() argument
300 struct bpf_raw_tp_test_run_info *info = data; in __bpf_prog_test_run_raw_tp()
303 info->retval = BPF_PROG_RUN(info->prog, info->ctx); in __bpf_prog_test_run_raw_tp()
311 void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_prog_test_run_raw_tp()
312 __u32 ctx_size_in = kattr->test.ctx_size_in; in bpf_prog_test_run_raw_tp()
314 int cpu = kattr->test.cpu, err = 0; in bpf_prog_test_run_raw_tp()
317 /* doesn't support data_in/out, ctx_out, duration, or repeat */ in bpf_prog_test_run_raw_tp()
318 if (kattr->test.data_in || kattr->test.data_out || in bpf_prog_test_run_raw_tp()
319 kattr->test.ctx_out || kattr->test.duration || in bpf_prog_test_run_raw_tp()
320 kattr->test.repeat) in bpf_prog_test_run_raw_tp()
321 return -EINVAL; in bpf_prog_test_run_raw_tp()
323 if (ctx_size_in < prog->aux->max_ctx_offset || in bpf_prog_test_run_raw_tp()
325 return -EINVAL; in bpf_prog_test_run_raw_tp()
327 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0) in bpf_prog_test_run_raw_tp()
328 return -EINVAL; in bpf_prog_test_run_raw_tp()
333 return -ENOMEM; in bpf_prog_test_run_raw_tp()
335 err = -EFAULT; in bpf_prog_test_run_raw_tp()
336 goto out; in bpf_prog_test_run_raw_tp()
345 if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 || in bpf_prog_test_run_raw_tp()
351 * space, let's do an extra quick check to filter out in bpf_prog_test_run_raw_tp()
354 err = -ENXIO; in bpf_prog_test_run_raw_tp()
362 copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32))) in bpf_prog_test_run_raw_tp()
363 err = -EFAULT; in bpf_prog_test_run_raw_tp()
365 out: in bpf_prog_test_run_raw_tp()
372 void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in); in bpf_ctx_init()
373 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_init()
374 u32 size = kattr->test.ctx_size_in; in bpf_ctx_init()
375 void *data; in bpf_ctx_init() local
381 data = kzalloc(max_size, GFP_USER); in bpf_ctx_init()
382 if (!data) in bpf_ctx_init()
383 return ERR_PTR(-ENOMEM); in bpf_ctx_init()
388 kfree(data); in bpf_ctx_init()
393 if (copy_from_user(data, data_in, size)) { in bpf_ctx_init()
394 kfree(data); in bpf_ctx_init()
395 return ERR_PTR(-EFAULT); in bpf_ctx_init()
398 return data; in bpf_ctx_init()
402 union bpf_attr __user *uattr, const void *data, in bpf_ctx_finish() argument
405 void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out); in bpf_ctx_finish()
406 int err = -EFAULT; in bpf_ctx_finish()
409 if (!data || !data_out) in bpf_ctx_finish()
412 if (copy_size > kattr->test.ctx_size_out) { in bpf_ctx_finish()
413 copy_size = kattr->test.ctx_size_out; in bpf_ctx_finish()
414 err = -ENOSPC; in bpf_ctx_finish()
417 if (copy_to_user(data_out, data, copy_size)) in bpf_ctx_finish()
418 goto out; in bpf_ctx_finish()
419 if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size))) in bpf_ctx_finish()
420 goto out; in bpf_ctx_finish()
421 if (err != -ENOSPC) in bpf_ctx_finish()
423 out: in bpf_ctx_finish()
428 * range_is_zero - test whether buffer is initialized
433 * This function returns true if the there is a non-zero byte
438 return !memchr_inv((u8 *)buf + from, 0, to - from); in range_is_zero()
443 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert___skb_to_skb()
445 if (!skb->len) in convert___skb_to_skb()
446 return -EINVAL; in convert___skb_to_skb()
453 return -EINVAL; in convert___skb_to_skb()
459 return -EINVAL; in convert___skb_to_skb()
465 return -EINVAL; in convert___skb_to_skb()
471 return -EINVAL; in convert___skb_to_skb()
477 return -EINVAL; in convert___skb_to_skb()
485 return -EINVAL; in convert___skb_to_skb()
491 return -EINVAL; in convert___skb_to_skb()
493 skb->mark = __skb->mark; in convert___skb_to_skb()
494 skb->priority = __skb->priority; in convert___skb_to_skb()
495 skb->tstamp = __skb->tstamp; in convert___skb_to_skb()
496 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN); in convert___skb_to_skb()
498 if (__skb->wire_len == 0) { in convert___skb_to_skb()
499 cb->pkt_len = skb->len; in convert___skb_to_skb()
501 if (__skb->wire_len < skb->len || in convert___skb_to_skb()
502 __skb->wire_len > GSO_MAX_SIZE) in convert___skb_to_skb()
503 return -EINVAL; in convert___skb_to_skb()
504 cb->pkt_len = __skb->wire_len; in convert___skb_to_skb()
507 if (__skb->gso_segs > GSO_MAX_SEGS) in convert___skb_to_skb()
508 return -EINVAL; in convert___skb_to_skb()
509 skb_shinfo(skb)->gso_segs = __skb->gso_segs; in convert___skb_to_skb()
510 skb_shinfo(skb)->gso_size = __skb->gso_size; in convert___skb_to_skb()
517 struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb; in convert_skb_to___skb()
522 __skb->mark = skb->mark; in convert_skb_to___skb()
523 __skb->priority = skb->priority; in convert_skb_to___skb()
524 __skb->ifindex = skb->dev->ifindex; in convert_skb_to___skb()
525 __skb->tstamp = skb->tstamp; in convert_skb_to___skb()
526 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN); in convert_skb_to___skb()
527 __skb->wire_len = cb->pkt_len; in convert_skb_to___skb()
528 __skb->gso_segs = skb_shinfo(skb)->gso_segs; in convert_skb_to___skb()
541 struct net *net = current->nsproxy->net_ns; in bpf_prog_test_run_skb()
542 struct net_device *dev = net->loopback_dev; in bpf_prog_test_run_skb()
543 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_skb()
544 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_skb()
550 void *data; in bpf_prog_test_run_skb() local
553 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_skb()
554 return -EINVAL; in bpf_prog_test_run_skb()
556 data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN, in bpf_prog_test_run_skb()
558 if (IS_ERR(data)) in bpf_prog_test_run_skb()
559 return PTR_ERR(data); in bpf_prog_test_run_skb()
563 kfree(data); in bpf_prog_test_run_skb()
567 switch (prog->type) { in bpf_prog_test_run_skb()
583 kfree(data); in bpf_prog_test_run_skb()
585 return -ENOMEM; in bpf_prog_test_run_skb()
589 skb = build_skb(data, 0); in bpf_prog_test_run_skb()
591 kfree(data); in bpf_prog_test_run_skb()
594 return -ENOMEM; in bpf_prog_test_run_skb()
596 skb->sk = sk; in bpf_prog_test_run_skb()
600 if (ctx && ctx->ifindex > 1) { in bpf_prog_test_run_skb()
601 dev = dev_get_by_index(net, ctx->ifindex); in bpf_prog_test_run_skb()
603 ret = -ENODEV; in bpf_prog_test_run_skb()
604 goto out; in bpf_prog_test_run_skb()
607 skb->protocol = eth_type_trans(skb, dev); in bpf_prog_test_run_skb()
610 switch (skb->protocol) { in bpf_prog_test_run_skb()
612 sk->sk_family = AF_INET; in bpf_prog_test_run_skb()
614 sk->sk_rcv_saddr = ip_hdr(skb)->saddr; in bpf_prog_test_run_skb()
615 sk->sk_daddr = ip_hdr(skb)->daddr; in bpf_prog_test_run_skb()
620 sk->sk_family = AF_INET6; in bpf_prog_test_run_skb()
622 sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; in bpf_prog_test_run_skb()
623 sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; in bpf_prog_test_run_skb()
637 goto out; in bpf_prog_test_run_skb()
640 goto out; in bpf_prog_test_run_skb()
643 int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); in bpf_prog_test_run_skb()
646 ret = -ENOMEM; in bpf_prog_test_run_skb()
647 goto out; in bpf_prog_test_run_skb()
654 size = skb->len; in bpf_prog_test_run_skb()
655 /* bpf program can never convert linear skb to non-linear */ in bpf_prog_test_run_skb()
658 ret = bpf_test_finish(kattr, uattr, skb->data, size, retval, duration); in bpf_prog_test_run_skb()
662 out: in bpf_prog_test_run_skb()
663 if (dev && dev != net->loopback_dev) in bpf_prog_test_run_skb()
676 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_xdp()
677 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_xdp()
682 void *data; in bpf_prog_test_run_xdp() local
685 if (prog->expected_attach_type == BPF_XDP_DEVMAP || in bpf_prog_test_run_xdp()
686 prog->expected_attach_type == BPF_XDP_CPUMAP) in bpf_prog_test_run_xdp()
687 return -EINVAL; in bpf_prog_test_run_xdp()
688 if (kattr->test.ctx_in || kattr->test.ctx_out) in bpf_prog_test_run_xdp()
689 return -EINVAL; in bpf_prog_test_run_xdp()
692 max_data_sz = 4096 - headroom - tailroom; in bpf_prog_test_run_xdp()
694 data = bpf_test_init(kattr, max_data_sz, headroom, tailroom); in bpf_prog_test_run_xdp()
695 if (IS_ERR(data)) in bpf_prog_test_run_xdp()
696 return PTR_ERR(data); in bpf_prog_test_run_xdp()
698 xdp.data_hard_start = data; in bpf_prog_test_run_xdp()
699 xdp.data = data + headroom; in bpf_prog_test_run_xdp()
700 xdp.data_meta = xdp.data; in bpf_prog_test_run_xdp()
701 xdp.data_end = xdp.data + size; in bpf_prog_test_run_xdp()
704 rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0); in bpf_prog_test_run_xdp()
705 xdp.rxq = &rxqueue->xdp_rxq; in bpf_prog_test_run_xdp()
709 goto out; in bpf_prog_test_run_xdp()
710 if (xdp.data != data + headroom || xdp.data_end != xdp.data + size) in bpf_prog_test_run_xdp()
711 size = xdp.data_end - xdp.data; in bpf_prog_test_run_xdp()
712 ret = bpf_test_finish(kattr, uattr, xdp.data, size, retval, duration); in bpf_prog_test_run_xdp()
713 out: in bpf_prog_test_run_xdp()
715 kfree(data); in bpf_prog_test_run_xdp()
723 return -EINVAL; in verify_user_bpf_flow_keys()
729 return -EINVAL; in verify_user_bpf_flow_keys()
739 u32 size = kattr->test.data_size_in; in bpf_prog_test_run_flow_dissector()
741 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_flow_dissector()
747 void *data; in bpf_prog_test_run_flow_dissector() local
750 if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR) in bpf_prog_test_run_flow_dissector()
751 return -EINVAL; in bpf_prog_test_run_flow_dissector()
753 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_flow_dissector()
754 return -EINVAL; in bpf_prog_test_run_flow_dissector()
757 return -EINVAL; in bpf_prog_test_run_flow_dissector()
759 data = bpf_test_init(kattr, size, 0, 0); in bpf_prog_test_run_flow_dissector()
760 if (IS_ERR(data)) in bpf_prog_test_run_flow_dissector()
761 return PTR_ERR(data); in bpf_prog_test_run_flow_dissector()
763 eth = (struct ethhdr *)data; in bpf_prog_test_run_flow_dissector()
770 kfree(data); in bpf_prog_test_run_flow_dissector()
776 goto out; in bpf_prog_test_run_flow_dissector()
777 flags = user_ctx->flags; in bpf_prog_test_run_flow_dissector()
781 ctx.data = data; in bpf_prog_test_run_flow_dissector()
782 ctx.data_end = (__u8 *)data + size; in bpf_prog_test_run_flow_dissector()
786 retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN, in bpf_prog_test_run_flow_dissector()
792 goto out; in bpf_prog_test_run_flow_dissector()
800 out: in bpf_prog_test_run_flow_dissector()
802 kfree(data); in bpf_prog_test_run_flow_dissector()
812 u32 repeat = kattr->test.repeat; in bpf_prog_test_run_sk_lookup()
815 int ret = -EINVAL; in bpf_prog_test_run_sk_lookup()
817 if (prog->type != BPF_PROG_TYPE_SK_LOOKUP) in bpf_prog_test_run_sk_lookup()
818 return -EINVAL; in bpf_prog_test_run_sk_lookup()
820 if (kattr->test.flags || kattr->test.cpu) in bpf_prog_test_run_sk_lookup()
821 return -EINVAL; in bpf_prog_test_run_sk_lookup()
823 if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out || in bpf_prog_test_run_sk_lookup()
824 kattr->test.data_size_out) in bpf_prog_test_run_sk_lookup()
825 return -EINVAL; in bpf_prog_test_run_sk_lookup()
835 return -EINVAL; in bpf_prog_test_run_sk_lookup()
837 if (user_ctx->sk) in bpf_prog_test_run_sk_lookup()
838 goto out; in bpf_prog_test_run_sk_lookup()
841 goto out; in bpf_prog_test_run_sk_lookup()
843 if (user_ctx->local_port > U16_MAX || user_ctx->remote_port > U16_MAX) { in bpf_prog_test_run_sk_lookup()
844 ret = -ERANGE; in bpf_prog_test_run_sk_lookup()
845 goto out; in bpf_prog_test_run_sk_lookup()
848 ctx.family = (u16)user_ctx->family; in bpf_prog_test_run_sk_lookup()
849 ctx.protocol = (u16)user_ctx->protocol; in bpf_prog_test_run_sk_lookup()
850 ctx.dport = (u16)user_ctx->local_port; in bpf_prog_test_run_sk_lookup()
851 ctx.sport = (__force __be16)user_ctx->remote_port; in bpf_prog_test_run_sk_lookup()
855 ctx.v4.daddr = (__force __be32)user_ctx->local_ip4; in bpf_prog_test_run_sk_lookup()
856 ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4; in bpf_prog_test_run_sk_lookup()
861 ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6; in bpf_prog_test_run_sk_lookup()
862 ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6; in bpf_prog_test_run_sk_lookup()
867 ret = -EAFNOSUPPORT; in bpf_prog_test_run_sk_lookup()
868 goto out; in bpf_prog_test_run_sk_lookup()
873 ret = -ENOMEM; in bpf_prog_test_run_sk_lookup()
874 goto out; in bpf_prog_test_run_sk_lookup()
877 progs->items[0].prog = prog; in bpf_prog_test_run_sk_lookup()
887 goto out; in bpf_prog_test_run_sk_lookup()
889 user_ctx->cookie = 0; in bpf_prog_test_run_sk_lookup()
891 if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { in bpf_prog_test_run_sk_lookup()
892 ret = -EOPNOTSUPP; in bpf_prog_test_run_sk_lookup()
893 goto out; in bpf_prog_test_run_sk_lookup()
896 user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); in bpf_prog_test_run_sk_lookup()
903 out: in bpf_prog_test_run_sk_lookup()