65 	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
209 	u8 *prog = *pprog;  in emit_ia32_mov_i()  local
230 	*pprog = prog;  in emit_ia32_mov_i()
237 	u8 *prog = *pprog;  in emit_ia32_mov_r()  local
251 	*pprog = prog;  in emit_ia32_mov_r()
288 	u8 *prog = *pprog;  in emit_ia32_mul_r()  local
314 	*pprog = prog;  in emit_ia32_mul_r()
321 	u8 *prog = *pprog;  in emit_ia32_to_le_r64()  local
362 	*pprog = prog;  in emit_ia32_to_le_r64()
369 	u8 *prog = *pprog;  in emit_ia32_to_be_r64()  local
428 	*pprog = prog;  in emit_ia32_to_be_r64()
438 	u8 *prog = *pprog;  in emit_ia32_div_mod_r()  local
475 	*pprog = prog;  in emit_ia32_div_mod_r()
485 	u8 *prog = *pprog;  in emit_ia32_shift_r()  local
516 	*pprog = prog;  in emit_ia32_shift_r()
527 	u8 *prog = *pprog;  in emit_ia32_alu_r()  local
573 	*pprog = prog;  in emit_ia32_alu_r()
582 	u8 *prog = *pprog;  in emit_ia32_alu_r64()  local
584 	emit_ia32_alu_r(is64, false, op, dst_lo, src_lo, dstk, sstk, &prog);  in emit_ia32_alu_r64()
587 				&prog);  in emit_ia32_alu_r64()
589 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in emit_ia32_alu_r64()
590 	*pprog = prog;  in emit_ia32_alu_r64()
601 	u8 *prog = *pprog;  in emit_ia32_alu_i()  local
673 	*pprog = prog;  in emit_ia32_alu_i()
682 	u8 *prog = *pprog;  in emit_ia32_alu_i64()  local
688 	emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);  in emit_ia32_alu_i64()
690 		emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);  in emit_ia32_alu_i64()
692 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in emit_ia32_alu_i64()
694 	*pprog = prog;  in emit_ia32_alu_i64()
700 	u8 *prog = *pprog;  in emit_ia32_neg64()  local
727 	*pprog = prog;  in emit_ia32_neg64()
734 	u8 *prog = *pprog;  in emit_ia32_lsh_r64()  local
780 	*pprog = prog;  in emit_ia32_lsh_r64()
787 	u8 *prog = *pprog;  in emit_ia32_arsh_r64()  local
833 	*pprog = prog;  in emit_ia32_arsh_r64()
840 	u8 *prog = *pprog;  in emit_ia32_rsh_r64()  local
886 	*pprog = prog;  in emit_ia32_rsh_r64()
893 	u8 *prog = *pprog;  in emit_ia32_lsh_i64()  local
934 	*pprog = prog;  in emit_ia32_lsh_i64()
941 	u8 *prog = *pprog;  in emit_ia32_rsh_i64()  local
983 	*pprog = prog;  in emit_ia32_rsh_i64()
990 	u8 *prog = *pprog;  in emit_ia32_arsh_i64()  local
1032 	*pprog = prog;  in emit_ia32_arsh_i64()
1038 	u8 *prog = *pprog;  in emit_ia32_mul_r64()  local
1109 	*pprog = prog;  in emit_ia32_mul_r64()
1115 	u8 *prog = *pprog;  in emit_ia32_mul_i64()  local
1169 	*pprog = prog;  in emit_ia32_mul_i64()
1202 	u8 *prog = *pprog;  in emit_prologue()  local
1241 	*pprog = prog;  in emit_prologue()
1247 	u8 *prog = *pprog;  in emit_epilogue()  local
1268 	*pprog = prog;  in emit_epilogue()
1273 	u8 *prog = *pprog;  in emit_jmp_edx()  local
1281 	*pprog = prog;  in emit_jmp_edx()
1301 	u8 *prog = *pprog;  in emit_bpf_tail_call()  local
1381 	cnt += emit_jmp_edx(&prog, ip + cnt);  in emit_bpf_tail_call()
1387 	*pprog = prog;  in emit_bpf_tail_call()
1393 	u8 *prog = *pprog;  in emit_push_r64()  local
1406 	*pprog = prog;  in emit_push_r64()
1487 	u8 *prog = temp;  in do_jit()  local
1489 	emit_prologue(&prog, bpf_prog->aux->stack_depth);  in do_jit()
1516 					emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1520 						  &prog, bpf_prog->aux);  in do_jit()
1525 						  dstk, &prog);  in do_jit()
1560 						  src, dstk, sstk, &prog,  in do_jit()
1565 						  imm32, dstk, &prog,  in do_jit()
1575 						sstk, &prog);  in do_jit()
1582 						false, &prog);  in do_jit()
1586 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1595 						  dstk, sstk, &prog);  in do_jit()
1603 						  &prog);  in do_jit()
1607 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1618 						    src_lo, dstk, sstk, &prog);  in do_jit()
1626 						    &prog);  in do_jit()
1630 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1646 					  false, &prog);  in do_jit()
1648 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1654 			emit_ia32_lsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1660 			emit_ia32_rsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1664 			emit_ia32_lsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1668 			emit_ia32_rsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1672 			emit_ia32_arsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1678 			emit_ia32_arsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1683 					dst_lo, 0, dstk, &prog);  in do_jit()
1685 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1689 			emit_ia32_neg64(dst, dstk, &prog);  in do_jit()
1696 				emit_ia32_mul_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1699 				emit_ia32_mul_i64(dst, imm32, dstk, &prog);  in do_jit()
1705 			emit_ia32_to_le_r64(dst, imm32, dstk, &prog,  in do_jit()
1710 			emit_ia32_to_be_r64(dst, imm32, dstk, &prog,  in do_jit()
1718 			emit_ia32_mov_i(dst_lo, lo, dstk, &prog);  in do_jit()
1719 			emit_ia32_mov_i(dst_hi, hi, dstk, &prog);  in do_jit()
1929 			emit_push_r64(r5, &prog);  in do_jit()
1930 			emit_push_r64(r4, &prog);  in do_jit()
1931 			emit_push_r64(r3, &prog);  in do_jit()
1932 			emit_push_r64(r2, &prog);  in do_jit()
1948 			emit_bpf_tail_call(&prog, image + addrs[i - 1]);  in do_jit()
2281 			emit_epilogue(&prog, bpf_prog->aux->stack_depth);  in do_jit()
2296 		ilen = prog - temp;  in do_jit()
2320 		prog = temp;  in do_jit()
2330 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)  in bpf_int_jit_compile()  argument
2333 	struct bpf_prog *tmp, *orig_prog = prog;  in bpf_int_jit_compile()
2342 	if (!prog->jit_requested)  in bpf_int_jit_compile()
2345 	tmp = bpf_jit_blind_constants(prog);  in bpf_int_jit_compile()
2352 	if (tmp != prog) {  in bpf_int_jit_compile()
2354 		prog = tmp;  in bpf_int_jit_compile()
2357 	addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL);  in bpf_int_jit_compile()
2359 		prog = orig_prog;  in bpf_int_jit_compile()
2367 	for (proglen = 0, i = 0; i < prog->len; i++) {  in bpf_int_jit_compile()
2380 		proglen = do_jit(prog, addrs, image, oldproglen, &ctx);  in bpf_int_jit_compile()
2386 			prog = orig_prog;  in bpf_int_jit_compile()
2401 				prog = orig_prog;  in bpf_int_jit_compile()
2410 		bpf_jit_dump(prog->len, proglen, pass + 1, image);  in bpf_int_jit_compile()
2414 		prog->bpf_func = (void *)image;  in bpf_int_jit_compile()
2415 		prog->jited = 1;  in bpf_int_jit_compile()
2416 		prog->jited_len = proglen;  in bpf_int_jit_compile()
2418 		prog = orig_prog;  in bpf_int_jit_compile()
2425 		bpf_jit_prog_release_other(prog, prog == orig_prog ?  in bpf_int_jit_compile()
2427 	return prog;  in bpf_int_jit_compile()