Lines Matching +full:0 +full:x48
33 do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
41 do { EMIT1(b1); EMIT(off, 4); } while (0)
43 do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
45 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
47 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
68 EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
69 } while (0)
82 return 0; in bpf_size_to_x86_bytes()
87 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
89 #define X86_JB 0x72
90 #define X86_JAE 0x73
91 #define X86_JE 0x74
92 #define X86_JNE 0x75
93 #define X86_JBE 0x76
94 #define X86_JA 0x77
95 #define X86_JL 0x7C
96 #define X86_JGE 0x7D
97 #define X86_JLE 0x7E
98 #define X86_JG 0x7F
115 [BPF_REG_0] = 0, /* RAX */
120 [BPF_REG_5] = 0, /* R8 */
210 memset(area, 0xcc, size); in jit_fill_hole()
237 int cnt = 0; in push_callee_regs()
239 if (callee_regs_used[0]) in push_callee_regs()
240 EMIT1(0x53); /* push rbx */ in push_callee_regs()
242 EMIT2(0x41, 0x55); /* push r13 */ in push_callee_regs()
244 EMIT2(0x41, 0x56); /* push r14 */ in push_callee_regs()
246 EMIT2(0x41, 0x57); /* push r15 */ in push_callee_regs()
253 int cnt = 0; in pop_callee_regs()
256 EMIT2(0x41, 0x5F); /* pop r15 */ in pop_callee_regs()
258 EMIT2(0x41, 0x5E); /* pop r14 */ in pop_callee_regs()
260 EMIT2(0x41, 0x5D); /* pop r13 */ in pop_callee_regs()
261 if (callee_regs_used[0]) in pop_callee_regs()
262 EMIT1(0x5B); /* pop rbx */ in pop_callee_regs()
284 EMIT2(0x31, 0xC0); /* xor eax, eax */ in emit_prologue()
286 EMIT2(0x66, 0x90); /* nop2 */ in emit_prologue()
288 EMIT1(0x55); /* push rbp */ in emit_prologue()
289 EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ in emit_prologue()
292 EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); in emit_prologue()
294 EMIT1(0x50); /* push rax */ in emit_prologue()
301 int cnt = 0; in emit_patch()
311 return 0; in emit_patch()
316 return emit_patch(pprog, func, ip, 0xE8); in emit_call()
321 return emit_patch(pprog, func, ip, 0xE9); in emit_jump()
364 ret = 0; in __bpf_arch_text_poke()
382 #define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8)
387 int cnt = 0; in emit_indirect_jump()
392 EMIT2(0xFF, 0xE0 + reg); in emit_indirect_jump()
397 EMIT2(0xFF, 0xE0 + reg); in emit_indirect_jump()
405 int cnt = 0; in emit_return()
410 EMIT1(0xC3); /* ret */ in emit_return()
412 EMIT1(0xCC); /* int3 */ in emit_return()
438 int cnt = 0, offset; in emit_bpf_tail_call_indirect()
450 EMIT2(0x89, 0xD2); /* mov edx, edx */ in emit_bpf_tail_call_indirect()
451 EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ in emit_bpf_tail_call_indirect()
461 EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ in emit_bpf_tail_call_indirect()
462 EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ in emit_bpf_tail_call_indirect()
466 EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ in emit_bpf_tail_call_indirect()
467 EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ in emit_bpf_tail_call_indirect()
470 EMIT4_off32(0x48, 0x8B, 0x8C, 0xD6, /* mov rcx, [rsi + rdx * 8 + offsetof(...)] */ in emit_bpf_tail_call_indirect()
477 EMIT3(0x48, 0x85, 0xC9); /* test rcx,rcx */ in emit_bpf_tail_call_indirect()
484 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_indirect()
486 EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ in emit_bpf_tail_call_indirect()
490 EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ in emit_bpf_tail_call_indirect()
492 EMIT4(0x48, 0x83, 0xC1, /* add rcx, X86_TAIL_CALL_OFFSET */ in emit_bpf_tail_call_indirect()
513 int cnt = 0, offset; in emit_bpf_tail_call_direct()
519 EMIT2_off32(0x8B, 0x85, tcc_off); /* mov eax, dword ptr [rbp - tcc_off] */ in emit_bpf_tail_call_direct()
520 EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ in emit_bpf_tail_call_direct()
524 EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ in emit_bpf_tail_call_direct()
525 EMIT2_off32(0x89, 0x85, tcc_off); /* mov dword ptr [rbp - tcc_off], eax */ in emit_bpf_tail_call_direct()
536 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_direct()
538 EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); in emit_bpf_tail_call_direct()
556 for (i = 0; i < prog->aux->size_poke_tab; i++) { in bpf_tail_call_direct_fixup()
580 BUG_ON(ret < 0); in bpf_tail_call_direct_fixup()
585 BUG_ON(ret < 0); in bpf_tail_call_direct_fixup()
597 int cnt = 0; in emit_mov_imm32()
603 if (sign_propagate && (s32)imm32 < 0) { in emit_mov_imm32()
605 b1 = add_1mod(0x48, dst_reg); in emit_mov_imm32()
606 b2 = 0xC7; in emit_mov_imm32()
607 b3 = 0xC0; in emit_mov_imm32()
616 if (imm32 == 0) { in emit_mov_imm32()
618 EMIT1(add_2mod(0x40, dst_reg, dst_reg)); in emit_mov_imm32()
619 b2 = 0x31; /* xor */ in emit_mov_imm32()
620 b3 = 0xC0; in emit_mov_imm32()
627 EMIT1(add_1mod(0x40, dst_reg)); in emit_mov_imm32()
628 EMIT1_off32(add_1reg(0xB8, dst_reg), imm32); in emit_mov_imm32()
637 int cnt = 0; in emit_mov_imm64()
649 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg)); in emit_mov_imm64()
660 int cnt = 0; in emit_mov_reg()
668 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in emit_mov_reg()
669 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg)); in emit_mov_reg()
679 int cnt = 0; in emit_ldx()
684 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); in emit_ldx()
688 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); in emit_ldx()
691 /* Emit 'mov eax, dword ptr [rax+0x14]' */ in emit_ldx()
693 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); in emit_ldx()
695 EMIT1(0x8B); in emit_ldx()
698 /* Emit 'mov rax, qword ptr [rax+0x14]' */ in emit_ldx()
699 EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); in emit_ldx()
703 * If insn->off == 0 we can save one extra byte, but in emit_ldx()
708 EMIT2(add_2reg(0x40, src_reg, dst_reg), off); in emit_ldx()
710 EMIT1_off32(add_2reg(0x80, src_reg, dst_reg), off); in emit_ldx()
718 int cnt = 0; in emit_stx()
725 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); in emit_stx()
727 EMIT1(0x88); in emit_stx()
731 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89); in emit_stx()
733 EMIT2(0x66, 0x89); in emit_stx()
737 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89); in emit_stx()
739 EMIT1(0x89); in emit_stx()
742 EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89); in emit_stx()
746 EMIT2(add_2reg(0x40, dst_reg, src_reg), off); in emit_stx()
748 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), off); in emit_stx()
759 *(unsigned long *)((void *)regs + reg) = 0; in ex_handler_bpf()
760 regs->ip += x->fixup & 0xff; in ex_handler_bpf()
773 regs_used[0] = true; in detect_reg_usage()
793 int i, cnt = 0, excnt = 0; in do_jit()
794 int proglen = 0; in do_jit()
805 bpf_prog->aux->func_idx != 0); in do_jit()
807 addrs[0] = prog - temp; in do_jit()
813 u8 b2 = 0, b3 = 0; in do_jit()
832 case BPF_ADD: b2 = 0x01; break; in do_jit()
833 case BPF_SUB: b2 = 0x29; break; in do_jit()
834 case BPF_AND: b2 = 0x21; break; in do_jit()
835 case BPF_OR: b2 = 0x09; break; in do_jit()
836 case BPF_XOR: b2 = 0x31; break; in do_jit()
839 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in do_jit()
841 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in do_jit()
842 EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
856 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
858 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
859 EMIT2(0xF7, add_1reg(0xD8, dst_reg)); in do_jit()
873 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
875 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
883 b3 = 0xC0; in do_jit()
884 b2 = 0x05; in do_jit()
887 b3 = 0xE8; in do_jit()
888 b2 = 0x2D; in do_jit()
891 b3 = 0xE0; in do_jit()
892 b2 = 0x25; in do_jit()
895 b3 = 0xC8; in do_jit()
896 b2 = 0x0D; in do_jit()
899 b3 = 0xF0; in do_jit()
900 b2 = 0x35; in do_jit()
905 EMIT3(0x83, add_1reg(b3, dst_reg), imm32); in do_jit()
909 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); in do_jit()
919 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); in do_jit()
933 EMIT1(0x50); /* push rax */ in do_jit()
934 EMIT1(0x52); /* push rdx */ in do_jit()
941 EMIT3_off32(0x49, 0xC7, 0xC3, imm32); in do_jit()
950 EMIT2(0x31, 0xd2); in do_jit()
954 EMIT3(0x49, 0xF7, 0xF3); in do_jit()
957 EMIT3(0x41, 0xF7, 0xF3); in do_jit()
961 EMIT3(0x49, 0x89, 0xD3); in do_jit()
964 EMIT3(0x49, 0x89, 0xC3); in do_jit()
966 EMIT1(0x5A); /* pop rdx */ in do_jit()
967 EMIT1(0x58); /* pop rax */ in do_jit()
981 EMIT1(0x50); /* push rax */ in do_jit()
983 EMIT1(0x52); /* push rdx */ in do_jit()
994 EMIT1(add_1mod(0x48, AUX_REG)); in do_jit()
996 EMIT1(add_1mod(0x40, AUX_REG)); in do_jit()
998 EMIT2(0xF7, add_1reg(0xE0, AUX_REG)); in do_jit()
1001 EMIT1(0x5A); /* pop rdx */ in do_jit()
1005 EMIT1(0x58); /* pop rax */ in do_jit()
1017 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
1019 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
1022 case BPF_LSH: b3 = 0xE0; break; in do_jit()
1023 case BPF_RSH: b3 = 0xE8; break; in do_jit()
1024 case BPF_ARSH: b3 = 0xF8; break; in do_jit()
1028 EMIT2(0xD1, add_1reg(b3, dst_reg)); in do_jit()
1030 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); in do_jit()
1048 EMIT1(0x51); /* push rcx */ in do_jit()
1056 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
1058 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
1061 case BPF_LSH: b3 = 0xE0; break; in do_jit()
1062 case BPF_RSH: b3 = 0xE8; break; in do_jit()
1063 case BPF_ARSH: b3 = 0xF8; break; in do_jit()
1065 EMIT2(0xD3, add_1reg(b3, dst_reg)); in do_jit()
1068 EMIT1(0x59); /* pop rcx */ in do_jit()
1079 EMIT1(0x66); in do_jit()
1081 EMIT1(0x41); in do_jit()
1082 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); in do_jit()
1086 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
1088 EMIT2(0x0F, 0xB7); in do_jit()
1089 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1094 EMIT2(0x41, 0x0F); in do_jit()
1096 EMIT1(0x0F); in do_jit()
1097 EMIT1(add_1reg(0xC8, dst_reg)); in do_jit()
1101 EMIT3(add_1mod(0x48, dst_reg), 0x0F, in do_jit()
1102 add_1reg(0xC8, dst_reg)); in do_jit()
1115 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
1117 EMIT2(0x0F, 0xB7); in do_jit()
1118 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1123 EMIT1(0x45); in do_jit()
1124 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1141 EMIT2(0x41, 0xC6); in do_jit()
1143 EMIT1(0xC6); in do_jit()
1147 EMIT3(0x66, 0x41, 0xC7); in do_jit()
1149 EMIT2(0x66, 0xC7); in do_jit()
1153 EMIT2(0x41, 0xC7); in do_jit()
1155 EMIT1(0xC7); in do_jit()
1158 EMIT2(add_1mod(0x48, dst_reg), 0xC7); in do_jit()
1161 EMIT2(add_1reg(0x40, dst_reg), insn->off); in do_jit()
1163 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off); in do_jit()
1223 * End result: x86 insn "mov rbx, qword ptr [rax+0x14]" in do_jit()
1234 EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); in do_jit()
1236 EMIT2(0xF0, 0x01); in do_jit()
1239 EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01); in do_jit()
1241 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off); in do_jit()
1243 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), in do_jit()
1252 EMIT3_off32(0x48, 0x8B, 0x85, in do_jit()
1300 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in do_jit()
1302 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in do_jit()
1303 EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
1310 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in do_jit()
1312 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in do_jit()
1313 EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
1320 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
1322 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
1323 EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); in do_jit()
1347 if (imm32 == 0) { in do_jit()
1349 EMIT1(add_2mod(0x48, dst_reg, dst_reg)); in do_jit()
1351 EMIT1(add_2mod(0x40, dst_reg, dst_reg)); in do_jit()
1352 EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1358 EMIT1(add_1mod(0x48, dst_reg)); in do_jit()
1360 EMIT1(add_1mod(0x40, dst_reg)); in do_jit()
1363 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); in do_jit()
1365 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); in do_jit()
1415 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
1440 EMIT2(0xEB, jmp_offset); in do_jit()
1442 EMIT1_off32(0xE9, jmp_offset); in do_jit()
1458 EMIT1(0xC9); /* leave */ in do_jit()
1513 * mov QWORD PTR [rbp-0x10],rdi in save_regs()
1514 * mov QWORD PTR [rbp-0x8],rsi in save_regs()
1516 for (i = 0; i < min(nr_args, 6); i++) in save_regs()
1530 * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10] in restore_regs()
1531 * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8] in restore_regs()
1533 for (i = 0; i < min(nr_args, 6); i++) in restore_regs()
1544 int cnt = 0; in invoke_bpf_prog()
1557 EMIT4(0x48, 0x8D, 0x7D, -stack_size); in invoke_bpf_prog()
1592 return 0; in invoke_bpf_prog()
1599 int cnt = 0; in emit_nops()
1601 while (len > 0) { in emit_nops()
1607 for (i = 0; i < noplen; i++) in emit_nops()
1629 int cnt = 0; in emit_cond_near_jump()
1637 EMIT2_off32(0x0F, jmp_cond + 0x10, offset); in emit_cond_near_jump()
1639 return 0; in emit_cond_near_jump()
1649 for (i = 0; i < tp->nr_progs; i++) { in invoke_bpf()
1655 return 0; in invoke_bpf()
1663 int i, cnt = 0; in invoke_bpf_mod_ret()
1666 * Set this to 0 to avoid confusing the program. in invoke_bpf_mod_ret()
1668 emit_mov_imm32(&prog, false, BPF_REG_0, 0); in invoke_bpf_mod_ret()
1670 for (i = 0; i < tp->nr_progs; i++) { in invoke_bpf_mod_ret()
1675 * if (*(u64 *)(rbp - 8) != 0) in invoke_bpf_mod_ret()
1678 /* cmp QWORD PTR [rbp - 0x8], 0x0 */ in invoke_bpf_mod_ret()
1679 EMIT4(0x48, 0x83, 0x7d, 0xf8); EMIT1(0x00); in invoke_bpf_mod_ret()
1691 return 0; in invoke_bpf_mod_ret()
1776 int ret, i, cnt = 0, nr_args = m->nr_args; in arch_prepare_bpf_trampoline()
1805 EMIT1(0x55); /* push rbp */ in arch_prepare_bpf_trampoline()
1806 EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ in arch_prepare_bpf_trampoline()
1807 EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */ in arch_prepare_bpf_trampoline()
1808 EMIT1(0x53); /* push rbx */ in arch_prepare_bpf_trampoline()
1864 for (i = 0; i < fmod_ret->nr_progs; i++) in arch_prepare_bpf_trampoline()
1895 EMIT1(0x5B); /* pop rbx */ in arch_prepare_bpf_trampoline()
1896 EMIT1(0xC9); /* leave */ in arch_prepare_bpf_trampoline()
1899 EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ in arch_prepare_bpf_trampoline()
1916 int pivot, err, jg_bytes = 1, cnt = 0; in emit_bpf_dispatcher()
1923 EMIT1(add_1mod(0x48, BPF_REG_3)); /* cmp rdx,func */ in emit_bpf_dispatcher()
1926 EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), in emit_bpf_dispatcher()
1937 return 0; in emit_bpf_dispatcher()
1944 EMIT1(add_1mod(0x48, BPF_REG_3)); /* cmp rdx,func */ in emit_bpf_dispatcher()
1947 EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), progs[a + pivot]); in emit_bpf_dispatcher()
1952 EMIT2_off32(0x0F, X86_JG + 0x10, 0); in emit_bpf_dispatcher()
1954 EMIT2(X86_JG, 0); in emit_bpf_dispatcher()
1978 return 0; in emit_bpf_dispatcher()
1990 return 0; in cmp_ips()
1997 sort(funcs, num_funcs, sizeof(funcs[0]), cmp_ips, NULL); in arch_prepare_bpf_dispatcher()
1998 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs); in arch_prepare_bpf_dispatcher()
2014 int proglen, oldproglen = 0; in bpf_int_jit_compile()
2066 for (proglen = 0, i = 0; i <= prog->len; i++) { in bpf_int_jit_compile()
2079 for (pass = 0; pass < 20 || image; pass++) { in bpf_int_jit_compile()
2081 if (proglen <= 0) { in bpf_int_jit_compile()