69 #define EMIT3(b1, b2, b3)	EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)  macro
78 	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
217 			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_mov_i()
243 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src));  in emit_ia32_mov_r()
246 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, sreg), STACK_VAR(dst));  in emit_ia32_mov_r()
294 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src));  in emit_ia32_mul_r()
298 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));  in emit_ia32_mul_r()
308 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_mul_r()
327 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_to_le_r64()
329 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_to_le_r64()
356 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_to_le_r64()
359 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_to_le_r64()
375 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_to_be_r64()
377 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_to_be_r64()
384 		EMIT3(0xC1, add_1reg(0xC8, dreg_lo), 8);  in emit_ia32_to_be_r64()
422 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_to_be_r64()
425 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_to_be_r64()
443 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),  in emit_ia32_div_mod_r()
451 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_div_mod_r()
464 			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_div_mod_r()
470 			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_div_mod_r()
492 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));  in emit_ia32_shift_r()
496 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src));  in emit_ia32_shift_r()
515 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), STACK_VAR(dst));  in emit_ia32_shift_r()
534 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src));  in emit_ia32_alu_r()
538 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(dst));  in emit_ia32_alu_r()
571 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg),  in emit_ia32_alu_r()
608 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst));  in emit_ia32_alu_i()
619 				EMIT3(0x83, add_1reg(0xD0, dreg), val);  in emit_ia32_alu_i()
624 				EMIT3(0x83, add_1reg(0xC0, dreg), val);  in emit_ia32_alu_i()
633 				EMIT3(0x83, add_1reg(0xD8, dreg), val);  in emit_ia32_alu_i()
638 				EMIT3(0x83, add_1reg(0xE8, dreg), val);  in emit_ia32_alu_i()
646 			EMIT3(0x83, add_1reg(0xC8, dreg), val);  in emit_ia32_alu_i()
653 			EMIT3(0x83, add_1reg(0xE0, dreg), val);  in emit_ia32_alu_i()
660 			EMIT3(0x83, add_1reg(0xF0, dreg), val);  in emit_ia32_alu_i()
671 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg),  in emit_ia32_alu_i()
706 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_neg64()
708 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_neg64()
715 	EMIT3(0x83, add_1reg(0xD0, dreg_hi), 0x00);  in emit_ia32_neg64()
721 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_neg64()
724 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_neg64()
740 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_lsh_r64()
742 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_lsh_r64()
748 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),  in emit_ia32_lsh_r64()
755 	EMIT3(0x0F, 0xA5, add_2reg(0xC0, dreg_hi, dreg_lo));  in emit_ia32_lsh_r64()
762 	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);  in emit_ia32_lsh_r64()
773 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_lsh_r64()
776 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_lsh_r64()
793 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_arsh_r64()
795 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_arsh_r64()
801 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),  in emit_ia32_arsh_r64()
808 	EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi));  in emit_ia32_arsh_r64()
815 	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);  in emit_ia32_arsh_r64()
822 	EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);  in emit_ia32_arsh_r64()
826 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_arsh_r64()
829 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_arsh_r64()
846 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_rsh_r64()
848 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_rsh_r64()
854 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),  in emit_ia32_rsh_r64()
861 	EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi));  in emit_ia32_rsh_r64()
868 	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32);  in emit_ia32_rsh_r64()
879 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_rsh_r64()
882 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_rsh_r64()
899 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_lsh_i64()
901 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_lsh_i64()
909 		EMIT3(0xC1, add_1reg(0xE0, dreg_lo), val);  in emit_ia32_lsh_i64()
914 		EMIT3(0xC1, add_1reg(0xE0, dreg_lo), value);  in emit_ia32_lsh_i64()
928 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_lsh_i64()
931 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_lsh_i64()
947 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_rsh_i64()
949 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_rsh_i64()
958 		EMIT3(0xC1, add_1reg(0xE8, dreg_hi), val);  in emit_ia32_rsh_i64()
963 		EMIT3(0xC1, add_1reg(0xE8, dreg_hi), value);  in emit_ia32_rsh_i64()
977 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_rsh_i64()
980 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_rsh_i64()
996 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_arsh_i64()
998 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in emit_ia32_arsh_i64()
1006 		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), val);  in emit_ia32_arsh_i64()
1011 		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), value);  in emit_ia32_arsh_i64()
1016 		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);  in emit_ia32_arsh_i64()
1019 		EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31);  in emit_ia32_arsh_i64()
1026 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo),  in emit_ia32_arsh_i64()
1029 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi),  in emit_ia32_arsh_i64()
1043 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_mul_r64()
1051 		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo));  in emit_ia32_mul_r64()
1061 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_mul_r64()
1069 		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_hi));  in emit_ia32_mul_r64()
1079 		EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_mul_r64()
1087 		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo));  in emit_ia32_mul_r64()
1097 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_mul_r64()
1100 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX),  in emit_ia32_mul_r64()
1124 		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_hi));  in emit_ia32_mul_i64()
1136 		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo));  in emit_ia32_mul_i64()
1147 		EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo));  in emit_ia32_mul_i64()
1157 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),  in emit_ia32_mul_i64()
1160 		EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX),  in emit_ia32_mul_i64()
1223 	EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 12);  in emit_prologue()
1228 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBP), STACK_VAR(fplo));  in emit_prologue()
1229 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(fphi));  in emit_prologue()
1233 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0]));  in emit_prologue()
1234 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(r1[1]));  in emit_prologue()
1237 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[0]));  in emit_prologue()
1238 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));  in emit_prologue()
1252 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r0[0]));  in emit_epilogue()
1254 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r0[1]));  in emit_epilogue()
1257 	EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 12);  in emit_epilogue()
1260 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12);  in emit_epilogue()
1262 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ESI), -8);  in emit_epilogue()
1264 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDI), -4);  in emit_epilogue()
1315 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r2[0]));  in emit_bpf_tail_call()
1317 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r3[0]));  in emit_bpf_tail_call()
1320 	EMIT3(0x39, add_2reg(0x40, IA32_EAX, IA32_EDX),  in emit_bpf_tail_call()
1331 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0]));  in emit_bpf_tail_call()
1332 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));  in emit_bpf_tail_call()
1335 	EMIT3(0x83, add_1reg(0xF8, IA32_EBX), hi);  in emit_bpf_tail_call()
1338 	EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo);  in emit_bpf_tail_call()
1344 	EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 0x01);  in emit_bpf_tail_call()
1346 	EMIT3(0x83, add_1reg(0xD0, IA32_EBX), 0x00);  in emit_bpf_tail_call()
1349 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0]));  in emit_bpf_tail_call()
1351 	EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1]));  in emit_bpf_tail_call()
1368 	EMIT3(0x8B, add_2reg(0x40, IA32_EDX, IA32_EDX),  in emit_bpf_tail_call()
1371 	EMIT3(0x83, add_1reg(0xC0, IA32_EDX), PROLOGUE_SIZE);  in emit_bpf_tail_call()
1374 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0]));  in emit_bpf_tail_call()
1397 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_hi));  in emit_push_r64()
1402 	EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo));  in emit_push_r64()
1728 				EMIT3(0x0F, 0xAE, 0xE8);  in do_jit()
1737 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
1777 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
1785 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in do_jit()
1811 					EMIT3(0x8B, add_2reg(0x40, IA32_EBP,  in do_jit()
1838 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
1863 				EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),  in do_jit()
1875 					EMIT3(0xC7, add_1reg(0x40, IA32_EBP),  in do_jit()
1889 					EMIT3(0x89,  in do_jit()
1923 			EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
1926 			EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX),  in do_jit()
1937 			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
1940 			EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX),  in do_jit()
1944 			EMIT3(0x83, add_1reg(0xC0, IA32_ESP), 32);  in do_jit()
1975 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
1978 					EMIT3(0x8B,  in do_jit()
1985 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),  in do_jit()
1988 					EMIT3(0x8B,  in do_jit()
2013 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
2015 				EMIT3(0x8B,  in do_jit()
2022 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),  in do_jit()
2024 				EMIT3(0x8B,  in do_jit()
2046 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
2049 					EMIT3(0x8B,  in do_jit()
2063 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX),  in do_jit()
2066 					EMIT3(0x8B,  in do_jit()
2091 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
2094 					EMIT3(0x8B,  in do_jit()
2147 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
2150 					EMIT3(0x8B,  in do_jit()
2194 				EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX),  in do_jit()
2196 				EMIT3(0x8B,  in do_jit()