206 static inline void emit_ia32_mov_i(const u8 dst, const u32 val, bool dstk,  in emit_ia32_mov_i()  argument
212 	if (dstk) {  in emit_ia32_mov_i()
234 static inline void emit_ia32_mov_r(const u8 dst, const u8 src, bool dstk,  in emit_ia32_mov_r()  argument
244 	if (dstk)  in emit_ia32_mov_r()
256 				     const u8 src[], bool dstk,  in emit_ia32_mov_r64()  argument
260 	emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog);  in emit_ia32_mov_r64()
263 		emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog);  in emit_ia32_mov_r64()
266 		emit_ia32_mov_i(dst_hi, 0, dstk, pprog);  in emit_ia32_mov_r64()
271 				     const u32 val, bool dstk, u8 **pprog)  in emit_ia32_mov_i64()  argument
277 	emit_ia32_mov_i(dst_lo, val, dstk, pprog);  in emit_ia32_mov_i64()
278 	emit_ia32_mov_i(dst_hi, hi, dstk, pprog);  in emit_ia32_mov_i64()
285 static inline void emit_ia32_mul_r(const u8 dst, const u8 src, bool dstk,  in emit_ia32_mul_r()  argument
296 	if (dstk)  in emit_ia32_mul_r()
306 	if (dstk)  in emit_ia32_mul_r()
318 					 bool dstk, u8 **pprog,  in emit_ia32_to_le_r64()  argument
323 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_to_le_r64()
324 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_to_le_r64()
326 	if (dstk && val != 64) {  in emit_ia32_to_le_r64()
354 	if (dstk && val != 64) {  in emit_ia32_to_le_r64()
366 				       bool dstk, u8 **pprog,  in emit_ia32_to_be_r64()  argument
371 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_to_be_r64()
372 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_to_be_r64()
374 	if (dstk) {  in emit_ia32_to_be_r64()
420 	if (dstk) {  in emit_ia32_to_be_r64()
436 				       bool dstk, bool sstk, u8 **pprog)  in emit_ia32_div_mod_r()  argument
449 	if (dstk)  in emit_ia32_div_mod_r()
463 		if (dstk)  in emit_ia32_div_mod_r()
469 		if (dstk)  in emit_ia32_div_mod_r()
483 				     bool dstk, bool sstk, u8 **pprog)  in emit_ia32_shift_r()  argument
487 	u8 dreg = dstk ? IA32_EAX : dst;  in emit_ia32_shift_r()
490 	if (dstk)  in emit_ia32_shift_r()
513 	if (dstk)  in emit_ia32_shift_r()
524 				   const u8 dst, const u8 src, bool dstk,  in emit_ia32_alu_r()  argument
530 	u8 dreg = dstk ? IA32_EDX : dst;  in emit_ia32_alu_r()
536 	if (dstk)  in emit_ia32_alu_r()
569 	if (dstk)  in emit_ia32_alu_r()
579 				     bool dstk,  bool sstk,  in emit_ia32_alu_r64()  argument
584 	emit_ia32_alu_r(is64, false, op, dst_lo, src_lo, dstk, sstk, &prog);  in emit_ia32_alu_r64()
586 		emit_ia32_alu_r(is64, true, op, dst_hi, src_hi, dstk, sstk,  in emit_ia32_alu_r64()
589 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in emit_ia32_alu_r64()
598 				   const u8 dst, const s32 val, bool dstk,  in emit_ia32_alu_i()  argument
603 	u8 dreg = dstk ? IA32_EAX : dst;  in emit_ia32_alu_i()
606 	if (dstk)  in emit_ia32_alu_i()
669 	if (dstk)  in emit_ia32_alu_i()
679 				     bool dstk, u8 **pprog,  in emit_ia32_alu_i64()  argument
688 	emit_ia32_alu_i(is64, false, op, dst_lo, val, dstk, &prog);  in emit_ia32_alu_i64()
690 		emit_ia32_alu_i(is64, true, op, dst_hi, hi, dstk, &prog);  in emit_ia32_alu_i64()
692 		emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in emit_ia32_alu_i64()
698 static inline void emit_ia32_neg64(const u8 dst[], bool dstk, u8 **pprog)  in emit_ia32_neg64()  argument
702 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_neg64()
703 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_neg64()
705 	if (dstk) {  in emit_ia32_neg64()
719 	if (dstk) {  in emit_ia32_neg64()
732 				     bool dstk, bool sstk, u8 **pprog)  in emit_ia32_lsh_r64()  argument
736 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_lsh_r64()
737 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_lsh_r64()
739 	if (dstk) {  in emit_ia32_lsh_r64()
771 	if (dstk) {  in emit_ia32_lsh_r64()
785 				      bool dstk, bool sstk, u8 **pprog)  in emit_ia32_arsh_r64()  argument
789 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_arsh_r64()
790 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_arsh_r64()
792 	if (dstk) {  in emit_ia32_arsh_r64()
824 	if (dstk) {  in emit_ia32_arsh_r64()
837 static inline void emit_ia32_rsh_r64(const u8 dst[], const u8 src[], bool dstk,  in emit_ia32_rsh_r64()  argument
842 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_rsh_r64()
843 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_rsh_r64()
845 	if (dstk) {  in emit_ia32_rsh_r64()
877 	if (dstk) {  in emit_ia32_rsh_r64()
891 				     bool dstk, u8 **pprog)  in emit_ia32_lsh_i64()  argument
895 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_lsh_i64()
896 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_lsh_i64()
898 	if (dstk) {  in emit_ia32_lsh_i64()
926 	if (dstk) {  in emit_ia32_lsh_i64()
939 				     bool dstk, u8 **pprog)  in emit_ia32_rsh_i64()  argument
943 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_rsh_i64()
944 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_rsh_i64()
946 	if (dstk) {  in emit_ia32_rsh_i64()
975 	if (dstk) {  in emit_ia32_rsh_i64()
988 				      bool dstk, u8 **pprog)  in emit_ia32_arsh_i64()  argument
992 	u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in emit_ia32_arsh_i64()
993 	u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in emit_ia32_arsh_i64()
995 	if (dstk) {  in emit_ia32_arsh_i64()
1024 	if (dstk) {  in emit_ia32_arsh_i64()
1035 static inline void emit_ia32_mul_r64(const u8 dst[], const u8 src[], bool dstk,  in emit_ia32_mul_r64()  argument
1041 	if (dstk)  in emit_ia32_mul_r64()
1059 	if (dstk)  in emit_ia32_mul_r64()
1077 	if (dstk)  in emit_ia32_mul_r64()
1095 	if (dstk) {  in emit_ia32_mul_r64()
1113 				     bool dstk, u8 **pprog)  in emit_ia32_mul_i64()  argument
1122 	if (dstk)  in emit_ia32_mul_i64()
1134 	if (dstk)  in emit_ia32_mul_i64()
1145 	if (dstk)  in emit_ia32_mul_i64()
1155 	if (dstk) {  in emit_ia32_mul_i64()
1494 		const bool dstk = insn->dst_reg != BPF_REG_AX;  in do_jit()  local
1516 					emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1519 				emit_ia32_mov_r64(is64, dst, src, dstk, sstk,  in do_jit()
1525 						  dstk, &prog);  in do_jit()
1560 						  src, dstk, sstk, &prog,  in do_jit()
1565 						  imm32, dstk, &prog,  in do_jit()
1574 				emit_ia32_mul_r(dst_lo, src_lo, dstk,  in do_jit()
1581 				emit_ia32_mul_r(dst_lo, IA32_ECX, dstk,  in do_jit()
1586 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1595 						  dstk, sstk, &prog);  in do_jit()
1602 						  IA32_ECX, dstk, false,  in do_jit()
1607 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1618 						    src_lo, dstk, sstk, &prog);  in do_jit()
1625 						    IA32_ECX, dstk, false,  in do_jit()
1630 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1645 			emit_ia32_shift_r(BPF_OP(code), dst_lo, IA32_ECX, dstk,  in do_jit()
1648 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1654 			emit_ia32_lsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1660 			emit_ia32_rsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1664 			emit_ia32_lsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1668 			emit_ia32_rsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1672 			emit_ia32_arsh_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1678 			emit_ia32_arsh_i64(dst, imm32, dstk, &prog);  in do_jit()
1683 					dst_lo, 0, dstk, &prog);  in do_jit()
1685 				emit_ia32_mov_i(dst_hi, 0, dstk, &prog);  in do_jit()
1689 			emit_ia32_neg64(dst, dstk, &prog);  in do_jit()
1696 				emit_ia32_mul_r64(dst, src, dstk, sstk, &prog);  in do_jit()
1699 				emit_ia32_mul_i64(dst, imm32, dstk, &prog);  in do_jit()
1705 			emit_ia32_to_le_r64(dst, imm32, dstk, &prog,  in do_jit()
1710 			emit_ia32_to_be_r64(dst, imm32, dstk, &prog,  in do_jit()
1718 			emit_ia32_mov_i(dst_lo, lo, dstk, &prog);  in do_jit()
1719 			emit_ia32_mov_i(dst_hi, hi, dstk, &prog);  in do_jit()
1735 			if (dstk)  in do_jit()
1775 			if (dstk)  in do_jit()
1861 			if (dstk)  in do_jit()
1874 				if (dstk) {  in do_jit()
1888 				if (dstk)  in do_jit()
1969 			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in do_jit()
1970 			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in do_jit()
1974 			if (dstk) {  in do_jit()
2007 			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in do_jit()
2008 			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in do_jit()
2012 			if (dstk) {  in do_jit()
2045 			if (dstk) {  in do_jit()
2090 			if (dstk) {  in do_jit()
2140 			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in do_jit()
2141 			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in do_jit()
2146 			if (dstk) {  in do_jit()
2187 			u8 dreg_lo = dstk ? IA32_EAX : dst_lo;  in do_jit()
2188 			u8 dreg_hi = dstk ? IA32_EDX : dst_hi;  in do_jit()
2193 			if (dstk) {  in do_jit()