111 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)  in reg_set_seen()  argument
115 	if (r1 >= 6 && r1 <= 15 && !jit->seen_reg[r1])  in reg_set_seen()
116 		jit->seen_reg[r1] = 1;  in reg_set_seen()
121 	reg_set_seen(jit, b1);					\
124 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
132 	if (jit->prg_buf)					\
133 		*(u16 *) (jit->prg_buf + jit->prg) = (op);	\
134 	jit->prg += 2;						\
146 	if (jit->prg_buf)					\
147 		*(u32 *) (jit->prg_buf + jit->prg) = (op);	\
148 	jit->prg += 4;						\
195 	int __rel = ((target) - jit->prg) / 2;			\
201 	if (jit->prg_buf) {					\
202 		*(u32 *) (jit->prg_buf + jit->prg) = (op1);	\
203 		*(u16 *) (jit->prg_buf + jit->prg + 4) = (op2);	\
205 	jit->prg += 6;						\
233 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
242 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
251 	int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2;	\
259 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
266 	unsigned int rel = (int)((target) - jit->prg) / 2;	\
290 	ret = jit->lit32;					\
291 	if (jit->prg_buf)					\
292 		*(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\
293 	jit->lit32 += 4;					\
299 	jit->seen |= SEEN_LITERAL;				\
300 	_EMIT_CONST_U32(val) - jit->base_ip;			\
306 	ret = jit->lit64;					\
307 	if (jit->prg_buf)					\
308 		*(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\
309 	jit->lit64 += 8;					\
315 	jit->seen |= SEEN_LITERAL;				\
316 	_EMIT_CONST_U64(val) - jit->base_ip;			\
332 static bool is_first_pass(struct bpf_jit *jit)  in is_first_pass()  argument
334 	return jit->size == 0;  in is_first_pass()
341 static bool is_codegen_pass(struct bpf_jit *jit)  in is_codegen_pass()  argument
343 	return jit->prg_buf;  in is_codegen_pass()
357 static bool can_use_rel(struct bpf_jit *jit, int off)  in can_use_rel()  argument
359 	return is_valid_rel(off - jit->prg);  in can_use_rel()
375 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)  in can_use_ldisp_for_lit32()  argument
377 	return is_valid_ldisp(jit->lit32 - jit->base_ip);  in can_use_ldisp_for_lit32()
384 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)  in can_use_ldisp_for_lit64()  argument
386 	return is_valid_ldisp(jit->lit64 - jit->base_ip);  in can_use_ldisp_for_lit64()
400 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)  in save_regs()  argument
415 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth)  in restore_regs()  argument
419 	if (jit->seen & SEEN_STACK)  in restore_regs()
433 static int get_start(struct bpf_jit *jit, int start)  in get_start()  argument
438 		if (jit->seen_reg[i])  in get_start()
447 static int get_end(struct bpf_jit *jit, int start)  in get_end()  argument
452 		if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])  in get_end()
455 	return jit->seen_reg[15] ? 15 : 14;  in get_end()
464 static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)  in save_restore_regs()  argument
469 	if (is_first_pass(jit)) {  in save_restore_regs()
474 		jit->prg += (last - re + 1) * save_restore_size;  in save_restore_regs()
479 		rs = get_start(jit, re);  in save_restore_regs()
482 		re = get_end(jit, rs + 1);  in save_restore_regs()
484 			save_regs(jit, rs, re);  in save_restore_regs()
486 			restore_regs(jit, rs, re, stack_depth);  in save_restore_regs()
491 static void bpf_skip(struct bpf_jit *jit, int size)  in bpf_skip()  argument
515 static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth)  in bpf_jit_prologue()  argument
517 	if (jit->seen & SEEN_TAIL_CALL) {  in bpf_jit_prologue()
525 		bpf_skip(jit, 6);  in bpf_jit_prologue()
528 	jit->tail_call_start = jit->prg;  in bpf_jit_prologue()
530 	save_restore_regs(jit, REGS_SAVE, stack_depth);  in bpf_jit_prologue()
532 	if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) {  in bpf_jit_prologue()
533 		if (!is_first_pass(jit) &&  in bpf_jit_prologue()
534 		    is_valid_ldisp(jit->size - (jit->prg + 2))) {  in bpf_jit_prologue()
537 			jit->base_ip = jit->prg;  in bpf_jit_prologue()
540 			EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start);  in bpf_jit_prologue()
541 			jit->base_ip = jit->lit32_start;  in bpf_jit_prologue()
545 	if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {  in bpf_jit_prologue()
546 		if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))  in bpf_jit_prologue()
553 		if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))  in bpf_jit_prologue()
563 static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)  in bpf_jit_epilogue()  argument
565 	jit->exit_ip = jit->prg;  in bpf_jit_epilogue()
569 	save_restore_regs(jit, REGS_RESTORE, stack_depth);  in bpf_jit_epilogue()
571 		jit->r14_thunk_ip = jit->prg;  in bpf_jit_epilogue()
575 			EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);  in bpf_jit_epilogue()
578 			EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);  in bpf_jit_epilogue()
589 	    (is_first_pass(jit) || (jit->seen & SEEN_FUNC))) {  in bpf_jit_epilogue()
590 		jit->r1_thunk_ip = jit->prg;  in bpf_jit_epilogue()
594 			EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);  in bpf_jit_epilogue()
641 static int bpf_jit_probe_mem(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_probe_mem()  argument
653 	insn = jit->prg_buf + probe_prg;  in bpf_jit_probe_mem()
661 		if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))  in bpf_jit_probe_mem()
664 		ex = &fp->aux->extable[jit->excnt];  in bpf_jit_probe_mem()
667 		delta = jit->prg_buf + prg - (u8 *)&ex->insn;  in bpf_jit_probe_mem()
676 		delta = jit->prg_buf + nop_prg - (u8 *)&ex->fixup;  in bpf_jit_probe_mem()
682 		jit->excnt++;  in bpf_jit_probe_mem()
693 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_insn()  argument
700 	u32 *addrs = jit->addrs;  in bpf_jit_insn()
710 		probe_prg = jit->prg;  in bpf_jit_insn()
882 		if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {  in bpf_jit_insn()
890 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
915 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
923 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
949 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
958 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
981 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
990 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
1015 		if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {  in bpf_jit_insn()
1024 			jit->seen |= SEEN_LITERAL;  in bpf_jit_insn()
1173 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1178 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1183 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1188 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1195 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1202 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1209 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1216 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1225 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1231 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1240 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1248 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1255 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1263 		jit->seen |= SEEN_MEM;  in bpf_jit_insn()
1281 		jit->seen |= SEEN_FUNC;  in bpf_jit_insn()
1286 			EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);  in bpf_jit_insn()
1304 		jit->seen |= SEEN_TAIL_CALL;  in bpf_jit_insn()
1316 		patch_1_clrj = jit->prg;  in bpf_jit_insn()
1318 				 jit->prg);  in bpf_jit_insn()
1325 		if (jit->seen & SEEN_STACK)  in bpf_jit_insn()
1334 		patch_2_clij = jit->prg;  in bpf_jit_insn()
1336 				 2, jit->prg);  in bpf_jit_insn()
1352 		patch_3_brc = jit->prg;  in bpf_jit_insn()
1353 		EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);  in bpf_jit_insn()
1358 		save_restore_regs(jit, REGS_RESTORE, stack_depth);  in bpf_jit_insn()
1368 		_EMIT4(0x47f01000 + jit->tail_call_start);  in bpf_jit_insn()
1370 		if (jit->prg_buf) {  in bpf_jit_insn()
1371 			*(u16 *)(jit->prg_buf + patch_1_clrj + 2) =  in bpf_jit_insn()
1372 				(jit->prg - patch_1_clrj) >> 1;  in bpf_jit_insn()
1373 			*(u16 *)(jit->prg_buf + patch_2_clij + 2) =  in bpf_jit_insn()
1374 				(jit->prg - patch_2_clij) >> 1;  in bpf_jit_insn()
1375 			*(u16 *)(jit->prg_buf + patch_3_brc + 2) =  in bpf_jit_insn()
1376 				(jit->prg - patch_3_brc) >> 1;  in bpf_jit_insn()
1384 		if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip))  in bpf_jit_insn()
1386 			EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip);  in bpf_jit_insn()
1389 			EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip);  in bpf_jit_insn()
1525 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1526 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1543 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1544 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1561 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1562 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1578 		if (!is_first_pass(jit) &&  in bpf_jit_insn()
1579 		    can_use_rel(jit, addrs[i + off + 1])) {  in bpf_jit_insn()
1603 		nop_prg = jit->prg;  in bpf_jit_insn()
1606 		err = bpf_jit_probe_mem(jit, fp, probe_prg, nop_prg);  in bpf_jit_insn()
1617 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i)  in bpf_is_new_addr_sane()  argument
1620 	if (is_first_pass(jit))  in bpf_is_new_addr_sane()
1624 	if (is_codegen_pass(jit))  in bpf_is_new_addr_sane()
1625 		return jit->addrs[i] == jit->prg;  in bpf_is_new_addr_sane()
1628 	return jit->addrs[i] >= jit->prg;  in bpf_is_new_addr_sane()
1634 static int bpf_set_addr(struct bpf_jit *jit, int i)  in bpf_set_addr()  argument
1638 	if (is_codegen_pass(jit)) {  in bpf_set_addr()
1639 		delta = jit->prg - jit->addrs[i];  in bpf_set_addr()
1641 			bpf_skip(jit, -delta);  in bpf_set_addr()
1643 	if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))  in bpf_set_addr()
1645 	jit->addrs[i] = jit->prg;  in bpf_set_addr()
1652 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,  in bpf_jit_prog()  argument
1657 	jit->lit32 = jit->lit32_start;  in bpf_jit_prog()
1658 	jit->lit64 = jit->lit64_start;  in bpf_jit_prog()
1659 	jit->prg = 0;  in bpf_jit_prog()
1660 	jit->excnt = 0;  in bpf_jit_prog()
1662 	bpf_jit_prologue(jit, stack_depth);  in bpf_jit_prog()
1663 	if (bpf_set_addr(jit, 0) < 0)  in bpf_jit_prog()
1666 		insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth);  in bpf_jit_prog()
1670 		if (bpf_set_addr(jit, i + insn_count) < 0)  in bpf_jit_prog()
1673 	bpf_jit_epilogue(jit, stack_depth);  in bpf_jit_prog()
1675 	lit32_size = jit->lit32 - jit->lit32_start;  in bpf_jit_prog()
1676 	lit64_size = jit->lit64 - jit->lit64_start;  in bpf_jit_prog()
1677 	jit->lit32_start = jit->prg;  in bpf_jit_prog()
1679 		jit->lit32_start = ALIGN(jit->lit32_start, 4);  in bpf_jit_prog()
1680 	jit->lit64_start = jit->lit32_start + lit32_size;  in bpf_jit_prog()
1682 		jit->lit64_start = ALIGN(jit->lit64_start, 8);  in bpf_jit_prog()
1683 	jit->size = jit->lit64_start + lit64_size;  in bpf_jit_prog()
1684 	jit->size_prg = jit->prg;  in bpf_jit_prog()
1687 			 jit->excnt != fp->aux->num_exentries))  in bpf_jit_prog()
1705 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,  in bpf_jit_alloc()  argument
1715 	code_size = roundup(jit->size,  in bpf_jit_alloc()
1719 	header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf,  in bpf_jit_alloc()
1724 		(jit->prg_buf + code_size);  in bpf_jit_alloc()
1739 	struct bpf_jit jit;  in bpf_int_jit_compile()  local
1767 		jit = jit_data->ctx;  in bpf_int_jit_compile()
1774 	memset(&jit, 0, sizeof(jit));  in bpf_int_jit_compile()
1775 	jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);  in bpf_int_jit_compile()
1776 	if (jit.addrs == NULL) {  in bpf_int_jit_compile()
1786 		if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {  in bpf_int_jit_compile()
1794 	header = bpf_jit_alloc(&jit, fp);  in bpf_int_jit_compile()
1800 	if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {  in bpf_int_jit_compile()
1806 		bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);  in bpf_int_jit_compile()
1807 		print_fn_code(jit.prg_buf, jit.size_prg);  in bpf_int_jit_compile()
1813 		jit_data->ctx = jit;  in bpf_int_jit_compile()
1816 	fp->bpf_func = (void *) jit.prg_buf;  in bpf_int_jit_compile()
1818 	fp->jited_len = jit.size;  in bpf_int_jit_compile()
1821 		bpf_prog_fill_jited_linfo(fp, jit.addrs + 1);  in bpf_int_jit_compile()
1823 		kvfree(jit.addrs);  in bpf_int_jit_compile()