123 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)  in emit_imm()  argument
144 			emit_lui(rd, upper, ctx);  in emit_imm()
147 			emit_li(rd, lower, ctx);  in emit_imm()
151 		emit_addiw(rd, rd, lower, ctx);  in emit_imm()
159 	emit_imm(rd, upper, ctx);  in emit_imm()
161 	emit_slli(rd, rd, shift, ctx);  in emit_imm()
163 		emit_addi(rd, rd, lower, ctx);  in emit_imm()
210 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,  in emit_bcc()  argument
215 		emit(rv_beq(rd, rs, rvoff >> 1), ctx);  in emit_bcc()
218 		emit(rv_bltu(rs, rd, rvoff >> 1), ctx);  in emit_bcc()
221 		emit(rv_bltu(rd, rs, rvoff >> 1), ctx);  in emit_bcc()
224 		emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);  in emit_bcc()
227 		emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);  in emit_bcc()
230 		emit(rv_bne(rd, rs, rvoff >> 1), ctx);  in emit_bcc()
233 		emit(rv_blt(rs, rd, rvoff >> 1), ctx);  in emit_bcc()
236 		emit(rv_blt(rd, rs, rvoff >> 1), ctx);  in emit_bcc()
239 		emit(rv_bge(rd, rs, rvoff >> 1), ctx);  in emit_bcc()
242 		emit(rv_bge(rs, rd, rvoff >> 1), ctx);  in emit_bcc()
246 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,  in emit_branch()  argument
252 		emit_bcc(cond, rd, rs, rvoff, ctx);  in emit_branch()
269 		emit_bcc(cond, rd, rs, 8, ctx);  in emit_branch()
280 	emit_bcc(cond, rd, rs, 12, ctx);  in emit_branch()
344 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,  in init_regs()  argument
356 		*rd = bpf_to_rv_reg(insn->dst_reg, ctx);  in init_regs()
365 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)  in emit_zext_32_rd_rs()  argument
367 	emit_mv(RV_REG_T2, *rd, ctx);  in emit_zext_32_rd_rs()
371 	*rd = RV_REG_T2;  in emit_zext_32_rd_rs()
375 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)  in emit_sext_32_rd_rs()  argument
377 	emit_addiw(RV_REG_T2, *rd, 0, ctx);  in emit_sext_32_rd_rs()
379 	*rd = RV_REG_T2;  in emit_sext_32_rd_rs()
383 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)  in emit_zext_32_rd_t1()  argument
385 	emit_mv(RV_REG_T2, *rd, ctx);  in emit_zext_32_rd_t1()
388 	*rd = RV_REG_T2;  in emit_zext_32_rd_t1()
391 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)  in emit_sext_32_rd()  argument
393 	emit_addiw(RV_REG_T2, *rd, 0, ctx);  in emit_sext_32_rd()
394 	*rd = RV_REG_T2;  in emit_sext_32_rd()
397 static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr,  in emit_jump_and_link()  argument
403 		emit(rv_jal(rd, rvoff >> 1), ctx);  in emit_jump_and_link()
409 		emit(rv_jalr(rd, RV_REG_T1, lower), ctx);  in emit_jump_and_link()
427 	u8 rd;  in emit_call()  local
438 	rd = bpf_to_rv_reg(BPF_REG_0, ctx);  in emit_call()
439 	emit_mv(rd, RV_REG_A0, ctx);  in emit_call()
450 	u8 rd = -1, rs = -1, code = insn->code;  in bpf_jit_emit_insn()  local
454 	init_regs(&rd, &rs, insn, ctx);  in bpf_jit_emit_insn()
462 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
465 		emit_mv(rd, rs, ctx);  in bpf_jit_emit_insn()
467 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
473 		emit_add(rd, rd, rs, ctx);  in bpf_jit_emit_insn()
475 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
480 			emit_sub(rd, rd, rs, ctx);  in bpf_jit_emit_insn()
482 			emit_subw(rd, rd, rs, ctx);  in bpf_jit_emit_insn()
485 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
489 		emit_and(rd, rd, rs, ctx);  in bpf_jit_emit_insn()
491 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
495 		emit_or(rd, rd, rs, ctx);  in bpf_jit_emit_insn()
497 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
501 		emit_xor(rd, rd, rs, ctx);  in bpf_jit_emit_insn()
503 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
507 		emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);  in bpf_jit_emit_insn()
509 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
513 		emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);  in bpf_jit_emit_insn()
515 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
519 		emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);  in bpf_jit_emit_insn()
521 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
525 		emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);  in bpf_jit_emit_insn()
527 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
531 		emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);  in bpf_jit_emit_insn()
533 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
537 		emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);  in bpf_jit_emit_insn()
539 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
545 		emit_sub(rd, RV_REG_ZERO, rd, ctx);  in bpf_jit_emit_insn()
547 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
554 			emit_slli(rd, rd, 48, ctx);  in bpf_jit_emit_insn()
555 			emit_srli(rd, rd, 48, ctx);  in bpf_jit_emit_insn()
559 				emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
570 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
573 		emit_srli(rd, rd, 8, ctx);  in bpf_jit_emit_insn()
577 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
580 		emit_srli(rd, rd, 8, ctx);  in bpf_jit_emit_insn()
582 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
585 		emit_srli(rd, rd, 8, ctx);  in bpf_jit_emit_insn()
589 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
592 		emit_srli(rd, rd, 8, ctx);  in bpf_jit_emit_insn()
594 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
597 		emit_srli(rd, rd, 8, ctx);  in bpf_jit_emit_insn()
599 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
602 		emit_srli(rd, rd, 8, ctx);  in bpf_jit_emit_insn()
604 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
607 		emit_srli(rd, rd, 8, ctx);  in bpf_jit_emit_insn()
609 		emit_andi(RV_REG_T1, rd, 0xff, ctx);  in bpf_jit_emit_insn()
612 		emit_mv(rd, RV_REG_T2, ctx);  in bpf_jit_emit_insn()
618 		emit_imm(rd, imm, ctx);  in bpf_jit_emit_insn()
620 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
627 			emit_addi(rd, rd, imm, ctx);  in bpf_jit_emit_insn()
630 			emit_add(rd, rd, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
633 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
638 			emit_addi(rd, rd, -imm, ctx);  in bpf_jit_emit_insn()
641 			emit_sub(rd, rd, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
644 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
649 			emit_andi(rd, rd, imm, ctx);  in bpf_jit_emit_insn()
652 			emit_and(rd, rd, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
655 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
660 			emit(rv_ori(rd, rd, imm), ctx);  in bpf_jit_emit_insn()
663 			emit_or(rd, rd, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
666 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
671 			emit(rv_xori(rd, rd, imm), ctx);  in bpf_jit_emit_insn()
674 			emit_xor(rd, rd, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
677 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
682 		emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :  in bpf_jit_emit_insn()
683 		     rv_mulw(rd, rd, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
685 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
690 		emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :  in bpf_jit_emit_insn()
691 		     rv_divuw(rd, rd, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
693 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
698 		emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :  in bpf_jit_emit_insn()
699 		     rv_remuw(rd, rd, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
701 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
705 		emit_slli(rd, rd, imm, ctx);  in bpf_jit_emit_insn()
708 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
713 			emit_srli(rd, rd, imm, ctx);  in bpf_jit_emit_insn()
715 			emit(rv_srliw(rd, rd, imm), ctx);  in bpf_jit_emit_insn()
718 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
723 			emit_srai(rd, rd, imm, ctx);  in bpf_jit_emit_insn()
725 			emit(rv_sraiw(rd, rd, imm), ctx);  in bpf_jit_emit_insn()
728 			emit_zext_32(rd, ctx);  in bpf_jit_emit_insn()
766 				emit_sext_32_rd_rs(&rd, &rs, ctx);  in bpf_jit_emit_insn()
768 				emit_zext_32_rd_rs(&rd, &rs, ctx);  in bpf_jit_emit_insn()
778 			emit_and(RV_REG_T1, rd, rs, ctx);  in bpf_jit_emit_insn()
782 			emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);  in bpf_jit_emit_insn()
818 				emit_sext_32_rd(&rd, ctx);  in bpf_jit_emit_insn()
820 				emit_zext_32_rd_t1(&rd, ctx);  in bpf_jit_emit_insn()
826 		emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);  in bpf_jit_emit_insn()
834 			emit_andi(RV_REG_T1, rd, imm, ctx);  in bpf_jit_emit_insn()
837 			emit_and(RV_REG_T1, rd, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
890 		emit_imm(rd, imm64, ctx);  in bpf_jit_emit_insn()
897 			emit(rv_lbu(rd, off, rs), ctx);  in bpf_jit_emit_insn()
903 		emit(rv_lbu(rd, 0, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
909 			emit(rv_lhu(rd, off, rs), ctx);  in bpf_jit_emit_insn()
915 		emit(rv_lhu(rd, 0, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
921 			emit(rv_lwu(rd, off, rs), ctx);  in bpf_jit_emit_insn()
927 		emit(rv_lwu(rd, 0, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
933 			emit_ld(rd, off, rs, ctx);  in bpf_jit_emit_insn()
939 		emit_ld(rd, 0, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
950 			emit(rv_sb(rd, off, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
955 		emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);  in bpf_jit_emit_insn()
962 			emit(rv_sh(rd, off, RV_REG_T1), ctx);  in bpf_jit_emit_insn()
967 		emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);  in bpf_jit_emit_insn()
973 			emit_sw(rd, off, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
978 		emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);  in bpf_jit_emit_insn()
984 			emit_sd(rd, off, RV_REG_T1, ctx);  in bpf_jit_emit_insn()
989 		emit_add(RV_REG_T2, RV_REG_T2, rd, ctx);  in bpf_jit_emit_insn()
996 			emit(rv_sb(rd, off, rs), ctx);  in bpf_jit_emit_insn()
1001 		emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);  in bpf_jit_emit_insn()
1006 			emit(rv_sh(rd, off, rs), ctx);  in bpf_jit_emit_insn()
1011 		emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);  in bpf_jit_emit_insn()
1016 			emit_sw(rd, off, rs, ctx);  in bpf_jit_emit_insn()
1021 		emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);  in bpf_jit_emit_insn()
1026 			emit_sd(rd, off, rs, ctx);  in bpf_jit_emit_insn()
1031 		emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);  in bpf_jit_emit_insn()
1040 				emit_addi(RV_REG_T1, rd, off, ctx);  in bpf_jit_emit_insn()
1043 				emit_add(RV_REG_T1, RV_REG_T1, rd, ctx);  in bpf_jit_emit_insn()
1046 			rd = RV_REG_T1;  in bpf_jit_emit_insn()
1050 		     rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) :  in bpf_jit_emit_insn()
1051 		     rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx);  in bpf_jit_emit_insn()