diff --git a/os/StarryOS/kernel/src/ebpf/bpf_insn.rs b/os/StarryOS/kernel/src/ebpf/bpf_insn.rs new file mode 100644 index 0000000000..74c00238a1 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/bpf_insn.rs @@ -0,0 +1,86 @@ +pub(crate) const BPF_LD: u8 = 0x00; +pub(crate) const BPF_LDX: u8 = 0x01; +pub(crate) const BPF_ST: u8 = 0x02; +pub(crate) const BPF_STX: u8 = 0x03; +pub(crate) const BPF_ALU: u8 = 0x04; +pub(crate) const BPF_JMP: u8 = 0x05; +pub(crate) const BPF_JMP32: u8 = 0x06; +pub(crate) const BPF_ALU64: u8 = 0x07; + +pub(crate) const BPF_W: u8 = 0x00; +pub(crate) const BPF_H: u8 = 0x08; +pub(crate) const BPF_B: u8 = 0x10; +pub(crate) const BPF_DW: u8 = 0x18; + +pub(crate) const BPF_IMM: u8 = 0x00; +pub(crate) const BPF_MEM: u8 = 0x60; + +pub(crate) const BPF_ADD: u8 = 0x00; +pub(crate) const BPF_SUB: u8 = 0x10; +pub(crate) const BPF_MUL: u8 = 0x20; +pub(crate) const BPF_DIV: u8 = 0x30; +pub(crate) const BPF_OR: u8 = 0x40; +pub(crate) const BPF_AND: u8 = 0x50; +pub(crate) const BPF_LSH: u8 = 0x60; +pub(crate) const BPF_RSH: u8 = 0x70; +pub(crate) const BPF_NEG: u8 = 0x80; +pub(crate) const BPF_MOD: u8 = 0x90; +pub(crate) const BPF_XOR: u8 = 0xa0; +pub(crate) const BPF_MOV: u8 = 0xb0; +pub(crate) const BPF_ARSH: u8 = 0xc0; +pub(crate) const BPF_END: u8 = 0xd0; + +pub(crate) const BPF_JA: u8 = 0x00; +pub(crate) const BPF_EXIT: u8 = 0x90; +pub(crate) const BPF_JEQ: u8 = 0x10; +pub(crate) const BPF_JGT: u8 = 0x20; +pub(crate) const BPF_JGE: u8 = 0x30; +pub(crate) const BPF_JSET: u8 = 0x40; +pub(crate) const BPF_JNE: u8 = 0x50; +pub(crate) const BPF_JSGT: u8 = 0x60; +pub(crate) const BPF_JSGE: u8 = 0x70; +pub(crate) const BPF_JLT: u8 = 0xa0; +pub(crate) const BPF_JLE: u8 = 0xb0; +pub(crate) const BPF_JSLT: u8 = 0xc0; +pub(crate) const BPF_JSLE: u8 = 0xd0; + +pub(crate) const BPF_X: u8 = 0x08; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct BpfInsn { + pub code: u8, + pub dst_src_reg: u8, + pub off: i16, + pub imm: i32, +} + +impl BpfInsn { + pub fn dst_reg(&self) -> u8 { + self.dst_src_reg & 0xf + } + + pub fn src_reg(&self) -> u8 { + (self.dst_src_reg >> 4) & 0xf + } + + pub fn class(&self) -> u8 { + self.code & 0x07 + } + + pub fn size(&self) -> u8 { + self.code & 0x18 + } + + pub fn mode(&self) -> u8 { + self.code & 0xe0 + } + + pub fn alu_op(&self) -> u8 { + self.code & 0xf0 + } + + pub fn is_ld_dw_imm(&self) -> bool { + self.code == (BPF_LD | BPF_IMM | BPF_DW) + } +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_aarch64.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_aarch64.rs new file mode 100644 index 0000000000..94a9cdd917 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_aarch64.rs @@ -0,0 +1,1102 @@ +use super::{ + HelperFn, JitBackend, JitBuffer, + bpf_insn::{ + BPF_ADD, BPF_ALU, BPF_ALU64, BPF_AND, BPF_ARSH, BPF_B, BPF_DIV, BPF_DW, BPF_END, BPF_EXIT, + BPF_H, BPF_JA, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, BPF_JNE, + BPF_JSET, BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_LD, BPF_LDX, BPF_LSH, BPF_MEM, + BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, BPF_OR, BPF_RSH, BPF_ST, BPF_STX, BPF_SUB, BPF_W, + BPF_X, BPF_XOR, BpfInsn, + }, +}; + +// ========================================================================== +// AArch64 register mapping +// ========================================================================== + +// BPF R0 = x0 (return value / helper arg 1) +// BPF R1 = x1 (context pointer / helper arg 2) +// BPF R2 = x2 (helper arg 3) +// BPF R3 = x3 (helper arg 4) +// BPF R4 = x4 (helper arg 5) +// BPF R5 = x5 (caller-saved) +// BPF R6 = x19 (callee-saved) +// BPF R7 = x20 (callee-saved) +// BPF R8 = x21 (callee-saved) +// BPF R9 = x22 (callee-saved) +// BPF R10 = x25 (frame pointer base) +// +// Temps: x6, x7, x9, x10, x11, x12, x16, x17 +// x8 = indirect result (not used) +// x29 = FP (callee-saved, saved for stack walking) +// x30 = LR (saved for return) +// xzr/sp = 31 (zero or stack pointer depending on context) + +const A64_X0: u32 = 0; +const A64_X1: u32 = 1; +const A64_X2: u32 = 2; +const A64_X3: u32 = 3; +const A64_X4: u32 = 4; +const A64_X5: u32 = 5; +const A64_X6: u32 = 6; +const A64_X7: u32 = 7; +const A64_X8: u32 = 8; +const A64_X9: u32 = 9; +const A64_X10: u32 = 10; +const A64_X11: u32 = 11; +const A64_X12: u32 = 12; +const A64_X16: u32 = 16; +const A64_X17: u32 = 17; +const A64_X19: u32 = 19; +const A64_X20: u32 = 20; +const A64_X21: u32 = 21; +const A64_X22: u32 = 22; +const A64_X25: u32 = 25; +const A64_X29: u32 = 29; +const A64_X30: u32 = 30; +// x31 is SP or XZR depending on instruction context +const A64_SP: u32 = 31; +const A64_XZR: u32 = 31; + +const BPF_STACK_SIZE: usize = 512; +/// Saved registers: x19-x22, x25 (5 regs) + x29, x30 = 7 regs, padded to 8 +const CALLEE_SAVED_SIZE: usize = 64; // 8 registers * 8 bytes +const FRAME_SIZE: usize = BPF_STACK_SIZE + CALLEE_SAVED_SIZE; + +fn bpf_to_a64(r: u8) -> u32 { + match r { + 0 => A64_X0, + 1 => A64_X1, + 2 => A64_X2, + 3 => A64_X3, + 4 => A64_X4, + 5 => A64_X5, + 6 => A64_X19, + 7 => A64_X20, + 8 => A64_X21, + 9 => A64_X22, + 10 => A64_X25, + _ => A64_XZR, + } +} + +// ========================================================================== +// AArch64 instruction encoding helpers +// ========================================================================== + +/// ADD (shifted register): Xd = Xn + Xm +fn a64_add(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | op=0 | S=0 | 01011 | shift=00 | 0 | Rm | imm6=000000 | Rn | Rd + buf.emit_u32(0x8B00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// ADD (shifted register, 32-bit): Wd = Wn + Wm +fn a64_addw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=0 | op=0 | S=0 | 01011 | shift=00 | 0 | Rm | imm6=000000 | Rn | Rd + buf.emit_u32(0x0B00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// SUB (shifted register): Xd = Xn - Xm +fn a64_sub(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | op=1 | S=0 | 01011 | shift=00 | 0 | Rm | imm6=000000 | Rn | Rd + buf.emit_u32(0xCB00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// SUB (shifted register, 32-bit): Wd = Wn - Wm +fn a64_subw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=0 | op=1 | S=0 | 01011 | shift=00 | 0 | Rm | imm6=000000 | Rn | Rd + buf.emit_u32(0x4B00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// AND (shifted register): Xd = Xn & Xm +fn a64_and(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=00 | 01010 | shift=00 | N=0 | Rm | imm6=000000 | Rn | Rd + buf.emit_u32(0x8A00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// AND (shifted register, 32-bit): Wd = Wn & Wm +fn a64_andw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x0A00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// ORR (shifted register): Xd = Xn | Xm +fn a64_orr(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=01 | 01010 | shift=00 | N=0 | Rm | imm6=000000 | Rn | Rd + buf.emit_u32(0xAA00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// ORR (shifted register, 32-bit): Wd = Wn | Wm +fn a64_orrw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x2A00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// EOR (shifted register): Xd = Xn ^ Xm +fn a64_eor(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=10 | 01010 | shift=00 | N=0 | Rm | imm6=000000 | Rn | Rd + buf.emit_u32(0xCA00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// EOR (shifted register, 32-bit): Wd = Wn ^ Wm +fn a64_eorw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x4A00_0000 | (rm << 16) | (rn << 5) | rd); +} + +/// ADD (immediate): Xd = Xn + #imm12 +fn a64_addi(buf: &mut JitBuffer, rd: u32, rn: u32, imm12: u32) { + // sf=1 | op=0 | S=0 | 100010 | sh=0 | imm12 | Rn | Rd + buf.emit_u32(0x9100_0000 | ((imm12 & 0xFFF) << 10) | (rn << 5) | rd); +} + +/// ADD (immediate, 32-bit): Wd = Wn + #imm12 +fn a64_addiw(buf: &mut JitBuffer, rd: u32, rn: u32, imm12: u32) { + buf.emit_u32(0x1100_0000 | ((imm12 & 0xFFF) << 10) | (rn << 5) | rd); +} + +/// SUB (immediate): Xd = Xn - #imm12 +fn a64_subi(buf: &mut JitBuffer, rd: u32, rn: u32, imm12: u32) { + // sf=1 | op=1 | S=0 | 100010 | sh=0 | imm12 | Rn | Rd + buf.emit_u32(0xD100_0000 | ((imm12 & 0xFFF) << 10) | (rn << 5) | rd); +} + +/// SUB (immediate, 32-bit): Wd = Wn - #imm12 +fn a64_subiw(buf: &mut JitBuffer, rd: u32, rn: u32, imm12: u32) { + buf.emit_u32(0x5100_0000 | ((imm12 & 0xFFF) << 10) | (rn << 5) | rd); +} + +/// MADD: Xd = Xa + Xn * Xm +fn a64_madd(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32, ra: u32) { + // sf=1 | op31=00 | 11000 | 000 | Rm | 0 | Ra | Rn | Rd + buf.emit_u32(0x9B00_0000 | (rm << 16) | (ra << 10) | (rn << 5) | rd); +} + +/// MADD (32-bit): Wd = Wa + Wn * Wm +fn a64_maddw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32, ra: u32) { + // sf=0 | op31=00 | 11000 | 000 | Rm | 0 | Ra | Rn | Rd + buf.emit_u32(0x1B00_0000 | (rm << 16) | (ra << 10) | (rn << 5) | rd); +} + +/// MUL: Xd = Xn * Xm (alias of MADD with XZR) +fn a64_mul(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + a64_madd(buf, rd, rn, rm, A64_XZR); +} + +/// MUL (32-bit): Wd = Wn * Wm +fn a64_mulw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + a64_maddw(buf, rd, rn, rm, A64_XZR); +} + +/// SDIV: Xd = Xn / Xm (signed) +fn a64_sdiv(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=11 | 11010110 | Rm | 000011 | Rn | Rd + buf.emit_u32(0x9AC0_0C00 | (rm << 16) | (rn << 5) | rd); +} + +/// UDIV: Xd = Xn / Xm (unsigned) +fn a64_udiv(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=10 | 11010110 | Rm | 000010 | Rn | Rd + buf.emit_u32(0x9AC0_0800 | (rm << 16) | (rn << 5) | rd); +} + +/// UDIV (32-bit, unsigned) +fn a64_udivw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=0 + buf.emit_u32(0x1AC0_0800 | (rm << 16) | (rn << 5) | rd); +} + +/// UBFM (used for LSL/LSR/zero-extend): Xd = Xn[imms:immr] +/// UBFM has sf | 1|0 | 100110 | N(1) | immr(6) | imms(6) | Rn | Rd +/// sf=1: 64-bit, sf=0: 32-bit, N=sf +fn a64_ubfm(buf: &mut JitBuffer, sf: u32, rd: u32, rn: u32, immr: u32, imms: u32) { + let enc = (sf << 31) + | (0b10 << 29) + | (0b100110 << 23) + | (sf << 22) + | (immr << 16) + | (imms << 10) + | (rn << 5) + | rd; + buf.emit_u32(enc); +} + +/// SBFM (used for ASR/sign-extend) +/// sf | 0|0 | 100110 | N(sf) | immr | imms | Rn | Rd +fn a64_sbfm(buf: &mut JitBuffer, sf: u32, rd: u32, rn: u32, immr: u32, imms: u32) { + let enc = (sf << 31) + | (0b00 << 29) + | (0b100110 << 23) + | (sf << 22) + | (immr << 16) + | (imms << 10) + | (rn << 5) + | rd; + buf.emit_u32(enc); +} + +/// LSL (immediate): Xd = Xn << sh (UBFM alias) +fn a64_lsl(buf: &mut JitBuffer, rd: u32, rn: u32, sh: u32) { + let immr = ((-(sh as i32)) & 0x3F) as u32; + let imms = 63 - sh; + a64_ubfm(buf, 1, rd, rn, immr, imms); +} + +/// LSL (immediate, 32-bit) +fn a64_lslw(buf: &mut JitBuffer, rd: u32, rn: u32, sh: u32) { + let immr = ((-(sh as i32)) & 0x1F) as u32; + let imms = 31 - sh; + a64_ubfm(buf, 0, rd, rn, immr, imms); +} + +/// LSR (immediate): Xd = Xn >> sh (UBFM alias) +fn a64_lsr(buf: &mut JitBuffer, rd: u32, rn: u32, sh: u32) { + a64_ubfm(buf, 1, rd, rn, sh, 63); +} + +/// LSR (immediate, 32-bit) +fn a64_lsrw(buf: &mut JitBuffer, rd: u32, rn: u32, sh: u32) { + a64_ubfm(buf, 0, rd, rn, sh, 31); +} + +/// ASR (immediate): Xd = Xn >>> sh (SBFM alias) +fn a64_asr(buf: &mut JitBuffer, rd: u32, rn: u32, sh: u32) { + a64_sbfm(buf, 1, rd, rn, sh, 63); +} + +/// ASR (immediate, 32-bit) +fn a64_asrw(buf: &mut JitBuffer, rd: u32, rn: u32, sh: u32) { + a64_sbfm(buf, 0, rd, rn, sh, 31); +} + +/// LSL (register): Xd = Xn << Xm +fn a64_lslv(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=10 | 11010110 | Rm | 0010 00 | Rn | Rd + buf.emit_u32(0x9AC0_2000 | (rm << 16) | (rn << 5) | rd); +} + +/// LSL (register, 32-bit) +fn a64_lslvw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=0 + buf.emit_u32(0x1AC0_2000 | (rm << 16) | (rn << 5) | rd); +} + +/// LSR (register): Xd = Xn >> Xm +fn a64_lsrv(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=01 | 11010110 | Rm | 0010 01 | Rn | Rd + buf.emit_u32(0x9AC0_2400 | (rm << 16) | (rn << 5) | rd); +} + +/// LSR (register, 32-bit) +fn a64_lsrvw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x1AC0_2400 | (rm << 16) | (rn << 5) | rd); +} + +/// ASR (register): Xd = Xn >>> Xm +fn a64_asrv(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + // sf=1 | opc=00 | 11010110 | Rm | 0010 10 | Rn | Rd + buf.emit_u32(0x9AC0_2800 | (rm << 16) | (rn << 5) | rd); +} + +/// ASR (register, 32-bit) +fn a64_asrvw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x1AC0_2800 | (rm << 16) | (rn << 5) | rd); +} + +/// MOV (register): Xd = Xn (alias of ORR with XZR) +fn a64_mov(buf: &mut JitBuffer, rd: u32, rn: u32) { + a64_orr(buf, rd, A64_XZR, rn); +} + +/// MOV (register, 32-bit): Wd = Wn +fn a64_movw(buf: &mut JitBuffer, rd: u32, rn: u32) { + a64_orrw(buf, rd, A64_XZR, rn); +} + +/// REV: Xd = ByteReverse(Xn) — reverse byte order in 64-bit register +fn a64_rev64(buf: &mut JitBuffer, rd: u32, rn: u32) { + // dp_1src: sf=1, opcode2=00000, opcode=000011, field=0000 + buf.emit_u32(0xDAC0_0C00 | (rn << 5) | rd); +} + +/// REV32: Wd = ByteReverse(Wn) — reverse byte order in 32-bit word +fn a64_rev32(buf: &mut JitBuffer, rd: u32, rn: u32) { + // dp_1src: sf=0, opcode2=00000, opcode=000010, field=0000 + buf.emit_u32(0x5AC0_0800 | (rn << 5) | rd); +} + +/// REV16: Wd = ReverseHalfwords(Wn) — reverse bytes in each 16-bit halfword +fn a64_rev16(buf: &mut JitBuffer, rd: u32, rn: u32) { + // dp_1src: sf=0, opcode2=00000, opcode=000001, field=0000 + buf.emit_u32(0x5AC0_0400 | (rn << 5) | rd); +} + +/// MOVZ: Xd = imm16 << (hw * 16), zeroing other bits +fn a64_movz(buf: &mut JitBuffer, rd: u32, imm16: u32, hw: u32) { + // sf=1 | 0 | 0 | 100101 | hw(2) | imm16(16) | Rd(5) + let enc = (1 << 31) | (0b100101 << 23) | ((hw & 3) << 21) | ((imm16 & 0xFFFF) << 5) | rd; + buf.emit_u32(enc); +} + +/// MOVK: Xd[hw*16+15:hw*16] = imm16, preserving other bits +fn a64_movk(buf: &mut JitBuffer, rd: u32, imm16: u32, hw: u32) { + // sf=1 | 1 | 1 | 100101 | hw(2) | imm16(16) | Rd(5) + let enc = (1 << 31) + | (0b11 << 29) + | (0b100101 << 23) + | ((hw & 3) << 21) + | ((imm16 & 0xFFFF) << 5) + | rd; + buf.emit_u32(enc); +} + +/// LDR (64-bit): Xt = [Xn + #offset] (offset is scaled by 8, no scaling in our encoding) +fn a64_ldr(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + // [31:30] 11 | [29:28] 01 | [27:24] 1101 | [23:22] depends on variant + // For unsigned offset: 11 01 1101 0 1 0 imm12(12) Rn(5) Rt(5) + let imm12 = off as u32; + buf.emit_u32(0xF940_0000 | ((imm12 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// LDR (32-bit, zero-extending): Wt = [Xn + #offset] +fn a64_ldrw(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + // 10 01 1101 0 1 0 imm12(12) Rn(5) Rt(5) + buf.emit_u32(0xB940_0000 | ((off as u32 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// LDRH (16-bit, zero-extending): Wt = [Xn + #offset] +fn a64_ldrh(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + // 01 01 1101 0 1 0 imm12(12) Rn(5) Rt(5) (imm12 is byte offset) + buf.emit_u32(0x7940_0000 | ((off as u32 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// LDRB (8-bit, zero-extending): Wt = [Xn + #offset] +fn a64_ldrb(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + // 00 01 1101 0 1 0 imm12(12) Rn(5) Rt(5) + buf.emit_u32(0x3940_0000 | ((off as u32 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// STR (64-bit): [Xn + #offset] = Xt +fn a64_str(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + // 11 01 1101 0 0 0 imm12(12) Rn(5) Rt(5) + buf.emit_u32(0xF900_0000 | ((off as u32 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// STR (32-bit): [Xn + #offset] = Wt +fn a64_strw(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + buf.emit_u32(0xB900_0000 | ((off as u32 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// STRH (16-bit): [Xn + #offset] = Wt +fn a64_strh(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + buf.emit_u32(0x7900_0000 | ((off as u32 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// STRB (8-bit): [Xn + #offset] = Wt +fn a64_strb(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + buf.emit_u32(0x3900_0000 | ((off as u32 & 0xFFF) << 10) | (rn << 5) | rt); +} + +/// STP (store pair): [Xn + #imm] = Xt1, [Xn + #imm + 8] = Xt2 +/// imm must be in range [-512, 504] and 8-byte aligned +fn a64_stp_pre(buf: &mut JitBuffer, rt1: u32, rt2: u32, rn: u32, imm: i32) { + // 10 1 0 100 0 1 | imm7(7) | Rt2(5) | Rn(5) | Rt1(5) + // imm7 is imm / 8, signed + let imm7 = ((imm / 8) & 0x7F) as u32; + buf.emit_u32(0xA980_0000 | (imm7 << 15) | (rt2 << 10) | (rn << 5) | rt1); +} + +/// LDP (load pair, post-index): Xt1, Xt2 = [Xn], Xn += imm +fn a64_ldp_post(buf: &mut JitBuffer, rt1: u32, rt2: u32, rn: u32, imm: i32) { + // 10 1 0 100 0 1 | imm7(7) | Rt2(5) | Rn(5) | Rt1(5) + // post-index variant: 10 1 0 100 0 0 | imm7 | Rt2 | Rn | Rt1 + let imm7 = ((imm / 8) & 0x7F) as u32; + buf.emit_u32(0xA8C0_0000 | (imm7 << 15) | (rt2 << 10) | (rn << 5) | rt1); +} + +/// ADR: Xd = PC + imm (imm is a signed 21-bit value) +fn a64_adr(buf: &mut JitBuffer, rd: u32, imm: i32) { + let imm = imm as u32; + let immlo = imm & 3; + let immhi = (imm >> 2) & 0x7FFFF; + // 0 | immhi(19) | 1 0000 | immlo(2) | Rd(5) + buf.emit_u32((immhi << 5) | (immlo << 29) | (0x10 << 24) | rd); +} + +/// B (unconditional branch): PC += imm26 * 4 +fn a64_b(buf: &mut JitBuffer, imm: i32) { + let imm26 = ((imm as u32) & 0x03FF_FFFF) >> 2; + buf.emit_u32(0x1400_0000 | imm26); +} + +/// B.cond: if condition, PC += imm19 * 4 +fn a64_bcond(buf: &mut JitBuffer, cond: u32, imm: i32) { + let imm19 = ((imm as u32) & 0x7FFFF) >> 2; + buf.emit_u32(0x5400_0000 | (imm19 << 5) | cond); +} + +/// RET: return to address in X30 (LR) +fn a64_ret(buf: &mut JitBuffer) { + // 1101011 0010 11111 000000 | 11110 | 00000 + buf.emit_u32(0xD65F_03C0); +} + +/// BR: unconditional branch to register Xn +fn a64_br(buf: &mut JitBuffer, rn: u32) { + // 1101011 0000 11111 000000 | Rn | 00000 + buf.emit_u32(0xD61F_0000 | (rn << 5)); +} + +/// BLR: call function at Xn, LR = return address +fn a64_blr(buf: &mut JitBuffer, rn: u32) { + // 1101011 0001 11111 000000 | Rn | 00000 + buf.emit_u32(0xD63F_0000 | (rn << 5)); +} + +/// NOP +fn a64_nop(buf: &mut JitBuffer) { + buf.emit_u32(0xD503_201F); +} + +/// TST: Xn & Xm, set flags (ANDS with XZR) +fn a64_tst(buf: &mut JitBuffer, rn: u32, rm: u32) { + // sf=1 | opc=11 | 01010 | shift=00 | N=0 | Rm | imm6=000000 | Rn | XZR + buf.emit_u32(0xEA00_0000 | (rm << 16) | (rn << 5) | A64_XZR); +} + +/// TST (32-bit) +fn a64_tstw(buf: &mut JitBuffer, rn: u32, rm: u32) { + buf.emit_u32(0x6A00_0000 | (rm << 16) | (rn << 5) | A64_XZR); +} + +/// CMP: Xn - Xm, set flags (SUBS with XZR) +fn a64_cmp(buf: &mut JitBuffer, rn: u32, rm: u32) { + // SUBS XZR, Xn, Xm + // sf=1 | op=1 | S=1 | 01011 | shift=00 | 0 | Rm | imm6=000000 | Rn | XZR + buf.emit_u32(0xEB00_0000 | (rm << 16) | (rn << 5) | A64_XZR); +} + +/// CMP (32-bit) +fn a64_cmpw(buf: &mut JitBuffer, rn: u32, rm: u32) { + buf.emit_u32(0x6B00_0000 | (rm << 16) | (rn << 5) | A64_XZR); +} + +// ========================================================================== +// Condition codes +// ========================================================================== +const COND_EQ: u32 = 0b0000; +const COND_NE: u32 = 0b0001; +const COND_HS: u32 = 0b0010; // unsigned >= +const COND_LO: u32 = 0b0011; // unsigned < +const COND_MI: u32 = 0b0100; +const COND_PL: u32 = 0b0101; +const COND_VS: u32 = 0b0110; +const COND_VC: u32 = 0b0111; +const COND_HI: u32 = 0b1000; // unsigned > +const COND_LS: u32 = 0b1001; // unsigned <= +const COND_GE: u32 = 0b1010; // signed >= +const COND_LT: u32 = 0b1011; // signed < +const COND_GT: u32 = 0b1100; // signed > +const COND_LE: u32 = 0b1101; // signed <= + +// ========================================================================== +// Higher-level codegen helpers +// ========================================================================== + +/// Load a 64-bit immediate into a register using up to 4 MOVZ/MOVK instructions +fn emit_load_imm64(buf: &mut JitBuffer, rd: u32, val: u64) { + // Handle small values inline + if val == 0 { + a64_mov(buf, rd, A64_XZR); + return; + } + if val <= 0xFFFF { + a64_movz(buf, rd, val as u32, 0); + return; + } + let mut first = true; + for hw in 0..4u32 { + let chunk = ((val >> (hw * 16)) & 0xFFFF) as u32; + if first { + a64_movz(buf, rd, chunk, hw); + first = false; + } else if chunk != 0 { + a64_movk(buf, rd, chunk, hw); + } + } +} + +/// Load a 32-bit signed immediate into a register +fn emit_load_imm32(buf: &mut JitBuffer, rd: u32, val: i32) { + if val == 0 { + a64_movw(buf, rd, A64_XZR); + return; + } + emit_load_imm64(buf, rd, val as u64); +} + +/// Load a 64-bit immediate with NOP padding to 24 bytes (6 instructions) +fn emit_load_imm64_padded(buf: &mut JitBuffer, rd: u32, val: u64) { + let start = buf.offset(); + emit_load_imm64(buf, rd, val); + let emitted = buf.offset() - start; + // Pad to 24 bytes (6 instructions) + let pad = if emitted < 24 { (24 - emitted) / 4 } else { 0 }; + for _ in 0..pad { + a64_nop(buf); + } +} + +/// Compute rd = rn + off, handling the case where off doesn't fit in 12-bit immediate +fn emit_add_offset(buf: &mut JitBuffer, rd: u32, rn: u32, off: i32) { + if off >= 0 && (off as u32) < 4096 { + a64_addi(buf, rd, rn, off as u32); + } else if off < 0 && off > -4096 { + a64_subi(buf, rd, rn, (-off) as u32); + } else { + emit_load_imm64(buf, A64_X6, off as u64); + a64_add(buf, rd, rn, A64_X6); + } +} + +/// Patch a 32-bit value at a given offset in the buffer. +/// Does nothing in counting (sizing) mode. +unsafe fn patch_u32(buf: &JitBuffer, offset: usize, val: u32) { + if buf.counting() { + return; + } + let ptr = buf.entry().add(offset) as *mut u32; + *ptr = val.to_le(); +} + +// ========================================================================== +// JIT Backend implementation +// ========================================================================== + +pub(crate) struct Aarch64Backend; + +impl JitBackend for Aarch64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + // Save frame pointer and link register + a64_stp_pre(buf, A64_X29, A64_X30, A64_SP, -16); + // Save callee-saved BPF registers: x19-x22, x25 + a64_stp_pre(buf, A64_X19, A64_X20, A64_SP, -16); + a64_stp_pre(buf, A64_X21, A64_X22, A64_SP, -16); + a64_stp_pre(buf, A64_X25, A64_XZR, A64_SP, -16); + // Allocate BPF stack space + a64_subi(buf, A64_SP, A64_SP, BPF_STACK_SIZE as u32); + // x25 = frame pointer base = SP + BPF_STACK_SIZE + 64 + a64_addi( + buf, + A64_X25, + A64_SP, + (BPF_STACK_SIZE + CALLEE_SAVED_SIZE) as u32, + ); + // Move context pointer: x0 → x1 (BPF R1), x0 (BPF R0) = 0 + a64_mov(buf, A64_X1, A64_X0); + a64_mov(buf, A64_X0, A64_XZR); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + // Deallocate BPF stack + a64_addi(buf, A64_SP, A64_SP, BPF_STACK_SIZE as u32); + // Restore callee-saved registers + a64_ldp_post(buf, A64_X25, A64_XZR, A64_SP, 16); + a64_ldp_post(buf, A64_X21, A64_X22, A64_SP, 16); + a64_ldp_post(buf, A64_X19, A64_X20, A64_SP, 16); + a64_ldp_post(buf, A64_X29, A64_X30, A64_SP, 16); + // Return + a64_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_a64(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { + A64_X6 + } else { + bpf_to_a64(insn.src_reg()) + }; + if use_imm { + if is_64 { + emit_load_imm64(buf, A64_X6, insn.imm as u64); + } else { + emit_load_imm32(buf, A64_X6, insn.imm); + } + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + a64_add(buf, dst, dst, src); + } else { + a64_addw(buf, dst, dst, src); + } + } + BPF_SUB => { + if is_64 { + a64_sub(buf, dst, dst, src); + } else { + a64_subw(buf, dst, dst, src); + } + } + BPF_MUL => { + if is_64 { + a64_mul(buf, dst, dst, src); + } else { + a64_mulw(buf, dst, dst, src); + } + } + BPF_DIV => { + // Division by zero: set result to 0 + let skip = buf.offset(); + a64_cmp(buf, src, A64_XZR); + a64_bcond(buf, COND_EQ, 0); // patched below + if is_64 { + a64_udiv(buf, dst, dst, src); + } else { + a64_udivw(buf, dst, dst, src); + } + let end_div = buf.offset(); + a64_b(buf, 8); + // Zero result (skip target) + a64_mov(buf, dst, A64_XZR); + let end_zero = buf.offset(); + unsafe { + // Patch the B.EQ: offset from skip to end_div (skip the branch) + let beq_off = (end_div - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((beq_off >> 2) << 5) | COND_EQ); + } + assert_eq!(end_zero - end_div, 8); + } + BPF_OR => { + if is_64 { + a64_orr(buf, dst, dst, src); + } else { + a64_orrw(buf, dst, dst, src); + } + } + BPF_AND => { + if is_64 { + a64_and(buf, dst, dst, src); + } else { + a64_andw(buf, dst, dst, src); + } + } + BPF_LSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + a64_lsl(buf, dst, dst, shamt); + } else { + a64_lslw(buf, dst, dst, shamt); + } + } else if is_64 { + a64_lslv(buf, dst, dst, src); + } else { + a64_lslvw(buf, dst, dst, src); + } + } + BPF_RSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + a64_lsr(buf, dst, dst, shamt); + } else { + a64_lsrw(buf, dst, dst, shamt); + } + } else if is_64 { + a64_lsrv(buf, dst, dst, src); + } else { + a64_lsrvw(buf, dst, dst, src); + } + } + BPF_NEG => { + if is_64 { + a64_sub(buf, dst, A64_XZR, dst); + } else { + a64_subw(buf, dst, A64_XZR, dst); + } + } + BPF_MOD => { + // Division by zero: result = dst (unchanged in BPF for MOD) + let skip = buf.offset(); + a64_cmp(buf, src, A64_XZR); + a64_bcond(buf, COND_EQ, 0); // patched below + // UDIV temp = dst / src; MSUB dst = dst - temp * src + if is_64 { + a64_udiv(buf, A64_X7, dst, src); // X7 = dst / src + a64_madd(buf, A64_X7, A64_X7, src, A64_XZR); // X7 = X7 * src + a64_sub(buf, dst, dst, A64_X7); // dst = dst - (dst/src)*src + } else { + a64_udivw(buf, A64_X7, dst, src); + a64_maddw(buf, A64_X7, A64_X7, src, A64_XZR); + a64_subw(buf, dst, dst, A64_X7); + } + let end = buf.offset(); + unsafe { + let beq_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((beq_off >> 2) << 5) | COND_EQ); + } + } + BPF_XOR => { + if is_64 { + a64_eor(buf, dst, dst, src); + } else { + a64_eorw(buf, dst, dst, src); + } + } + BPF_MOV => { + if is_64 { + a64_mov(buf, dst, src); + } else { + a64_movw(buf, dst, src); + } + } + BPF_ARSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + a64_asr(buf, dst, dst, shamt); + } else { + a64_asrw(buf, dst, dst, shamt); + } + } else if is_64 { + a64_asrv(buf, dst, dst, src); + } else { + a64_asrvw(buf, dst, dst, src); + } + } + BPF_END => { + // BPF_TO_BE: byte swap to big-endian (AArch64 is little-endian native) + let to_be = (insn.code & BPF_X) != 0; + match (to_be, insn.imm) { + (true, 16) => { + // 16-bit: REV16 Wd, Wn — swap bytes in each 16-bit halfword + a64_rev16(buf, dst, dst); + } + (true, 32) => { + // 32-bit: REV32 Wd, Wn — swap bytes in 32-bit word + a64_rev32(buf, dst, dst); + } + (true, 64) => { + // 64-bit: REV Xd, Xn — swap all 8 bytes + a64_rev64(buf, dst, dst); + } + // BPF_TO_LE on AArch64: no-op (native little-endian) + _ => {} + } + } + _ => {} + } + + // For 32-bit ALU ops (except ARSH, MOV), result is already zero-extended + // by the W-form instruction. No explicit zext needed on AArch64. + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + // Unconditional jump (BPF_JA) + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let target_offset = offsets[target_pc] as isize - buf.offset() as isize; + // Load target offset and branch + emit_load_imm64_padded(buf, A64_X16, target_offset as u64); + // ADR x17, . ; ADD x17, x17, x16 ; BR x17 + a64_adr(buf, A64_X17, 0); + a64_add(buf, A64_X17, A64_X17, A64_X16); + a64_br(buf, A64_X17); + } + return; + } + + // BPF_CALL is handled in compile() + if op == 0x80 { + return; + } + + let dst = bpf_to_a64(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src_reg = if use_imm { + A64_X6 + } else { + bpf_to_a64(insn.src_reg()) + }; + + if use_imm { + if is_64 { + emit_load_imm64(buf, A64_X6, insn.imm as u64); + } else { + emit_load_imm32(buf, A64_X6, insn.imm); + } + } + + // Compare and set flags + if is_64 { + a64_cmp(buf, dst, src_reg); + } else { + a64_cmpw(buf, dst, src_reg); + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + + // Pattern: conditional branch to skip the jump, then jump to target + fn emit_jump_to_target(buf: &mut JitBuffer, offsets: &[usize], target_pc: usize) { + let target_offset = offsets[target_pc] as isize - buf.offset() as isize; + emit_load_imm64_padded(buf, A64_X16, target_offset as u64); + a64_adr(buf, A64_X17, 0); + a64_add(buf, A64_X17, A64_X17, A64_X16); + a64_br(buf, A64_X17); + } + + match op { + BPF_JEQ => { + // Jump to target if dst == src → skip jump if dst != src + let skip = buf.offset(); + a64_bcond(buf, COND_NE, 0); // patched + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let bne_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((bne_off >> 2) << 5) | COND_NE); + } + } + BPF_JGT => { + // Jump if dst > src (unsigned) → skip jump if dst <= src + let skip = buf.offset(); + a64_bcond(buf, COND_LS, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let bls_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((bls_off >> 2) << 5) | COND_LS); + } + } + BPF_JGE => { + // Jump if dst >= src (unsigned) → skip jump if dst < src + let skip = buf.offset(); + a64_bcond(buf, COND_LO, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let blo_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((blo_off >> 2) << 5) | COND_LO); + } + } + BPF_JSET => { + // Jump if dst & src != 0 → skip jump if dst & src == 0 + if is_64 { + a64_tst(buf, dst, src_reg); + } else { + a64_tstw(buf, dst, src_reg); + } + let skip = buf.offset(); + a64_bcond(buf, COND_EQ, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let beq_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((beq_off >> 2) << 5) | COND_EQ); + } + } + BPF_JNE => { + let skip = buf.offset(); + a64_bcond(buf, COND_EQ, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let beq_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((beq_off >> 2) << 5) | COND_EQ); + } + } + BPF_JSGT => { + // Signed > + let skip = buf.offset(); + a64_bcond(buf, COND_LE, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let ble_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((ble_off >> 2) << 5) | COND_LE); + } + } + BPF_JSGE => { + // Signed >= + let skip = buf.offset(); + a64_bcond(buf, COND_LT, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let blt_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((blt_off >> 2) << 5) | COND_LT); + } + } + BPF_JLT => { + // Unsigned < + let skip = buf.offset(); + a64_bcond(buf, COND_HS, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let bhs_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((bhs_off >> 2) << 5) | COND_HS); + } + } + BPF_JLE => { + // Unsigned <= + let skip = buf.offset(); + a64_bcond(buf, COND_HI, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let bhi_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((bhi_off >> 2) << 5) | COND_HI); + } + } + BPF_JSLT => { + // Signed < + let skip = buf.offset(); + a64_bcond(buf, COND_GE, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let bge_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((bge_off >> 2) << 5) | COND_GE); + } + } + BPF_JSLE => { + // Signed <= + let skip = buf.offset(); + a64_bcond(buf, COND_GT, 0); + emit_jump_to_target(buf, offsets, target_pc); + let end = buf.offset(); + unsafe { + let bgt_off = (end - skip) as u32; + patch_u32(buf, skip, 0x5400_0000 | ((bgt_off >> 2) << 5) | COND_GT); + } + } + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_a64(insn.dst_reg()); + let adjusted_off = if base == A64_X25 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + // Classic BPF: ST [dst + off] = imm + // eBPF: ST [dst + off] = imm (only BPF_ST size variants exist) + emit_add_offset(buf, A64_X7, base, adjusted_off); + let val = insn.imm as u64; + match insn.size() { + BPF_B => { + emit_load_imm32(buf, A64_X6, val as i32); + a64_strb(buf, A64_X6, A64_X7, 0); + } + BPF_H => { + emit_load_imm32(buf, A64_X6, val as i32); + a64_strh(buf, A64_X6, A64_X7, 0); + } + BPF_W => { + emit_load_imm32(buf, A64_X6, val as i32); + a64_strw(buf, A64_X6, A64_X7, 0); + } + BPF_DW => { + emit_load_imm64(buf, A64_X6, val); + a64_str(buf, A64_X6, A64_X7, 0); + } + _ => {} + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_a64(insn.src_reg()); + let base = bpf_to_a64(insn.dst_reg()); + let adjusted_off = if base == A64_X25 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, A64_X7, base, adjusted_off); + match insn.size() { + BPF_B => a64_strb(buf, src, A64_X7, 0), + BPF_H => a64_strh(buf, src, A64_X7, 0), + BPF_W => a64_strw(buf, src, A64_X7, 0), + BPF_DW => a64_str(buf, src, A64_X7, 0), + _ => {} + } + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_a64(insn.src_reg()); + let dst = bpf_to_a64(insn.dst_reg()); + let adjusted_off = if src == A64_X25 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, A64_X7, src, adjusted_off); + match insn.size() { + BPF_B => a64_ldrb(buf, dst, A64_X7, 0), + BPF_H => a64_ldrh(buf, dst, A64_X7, 0), + BPF_W => a64_ldrw(buf, dst, A64_X7, 0), + BPF_DW => a64_ldr(buf, dst, A64_X7, 0), + _ => {} + } + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_a64(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + let start = buf.offset(); + emit_load_imm64(buf, dst, val); + let emitted = buf.offset() - start; + // Pad to 24 bytes (6 instructions) + let pad = if emitted < 24 { (24 - emitted) / 4 } else { 0 }; + for _ in 0..pad { + a64_nop(buf); + } + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + // Save BPF R5 (x5) before rearranging args + a64_mov(buf, A64_X6, A64_X5); + // Rearrange: BPF R1-R5 → helper args (x0-x4) + // x0 = BPF R1 (x1), x1 = BPF R2 (x2), x2 = BPF R3 (x3), x3 = BPF R4 (x4), x4 = BPF R5 (x5) + a64_mov(buf, A64_X0, A64_X1); + a64_mov(buf, A64_X1, A64_X2); + a64_mov(buf, A64_X2, A64_X3); + a64_mov(buf, A64_X3, A64_X4); + a64_mov(buf, A64_X4, A64_X6); + // Load helper fn address and call + emit_load_imm64_padded(buf, A64_X16, helper_fn as u64); + a64_blr(buf, A64_X16); + } +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_riscv64.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_riscv64.rs new file mode 100644 index 0000000000..e4eee6e804 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_riscv64.rs @@ -0,0 +1,1311 @@ +use super::{ + BpfInsn, HelperFn, JitBackend, JitBuffer, + bpf_insn::{ + BPF_ADD, BPF_ALU, BPF_ALU64, BPF_AND, BPF_ARSH, BPF_B, BPF_DIV, BPF_DW, BPF_END, BPF_EXIT, + BPF_H, BPF_JA, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, BPF_JNE, + BPF_JSET, BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_LD, BPF_LDX, BPF_LSH, BPF_MEM, + BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, BPF_OR, BPF_RSH, BPF_ST, BPF_STX, BPF_SUB, BPF_W, + BPF_X, BPF_XOR, + }, +}; + +const RV_ZERO: u32 = 0; +const RV_RA: u32 = 1; +const RV_SP: u32 = 2; +const RV_T1: u32 = 6; +const RV_T2: u32 = 7; +const RV_S1: u32 = 9; +const RV_A0: u32 = 10; +const RV_A1: u32 = 11; +const RV_A2: u32 = 12; +const RV_A3: u32 = 13; +const RV_A4: u32 = 14; +const RV_A5: u32 = 15; +const RV_S2: u32 = 18; +const RV_S3: u32 = 19; +const RV_S4: u32 = 20; +const RV_S5: u32 = 21; +const RV_T6: u32 = 31; + +const BPF_STACK_SIZE: usize = 512; +const CALLEE_SAVED_SIZE: usize = 48; +const FRAME_SIZE: usize = BPF_STACK_SIZE + CALLEE_SAVED_SIZE; + +fn bpf_to_rv(r: u8) -> u32 { + match r { + 0 => RV_A0, + 1 => RV_A1, + 2 => RV_A2, + 3 => RV_A3, + 4 => RV_A4, + 5 => RV_A5, + 6 => RV_S1, + 7 => RV_S2, + 8 => RV_S3, + 9 => RV_S4, + 10 => RV_S5, + _ => RV_ZERO, + } +} + +fn rv_r(funct7: u32, rs2: u32, rs1: u32, funct3: u32, rd: u32) -> u32 { + (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 0x33 +} + +fn rv_rw(funct7: u32, rs2: u32, rs1: u32, funct3: u32, rd: u32) -> u32 { + rv_r(funct7, rs2, rs1, funct3, rd) | (0x3b ^ 0x33) +} + +fn rv_i(imm: u32, rs1: u32, funct3: u32, rd: u32, opcode: u32) -> u32 { + (imm << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | opcode +} + +fn rv_s(imm: u32, rs2: u32, rs1: u32, funct3: u32) -> u32 { + ((imm >> 5) << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | ((imm & 0x1f) << 7) | 0x23 +} + +fn rv_b(imm: u32, rs2: u32, rs1: u32, funct3: u32) -> u32 { + let bit12 = (imm >> 12) & 1; + let bits10_5 = (imm >> 5) & 0x3f; + let bits4_1 = (imm >> 1) & 0xf; + let bit11 = (imm >> 11) & 1; + (bit12 << 31) + | (bits10_5 << 25) + | (rs2 << 20) + | (rs1 << 15) + | (funct3 << 12) + | (bits4_1 << 8) + | (bit11 << 7) + | 0x63 +} + +fn rv_u(imm: u32, rd: u32, opcode: u32) -> u32 { + (imm << 12) | (rd << 7) | opcode +} + +fn emit_add(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 0, rd)); +} + +fn emit_addw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 0, rd)); +} + +fn emit_sub(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 0, rd)); +} + +fn emit_subw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0x20, rs2, rs1, 0, rd)); +} + +fn emit_addi(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 0, rd, 0x13)); +} + +fn emit_addiw(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 0, rd, 0x1b)); +} + +fn emit_and(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 7, rd)); +} + +fn emit_andw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 7, rd)); +} + +fn emit_or(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 6, rd)); +} + +fn emit_orw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 6, rd)); +} + +fn emit_xor(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 4, rd)); +} + +fn emit_xorw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 4, rd)); +} + +fn emit_sll(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 1, rd)); +} + +fn emit_sllw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 1, rd)); +} + +fn emit_srl(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 5, rd)); +} + +fn emit_srlw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 5, rd)); +} + +fn emit_sra(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 5, rd)); +} + +fn emit_sraw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0x20, rs2, rs1, 5, rd)); +} + +fn emit_mul(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 0, rd)); +} + +fn emit_mulw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(1, rs2, rs1, 0, rd)); +} + +fn emit_divu(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 5, rd)); +} + +fn emit_divuw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(1, rs2, rs1, 5, rd)); +} + +fn emit_remu(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 7, rd)); +} + +fn emit_remuw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(1, rs2, rs1, 7, rd)); +} + +fn emit_andi(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 7, rd, 0x13)); +} + +fn emit_ori(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 6, rd, 0x13)); +} + +fn emit_xori(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 4, rd, 0x13)); +} + +fn emit_zext32(buf: &mut JitBuffer, rd: u32) { + emit_slli(buf, rd, rd, 32); + emit_srli(buf, rd, rd, 32); +} + +/// Byte-reverse within 8 bytes (rev8 rd, rs1) +/// Encoding: grevi rd, rs1, 24 (RV64 Zbb) +fn emit_rev8(buf: &mut JitBuffer, rd: u32, rs: u32) { + buf.emit_u32(rv_i(24, rs, 5, rd, 0x13)); +} + +fn emit_slli(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 1, rd, 0x13)); +} + +fn emit_srli(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 5, rd, 0x13)); +} + +fn emit_srai(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(0x400 | shamt, rs1, 5, rd, 0x13)); +} + +fn emit_slliw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 1, rd, 0x1b)); +} + +fn emit_srliw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 5, rd, 0x1b)); +} + +fn emit_sraiw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(0x400 | shamt, rs1, 5, rd, 0x1b)); +} + +fn emit_lui(buf: &mut JitBuffer, rd: u32, imm: u32) { + buf.emit_u32(rv_u(imm, rd, 0x37)); +} + +fn emit_ld(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 3, rd, 0x03)); +} + +fn emit_lwu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 6, rd, 0x03)); +} + +fn emit_lw(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 2, rd, 0x03)); +} + +fn emit_lhu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 5, rd, 0x03)); +} + +fn emit_lh(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 1, rd, 0x03)); +} + +fn emit_lbu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 4, rd, 0x03)); +} + +fn emit_lb(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 0, rd, 0x03)); +} + +fn emit_sd(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 3)); +} + +fn emit_sw(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 2)); +} + +fn emit_sh(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 1)); +} + +fn emit_sb(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 0)); +} + +fn emit_beq(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 0)); +} + +fn emit_bne(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 1)); +} + +fn emit_blt(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 4)); +} + +fn emit_bge(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 5)); +} + +fn emit_bltu(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 6)); +} + +fn emit_bgeu(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 7)); +} + +fn emit_jalr(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 0, rd, 0x67)); +} + +fn emit_auipc(buf: &mut JitBuffer, rd: u32, imm: u32) { + buf.emit_u32(rv_u(imm, rd, 0x17)); +} + +fn emit_jal(buf: &mut JitBuffer, rd: u32, imm: i32) { + let imm = imm as u32; + let bit20 = (imm >> 20) & 1; + let bits10_1 = (imm >> 1) & 0x3ff; + let bit11 = (imm >> 11) & 1; + let bits19_12 = (imm >> 12) & 0xff; + buf.emit_u32( + (bit20 << 31) | (bits10_1 << 21) | (bit11 << 20) | (bits19_12 << 12) | (rd << 7) | 0x6f, + ); +} + +fn emit_ret(buf: &mut JitBuffer) { + emit_jalr(buf, RV_ZERO, RV_RA, 0); +} + +fn emit_mv(buf: &mut JitBuffer, rd: u32, rs: u32) { + emit_addi(buf, rd, rs, 0); +} + +fn emit_nop(buf: &mut JitBuffer) { + buf.emit_u32(0x00000013); +} + +fn emit_load_imm64_padded(buf: &mut JitBuffer, rd: u32, val: u64) { + let start = buf.offset(); + emit_load_imm64(buf, rd, val); + let emitted = buf.offset() - start; + for _ in 0..((24 - emitted) / 4) { + emit_nop(buf); + } +} + +fn emit_load_imm64(buf: &mut JitBuffer, rd: u32, val: u64) { + let val_i = val as i64; + if val_i >= -2048 && val_i < 2048 { + emit_addi(buf, rd, RV_ZERO, val_i as i32); + return; + } + if (val as u32 as i32) as i64 == val_i { + let lo32 = val as u32; + let lo12 = (lo32 << 20) >> 20; + let hi20 = (lo32.wrapping_sub(lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, rd, hi20 & 0xFFFFF); + if lo12 != 0 { + emit_addiw(buf, rd, rd, lo12 as i32); + } + return; + } + let upper = (val >> 32) as u32; + let upper_lo12 = (upper << 20) >> 20; + let upper_hi20 = (upper.wrapping_sub(upper_lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, rd, upper_hi20 & 0xFFFFF); + emit_addiw(buf, rd, rd, upper_lo12 as i32); + emit_slli(buf, rd, rd, 32); + let lower = val as u32; + let lower_lo12 = (lower << 20) >> 20; + let lower_hi20 = (lower.wrapping_sub(lower_lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, RV_T1, lower_hi20 & 0xFFFFF); + if lower_lo12 != 0 { + emit_addiw(buf, RV_T1, RV_T1, lower_lo12 as i32); + } + emit_add(buf, rd, rd, RV_T1); +} + +fn emit_load_imm32(buf: &mut JitBuffer, rd: u32, val: i32) { + let needs_upper = val < -2048 || val >= 2048; + if !needs_upper { + emit_addi(buf, rd, RV_ZERO, val); + } else { + let val_u = val as u32; + let lo12 = (val_u << 20) >> 20; + let hi20 = (val_u.wrapping_sub(lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, rd, hi20 & 0xFFFFF); + if lo12 != 0 { + emit_addiw(buf, rd, rd, lo12 as i32); + } + } +} + +fn emit_add_offset(buf: &mut JitBuffer, rd: u32, rs: u32, off: i32) { + if off >= -2048 && off < 2048 { + emit_addi(buf, rd, rs, off); + } else { + emit_load_imm32(buf, RV_T1, off); + emit_add(buf, rd, rs, RV_T1); + } +} + +pub(crate) struct Riscv64Backend; + +impl JitBackend for Riscv64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + emit_addi(buf, RV_SP, RV_SP, -(FRAME_SIZE as i32)); + emit_sd(buf, RV_RA, RV_SP, 0); + emit_sd(buf, RV_S1, RV_SP, 8); + emit_sd(buf, RV_S2, RV_SP, 16); + emit_sd(buf, RV_S3, RV_SP, 24); + emit_sd(buf, RV_S4, RV_SP, 32); + emit_sd(buf, RV_S5, RV_SP, 40); + emit_addi(buf, RV_S5, RV_SP, FRAME_SIZE as i32); + emit_mv(buf, RV_A1, RV_A0); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + emit_ld(buf, RV_RA, RV_SP, 0); + emit_ld(buf, RV_S1, RV_SP, 8); + emit_ld(buf, RV_S2, RV_SP, 16); + emit_ld(buf, RV_S3, RV_SP, 24); + emit_ld(buf, RV_S4, RV_SP, 32); + emit_ld(buf, RV_S5, RV_SP, 40); + emit_addi(buf, RV_SP, RV_SP, FRAME_SIZE as i32); + emit_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_rv(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { + RV_T1 + } else { + bpf_to_rv(insn.src_reg()) + }; + if use_imm { + emit_load_imm64(buf, RV_T1, insn.imm as u64); + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + emit_add(buf, dst, dst, src); + } else { + emit_addw(buf, dst, dst, src); + } + } + BPF_SUB => { + if is_64 { + emit_sub(buf, dst, dst, src); + } else { + emit_subw(buf, dst, dst, src); + } + } + BPF_MUL => { + if is_64 { + emit_mul(buf, dst, dst, src); + } else { + emit_mulw(buf, dst, dst, src); + } + } + BPF_DIV => { + let skip = buf.offset(); + if is_64 { + emit_beq(buf, src, RV_ZERO, 0); + emit_divu(buf, dst, dst, src); + } else { + emit_beq(buf, src, RV_ZERO, 0); + emit_divuw(buf, dst, dst, src); + } + emit_jal(buf, RV_ZERO, 8); + emit_addi(buf, dst, RV_ZERO, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let beq_ptr = buf.entry().add(skip) as *mut u32; + // beq must jump to addi dst,zero,0 (one instruction before end), + // not to end itself. Offset from beq to addi = end - skip - 4. + let beq_off = (end - skip - 4) as u32; + *beq_ptr = rv_b(beq_off, RV_ZERO, src, 0); + } + } + } + BPF_OR => { + if is_64 { + emit_or(buf, dst, dst, src); + } else { + emit_orw(buf, dst, dst, src); + } + } + BPF_AND => { + if is_64 { + emit_and(buf, dst, dst, src); + } else { + emit_andw(buf, dst, dst, src); + } + } + BPF_LSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_slli(buf, dst, dst, shamt); + } else { + emit_slliw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_sll(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_sllw(buf, dst, dst, RV_T2); + } + } + BPF_RSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_srli(buf, dst, dst, shamt); + } else { + emit_srliw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_srl(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_srlw(buf, dst, dst, RV_T2); + } + } + BPF_NEG => { + if is_64 { + emit_sub(buf, dst, RV_ZERO, dst); + } else { + emit_subw(buf, dst, RV_ZERO, dst); + } + } + BPF_MOD => { + let skip = buf.offset(); + if is_64 { + emit_beq(buf, src, RV_ZERO, 0); + emit_remu(buf, dst, dst, src); + } else { + emit_beq(buf, src, RV_ZERO, 0); + emit_remuw(buf, dst, dst, src); + } + let end = buf.offset(); + if !buf.counting() { + unsafe { + let beq_ptr = buf.entry().add(skip) as *mut u32; + let beq_off = (end - skip) as u32; + *beq_ptr = rv_b(beq_off, RV_ZERO, src, 0); + } + } + } + BPF_XOR => { + if is_64 { + emit_xor(buf, dst, dst, src); + } else { + emit_xorw(buf, dst, dst, src); + } + } + BPF_MOV => { + if is_64 { + emit_mv(buf, dst, src); + } else { + emit_addiw(buf, dst, src, 0); + } + } + BPF_ARSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_srai(buf, dst, dst, shamt); + } else { + emit_sraiw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_sra(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_sraw(buf, dst, dst, RV_T2); + } + } + BPF_END => { + // BPF_TO_BE: byte swap to big-endian (RISC-V is little-endian native) + let to_be = (insn.code & BPF_X) != 0; + match (to_be, insn.imm) { + (true, 16) => { + // 16-bit: rev8 then shift + emit_rev8(buf, dst, dst); + emit_srli(buf, dst, dst, 48); + } + (true, 32) => { + // 32-bit: rev8 then shift + emit_rev8(buf, dst, dst); + emit_srli(buf, dst, dst, 32); + } + (true, 64) => { + // 64-bit: rev8 + emit_rev8(buf, dst, dst); + } + // BPF_TO_LE on RISC-V: no-op (native little-endian) + _ => {} + } + } + _ => {} + } + + if !is_64 && insn.alu_op() != BPF_ARSH { + emit_zext32(buf, dst); + } + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let off = offsets[target_pc] as isize - buf.offset() as isize; + emit_auipc(buf, RV_T6, 0); + emit_load_imm64_padded(buf, RV_T1, off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + } + return; + } + + // BPF_CALL (0x80) is handled separately in compile() + if op == 0x80 { + return; + } + + let dst = bpf_to_rv(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { + RV_T1 + } else { + bpf_to_rv(insn.src_reg()) + }; + + if use_imm { + if is_64 { + emit_load_imm64(buf, RV_T1, insn.imm as u64); + } else { + emit_load_imm32(buf, RV_T1, insn.imm); + } + } + + if !is_64 { + emit_zext32(buf, dst); + emit_zext32(buf, src); + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + + match op { + BPF_JEQ => { + let start = buf.offset(); + emit_bne(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 1); + } + } + } + BPF_JGT => { + let start = buf.offset(); + emit_bgeu(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 7); + } + } + } + BPF_JGE => { + let start = buf.offset(); + emit_bltu(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 6); + } + } + } + BPF_JSET => { + emit_and(buf, RV_T2, dst, src); + let start = buf.offset(); + emit_beq(buf, RV_T2, RV_ZERO, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, RV_ZERO, RV_T2, 0); + } + } + } + BPF_JNE => { + let start = buf.offset(); + emit_beq(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 0); + } + } + } + BPF_JSGT => { + let start = buf.offset(); + emit_bge(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 5); + } + } + } + BPF_JSGE => { + let start = buf.offset(); + emit_blt(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 4); + } + } + } + BPF_JLT => { + let start = buf.offset(); + emit_bgeu(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 7); + } + } + } + BPF_JLE => { + let start = buf.offset(); + emit_bltu(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 6); + } + } + } + BPF_JSLT => { + let start = buf.offset(); + emit_bge(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 5); + } + } + } + BPF_JSLE => { + let start = buf.offset(); + emit_blt(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 4); + } + } + } + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_rv(insn.dst_reg()); + let adjusted_off = if base == RV_S5 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, RV_T1, base, adjusted_off); + let val = insn.imm as u64; + match insn.size() { + BPF_B => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sb(buf, RV_T2, RV_T1, 0); + } + BPF_H => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sh(buf, RV_T2, RV_T1, 0); + } + BPF_W => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sw(buf, RV_T2, RV_T1, 0); + } + BPF_DW => { + emit_load_imm64(buf, RV_T2, val); + emit_sd(buf, RV_T2, RV_T1, 0); + } + _ => {} + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_rv(insn.src_reg()); + let base = bpf_to_rv(insn.dst_reg()); + let adjusted_off = if base == RV_S5 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, RV_T1, base, adjusted_off); + match insn.size() { + BPF_B => emit_sb(buf, src, RV_T1, 0), + BPF_H => emit_sh(buf, src, RV_T1, 0), + BPF_W => emit_sw(buf, src, RV_T1, 0), + BPF_DW => emit_sd(buf, src, RV_T1, 0), + _ => {} + } + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_rv(insn.src_reg()); + let dst = bpf_to_rv(insn.dst_reg()); + let adjusted_off = if src == RV_S5 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, RV_T1, src, adjusted_off); + match insn.size() { + BPF_B => { + emit_lbu(buf, dst, RV_T1, 0); + } + BPF_H => { + emit_lhu(buf, dst, RV_T1, 0); + } + BPF_W => { + emit_lwu(buf, dst, RV_T1, 0); + } + BPF_DW => { + emit_ld(buf, dst, RV_T1, 0); + } + _ => {} + } + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_rv(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + let start = buf.offset(); + emit_load_imm64(buf, dst, val); + let emitted = buf.offset() - start; + for _ in 0..((24 - emitted) / 4) { + emit_nop(buf); + } + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + emit_mv(buf, RV_T2, RV_A5); + emit_mv(buf, RV_A0, RV_A1); + emit_mv(buf, RV_A1, RV_A2); + emit_mv(buf, RV_A2, RV_A3); + emit_mv(buf, RV_A3, RV_A4); + emit_mv(buf, RV_A4, RV_T2); + + emit_load_imm64_padded(buf, RV_T1, helper_fn as u64); + emit_jalr(buf, RV_RA, RV_T1, 0); + } +} + +#[cfg(test)] +mod tests_arch_independent { + fn compute_hi20_lo12(val: u32) -> (u32, u32) { + let lo12 = (val << 20) >> 20; + let hi20 = (val.wrapping_sub(lo12).wrapping_add(0x800)) >> 12; + (hi20 & 0xFFFFF, lo12) + } + + fn reconstruct_from_hi20_lo12(hi20: u32, lo12: u32) -> u64 { + let lui_val = ((hi20 as i32).wrapping_shl(12)) as i64; + (lui_val.wrapping_add(lo12 as i32 as i64)) as u64 + } + + fn reconstruct_64bit( + upper_hi20: u32, + upper_lo12: u32, + lower_hi20: u32, + lower_lo12: u32, + ) -> u64 { + let upper = reconstruct_from_hi20_lo12(upper_hi20, upper_lo12); + let lower = reconstruct_from_hi20_lo12(lower_hi20, lower_lo12); + ((upper as u64) << 32).wrapping_add(lower) + } + + #[test] + fn test_hi20_lo12_identity() { + for val in [ + 0x00000000u32, + 0x00000001, + 0x00000FFF, + 0xFFFFF000, + 0xFFFFFFFF, + ] { + let (hi20, lo12) = compute_hi20_lo12(val); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!( + reconstructed, val as u64, + "val={val:#010x}: hi20={hi20:#010x} lo12={lo12:#010x}" + ); + } + } + + #[test] + fn test_hi20_lo12_bit31_set() { + let val: u32 = 0x80000000; + let (hi20, lo12) = compute_hi20_lo12(val); + assert_eq!(hi20, 0x80000, "hi20 should be 0x80000"); + assert_eq!(lo12, 0, "lo12 should be 0"); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!(reconstructed, val as u64); + } + + #[test] + fn test_hi20_lo12_large_negative() { + let val: u32 = 0xC0000000; + let (hi20, lo12) = compute_hi20_lo12(val); + assert_ne!(hi20, 0, "hi20 should not be 0"); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!(reconstructed, val as u64); + } + + #[test] + fn test_hi20_lo12_0x7FFFFFFF() { + let val: u32 = 0x7FFFFFFF; + let (hi20, lo12) = compute_hi20_lo12(val); + assert_ne!(hi20, 0, "hi20 should not be 0"); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!(reconstructed, val as u64); + } + + #[test] + fn test_64bit_reconstruct_0x1_80000000() { + let val: u64 = 0x0000_0001_8000_0000; + let upper = (val >> 32) as u32; + let lower = val as u32; + let (upper_hi20, upper_lo12) = compute_hi20_lo12(upper); + let (lower_hi20, lower_lo12) = compute_hi20_lo12(lower); + assert_ne!(lower_hi20, 0, "lower_hi20 should not be 0 for 0x80000000"); + let reconstructed = reconstruct_64bit(upper_hi20, upper_lo12, lower_hi20, lower_lo12); + assert_eq!(reconstructed, val, "val={val:#018x}"); + } + + #[test] + fn test_64bit_reconstruct_all_ones() { + let val: u64 = 0xFFFF_FFFF_FFFF_FFFF; + let upper = (val >> 32) as u32; + let lower = val as u32; + let (upper_hi20, upper_lo12) = compute_hi20_lo12(upper); + let (lower_hi20, lower_lo12) = compute_hi20_lo12(lower); + let reconstructed = reconstruct_64bit(upper_hi20, upper_lo12, lower_hi20, lower_lo12); + assert_eq!(reconstructed, val); + } + + #[test] + fn test_64bit_reconstruct_alternating() { + let val: u64 = 0x5555_5555_AAAA_AAAA; + let upper = (val >> 32) as u32; + let lower = val as u32; + let (upper_hi20, upper_lo12) = compute_hi20_lo12(upper); + let (lower_hi20, lower_lo12) = compute_hi20_lo12(lower); + let reconstructed = reconstruct_64bit(upper_hi20, upper_lo12, lower_hi20, lower_lo12); + assert_eq!(reconstructed, val); + } + + #[test] + fn test_hi20_masked_to_20_bits() { + for val in 0u32..4096 { + let (hi20, _) = compute_hi20_lo12(val * 0x1000); + assert_eq!( + hi20 & !0xFFFFF, + 0, + "hi20 must fit in 20 bits for val={val:#010x}" + ); + } + } +} + +#[cfg(all(test, target_arch = "riscv64"))] +mod tests { + use super::*; + + fn new_buf() -> JitBuffer { + JitBuffer::new(4096).unwrap() + } + + fn buf_as_u32_slice(buf: &JitBuffer) -> &[u32] { + unsafe { core::slice::from_raw_parts(buf.entry() as *const u32, buf.offset() / 4) } + } + + fn decode_lui(insn: u32) -> (u32, u32) { + let rd = (insn >> 7) & 0x1f; + let imm = (insn >> 12) & 0xfffff; + (rd, imm) + } + + fn decode_addiw(insn: u32) -> (u32, u32, i32) { + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let imm = ((insn as i32) << 20) >> 20; + (rd, rs1, imm) + } + + fn decode_slli(insn: u32) -> (u32, u32, u32) { + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let shamt = (insn >> 20) & 0x3f; + (rd, rs1, shamt) + } + + fn decode_add(insn: u32) -> (u32, u32, u32) { + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let rs2 = (insn >> 20) & 0x1f; + (rd, rs1, rs2) + } + + fn decode_addi(insn: u32) -> (u32, u32, i32) { + decode_addiw(insn) + } + + fn sign_extend_20(imm20: u32) -> i64 { + ((imm20 << 12) as i32 as i64) << 12 >> 12 + } + + fn reconstruct_load_imm64(insns: &[u32]) -> u64 { + let mut idx = 0; + let first_opcode = insns[idx] & 0x7f; + if first_opcode == 0x13 { + let (_, _, imm) = decode_addi(insns[idx]); + return imm as u64; + } + assert_eq!(first_opcode, 0x37, "expected LUI"); + let (rd1, imm20_hi) = decode_lui(insns[idx]); + idx += 1; + let upper_val; + if idx < insns.len() && (insns[idx] & 0x7f) == 0x1b { + let (rd2, rs1, lo12) = decode_addiw(insns[idx]); + assert_eq!(rd2, rd1); + assert_eq!(rs1, rd1); + upper_val = ((imm20_hi as i32) << 12).wrapping_add(lo12); + idx += 1; + } else { + upper_val = (imm20_hi as i32) << 12; + } + if idx >= insns.len() || (insns[idx] & 0x7f) != 0x13 { + return upper_val as u64; + } + let (rd3, rs1_3, shamt) = decode_slli(insns[idx]); + assert_eq!(rd3, rd1); + assert_eq!(rs1_3, rd1); + assert_eq!(shamt, 32); + let shifted = (upper_val as u64) << 32; + idx += 1; + let (_, imm20_lo) = decode_lui(insns[idx]); + idx += 1; + let (rd_t1, rs1_t1, lo12_lo) = decode_addiw(insns[idx]); + let lower_val = ((imm20_lo as i32) << 12).wrapping_add(lo12_lo); + let lower_extended = lower_val as i64 as u64; + shifted.wrapping_add(lower_extended) + } + + #[test] + fn test_load_imm64_small_positive() { + let mut buf = new_buf(); + emit_load_imm64(&mut buf, RV_A0, 42); + assert_eq!(buf.offset(), 4); + let insns = buf_as_u32_slice(&buf); + let (_, _, imm) = decode_addi(insns[0]); + assert_eq!(imm, 42); + } + + #[test] + fn test_load_imm64_small_negative() { + let mut buf = new_buf(); + emit_load_imm64(&mut buf, RV_A0, (-1i64) as u64); + assert_eq!(buf.offset(), 4); + let insns = buf_as_u32_slice(&buf); + let (_, _, imm) = decode_addi(insns[0]); + assert_eq!(imm, -1); + } + + #[test] + fn test_load_imm64_32bit_value() { + let mut buf = new_buf(); + let val: u64 = 0x12345000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_bit31_set() { + let mut buf = new_buf(); + let val: u64 = 0x0000_0001_8000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_all_ones() { + let mut buf = new_buf(); + let val: u64 = 0xFFFF_FFFF_FFFF_FFFF; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_high_bit_only() { + let mut buf = new_buf(); + let val: u64 = 0x8000_0000_0000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_max_positive() { + let mut buf = new_buf(); + let val: u64 = 0x7FFF_FFFF_FFFF_FFFF; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_alternating_bits() { + let mut buf = new_buf(); + let val: u64 = 0x5555_5555_AAAA_AAAA; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_upper_ffff_lower_0() { + let mut buf = new_buf(); + let val: u64 = 0xFFFF_FFFF_0000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_zero_lower() { + let mut buf = new_buf(); + let val: u64 = 0x1234_5678_0000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_alu_add32_zext() { + let mut buf = new_buf(); + let mut insn = BpfInsn::default(); + insn.code = BPF_ALU | BPF_ADD | BPF_X; + insn.dst_src_reg = 1 | (2 << 4); // dst=1, src=2 + Riscv64Backend::emit_alu(&mut buf, &insn, false); + let insns = buf_as_u32_slice(&buf); + let last = insns[insns.len() - 1]; + let prev = insns[insns.len() - 2]; + let (_, _, shamt1) = decode_slli(prev); + let (_, _, shamt2) = decode_slli_with_func(last); + assert_eq!(shamt1, 32); + assert_eq!(shamt2, 32); + } + + fn decode_slli_with_func(insn: u32) -> (u32, u32, u32) { + let funct3 = (insn >> 12) & 0x7; + let opcode = insn & 0x7f; + assert_eq!(opcode, 0x13); + assert!(funct3 == 1 || funct3 == 5); + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let shamt = (insn >> 20) & 0x3f; + (rd, rs1, shamt) + } + + #[test] + fn test_alu_mov32_zext() { + let mut buf = new_buf(); + let mut insn = BpfInsn::default(); + insn.code = BPF_ALU | BPF_MOV | BPF_X; + insn.dst_src_reg = 1 | (2 << 4); // dst=1, src=2 + Riscv64Backend::emit_alu(&mut buf, &insn, false); + assert!(buf.offset() > 0); + } + + #[test] + fn test_arsh32_no_zext() { + let mut buf = new_buf(); + let mut insn = BpfInsn::default(); + insn.code = BPF_ALU | BPF_ARSH; + insn.dst_src_reg = 1; // dst=1 + insn.imm = 1; + Riscv64Backend::emit_alu(&mut buf, &insn, false); + assert!(buf.offset() > 0); + } +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_x86_64.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_x86_64.rs new file mode 100644 index 0000000000..f4be8c0a45 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_x86_64.rs @@ -0,0 +1,904 @@ +use super::{ + super::{ + HelperFn, + bpf_insn::{ + BPF_ADD, BPF_AND, BPF_ARSH, BPF_B, BPF_DIV, BPF_DW, BPF_END, BPF_H, BPF_JA, BPF_JEQ, + BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, BPF_JNE, BPF_JSET, BPF_JSGE, + BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_LSH, BPF_MEM, BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, + BPF_OR, BPF_RSH, BPF_SUB, BPF_W, BPF_X, BPF_XOR, BpfInsn, + }, + }, + JitBackend, JitBuffer, +}; + +const X86_RAX: u8 = 0; +const X86_RCX: u8 = 1; +const X86_RDX: u8 = 2; +const X86_RBX: u8 = 3; +const X86_RSP: u8 = 4; +const X86_RBP: u8 = 5; +const X86_RSI: u8 = 6; +const X86_RDI: u8 = 7; +const X86_R8: u8 = 8; +const _X86_R9: u8 = 9; +const X86_R10: u8 = 10; +const X86_R11: u8 = 11; +const _X86_R12: u8 = 12; +const X86_R13: u8 = 13; +const X86_R14: u8 = 14; +const X86_R15: u8 = 15; + +fn bpf_to_x86(r: u8) -> u8 { + match r { + 0 => X86_RAX, + 1 => X86_RDI, + 2 => X86_RSI, + 3 => X86_RDX, + 4 => X86_RCX, + 5 => X86_R8, + 6 => X86_RBX, + 7 => X86_R13, + 8 => X86_R14, + 9 => X86_R15, + 10 => X86_RBP, + _ => X86_RAX, + } +} + +fn need_rex(r: u8) -> bool { + r >= 8 +} + +fn emit_rex(buf: &mut JitBuffer, w: bool, r: u8, x: bool, b: u8) { + let mut rex: u8 = 0x40; + if w { + rex |= 0x08; + } + if need_rex(r) { + rex |= 0x04; + } + if x { + rex |= 0x02; + } + if need_rex(b) { + rex |= 0x01; + } + buf.emit_u8(rex); +} + +fn emit_modrm(buf: &mut JitBuffer, mod_bits: u8, reg: u8, rm: u8) { + buf.emit_u8((mod_bits << 6) | ((reg & 7) << 3) | (rm & 7)); +} + +fn emit_rex_if(buf: &mut JitBuffer, r: u8, b: u8) { + if need_rex(r) || need_rex(b) { + emit_rex(buf, false, r, false, b); + } +} + +fn emit_rex_w(buf: &mut JitBuffer, r: u8, b: u8) { + emit_rex(buf, true, r, false, b); +} + +fn emit_modrm_disp(buf: &mut JitBuffer, reg: u8, rm: u8, disp: i32) { + if disp == 0 && (rm & 7) != X86_RBP { + emit_modrm(buf, 0, reg, rm); + } else if (-128..=127).contains(&disp) { + emit_modrm(buf, 1, reg, rm); + buf.emit_u8(disp as u8); + } else { + emit_modrm(buf, 2, reg, rm); + buf.emit_u32(disp as u32); + } +} + +fn emit_push(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0x50 | (r & 7)); +} + +fn emit_pop(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0x58 | (r & 7)); +} + +fn emit_mov_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x89); + emit_modrm(buf, 3, src, dst); +} + +fn emit_mov_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x89); + emit_modrm(buf, 3, src, dst); +} + +fn emit_mov_imm64(buf: &mut JitBuffer, dst: u8, imm: u64) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xB8 | (dst & 7)); + buf.emit_u32(imm as u32); + buf.emit_u32((imm >> 32) as u32); +} + +fn emit_mov_imm32(buf: &mut JitBuffer, dst: u8, imm: i32) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC7); + emit_modrm(buf, 3, 0, dst); + buf.emit_u32(imm as u32); +} + +fn emit_add_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x01); + emit_modrm(buf, 3, src, dst); +} + +fn emit_sub_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x29); + emit_modrm(buf, 3, src, dst); +} + +fn emit_add_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x01); + emit_modrm(buf, 3, src, dst); +} + +fn emit_sub_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x29); + emit_modrm(buf, 3, src, dst); +} + +fn emit_imul_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, dst, src); + buf.emit_u8(0x0F); + buf.emit_u8(0xAF); + emit_modrm(buf, 3, dst, src); +} + +fn emit_imul_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, dst, src); + buf.emit_u8(0x0F); + buf.emit_u8(0xAF); + emit_modrm(buf, 3, dst, src); +} + +fn emit_xor_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x31); + emit_modrm(buf, 3, src, dst); +} + +fn emit_xor_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x31); + emit_modrm(buf, 3, src, dst); +} + +fn emit_or_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x09); + emit_modrm(buf, 3, src, dst); +} + +fn emit_or_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x09); + emit_modrm(buf, 3, src, dst); +} + +fn emit_and_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x21); + emit_modrm(buf, 3, src, dst); +} + +fn emit_and_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x21); + emit_modrm(buf, 3, src, dst); +} + +fn emit_neg_reg64(buf: &mut JitBuffer, r: u8) { + emit_rex_w(buf, 0, r); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 3, r); +} + +fn emit_neg_reg32(buf: &mut JitBuffer, r: u8) { + emit_rex_if(buf, 0, r); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 3, r); +} + +fn emit_shl_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 4, dst); +} + +fn emit_shr_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 5, dst); +} + +fn emit_sar_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 7, dst); +} + +fn emit_shl_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 4, dst); +} + +fn emit_shr_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 5, dst); +} + +fn emit_sar_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 7, dst); +} + +fn emit_shl_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 4, dst); + buf.emit_u8(imm); +} + +fn emit_shr_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 5, dst); + buf.emit_u8(imm); +} + +fn emit_sar_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 7, dst); + buf.emit_u8(imm); +} + +fn emit_shl_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 4, dst); + buf.emit_u8(imm); +} + +fn emit_shr_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 5, dst); + buf.emit_u8(imm); +} + +fn emit_sar_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 7, dst); + buf.emit_u8(imm); +} + +fn emit_test_reg64(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_w(buf, r2, r1); + buf.emit_u8(0x85); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_cmp_reg64(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_w(buf, r2, r1); + buf.emit_u8(0x39); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_cmp_reg32(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_if(buf, r2, r1); + buf.emit_u8(0x39); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_je(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x84); + buf.emit_u32(off as u32); +} + +fn emit_jne(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x85); + buf.emit_u32(off as u32); +} + +fn emit_ja(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x87); + buf.emit_u32(off as u32); +} + +fn emit_jae(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x83); + buf.emit_u32(off as u32); +} + +fn emit_jb(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x82); + buf.emit_u32(off as u32); +} + +fn emit_jbe(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x86); + buf.emit_u32(off as u32); +} + +fn emit_jg(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8F); + buf.emit_u32(off as u32); +} + +fn emit_jge(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8D); + buf.emit_u32(off as u32); +} + +fn emit_jl(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8C); + buf.emit_u32(off as u32); +} + +fn emit_jle(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8E); + buf.emit_u32(off as u32); +} + +fn emit_jmp_rel32(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0xE9); + buf.emit_u32(off as u32); +} + +fn emit_call_reg(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0xFF); + emit_modrm(buf, 3, 2, r); +} + +fn emit_ret(buf: &mut JitBuffer) { + buf.emit_u8(0xC3); +} + +fn emit_store_mem(buf: &mut JitBuffer, base: u8, off: i32, src: u8, size: u8) { + match size { + BPF_B => { + emit_rex_if(buf, src, base); + buf.emit_u8(0x88); + emit_modrm_disp(buf, src, base, off); + } + BPF_H => { + buf.emit_u8(0x66); + emit_rex_if(buf, src, base); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + BPF_W => { + emit_rex_if(buf, src, base); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + BPF_DW => { + emit_rex_w(buf, src, base); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + _ => {} + } +} + +fn emit_load_mem(buf: &mut JitBuffer, dst: u8, base: u8, off: i32, size: u8) { + match size { + BPF_B => { + emit_rex_if(buf, dst, base); + buf.emit_u8(0x0F); + buf.emit_u8(0xB6); + emit_modrm_disp(buf, dst, base, off); + } + BPF_H => { + emit_rex_if(buf, dst, base); + buf.emit_u8(0x0F); + buf.emit_u8(0xB7); + emit_modrm_disp(buf, dst, base, off); + } + BPF_W => { + emit_rex_if(buf, dst, base); + buf.emit_u8(0x8B); + emit_modrm_disp(buf, dst, base, off); + } + BPF_DW => { + emit_rex_w(buf, dst, base); + buf.emit_u8(0x8B); + emit_modrm_disp(buf, dst, base, off); + } + _ => {} + } +} + +fn emit_zext32(buf: &mut JitBuffer, r: u8) { + emit_rex_if(buf, r, r); + buf.emit_u8(0x23); + buf.emit_u8(0xC0 | ((r & 7) << 3) | (r & 7)); +} + +fn emit_divmod(buf: &mut JitBuffer, dst: u8, src: u8, is_div: bool, is_64: bool) { + emit_push(buf, X86_RCX); + + if is_64 { + emit_mov_reg64(buf, X86_R11, src); + emit_mov_reg64(buf, X86_R10, dst); + // For DIV with src==0, dst must become 0 → pre-zero RAX. + // For MOD with src==0, dst must stay unchanged → do not pre-zero. + if is_div { + emit_xor_reg64(buf, X86_RAX, X86_RAX); + } + emit_test_reg64(buf, X86_R11, X86_R11); + let skip = buf.offset(); + emit_je(buf, 0); + emit_mov_reg64(buf, X86_RAX, X86_R10); + emit_xor_reg64(buf, X86_RDX, X86_RDX); + emit_rex_w(buf, 0, X86_R11); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 6, X86_R11); + // For MOD, assign result inside the non-zero path so that the + // src==0 skip leaves dst unchanged. + if !is_div { + emit_mov_reg64(buf, dst, X86_RDX); + } + let after = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(skip) as *mut u8; + let off = (after - skip - 6) as i32; + core::ptr::copy_nonoverlapping(off.to_le_bytes().as_ptr(), ptr.add(2), 4); + } + } + emit_pop(buf, X86_RCX); + if is_div { + emit_mov_reg64(buf, dst, X86_RAX); + } + } else { + emit_mov_reg32(buf, X86_R11, src); + emit_mov_reg32(buf, X86_R10, dst); + if is_div { + emit_xor_reg32(buf, X86_RAX, X86_RAX); + } + emit_test_reg64(buf, X86_R11, X86_R11); + let skip = buf.offset(); + emit_je(buf, 0); + emit_mov_reg32(buf, X86_RAX, X86_R10); + emit_zext32(buf, X86_RAX); + emit_xor_reg32(buf, X86_RDX, X86_RDX); + emit_rex_if(buf, 0, X86_R11); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 6, X86_R11); + if !is_div { + emit_zext32(buf, X86_RDX); + emit_mov_reg32(buf, dst, X86_RDX); + } + let after = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(skip) as *mut u8; + let off = (after - skip - 6) as i32; + core::ptr::copy_nonoverlapping(off.to_le_bytes().as_ptr(), ptr.add(2), 4); + } + } + emit_pop(buf, X86_RCX); + if is_div { + emit_mov_reg32(buf, dst, X86_RAX); + } + } +} + +pub(crate) struct X86_64Backend; + +impl JitBackend for X86_64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + emit_push(buf, X86_RBP); + emit_mov_reg64(buf, X86_RBP, X86_RSP); + emit_push(buf, X86_RBX); + emit_push(buf, X86_R13); + emit_push(buf, X86_R14); + emit_push(buf, X86_R15); + buf.emit_u8(0x48); + buf.emit_u8(0x81); + buf.emit_u8(0xEC); + buf.emit_u32(512); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + buf.emit_u8(0x48); + buf.emit_u8(0x81); + buf.emit_u8(0xC4); + buf.emit_u32(512); + emit_pop(buf, X86_R15); + emit_pop(buf, X86_R14); + emit_pop(buf, X86_R13); + emit_pop(buf, X86_RBX); + emit_pop(buf, X86_RBP); + emit_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_x86(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let imm_src = if dst == X86_RCX { X86_R11 } else { X86_RCX }; + let src = if use_imm { + imm_src + } else { + bpf_to_x86(insn.src_reg()) + }; + + let alu_op = insn.alu_op(); + if use_imm + && alu_op != BPF_MOV + && alu_op != BPF_NEG + && alu_op != BPF_LSH + && alu_op != BPF_RSH + && alu_op != BPF_ARSH + { + let imm = insn.imm; + if is_64 { + emit_mov_imm64(buf, imm_src, insn.imm as u64); + } else { + emit_mov_imm32(buf, imm_src, imm); + } + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + emit_add_reg64(buf, dst, src); + } else { + emit_add_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_SUB => { + if is_64 { + emit_sub_reg64(buf, dst, src); + } else { + emit_sub_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_MUL => { + if is_64 { + emit_imul_reg64(buf, dst, src); + } else { + emit_imul_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_DIV => { + emit_divmod(buf, dst, src, true, is_64); + } + BPF_OR => { + if is_64 { + emit_or_reg64(buf, dst, src); + } else { + emit_or_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_AND => { + if is_64 { + emit_and_reg64(buf, dst, src); + } else { + emit_and_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_LSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_shl_imm64(buf, dst, shamt); + } else { + emit_shl_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if dst == X86_RCX { + emit_mov_reg64(buf, X86_R10, dst); + emit_mov_reg64(buf, X86_RCX, src); + if is_64 { + emit_shl_reg64(buf, X86_R10); + } else { + emit_shl_reg32(buf, X86_R10); + emit_zext32(buf, X86_R10); + } + emit_mov_reg64(buf, dst, X86_R10); + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_shl_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_shl_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_RSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_shr_imm64(buf, dst, shamt); + } else { + emit_shr_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if dst == X86_RCX { + emit_mov_reg64(buf, X86_R10, dst); + emit_mov_reg64(buf, X86_RCX, src); + if is_64 { + emit_shr_reg64(buf, X86_R10); + } else { + emit_shr_reg32(buf, X86_R10); + emit_zext32(buf, X86_R10); + } + emit_mov_reg64(buf, dst, X86_R10); + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_shr_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_shr_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_NEG => { + if is_64 { + emit_neg_reg64(buf, dst); + } else { + emit_neg_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_MOD => { + emit_divmod(buf, dst, src, false, is_64); + } + BPF_XOR => { + if is_64 { + emit_xor_reg64(buf, dst, src); + } else { + emit_xor_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_MOV => { + if is_64 { + if use_imm { + emit_mov_imm64(buf, dst, insn.imm as u64); + } else { + emit_mov_reg64(buf, dst, src); + } + } else { + if use_imm { + emit_mov_imm32(buf, dst, insn.imm); + } else { + emit_mov_reg32(buf, dst, src); + } + emit_zext32(buf, dst); + } + } + BPF_ARSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_sar_imm64(buf, dst, shamt); + } else { + emit_sar_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if dst == X86_RCX { + emit_mov_reg64(buf, X86_R10, dst); + emit_mov_reg64(buf, X86_RCX, src); + if is_64 { + emit_sar_reg64(buf, X86_R10); + } else { + emit_sar_reg32(buf, X86_R10); + } + emit_mov_reg64(buf, dst, X86_R10); + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_sar_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_sar_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_END => { + // BPF_TO_BE: byte swap to big-endian (x86_64 is little-endian native) + let to_be = (insn.code & BPF_X) != 0; + match (to_be, insn.imm) { + (true, 16) => { + // 16-bit: rol reg16, 8 + buf.emit_u8(0x66); // 16-bit operand size + if need_rex(dst) { + buf.emit_u8(0x41); // REX.B for r8-r15 + } + buf.emit_u8(0xC1); // ROL r/m, imm8 + emit_modrm(buf, 3, 0, dst); // mod=11, /0, rm=dst + buf.emit_u8(0x08); // rotate by 8 bits + } + (true, 32) => { + // 32-bit: bswap reg32 + if need_rex(dst) { + buf.emit_u8(0x41); // REX.B + } + buf.emit_u8(0x0F); + buf.emit_u8(0xC8 | (dst & 7)); + } + (true, 64) => { + // 64-bit: bswap reg64 (REX.W + 0F C8+r) + if need_rex(dst) { + buf.emit_u8(0x49); // REX.WB + } else { + buf.emit_u8(0x48); // REX.W + } + buf.emit_u8(0x0F); + buf.emit_u8(0xC8 | (dst & 7)); + } + // BPF_TO_LE on x86_64: no-op (native little-endian) + _ => {} + } + } + _ => {} + } + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let off = offsets[target_pc] as isize - buf.offset() as isize - 5; + emit_jmp_rel32(buf, off as i32); + } + return; + } + + let dst = bpf_to_x86(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let imm_src = if dst == X86_RCX { X86_R11 } else { X86_RCX }; + let src = if use_imm { + imm_src + } else { + bpf_to_x86(insn.src_reg()) + }; + + if use_imm { + if is_64 { + emit_mov_imm64(buf, imm_src, insn.imm as u64); + } else { + emit_mov_imm32(buf, imm_src, insn.imm); + } + } + + if op != BPF_JSET { + if is_64 { + emit_cmp_reg64(buf, dst, src); + } else { + emit_cmp_reg32(buf, dst, src); + } + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + let target_off = if target_pc < offsets.len() { + (offsets[target_pc] as isize - buf.offset() as isize - 6) as i32 + } else { + 0 + }; + + match op { + BPF_JEQ => emit_je(buf, target_off), + BPF_JGT => emit_ja(buf, target_off), + BPF_JGE => emit_jae(buf, target_off), + BPF_JSET => { + if is_64 { + emit_test_reg64(buf, dst, src); + } else { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x85); + emit_modrm(buf, 3, src, dst); + } + emit_jne(buf, target_off); + } + BPF_JNE => emit_jne(buf, target_off), + BPF_JSGT => emit_jg(buf, target_off), + BPF_JSGE => emit_jge(buf, target_off), + BPF_JLT => emit_jb(buf, target_off), + BPF_JLE => emit_jbe(buf, target_off), + BPF_JSLT => emit_jl(buf, target_off), + BPF_JSLE => emit_jle(buf, target_off), + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let imm = insn.imm as i64; + let base = bpf_to_x86(insn.dst_reg()); + let adjusted_off = if base == X86_RBP { off - 32 } else { off }; + let scratch = if base == X86_RCX { X86_R11 } else { X86_RCX }; + if insn.size() == BPF_DW { + emit_mov_imm64(buf, scratch, imm as u64); + emit_store_mem(buf, base, adjusted_off, scratch, BPF_DW); + } else { + emit_mov_imm32(buf, scratch, imm as i32); + emit_store_mem(buf, base, adjusted_off, scratch, insn.size()); + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_x86(insn.dst_reg()); + let src = bpf_to_x86(insn.src_reg()); + let adjusted_off = if base == X86_RBP { off - 32 } else { off }; + emit_store_mem(buf, base, adjusted_off, src, insn.size()); + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_x86(insn.src_reg()); + let dst = bpf_to_x86(insn.dst_reg()); + let adjusted_off = if base == X86_RBP { off - 32 } else { off }; + emit_load_mem(buf, dst, base, adjusted_off, insn.size()); + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_x86(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + emit_mov_imm64(buf, dst, val); + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + emit_mov_imm64(buf, X86_RAX, helper_fn as usize as u64); + emit_call_reg(buf, X86_RAX); + } +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/mod.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/mod.rs new file mode 100644 index 0000000000..efb7b441b0 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/mod.rs @@ -0,0 +1,352 @@ +use alloc::{ + alloc::{Layout, alloc_zeroed, dealloc}, + collections::BTreeMap, + vec, + vec::Vec, +}; + +#[cfg(target_arch = "aarch64")] +use ax_memory_addr::VirtAddr; + +pub(crate) use super::HelperFn; +#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] +pub(crate) use super::bpf_insn; +use super::bpf_insn::{ + BPF_ALU, BPF_ALU64, BPF_EXIT, BPF_JMP, BPF_JMP32, BPF_LD, BPF_LDX, BPF_ST, BPF_STX, BpfInsn, +}; + +#[cfg(target_arch = "aarch64")] +mod jit_aarch64; +#[cfg(target_arch = "riscv64")] +mod jit_riscv64; +#[cfg(target_arch = "x86_64")] +mod jit_x86_64; + +#[cfg(target_arch = "aarch64")] +use jit_aarch64::Aarch64Backend as Backend; +#[cfg(target_arch = "riscv64")] +use jit_riscv64::Riscv64Backend as Backend; +#[cfg(target_arch = "x86_64")] +use jit_x86_64::X86_64Backend as Backend; + +pub struct JitBuffer { + ptr: *mut u8, + size: usize, + pos: usize, + /// When true, emit methods only count bytes without writing to memory. + counting: bool, +} + +// SAFETY: JitBuffer owns a single heap allocation. After finalize(), the +// buffer is read-only. The Send/Sync impls are safe because the buffer is +// never mutated concurrently. +unsafe impl Send for JitBuffer {} +unsafe impl Sync for JitBuffer {} + +impl core::fmt::Debug for JitBuffer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("JitBuffer") + .field("size", &self.size) + .field("pos", &self.pos) + .finish() + } +} + +impl JitBuffer { + pub fn new(requested_size: usize) -> Result { + let size = (requested_size + 4095) & !4095; + if size == 0 { + return Err("jit buffer size is zero"); + } + let layout = Layout::from_size_align(size, 4096).map_err(|_| "invalid layout")?; + let ptr = unsafe { alloc_zeroed(layout) }; + if ptr.is_null() { + return Err("jit buffer allocation failed"); + } + Ok(Self { + ptr, + size, + pos: 0, + counting: false, + }) + } + + /// Creates a counting-only buffer used during the sizing pass. + /// All emit calls only count bytes without writing to memory. + pub fn new_sizing() -> Self { + Self { + ptr: core::ptr::null_mut(), + size: usize::MAX, + pos: 0, + counting: true, + } + } + + pub fn emit_u8(&mut self, val: u8) { + if !self.counting { + assert!( + self.pos < self.size, + "JitBuffer overflow at offset {} (size {})", + self.pos, + self.size + ); + unsafe { + let dst = self.ptr.add(self.pos); + *dst = val; + } + } + self.pos += 1; + } + + pub fn emit_u32(&mut self, val: u32) { + if !self.counting { + assert!( + self.pos + 4 <= self.size, + "JitBuffer overflow at offset {} (size {}, need 4 bytes)", + self.pos, + self.size + ); + unsafe { + let dst = self.ptr.add(self.pos) as *mut u32; + *dst = val.to_le(); + } + } + self.pos += 4; + } + + pub fn offset(&self) -> usize { + self.pos + } + + pub fn entry(&self) -> *const u8 { + self.ptr + } + + /// Returns true when the buffer is in counting mode (sizing pass). + /// Backends should skip direct memory writes when this returns true. + pub fn counting(&self) -> bool { + self.counting + } + + pub fn finalize(&mut self) { + #[cfg(target_arch = "aarch64")] + { + let vaddr = VirtAddr::from_usize(self.ptr as usize); + ax_runtime::hal::cpu::asm::clean_dcache_range_to_pou(vaddr, self.pos); + } + ax_runtime::hal::cpu::asm::flush_icache_all(); + } +} + +impl Drop for JitBuffer { + fn drop(&mut self) { + if !self.ptr.is_null() && self.size > 0 { + let layout = Layout::from_size_align(self.size, 4096).unwrap(); + unsafe { + dealloc(self.ptr, layout); + } + } + } +} + +pub(crate) trait JitBackend { + fn emit_prologue(buf: &mut JitBuffer) -> usize; + fn emit_epilogue(buf: &mut JitBuffer); + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool); + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool); + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32); + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn); +} + +struct JitCompiler<'a> { + insns: &'a [BpfInsn], + offsets: Vec, + helpers: &'a BTreeMap, +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +))] +impl<'a> JitCompiler<'a> { + fn new(insns: &'a [BpfInsn], helpers: &'a BTreeMap) -> Self { + let offsets = vec![0; insns.len()]; + Self { + insns, + offsets, + helpers, + } + } + + fn pass1_sizing(&mut self) -> usize { + let mut buf = JitBuffer::new_sizing(); + let num_insns = self.insns.len(); + let mut pc: usize = 0; + while pc < num_insns { + self.offsets[pc] = buf.offset(); + let insn = &self.insns[pc]; + let class = insn.class(); + + match class { + BPF_ALU | BPF_ALU64 => { + Backend::emit_alu(&mut buf, insn, class == BPF_ALU64); + pc += 1; + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + Backend::emit_epilogue(&mut buf); + pc += 1; + } else if op == 0x80 { + let helper_id = insn.imm as u32; + if let Some(&helper_fn) = self.helpers.get(&helper_id) { + Backend::emit_call(&mut buf, helper_fn); + } else { + Backend::emit_call(&mut buf, |_a1, _a2, _a3, _a4, _a5| u64::MAX); + } + pc += 1; + } else { + Backend::emit_jmp(&mut buf, insn, &self.offsets, pc, class == BPF_JMP); + pc += 1; + } + } + BPF_ST => { + Backend::emit_st(&mut buf, insn); + pc += 1; + } + BPF_STX => { + Backend::emit_stx(&mut buf, insn); + pc += 1; + } + BPF_LDX => { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + BPF_LD => { + if insn.is_ld_dw_imm() { + if pc + 1 < num_insns { + self.offsets[pc + 1] = buf.offset(); + } + Backend::emit_ld_imm64(&mut buf, insn, 0); + pc += 2; + } else { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + } + _ => return 0, + } + } + buf.offset() + } + + fn compile(&mut self) -> Result { + if self.insns.is_empty() { + return Err("no instructions to compile"); + } + + let insn_size_total = self.pass1_sizing(); + + let estimated = 128 + insn_size_total + 128 + 256; + let mut buf = JitBuffer::new(estimated)?; + + let prologue_size = Backend::emit_prologue(&mut buf); + + for i in 0..self.offsets.len() { + self.offsets[i] += prologue_size; + } + + let num_insns = self.insns.len(); + let mut pc: usize = 0; + while pc < num_insns { + let insn = &self.insns[pc]; + let class = insn.class(); + + match class { + BPF_ALU | BPF_ALU64 => { + let is_64 = class == BPF_ALU64; + Backend::emit_alu(&mut buf, insn, is_64); + pc += 1; + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + Backend::emit_epilogue(&mut buf); + pc += 1; + } else if op == 0x80 { + let helper_id = insn.imm as u32; + if let Some(&helper_fn) = self.helpers.get(&helper_id) { + Backend::emit_call(&mut buf, helper_fn); + } else { + Backend::emit_call(&mut buf, |_a1, _a2, _a3, _a4, _a5| u64::MAX); + } + pc += 1; + } else { + let is_64 = class == BPF_JMP; + Backend::emit_jmp(&mut buf, insn, &self.offsets, pc, is_64); + pc += 1; + } + } + BPF_ST => { + Backend::emit_st(&mut buf, insn); + pc += 1; + } + BPF_STX => { + Backend::emit_stx(&mut buf, insn); + pc += 1; + } + BPF_LDX => { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + BPF_LD => { + if insn.is_ld_dw_imm() { + let next_imm = if pc + 1 < num_insns { + self.insns[pc + 1].imm + } else { + 0 + }; + Backend::emit_ld_imm64(&mut buf, insn, next_imm); + pc += 2; + } else { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + } + _ => { + return Err("unsupported instruction class"); + } + } + } + + Backend::emit_epilogue(&mut buf); + buf.finalize(); + Ok(buf) + } +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +))] +pub fn try_jit_compile(insns: &[BpfInsn], helpers: &BTreeMap) -> Option { + let mut compiler = JitCompiler::new(insns, helpers); + compiler.compile().ok() +} + +#[cfg(not(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +)))] +pub fn try_jit_compile( + _insns: &[BpfInsn], + _helpers: &BTreeMap, +) -> Option { + None +} diff --git a/os/StarryOS/kernel/src/ebpf/mod.rs b/os/StarryOS/kernel/src/ebpf/mod.rs index e861046945..c8dc13e271 100644 --- a/os/StarryOS/kernel/src/ebpf/mod.rs +++ b/os/StarryOS/kernel/src/ebpf/mod.rs @@ -31,11 +31,15 @@ use kbpf_basic::{ raw_tracepoint::BpfRawTracePointArg, }; +pub(crate) mod bpf_insn; +pub mod ebpf_jit; pub(crate) mod error; pub mod map; pub mod prog; pub mod transform; +pub(crate) type HelperFn = fn(u64, u64, u64, u64, u64) -> u64; + pub use transform::EbpfKernelAuxiliary; use crate::{ diff --git a/os/StarryOS/kernel/src/perf/bpf.rs b/os/StarryOS/kernel/src/perf/bpf.rs index 321adc45af..a04379c67e 100644 --- a/os/StarryOS/kernel/src/perf/bpf.rs +++ b/os/StarryOS/kernel/src/perf/bpf.rs @@ -26,7 +26,7 @@ use rbpf::EbpfVmRaw; use super::PerfEventOps; use crate::{ - ebpf::{BPF_HELPER_FUN_SET, error::BpfResultExt, prog::BpfProg}, + ebpf::{BPF_HELPER_FUN_SET, bpf_insn::BpfInsn, ebpf_jit, error::BpfResultExt, prog::BpfProg}, file::FileLike, }; @@ -188,36 +188,89 @@ pub fn perf_event_open_bpf(args: PerfProbeArgs) -> BpfPerfEventWrapper { BpfPerfEventWrapper::new(BpfPerfEvent::new(args)) } -/// A loaded BPF program bundled with an `rbpf` interpreter that borrows -/// into the program's instruction buffer. +/// Execution backend for a loaded BPF program: JIT-compiled native code +/// or the `rbpf` interpreter as fallback. +enum EbpfExecutor { + /// JIT-compiled native code. + Jit { + entry: unsafe extern "C" fn(*mut u8) -> u64, + _jit_buf: ebpf_jit::JitBuffer, + }, + /// Interpreted execution via `rbpf::EbpfVmRaw`. + Interpreter(EbpfVmRaw<'static>), +} + +/// A loaded BPF program bundled with an execution backend (JIT or +/// interpreter). /// -/// Soundness: the interpreter holds a `'static`-typed slice into the -/// instruction bytes owned by `_prog`; the only thing keeping those bytes -/// alive is the [`Arc`] in `_prog`. Field order in this struct is -/// therefore load-bearing — `vm` is declared first, `_prog` last, so the -/// struct's drop glue runs `vm`'s destructor before `_prog`'s, and the -/// instruction buffer is freed strictly after the borrower is gone. Do not -/// reorder the fields. +/// Soundness: both the interpreter and the JIT buffer reference the +/// instruction bytes owned by `_prog`. Field order is load-bearing — +/// `executor` is declared first, `_prog` last, so the struct's drop glue +/// runs the executor's destructor before `_prog`'s. +/// Do not reorder the fields. pub struct OwnedEbpfVm { - vm: EbpfVmRaw<'static>, - /// MUST be declared after `vm` (drop order). Keeps the instruction - /// buffer alive for the entire lifetime of `vm`. + executor: EbpfExecutor, + /// MUST be declared after `executor` (drop order). _prog: Arc, } impl OwnedEbpfVm { - /// Build an `rbpf::EbpfVmRaw` around the program's instruction stream - /// and register the kernel helper table on it. The returned value owns - /// both the VM and the [`Arc`] backing its instruction buffer. + /// Build an execution backend for the BPF program. Tries JIT + /// compilation first; falls back to the `rbpf` interpreter. pub fn new(bpf_prog: Arc) -> AxResult { let prog = bpf_prog .into_any_arc() .downcast::() .map_err(|_| AxError::InvalidInput)?; - // Extend the borrow of `prog.insns()` to `'static`. SAFETY: the - // Arc is moved into the returned `OwnedEbpfVm` together - // with the VM, and the struct's field drop order (vm before _prog) - // guarantees the borrower is destroyed before the buffer is freed. + + let executor = if let Some(jit_executor) = Self::try_jit(&prog) { + jit_executor + } else { + Self::build_interpreter(&prog)? + }; + + Ok(Self { + executor, + _prog: prog, + }) + } + + fn try_jit(prog: &Arc) -> Option { + let prog_slice = prog.insns(); + if !prog_slice + .len() + .is_multiple_of(core::mem::size_of::()) + { + warn!("eBPF JIT: bytecode length not aligned to BpfInsn size"); + return None; + } + let insn_count = prog_slice.len() / core::mem::size_of::(); + if insn_count == 0 { + return None; + } + // SAFETY: BpfInsn is #[repr(C)] and 8 bytes; the byte slice is + // byte-swapped (little-endian) by kbpf-basic preprocessor. + let insns = unsafe { + core::slice::from_raw_parts(prog_slice.as_ptr() as *const BpfInsn, insn_count) + }; + let helpers = BPF_HELPER_FUN_SET.get()?; + let jit_buf = ebpf_jit::try_jit_compile(insns, helpers)?; + // SAFETY: the JIT buffer is page-aligned and holds valid native + // code for the target architecture. + let entry: unsafe extern "C" fn(*mut u8) -> u64 = + unsafe { core::mem::transmute(jit_buf.entry()) }; + info!( + "eBPF JIT: compiled {} instructions into {} bytes of native code", + insn_count, + jit_buf.offset() + ); + Some(EbpfExecutor::Jit { + entry, + _jit_buf: jit_buf, + }) + } + + fn build_interpreter(prog: &Arc) -> AxResult { let prog_slice = prog.insns(); let prog_slice = unsafe { core::slice::from_raw_parts(prog_slice.as_ptr(), prog_slice.len()) }; @@ -225,60 +278,53 @@ impl OwnedEbpfVm { error!("rbpf::EbpfVmRaw::new failed: {e:?}"); AxError::InvalidInput })?; - if let Some(table) = BPF_HELPER_FUN_SET.get() { for (key, value) in table.iter() { let _ = vm.register_helper(*key, *value); } } - // TODO: not all of the address space is accessible to a BPF program; - // allowing the full `0..u64::MAX` range disables rbpf's bounds check - // and lets a buggy/hostile program read arbitrary kernel memory via - // direct loads. Narrow this to the legitimately-reachable context / - // map / stack ranges once kbpf-basic exposes the per-program bounds. vm.register_allowed_memory(0..u64::MAX); - - Ok(Self { vm, _prog: prog }) + Ok(EbpfExecutor::Interpreter(vm)) } - /// Execute the wrapped BPF program with the supplied context bytes. - /// - /// Takes `&self`: `rbpf::EbpfVmRaw::execute_program` is itself `&self` - /// (the interpreter keeps its scratch state on the local stack), so no - /// exterior mutability — and therefore no lock — is required around an - /// `OwnedEbpfVm`. pub fn execute_program(&self, ctx: &mut [u8]) -> Result { - self.vm.execute_program(ctx) + match &self.executor { + EbpfExecutor::Jit { entry, .. } => Ok(unsafe { entry(ctx.as_mut_ptr()) }), + EbpfExecutor::Interpreter(vm) => vm.execute_program(ctx), + } } - /// Execute the wrapped BPF program with a `PtRegs` as the single-pointer - /// context argument the kprobe/kretprobe ABI expects. pub fn execute_with_ptregs(&self, pt_regs: &mut PtRegs) -> Result { - // SAFETY: kbpf-basic's kprobe-context contract passes a raw - // pointer to `PtRegs` as the program context; we hand the same - // bytes here. - let probe_context = unsafe { - core::slice::from_raw_parts_mut( - pt_regs as *mut PtRegs as *mut u8, - core::mem::size_of::(), - ) - }; - self.vm.execute_program(probe_context) + match &self.executor { + EbpfExecutor::Jit { entry, .. } => { + Ok(unsafe { entry(pt_regs as *mut PtRegs as *mut u8) }) + } + EbpfExecutor::Interpreter(vm) => { + let probe_context = unsafe { + core::slice::from_raw_parts_mut( + pt_regs as *mut PtRegs as *mut u8, + core::mem::size_of::(), + ) + }; + vm.execute_program(probe_context) + } + } } } impl Debug for OwnedEbpfVm { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "OwnedEbpfVm") + match &self.executor { + EbpfExecutor::Jit { .. } => write!(f, "OwnedEbpfVm(jit)"), + EbpfExecutor::Interpreter(_) => write!(f, "OwnedEbpfVm(interp)"), + } } } -// SAFETY: the bundled `EbpfVmRaw<'static>` is an interpreter over an immutable -// instruction slice; the `Arc` backing that slice is `Send + Sync`. -// `execute_program` runs entirely off `&self` and a private stack, so it is -// re-entrant and may be driven concurrently from probe-fire paths on several -// CPUs without data races. The raw-pointer fields rbpf carries internally are -// never mutated after construction, so promoting the bundle to `Send + Sync` -// is sound. +// SAFETY: both execution backends operate over an immutable instruction +// slice / JIT buffer backed by the `Arc`; `execute_program` runs +// entirely off `&self` and a private stack, so it is re-entrant and may be +// driven concurrently from probe-fire paths on several CPUs without data +// races. The JIT code is read-only after compilation. unsafe impl Send for OwnedEbpfVm {} unsafe impl Sync for OwnedEbpfVm {}