diff --git a/os/StarryOS/kernel/src/ebpf/bpf_insn.rs b/os/StarryOS/kernel/src/ebpf/bpf_insn.rs new file mode 100644 index 0000000000..05a625d27a --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/bpf_insn.rs @@ -0,0 +1,86 @@ +pub const BPF_LD: u8 = 0x00; +pub const BPF_LDX: u8 = 0x01; +pub const BPF_ST: u8 = 0x02; +pub const BPF_STX: u8 = 0x03; +pub const BPF_ALU: u8 = 0x04; +pub const BPF_JMP: u8 = 0x05; +pub const BPF_JMP32: u8 = 0x06; +pub const BPF_ALU64: u8 = 0x07; + +pub const BPF_W: u8 = 0x00; +pub const BPF_H: u8 = 0x08; +pub const BPF_B: u8 = 0x10; +pub const BPF_DW: u8 = 0x18; + +pub const BPF_IMM: u8 = 0x00; +pub const BPF_MEM: u8 = 0x60; + +pub const BPF_ADD: u8 = 0x00; +pub const BPF_SUB: u8 = 0x10; +pub const BPF_MUL: u8 = 0x20; +pub const BPF_DIV: u8 = 0x30; +pub const BPF_OR: u8 = 0x40; +pub const BPF_AND: u8 = 0x50; +pub const BPF_LSH: u8 = 0x60; +pub const BPF_RSH: u8 = 0x70; +pub const BPF_NEG: u8 = 0x80; +pub const BPF_MOD: u8 = 0x90; +pub const BPF_XOR: u8 = 0xa0; +pub const BPF_MOV: u8 = 0xb0; +pub const BPF_ARSH: u8 = 0xc0; +pub const BPF_END: u8 = 0xd0; + +pub const BPF_JA: u8 = 0x00; +pub const BPF_EXIT: u8 = 0x90; +pub const BPF_JEQ: u8 = 0x10; +pub const BPF_JGT: u8 = 0x20; +pub const BPF_JGE: u8 = 0x30; +pub const BPF_JSET: u8 = 0x40; +pub const BPF_JNE: u8 = 0x50; +pub const BPF_JSGT: u8 = 0x60; +pub const BPF_JSGE: u8 = 0x70; +pub const BPF_JLT: u8 = 0xa0; +pub const BPF_JLE: u8 = 0xb0; +pub const BPF_JSLT: u8 = 0xc0; +pub const BPF_JSLE: u8 = 0xd0; + +pub const BPF_X: u8 = 0x08; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct BpfInsn { + pub code: u8, + pub dst_src_reg: u8, + pub off: i16, + pub imm: i32, +} + +impl BpfInsn { + pub fn dst_reg(&self) -> u8 { + self.dst_src_reg & 0xf + } + + pub fn src_reg(&self) -> u8 { + (self.dst_src_reg >> 4) & 0xf + } + + pub fn class(&self) -> u8 { + self.code & 0x07 + } + + pub fn size(&self) -> u8 { + self.code & 0x18 + } + + pub fn mode(&self) -> u8 { + self.code & 0xe0 + } + + pub fn alu_op(&self) -> u8 { + self.code & 0xf0 + } + + pub fn is_ld_dw_imm(&self) -> bool { + self.code == (BPF_LD | BPF_IMM | BPF_DW) + } +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_aarch64.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_aarch64.rs new file mode 100644 index 0000000000..fb28e3eef3 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_aarch64.rs @@ -0,0 +1,24 @@ +use super::{HelperFn, JitBackend, JitBuffer, bpf_insn::BpfInsn}; + +pub(crate) struct Aarch64Backend; + +impl JitBackend for Aarch64Backend { + fn emit_prologue(_buf: &mut JitBuffer) -> usize { + 0 + } + fn emit_epilogue(_buf: &mut JitBuffer) {} + fn emit_alu(_buf: &mut JitBuffer, _insn: &BpfInsn, _is_64: bool) {} + fn emit_jmp( + _buf: &mut JitBuffer, + _insn: &BpfInsn, + _offsets: &[usize], + _pc: usize, + _is_64: bool, + ) { + } + fn emit_st(_buf: &mut JitBuffer, _insn: &BpfInsn) {} + fn emit_stx(_buf: &mut JitBuffer, _insn: &BpfInsn) {} + fn emit_ldx(_buf: &mut JitBuffer, _insn: &BpfInsn) {} + fn emit_ld_imm64(_buf: &mut JitBuffer, _insn: &BpfInsn, _next_imm: i32) {} + fn emit_call(_buf: &mut JitBuffer, _helper_fn: HelperFn) {} +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_riscv64.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_riscv64.rs new file mode 100644 index 0000000000..c61425b10f --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_riscv64.rs @@ -0,0 +1,1284 @@ +use super::{ + BpfInsn, HelperFn, JitBackend, JitBuffer, + bpf_insn::{ + BPF_ADD, BPF_ALU, BPF_ALU64, BPF_AND, BPF_ARSH, BPF_B, BPF_DIV, BPF_DW, BPF_END, BPF_EXIT, + BPF_H, BPF_JA, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, BPF_JNE, + BPF_JSET, BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_LD, BPF_LDX, BPF_LSH, BPF_MEM, + BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, BPF_OR, BPF_RSH, BPF_ST, BPF_STX, BPF_SUB, BPF_W, + BPF_X, BPF_XOR, + }, +}; + +const RV_ZERO: u32 = 0; +const RV_RA: u32 = 1; +const RV_SP: u32 = 2; +const RV_T1: u32 = 6; +const RV_T2: u32 = 7; +const RV_S1: u32 = 9; +const RV_A0: u32 = 10; +const RV_A1: u32 = 11; +const RV_A2: u32 = 12; +const RV_A3: u32 = 13; +const RV_A4: u32 = 14; +const RV_A5: u32 = 15; +const RV_S2: u32 = 18; +const RV_S3: u32 = 19; +const RV_S4: u32 = 20; +const RV_S5: u32 = 21; +const RV_T6: u32 = 31; + +const BPF_STACK_SIZE: usize = 512; +const CALLEE_SAVED_SIZE: usize = 48; +const FRAME_SIZE: usize = BPF_STACK_SIZE + CALLEE_SAVED_SIZE; + +fn bpf_to_rv(r: u8) -> u32 { + match r { + 0 => RV_A0, + 1 => RV_A1, + 2 => RV_A2, + 3 => RV_A3, + 4 => RV_A4, + 5 => RV_A5, + 6 => RV_S1, + 7 => RV_S2, + 8 => RV_S3, + 9 => RV_S4, + 10 => RV_S5, + _ => RV_ZERO, + } +} + +fn rv_r(funct7: u32, rs2: u32, rs1: u32, funct3: u32, rd: u32) -> u32 { + (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 0x33 +} + +fn rv_rw(funct7: u32, rs2: u32, rs1: u32, funct3: u32, rd: u32) -> u32 { + rv_r(funct7, rs2, rs1, funct3, rd) | (0x3b ^ 0x33) +} + +fn rv_i(imm: u32, rs1: u32, funct3: u32, rd: u32, opcode: u32) -> u32 { + (imm << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | opcode +} + +fn rv_s(imm: u32, rs2: u32, rs1: u32, funct3: u32) -> u32 { + ((imm >> 5) << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | ((imm & 0x1f) << 7) | 0x23 +} + +fn rv_b(imm: u32, rs2: u32, rs1: u32, funct3: u32) -> u32 { + let bit12 = (imm >> 12) & 1; + let bits10_5 = (imm >> 5) & 0x3f; + let bits4_1 = (imm >> 1) & 0xf; + let bit11 = (imm >> 11) & 1; + (bit12 << 31) + | (bits10_5 << 25) + | (rs2 << 20) + | (rs1 << 15) + | (funct3 << 12) + | (bits4_1 << 8) + | (bit11 << 7) + | 0x63 +} + +fn rv_u(imm: u32, rd: u32, opcode: u32) -> u32 { + (imm << 12) | (rd << 7) | opcode +} + +fn emit_add(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 0, rd)); +} + +fn emit_addw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 0, rd)); +} + +fn emit_sub(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 0, rd)); +} + +fn emit_subw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0x20, rs2, rs1, 0, rd)); +} + +fn emit_addi(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 0, rd, 0x13)); +} + +fn emit_addiw(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 0, rd, 0x1b)); +} + +fn emit_and(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 7, rd)); +} + +fn emit_andw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 7, rd)); +} + +fn emit_or(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 6, rd)); +} + +fn emit_orw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 6, rd)); +} + +fn emit_xor(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 4, rd)); +} + +fn emit_xorw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 4, rd)); +} + +fn emit_sll(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 1, rd)); +} + +fn emit_sllw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 1, rd)); +} + +fn emit_srl(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 5, rd)); +} + +fn emit_srlw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0, rs2, rs1, 5, rd)); +} + +fn emit_sra(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 5, rd)); +} + +fn emit_sraw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(0x20, rs2, rs1, 5, rd)); +} + +fn emit_mul(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 0, rd)); +} + +fn emit_mulw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(1, rs2, rs1, 0, rd)); +} + +fn emit_divu(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 5, rd)); +} + +fn emit_divuw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(1, rs2, rs1, 5, rd)); +} + +fn emit_remu(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 7, rd)); +} + +fn emit_remuw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_rw(1, rs2, rs1, 7, rd)); +} + +fn emit_andi(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 7, rd, 0x13)); +} + +fn emit_ori(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 6, rd, 0x13)); +} + +fn emit_xori(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 4, rd, 0x13)); +} + +fn emit_zext32(buf: &mut JitBuffer, rd: u32) { + emit_slli(buf, rd, rd, 32); + emit_srli(buf, rd, rd, 32); +} + +/// Byte-reverse within 8 bytes (rev8 rd, rs1) +/// Encoding: grevi rd, rs1, 24 (RV64 Zbb) +fn emit_rev8(buf: &mut JitBuffer, rd: u32, rs: u32) { + buf.emit_u32(rv_i(24, rs, 5, rd, 0x13)); +} + +fn emit_slli(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 1, rd, 0x13)); +} + +fn emit_srli(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 5, rd, 0x13)); +} + +fn emit_srai(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(0x400 | shamt, rs1, 5, rd, 0x13)); +} + +fn emit_slliw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 1, rd, 0x1b)); +} + +fn emit_srliw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 5, rd, 0x1b)); +} + +fn emit_sraiw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(0x400 | shamt, rs1, 5, rd, 0x1b)); +} + +fn emit_lui(buf: &mut JitBuffer, rd: u32, imm: u32) { + buf.emit_u32(rv_u(imm, rd, 0x37)); +} + +fn emit_ld(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 3, rd, 0x03)); +} + +fn emit_lwu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 6, rd, 0x03)); +} + +fn emit_lw(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 2, rd, 0x03)); +} + +fn emit_lhu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 5, rd, 0x03)); +} + +fn emit_lh(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 1, rd, 0x03)); +} + +fn emit_lbu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 4, rd, 0x03)); +} + +fn emit_lb(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 0, rd, 0x03)); +} + +fn emit_sd(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 3)); +} + +fn emit_sw(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 2)); +} + +fn emit_sh(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 1)); +} + +fn emit_sb(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 0)); +} + +fn emit_beq(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 0)); +} + +fn emit_bne(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 1)); +} + +fn emit_blt(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 4)); +} + +fn emit_bge(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 5)); +} + +fn emit_bltu(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 6)); +} + +fn emit_bgeu(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 7)); +} + +fn emit_jalr(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 0, rd, 0x67)); +} + +fn emit_auipc(buf: &mut JitBuffer, rd: u32, imm: u32) { + buf.emit_u32(rv_u(imm, rd, 0x17)); +} + +fn emit_jal(buf: &mut JitBuffer, rd: u32, imm: i32) { + let imm = imm as u32; + let bit20 = (imm >> 20) & 1; + let bits10_1 = (imm >> 1) & 0x3ff; + let bit11 = (imm >> 11) & 1; + let bits19_12 = (imm >> 12) & 0xff; + buf.emit_u32( + (bit20 << 31) | (bits10_1 << 21) | (bit11 << 20) | (bits19_12 << 12) | (rd << 7) | 0x6f, + ); +} + +fn emit_ret(buf: &mut JitBuffer) { + emit_jalr(buf, RV_ZERO, RV_RA, 0); +} + +fn emit_mv(buf: &mut JitBuffer, rd: u32, rs: u32) { + emit_addi(buf, rd, rs, 0); +} + +fn emit_nop(buf: &mut JitBuffer) { + buf.emit_u32(0x00000013); +} + +fn emit_load_imm64_padded(buf: &mut JitBuffer, rd: u32, val: u64) { + let start = buf.offset(); + emit_load_imm64(buf, rd, val); + let emitted = buf.offset() - start; + for _ in 0..((24 - emitted) / 4) { + emit_nop(buf); + } +} + +fn emit_load_imm64(buf: &mut JitBuffer, rd: u32, val: u64) { + let val_i = val as i64; + if val_i >= -2048 && val_i < 2048 { + emit_addi(buf, rd, RV_ZERO, val_i as i32); + return; + } + if (val as u32 as i32) as i64 == val_i { + let lo32 = val as u32; + let lo12 = (lo32 << 20) >> 20; + let hi20 = (lo32.wrapping_sub(lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, rd, hi20 & 0xFFFFF); + if lo12 != 0 { + emit_addiw(buf, rd, rd, lo12 as i32); + } + return; + } + let upper = (val >> 32) as u32; + let upper_lo12 = (upper << 20) >> 20; + let upper_hi20 = (upper.wrapping_sub(upper_lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, rd, upper_hi20 & 0xFFFFF); + emit_addiw(buf, rd, rd, upper_lo12 as i32); + emit_slli(buf, rd, rd, 32); + let lower = val as u32; + let lower_lo12 = (lower << 20) >> 20; + let lower_hi20 = (lower.wrapping_sub(lower_lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, RV_T1, lower_hi20 & 0xFFFFF); + if lower_lo12 != 0 { + emit_addiw(buf, RV_T1, RV_T1, lower_lo12 as i32); + } + emit_add(buf, rd, rd, RV_T1); +} + +fn emit_load_imm32(buf: &mut JitBuffer, rd: u32, val: i32) { + let needs_upper = (val as i32) < -2048 || (val as i32) >= 2048; + if !needs_upper { + emit_addi(buf, rd, RV_ZERO, val); + } else { + let val_u = val as u32; + let lo12 = (val_u << 20) >> 20; + let hi20 = (val_u.wrapping_sub(lo12).wrapping_add(0x800)) >> 12; + emit_lui(buf, rd, hi20 & 0xFFFFF); + if lo12 != 0 { + emit_addiw(buf, rd, rd, lo12 as i32); + } + } +} + +fn emit_add_offset(buf: &mut JitBuffer, rd: u32, rs: u32, off: i32) { + if off >= -2048 && off < 2048 { + emit_addi(buf, rd, rs, off); + } else { + emit_load_imm32(buf, RV_T1, off); + emit_add(buf, rd, rs, RV_T1); + } +} + +pub(crate) struct Riscv64Backend; + +impl JitBackend for Riscv64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + emit_addi(buf, RV_SP, RV_SP, -(FRAME_SIZE as i32)); + emit_sd(buf, RV_RA, RV_SP, 0); + emit_sd(buf, RV_S1, RV_SP, 8); + emit_sd(buf, RV_S2, RV_SP, 16); + emit_sd(buf, RV_S3, RV_SP, 24); + emit_sd(buf, RV_S4, RV_SP, 32); + emit_sd(buf, RV_S5, RV_SP, 40); + emit_addi(buf, RV_S5, RV_SP, FRAME_SIZE as i32); + emit_mv(buf, RV_A1, RV_A0); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + emit_ld(buf, RV_RA, RV_SP, 0); + emit_ld(buf, RV_S1, RV_SP, 8); + emit_ld(buf, RV_S2, RV_SP, 16); + emit_ld(buf, RV_S3, RV_SP, 24); + emit_ld(buf, RV_S4, RV_SP, 32); + emit_ld(buf, RV_S5, RV_SP, 40); + emit_addi(buf, RV_SP, RV_SP, FRAME_SIZE as i32); + emit_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_rv(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { + RV_T1 + } else { + bpf_to_rv(insn.src_reg()) + }; + if use_imm { + emit_load_imm64(buf, RV_T1, insn.imm as u64); + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + emit_add(buf, dst, dst, src); + } else { + emit_addw(buf, dst, dst, src); + } + } + BPF_SUB => { + if is_64 { + emit_sub(buf, dst, dst, src); + } else { + emit_subw(buf, dst, dst, src); + } + } + BPF_MUL => { + if is_64 { + emit_mul(buf, dst, dst, src); + } else { + emit_mulw(buf, dst, dst, src); + } + } + BPF_DIV => { + let skip = buf.offset(); + if is_64 { + emit_beq(buf, src, RV_ZERO, 0); + emit_divu(buf, dst, dst, src); + } else { + emit_beq(buf, src, RV_ZERO, 0); + emit_divuw(buf, dst, dst, src); + } + emit_jal(buf, RV_ZERO, 8); + emit_addi(buf, dst, RV_ZERO, 0); + unsafe { + let beq_ptr = buf.entry().add(skip) as *mut u32; + let beq_off = 12u32; + *beq_ptr = rv_b(beq_off, RV_ZERO, src, 0); + } + } + BPF_OR => { + if is_64 { + emit_or(buf, dst, dst, src); + } else { + emit_orw(buf, dst, dst, src); + } + } + BPF_AND => { + if is_64 { + emit_and(buf, dst, dst, src); + } else { + emit_andw(buf, dst, dst, src); + } + } + BPF_LSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_slli(buf, dst, dst, shamt); + } else { + emit_slliw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_sll(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_sllw(buf, dst, dst, RV_T2); + } + } + BPF_RSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_srli(buf, dst, dst, shamt); + } else { + emit_srliw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_srl(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_srlw(buf, dst, dst, RV_T2); + } + } + BPF_NEG => { + if is_64 { + emit_sub(buf, dst, RV_ZERO, dst); + } else { + emit_subw(buf, dst, RV_ZERO, dst); + } + } + BPF_MOD => { + let skip = buf.offset(); + if is_64 { + emit_beq(buf, src, RV_ZERO, 0); + emit_remu(buf, dst, dst, src); + } else { + emit_beq(buf, src, RV_ZERO, 0); + emit_remuw(buf, dst, dst, src); + } + let end = buf.offset(); + unsafe { + let beq_ptr = buf.entry().add(skip) as *mut u32; + let beq_off = (end - skip) as u32; + *beq_ptr = rv_b(beq_off, RV_ZERO, src, 0); + } + } + BPF_XOR => { + if is_64 { + emit_xor(buf, dst, dst, src); + } else { + emit_xorw(buf, dst, dst, src); + } + } + BPF_MOV => { + if is_64 { + emit_mv(buf, dst, src); + } else { + emit_addiw(buf, dst, src, 0); + } + } + BPF_ARSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_srai(buf, dst, dst, shamt); + } else { + emit_sraiw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_sra(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_sraw(buf, dst, dst, RV_T2); + } + } + BPF_END => { + // BPF_TO_BE: byte swap to big-endian (RISC-V is little-endian native) + let to_be = (insn.code & BPF_X) != 0; + match (to_be, insn.imm) { + (true, 16) => { + // 16-bit: rev8 then shift + emit_rev8(buf, dst, dst); + emit_srli(buf, dst, dst, 48); + } + (true, 32) => { + // 32-bit: rev8 then shift + emit_rev8(buf, dst, dst); + emit_srli(buf, dst, dst, 32); + } + (true, 64) => { + // 64-bit: rev8 + emit_rev8(buf, dst, dst); + } + // BPF_TO_LE on RISC-V: no-op (native little-endian) + _ => {} + } + } + _ => {} + } + + if !is_64 && insn.alu_op() != BPF_ARSH { + emit_zext32(buf, dst); + } + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let off = offsets[target_pc] as isize - buf.offset() as isize; + emit_auipc(buf, RV_T6, 0); + emit_load_imm64_padded(buf, RV_T1, off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + } + return; + } + + // BPF_CALL (0x80) is handled separately in compile() + if op == 0x80 { + return; + } + + let dst = bpf_to_rv(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { + RV_T1 + } else { + bpf_to_rv(insn.src_reg()) + }; + + if use_imm { + if is_64 { + emit_load_imm64(buf, RV_T1, insn.imm as u64); + } else { + emit_load_imm32(buf, RV_T1, insn.imm); + } + } + + if !is_64 { + emit_zext32(buf, dst); + emit_zext32(buf, src); + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + + match op { + BPF_JEQ => { + let start = buf.offset(); + emit_bne(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 1); + } + } + BPF_JGT => { + let start = buf.offset(); + emit_bgeu(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 7); + } + } + BPF_JGE => { + let start = buf.offset(); + emit_bltu(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 6); + } + } + BPF_JSET => { + emit_and(buf, RV_T2, dst, src); + let start = buf.offset(); + emit_beq(buf, RV_T2, RV_ZERO, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, RV_ZERO, RV_T2, 0); + } + } + BPF_JNE => { + let start = buf.offset(); + emit_beq(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 0); + } + } + BPF_JSGT => { + let start = buf.offset(); + emit_bge(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 5); + } + } + BPF_JSGE => { + let start = buf.offset(); + emit_blt(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 4); + } + } + BPF_JLT => { + let start = buf.offset(); + emit_bgeu(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 7); + } + } + BPF_JLE => { + let start = buf.offset(); + emit_bltu(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 6); + } + } + BPF_JSLT => { + let start = buf.offset(); + emit_bge(buf, dst, src, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, src, dst, 5); + } + } + BPF_JSLE => { + let start = buf.offset(); + emit_blt(buf, src, dst, 0); + let auipc_pos = buf.offset(); + emit_auipc(buf, RV_T6, 0); + let branch_off = (offsets[target_pc] as isize - auipc_pos as isize) as i32; + emit_load_imm64_padded(buf, RV_T1, branch_off as u64); + emit_add(buf, RV_T6, RV_T6, RV_T1); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + let end = buf.offset(); + unsafe { + let ptr = buf.entry().add(start) as *mut u32; + *ptr = rv_b((end - start) as u32, dst, src, 4); + } + } + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_rv(insn.dst_reg()); + let adjusted_off = if base == RV_S5 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, RV_T1, base, adjusted_off); + let val = insn.imm as u64; + match insn.size() { + BPF_B => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sb(buf, RV_T2, RV_T1, 0); + } + BPF_H => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sh(buf, RV_T2, RV_T1, 0); + } + BPF_W => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sw(buf, RV_T2, RV_T1, 0); + } + BPF_DW => { + emit_load_imm64(buf, RV_T2, val); + emit_sd(buf, RV_T2, RV_T1, 0); + } + _ => {} + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_rv(insn.src_reg()); + let base = bpf_to_rv(insn.dst_reg()); + let adjusted_off = if base == RV_S5 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, RV_T1, base, adjusted_off); + match insn.size() { + BPF_B => emit_sb(buf, src, RV_T1, 0), + BPF_H => emit_sh(buf, src, RV_T1, 0), + BPF_W => emit_sw(buf, src, RV_T1, 0), + BPF_DW => emit_sd(buf, src, RV_T1, 0), + _ => {} + } + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_rv(insn.src_reg()); + let dst = bpf_to_rv(insn.dst_reg()); + let adjusted_off = if src == RV_S5 { + off - CALLEE_SAVED_SIZE as i32 + } else { + off + }; + emit_add_offset(buf, RV_T1, src, adjusted_off); + match insn.size() { + BPF_B => { + emit_lbu(buf, dst, RV_T1, 0); + } + BPF_H => { + emit_lhu(buf, dst, RV_T1, 0); + } + BPF_W => { + emit_lwu(buf, dst, RV_T1, 0); + } + BPF_DW => { + emit_ld(buf, dst, RV_T1, 0); + } + _ => {} + } + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_rv(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + let start = buf.offset(); + emit_load_imm64(buf, dst, val); + let emitted = buf.offset() - start; + for _ in 0..((24 - emitted) / 4) { + emit_nop(buf); + } + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + emit_mv(buf, RV_T2, RV_A5); + emit_mv(buf, RV_A0, RV_A1); + emit_mv(buf, RV_A1, RV_A2); + emit_mv(buf, RV_A2, RV_A3); + emit_mv(buf, RV_A3, RV_A4); + emit_mv(buf, RV_A4, RV_T2); + + emit_load_imm64_padded(buf, RV_T1, helper_fn as u64); + emit_jalr(buf, RV_RA, RV_T1, 0); + } +} + +#[cfg(test)] +mod tests_arch_independent { + fn compute_hi20_lo12(val: u32) -> (u32, u32) { + let lo12 = (val << 20) >> 20; + let hi20 = (val.wrapping_sub(lo12).wrapping_add(0x800)) >> 12; + (hi20 & 0xFFFFF, lo12) + } + + fn reconstruct_from_hi20_lo12(hi20: u32, lo12: u32) -> u64 { + let lui_val = ((hi20 as i32).wrapping_shl(12)) as i64; + (lui_val.wrapping_add(lo12 as i32 as i64)) as u64 + } + + fn reconstruct_64bit( + upper_hi20: u32, + upper_lo12: u32, + lower_hi20: u32, + lower_lo12: u32, + ) -> u64 { + let upper = reconstruct_from_hi20_lo12(upper_hi20, upper_lo12); + let lower = reconstruct_from_hi20_lo12(lower_hi20, lower_lo12); + ((upper as u64) << 32).wrapping_add(lower) + } + + #[test] + fn test_hi20_lo12_identity() { + for val in [ + 0x00000000u32, + 0x00000001, + 0x00000FFF, + 0xFFFFF000, + 0xFFFFFFFF, + ] { + let (hi20, lo12) = compute_hi20_lo12(val); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!( + reconstructed, val as u64, + "val={val:#010x}: hi20={hi20:#010x} lo12={lo12:#010x}" + ); + } + } + + #[test] + fn test_hi20_lo12_bit31_set() { + let val: u32 = 0x80000000; + let (hi20, lo12) = compute_hi20_lo12(val); + assert_eq!(hi20, 0x80000, "hi20 should be 0x80000"); + assert_eq!(lo12, 0, "lo12 should be 0"); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!(reconstructed, val as u64); + } + + #[test] + fn test_hi20_lo12_large_negative() { + let val: u32 = 0xC0000000; + let (hi20, lo12) = compute_hi20_lo12(val); + assert_ne!(hi20, 0, "hi20 should not be 0"); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!(reconstructed, val as u64); + } + + #[test] + fn test_hi20_lo12_0x7FFFFFFF() { + let val: u32 = 0x7FFFFFFF; + let (hi20, lo12) = compute_hi20_lo12(val); + assert_ne!(hi20, 0, "hi20 should not be 0"); + let reconstructed = reconstruct_from_hi20_lo12(hi20, lo12); + assert_eq!(reconstructed, val as u64); + } + + #[test] + fn test_64bit_reconstruct_0x1_80000000() { + let val: u64 = 0x0000_0001_8000_0000; + let upper = (val >> 32) as u32; + let lower = val as u32; + let (upper_hi20, upper_lo12) = compute_hi20_lo12(upper); + let (lower_hi20, lower_lo12) = compute_hi20_lo12(lower); + assert_ne!(lower_hi20, 0, "lower_hi20 should not be 0 for 0x80000000"); + let reconstructed = reconstruct_64bit(upper_hi20, upper_lo12, lower_hi20, lower_lo12); + assert_eq!(reconstructed, val, "val={val:#018x}"); + } + + #[test] + fn test_64bit_reconstruct_all_ones() { + let val: u64 = 0xFFFF_FFFF_FFFF_FFFF; + let upper = (val >> 32) as u32; + let lower = val as u32; + let (upper_hi20, upper_lo12) = compute_hi20_lo12(upper); + let (lower_hi20, lower_lo12) = compute_hi20_lo12(lower); + let reconstructed = reconstruct_64bit(upper_hi20, upper_lo12, lower_hi20, lower_lo12); + assert_eq!(reconstructed, val); + } + + #[test] + fn test_64bit_reconstruct_alternating() { + let val: u64 = 0x5555_5555_AAAA_AAAA; + let upper = (val >> 32) as u32; + let lower = val as u32; + let (upper_hi20, upper_lo12) = compute_hi20_lo12(upper); + let (lower_hi20, lower_lo12) = compute_hi20_lo12(lower); + let reconstructed = reconstruct_64bit(upper_hi20, upper_lo12, lower_hi20, lower_lo12); + assert_eq!(reconstructed, val); + } + + #[test] + fn test_hi20_masked_to_20_bits() { + for val in 0u32..4096 { + let (hi20, _) = compute_hi20_lo12(val * 0x1000); + assert_eq!( + hi20 & !0xFFFFF, + 0, + "hi20 must fit in 20 bits for val={val:#010x}" + ); + } + } +} + +#[cfg(all(test, target_arch = "riscv64"))] +mod tests { + use super::*; + + fn new_buf() -> JitBuffer { + JitBuffer::new(4096).unwrap() + } + + fn buf_as_u32_slice(buf: &JitBuffer) -> &[u32] { + unsafe { core::slice::from_raw_parts(buf.entry() as *const u32, buf.offset() / 4) } + } + + fn decode_lui(insn: u32) -> (u32, u32) { + let rd = (insn >> 7) & 0x1f; + let imm = (insn >> 12) & 0xfffff; + (rd, imm) + } + + fn decode_addiw(insn: u32) -> (u32, u32, i32) { + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let imm = ((insn as i32) << 20) >> 20; + (rd, rs1, imm) + } + + fn decode_slli(insn: u32) -> (u32, u32, u32) { + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let shamt = (insn >> 20) & 0x3f; + (rd, rs1, shamt) + } + + fn decode_add(insn: u32) -> (u32, u32, u32) { + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let rs2 = (insn >> 20) & 0x1f; + (rd, rs1, rs2) + } + + fn decode_addi(insn: u32) -> (u32, u32, i32) { + decode_addiw(insn) + } + + fn sign_extend_20(imm20: u32) -> i64 { + ((imm20 << 12) as i32 as i64) << 12 >> 12 + } + + fn reconstruct_load_imm64(insns: &[u32]) -> u64 { + let mut idx = 0; + let first_opcode = insns[idx] & 0x7f; + if first_opcode == 0x13 { + let (_, _, imm) = decode_addi(insns[idx]); + return imm as u64; + } + assert_eq!(first_opcode, 0x37, "expected LUI"); + let (rd1, imm20_hi) = decode_lui(insns[idx]); + idx += 1; + let upper_val; + if idx < insns.len() && (insns[idx] & 0x7f) == 0x1b { + let (rd2, rs1, lo12) = decode_addiw(insns[idx]); + assert_eq!(rd2, rd1); + assert_eq!(rs1, rd1); + upper_val = ((imm20_hi as i32) << 12).wrapping_add(lo12); + idx += 1; + } else { + upper_val = (imm20_hi as i32) << 12; + } + if idx >= insns.len() || (insns[idx] & 0x7f) != 0x13 { + return upper_val as u64; + } + let (rd3, rs1_3, shamt) = decode_slli(insns[idx]); + assert_eq!(rd3, rd1); + assert_eq!(rs1_3, rd1); + assert_eq!(shamt, 32); + let shifted = (upper_val as u64) << 32; + idx += 1; + let (_, imm20_lo) = decode_lui(insns[idx]); + idx += 1; + let (rd_t1, rs1_t1, lo12_lo) = decode_addiw(insns[idx]); + let lower_val = ((imm20_lo as i32) << 12).wrapping_add(lo12_lo); + let lower_extended = lower_val as i64 as u64; + shifted.wrapping_add(lower_extended) + } + + #[test] + fn test_load_imm64_small_positive() { + let mut buf = new_buf(); + emit_load_imm64(&mut buf, RV_A0, 42); + assert_eq!(buf.offset(), 4); + let insns = buf_as_u32_slice(&buf); + let (_, _, imm) = decode_addi(insns[0]); + assert_eq!(imm, 42); + } + + #[test] + fn test_load_imm64_small_negative() { + let mut buf = new_buf(); + emit_load_imm64(&mut buf, RV_A0, (-1i64) as u64); + assert_eq!(buf.offset(), 4); + let insns = buf_as_u32_slice(&buf); + let (_, _, imm) = decode_addi(insns[0]); + assert_eq!(imm, -1); + } + + #[test] + fn test_load_imm64_32bit_value() { + let mut buf = new_buf(); + let val: u64 = 0x12345000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_bit31_set() { + let mut buf = new_buf(); + let val: u64 = 0x0000_0001_8000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_all_ones() { + let mut buf = new_buf(); + let val: u64 = 0xFFFF_FFFF_FFFF_FFFF; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_high_bit_only() { + let mut buf = new_buf(); + let val: u64 = 0x8000_0000_0000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_max_positive() { + let mut buf = new_buf(); + let val: u64 = 0x7FFF_FFFF_FFFF_FFFF; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_alternating_bits() { + let mut buf = new_buf(); + let val: u64 = 0x5555_5555_AAAA_AAAA; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_upper_ffff_lower_0() { + let mut buf = new_buf(); + let val: u64 = 0xFFFF_FFFF_0000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_load_imm64_zero_lower() { + let mut buf = new_buf(); + let val: u64 = 0x1234_5678_0000_0000; + emit_load_imm64(&mut buf, RV_A0, val); + let insns = buf_as_u32_slice(&buf); + assert_eq!(reconstruct_load_imm64(insns), val); + } + + #[test] + fn test_alu_add32_zext() { + let mut buf = new_buf(); + let mut insn = BpfInsn::default(); + insn.code = BPF_ALU | BPF_ADD | BPF_X; + insn.dst_reg = 1; + insn.src_reg = 2; + Riscv64Backend::emit_alu(&mut buf, &insn, false); + let insns = buf_as_u32_slice(&buf); + let last = insns[insns.len() - 1]; + let prev = insns[insns.len() - 2]; + let (_, _, shamt1) = decode_slli(prev); + let (_, _, shamt2) = decode_slli_with_func(last); + assert_eq!(shamt1, 32); + assert_eq!(shamt2, 32); + } + + fn decode_slli_with_func(insn: u32) -> (u32, u32, u32) { + let funct3 = (insn >> 12) & 0x7; + let opcode = insn & 0x7f; + assert_eq!(opcode, 0x13); + assert!(funct3 == 1 || funct3 == 5); + let rd = (insn >> 7) & 0x1f; + let rs1 = (insn >> 15) & 0x1f; + let shamt = (insn >> 20) & 0x3f; + (rd, rs1, shamt) + } + + #[test] + fn test_alu_mov32_zext() { + let mut buf = new_buf(); + let mut insn = BpfInsn::default(); + insn.code = BPF_ALU | BPF_MOV | BPF_X; + insn.dst_reg = 1; + insn.src_reg = 2; + Riscv64Backend::emit_alu(&mut buf, &insn, false); + assert!(buf.offset() > 0); + } + + #[test] + fn test_arsh32_no_zext() { + let mut buf = new_buf(); + let mut insn = BpfInsn::default(); + insn.code = BPF_ALU | BPF_ARSH; + insn.dst_reg = 1; + insn.imm = 1; + Riscv64Backend::emit_alu(&mut buf, &insn, false); + assert!(buf.offset() > 0); + } +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_x86_64.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_x86_64.rs new file mode 100644 index 0000000000..030f537a80 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/jit_x86_64.rs @@ -0,0 +1,904 @@ +use super::{ + super::{ + HelperFn, + bpf_insn::{ + BPF_ADD, BPF_AND, BPF_ARSH, BPF_B, BPF_DIV, BPF_DW, BPF_END, BPF_H, BPF_JA, BPF_JEQ, + BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, BPF_JNE, BPF_JSET, BPF_JSGE, + BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_LSH, BPF_MEM, BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, + BPF_OR, BPF_RSH, BPF_SUB, BPF_W, BPF_X, BPF_XOR, BpfInsn, + }, + }, + JitBackend, JitBuffer, +}; + +const X86_RAX: u8 = 0; +const X86_RCX: u8 = 1; +const X86_RDX: u8 = 2; +const X86_RBX: u8 = 3; +const X86_RSP: u8 = 4; +const X86_RBP: u8 = 5; +const X86_RSI: u8 = 6; +const X86_RDI: u8 = 7; +const X86_R8: u8 = 8; +const _X86_R9: u8 = 9; +const X86_R10: u8 = 10; +const X86_R11: u8 = 11; +const _X86_R12: u8 = 12; +const X86_R13: u8 = 13; +const X86_R14: u8 = 14; +const X86_R15: u8 = 15; + +fn bpf_to_x86(r: u8) -> u8 { + match r { + 0 => X86_RAX, + 1 => X86_RDI, + 2 => X86_RSI, + 3 => X86_RDX, + 4 => X86_RCX, + 5 => X86_R8, + 6 => X86_RBX, + 7 => X86_R13, + 8 => X86_R14, + 9 => X86_R15, + 10 => X86_RBP, + _ => X86_RAX, + } +} + +fn need_rex(r: u8) -> bool { + r >= 8 +} + +fn emit_rex(buf: &mut JitBuffer, w: bool, r: u8, x: bool, b: u8) { + let mut rex: u8 = 0x40; + if w { + rex |= 0x08; + } + if need_rex(r) { + rex |= 0x04; + } + if x { + rex |= 0x02; + } + if need_rex(b) { + rex |= 0x01; + } + buf.emit_u8(rex); +} + +fn emit_modrm(buf: &mut JitBuffer, mod_bits: u8, reg: u8, rm: u8) { + buf.emit_u8((mod_bits << 6) | ((reg & 7) << 3) | (rm & 7)); +} + +fn emit_rex_if(buf: &mut JitBuffer, r: u8, b: u8) { + if need_rex(r) || need_rex(b) { + emit_rex(buf, false, r, false, b); + } +} + +fn emit_rex_w(buf: &mut JitBuffer, r: u8, b: u8) { + emit_rex(buf, true, r, false, b); +} + +fn emit_modrm_disp(buf: &mut JitBuffer, reg: u8, rm: u8, disp: i32) { + if disp == 0 && (rm & 7) != X86_RBP { + emit_modrm(buf, 0, reg, rm); + } else if (-128..=127).contains(&disp) { + emit_modrm(buf, 1, reg, rm); + buf.emit_u8(disp as u8); + } else { + emit_modrm(buf, 2, reg, rm); + buf.emit_u32(disp as u32); + } +} + +fn emit_push(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0x50 | (r & 7)); +} + +fn emit_pop(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0x58 | (r & 7)); +} + +fn emit_mov_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x89); + emit_modrm(buf, 3, src, dst); +} + +fn emit_mov_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x89); + emit_modrm(buf, 3, src, dst); +} + +fn emit_mov_imm64(buf: &mut JitBuffer, dst: u8, imm: u64) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xB8 | (dst & 7)); + buf.emit_u32(imm as u32); + buf.emit_u32((imm >> 32) as u32); +} + +fn emit_mov_imm32(buf: &mut JitBuffer, dst: u8, imm: i32) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC7); + emit_modrm(buf, 3, 0, dst); + buf.emit_u32(imm as u32); +} + +fn emit_add_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x01); + emit_modrm(buf, 3, src, dst); +} + +fn emit_sub_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x29); + emit_modrm(buf, 3, src, dst); +} + +fn emit_add_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x01); + emit_modrm(buf, 3, src, dst); +} + +fn emit_sub_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x29); + emit_modrm(buf, 3, src, dst); +} + +fn emit_imul_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, dst, src); + buf.emit_u8(0x0F); + buf.emit_u8(0xAF); + emit_modrm(buf, 3, dst, src); +} + +fn emit_imul_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, dst, src); + buf.emit_u8(0x0F); + buf.emit_u8(0xAF); + emit_modrm(buf, 3, dst, src); +} + +fn emit_xor_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x31); + emit_modrm(buf, 3, src, dst); +} + +fn emit_xor_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x31); + emit_modrm(buf, 3, src, dst); +} + +fn emit_or_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x09); + emit_modrm(buf, 3, src, dst); +} + +fn emit_or_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x09); + emit_modrm(buf, 3, src, dst); +} + +fn emit_and_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x21); + emit_modrm(buf, 3, src, dst); +} + +fn emit_and_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x21); + emit_modrm(buf, 3, src, dst); +} + +fn emit_neg_reg64(buf: &mut JitBuffer, r: u8) { + emit_rex_w(buf, 0, r); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 3, r); +} + +fn emit_neg_reg32(buf: &mut JitBuffer, r: u8) { + emit_rex_if(buf, 0, r); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 3, r); +} + +fn emit_shl_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 4, dst); +} + +fn emit_shr_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 5, dst); +} + +fn emit_sar_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 7, dst); +} + +fn emit_shl_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 4, dst); +} + +fn emit_shr_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 5, dst); +} + +fn emit_sar_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 7, dst); +} + +fn emit_shl_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 4, dst); + buf.emit_u8(imm); +} + +fn emit_shr_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 5, dst); + buf.emit_u8(imm); +} + +fn emit_sar_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 7, dst); + buf.emit_u8(imm); +} + +fn emit_shl_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 4, dst); + buf.emit_u8(imm); +} + +fn emit_shr_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 5, dst); + buf.emit_u8(imm); +} + +fn emit_sar_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 7, dst); + buf.emit_u8(imm); +} + +fn emit_test_reg64(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_w(buf, r2, r1); + buf.emit_u8(0x85); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_cmp_reg64(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_w(buf, r2, r1); + buf.emit_u8(0x39); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_cmp_reg32(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_if(buf, r2, r1); + buf.emit_u8(0x39); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_je(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x84); + buf.emit_u32(off as u32); +} + +fn emit_jne(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x85); + buf.emit_u32(off as u32); +} + +fn emit_ja(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x87); + buf.emit_u32(off as u32); +} + +fn emit_jae(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x83); + buf.emit_u32(off as u32); +} + +fn emit_jb(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x82); + buf.emit_u32(off as u32); +} + +fn emit_jbe(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x86); + buf.emit_u32(off as u32); +} + +fn emit_jg(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8F); + buf.emit_u32(off as u32); +} + +fn emit_jge(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8D); + buf.emit_u32(off as u32); +} + +fn emit_jl(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8C); + buf.emit_u32(off as u32); +} + +fn emit_jle(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8E); + buf.emit_u32(off as u32); +} + +fn emit_jmp_rel32(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0xE9); + buf.emit_u32(off as u32); +} + +fn emit_call_reg(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0xFF); + emit_modrm(buf, 3, 2, r); +} + +fn emit_ret(buf: &mut JitBuffer) { + buf.emit_u8(0xC3); +} + +fn emit_store_mem(buf: &mut JitBuffer, base: u8, off: i32, src: u8, size: u8) { + match size { + BPF_B => { + emit_rex_if(buf, src, base); + buf.emit_u8(0x88); + emit_modrm_disp(buf, src, base, off); + } + BPF_H => { + buf.emit_u8(0x66); + emit_rex_if(buf, src, base); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + BPF_W => { + emit_rex_if(buf, src, base); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + BPF_DW => { + emit_rex_w(buf, src, base); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + _ => {} + } +} + +fn emit_load_mem(buf: &mut JitBuffer, dst: u8, base: u8, off: i32, size: u8) { + match size { + BPF_B => { + emit_rex_if(buf, dst, base); + buf.emit_u8(0x0F); + buf.emit_u8(0xB6); + emit_modrm_disp(buf, dst, base, off); + } + BPF_H => { + emit_rex_if(buf, dst, base); + buf.emit_u8(0x0F); + buf.emit_u8(0xB7); + emit_modrm_disp(buf, dst, base, off); + } + BPF_W => { + emit_rex_if(buf, dst, base); + buf.emit_u8(0x8B); + emit_modrm_disp(buf, dst, base, off); + } + BPF_DW => { + emit_rex_w(buf, dst, base); + buf.emit_u8(0x8B); + emit_modrm_disp(buf, dst, base, off); + } + _ => {} + } +} + +fn emit_zext32(buf: &mut JitBuffer, r: u8) { + emit_rex_if(buf, r, r); + buf.emit_u8(0x23); + buf.emit_u8(0xC0 | ((r & 7) << 3) | (r & 7)); +} + +fn emit_divmod(buf: &mut JitBuffer, dst: u8, src: u8, is_div: bool, is_64: bool) { + emit_push(buf, X86_RCX); + + if is_64 { + emit_mov_reg64(buf, X86_R11, src); + emit_mov_reg64(buf, X86_R10, dst); + // For DIV with src==0, dst must become 0 → pre-zero RAX. + // For MOD with src==0, dst must stay unchanged → do not pre-zero. + if is_div { + emit_xor_reg64(buf, X86_RAX, X86_RAX); + } + emit_test_reg64(buf, X86_R11, X86_R11); + let skip = buf.offset(); + emit_je(buf, 0); + emit_mov_reg64(buf, X86_RAX, X86_R10); + emit_xor_reg64(buf, X86_RDX, X86_RDX); + emit_rex_w(buf, 0, X86_R11); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 6, X86_R11); + // For MOD, assign result inside the non-zero path so that the + // src==0 skip leaves dst unchanged. + if !is_div { + emit_mov_reg64(buf, dst, X86_RDX); + } + let after = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(skip) as *mut u8; + let off = (after - skip - 6) as i32; + core::ptr::copy_nonoverlapping(off.to_le_bytes().as_ptr(), ptr.add(2), 4); + } + } + emit_pop(buf, X86_RCX); + if is_div { + emit_mov_reg64(buf, dst, X86_RAX); + } + } else { + emit_mov_reg32(buf, X86_R11, src); + emit_mov_reg32(buf, X86_R10, dst); + if is_div { + emit_xor_reg32(buf, X86_RAX, X86_RAX); + } + emit_test_reg64(buf, X86_R11, X86_R11); + let skip = buf.offset(); + emit_je(buf, 0); + emit_mov_reg32(buf, X86_RAX, X86_R10); + emit_zext32(buf, X86_RAX); + emit_xor_reg32(buf, X86_RDX, X86_RDX); + emit_rex_if(buf, 0, X86_R11); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 6, X86_R11); + if !is_div { + emit_zext32(buf, X86_RDX); + emit_mov_reg32(buf, dst, X86_RDX); + } + let after = buf.offset(); + if !buf.counting() { + unsafe { + let ptr = buf.entry().add(skip) as *mut u8; + let off = (after - skip - 6) as i32; + core::ptr::copy_nonoverlapping(off.to_le_bytes().as_ptr(), ptr.add(2), 4); + } + } + emit_pop(buf, X86_RCX); + if is_div { + emit_mov_reg32(buf, dst, X86_RAX); + } + } +} + +pub(crate) struct X86_64Backend; + +impl JitBackend for X86_64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + emit_push(buf, X86_RBP); + emit_mov_reg64(buf, X86_RBP, X86_RSP); + emit_push(buf, X86_RBX); + emit_push(buf, X86_R13); + emit_push(buf, X86_R14); + emit_push(buf, X86_R15); + buf.emit_u8(0x48); + buf.emit_u8(0x81); + buf.emit_u8(0xEC); + buf.emit_u32(512); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + buf.emit_u8(0x48); + buf.emit_u8(0x81); + buf.emit_u8(0xC4); + buf.emit_u32(512); + emit_pop(buf, X86_R15); + emit_pop(buf, X86_R14); + emit_pop(buf, X86_R13); + emit_pop(buf, X86_RBX); + emit_pop(buf, X86_RBP); + emit_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_x86(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let imm_src = if dst == X86_RCX { X86_R11 } else { X86_RCX }; + let src = if use_imm { + imm_src + } else { + bpf_to_x86(insn.src_reg()) + }; + + let alu_op = insn.alu_op(); + if use_imm + && alu_op != BPF_MOV + && alu_op != BPF_NEG + && alu_op != BPF_LSH + && alu_op != BPF_RSH + && alu_op != BPF_ARSH + { + let imm = insn.imm; + if is_64 { + emit_mov_imm64(buf, imm_src, insn.imm as u64); + } else { + emit_mov_imm32(buf, imm_src, imm); + } + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + emit_add_reg64(buf, dst, src); + } else { + emit_add_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_SUB => { + if is_64 { + emit_sub_reg64(buf, dst, src); + } else { + emit_sub_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_MUL => { + if is_64 { + emit_imul_reg64(buf, dst, src); + } else { + emit_imul_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_DIV => { + emit_divmod(buf, dst, src, true, is_64); + } + BPF_OR => { + if is_64 { + emit_or_reg64(buf, dst, src); + } else { + emit_or_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_AND => { + if is_64 { + emit_and_reg64(buf, dst, src); + } else { + emit_and_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_LSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_shl_imm64(buf, dst, shamt); + } else { + emit_shl_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if dst == X86_RCX { + emit_mov_reg64(buf, X86_R10, dst); + emit_mov_reg64(buf, X86_RCX, src); + if is_64 { + emit_shl_reg64(buf, X86_R10); + } else { + emit_shl_reg32(buf, X86_R10); + emit_zext32(buf, X86_R10); + } + emit_mov_reg64(buf, dst, X86_R10); + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_shl_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_shl_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_RSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_shr_imm64(buf, dst, shamt); + } else { + emit_shr_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if dst == X86_RCX { + emit_mov_reg64(buf, X86_R10, dst); + emit_mov_reg64(buf, X86_RCX, src); + if is_64 { + emit_shr_reg64(buf, X86_R10); + } else { + emit_shr_reg32(buf, X86_R10); + emit_zext32(buf, X86_R10); + } + emit_mov_reg64(buf, dst, X86_R10); + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_shr_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_shr_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_NEG => { + if is_64 { + emit_neg_reg64(buf, dst); + } else { + emit_neg_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_MOD => { + emit_divmod(buf, dst, src, false, is_64); + } + BPF_XOR => { + if is_64 { + emit_xor_reg64(buf, dst, src); + } else { + emit_xor_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_MOV => { + if is_64 { + if use_imm { + emit_mov_imm64(buf, dst, insn.imm as u64); + } else { + emit_mov_reg64(buf, dst, src); + } + } else { + if use_imm { + emit_mov_imm32(buf, dst, insn.imm); + } else { + emit_mov_reg32(buf, dst, src); + } + emit_zext32(buf, dst); + } + } + BPF_ARSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_sar_imm64(buf, dst, shamt); + } else { + emit_sar_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if dst == X86_RCX { + emit_mov_reg64(buf, X86_R10, dst); + emit_mov_reg64(buf, X86_RCX, src); + if is_64 { + emit_sar_reg64(buf, X86_R10); + } else { + emit_sar_reg32(buf, X86_R10); + } + emit_mov_reg64(buf, dst, X86_R10); + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_sar_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_sar_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_END => { + // BPF_TO_BE: byte swap to big-endian (x86_64 is little-endian native) + let to_be = (insn.code & BPF_X) == 0; + match (to_be, insn.imm) { + (true, 16) => { + // 16-bit: rol reg16, 8 + buf.emit_u8(0x66); // 16-bit operand size + if need_rex(dst) { + buf.emit_u8(0x41); // REX.B for r8-r15 + } + buf.emit_u8(0xC1); // ROL r/m, imm8 + emit_modrm(buf, 3, 0, dst); // mod=11, /0, rm=dst + buf.emit_u8(0x08); // rotate by 8 bits + } + (true, 32) => { + // 32-bit: bswap reg32 + if need_rex(dst) { + buf.emit_u8(0x41); // REX.B + } + buf.emit_u8(0x0F); + buf.emit_u8(0xC8 | (dst & 7)); + } + (true, 64) => { + // 64-bit: bswap reg64 (REX.W + 0F C8+r) + if need_rex(dst) { + buf.emit_u8(0x49); // REX.WB + } else { + buf.emit_u8(0x48); // REX.W + } + buf.emit_u8(0x0F); + buf.emit_u8(0xC8 | (dst & 7)); + } + // BPF_TO_LE on x86_64: no-op (native little-endian) + _ => {} + } + } + _ => {} + } + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let off = offsets[target_pc] as isize - buf.offset() as isize - 5; + emit_jmp_rel32(buf, off as i32); + } + return; + } + + let dst = bpf_to_x86(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let imm_src = if dst == X86_RCX { X86_R11 } else { X86_RCX }; + let src = if use_imm { + imm_src + } else { + bpf_to_x86(insn.src_reg()) + }; + + if use_imm { + if is_64 { + emit_mov_imm64(buf, imm_src, insn.imm as u64); + } else { + emit_mov_imm32(buf, imm_src, insn.imm); + } + } + + if op != BPF_JSET { + if is_64 { + emit_cmp_reg64(buf, dst, src); + } else { + emit_cmp_reg32(buf, dst, src); + } + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + let target_off = if target_pc < offsets.len() { + (offsets[target_pc] as isize - buf.offset() as isize - 6) as i32 + } else { + 0 + }; + + match op { + BPF_JEQ => emit_je(buf, target_off), + BPF_JGT => emit_ja(buf, target_off), + BPF_JGE => emit_jae(buf, target_off), + BPF_JSET => { + if is_64 { + emit_test_reg64(buf, dst, src); + } else { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x85); + emit_modrm(buf, 3, src, dst); + } + emit_jne(buf, target_off); + } + BPF_JNE => emit_jne(buf, target_off), + BPF_JSGT => emit_jg(buf, target_off), + BPF_JSGE => emit_jge(buf, target_off), + BPF_JLT => emit_jb(buf, target_off), + BPF_JLE => emit_jbe(buf, target_off), + BPF_JSLT => emit_jl(buf, target_off), + BPF_JSLE => emit_jle(buf, target_off), + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let imm = insn.imm as i64; + let base = bpf_to_x86(insn.dst_reg()); + let adjusted_off = if base == X86_RBP { off - 32 } else { off }; + let scratch = if base == X86_RCX { X86_R11 } else { X86_RCX }; + if insn.size() == BPF_DW { + emit_mov_imm64(buf, scratch, imm as u64); + emit_store_mem(buf, base, adjusted_off, scratch, BPF_DW); + } else { + emit_mov_imm32(buf, scratch, imm as i32); + emit_store_mem(buf, base, adjusted_off, scratch, insn.size()); + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_x86(insn.dst_reg()); + let src = bpf_to_x86(insn.src_reg()); + let adjusted_off = if base == X86_RBP { off - 32 } else { off }; + emit_store_mem(buf, base, adjusted_off, src, insn.size()); + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_x86(insn.src_reg()); + let dst = bpf_to_x86(insn.dst_reg()); + let adjusted_off = if base == X86_RBP { off - 32 } else { off }; + emit_load_mem(buf, dst, base, adjusted_off, insn.size()); + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_x86(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + emit_mov_imm64(buf, dst, val); + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + emit_mov_imm64(buf, X86_RAX, helper_fn as usize as u64); + emit_call_reg(buf, X86_RAX); + } +} diff --git a/os/StarryOS/kernel/src/ebpf/ebpf_jit/mod.rs b/os/StarryOS/kernel/src/ebpf/ebpf_jit/mod.rs new file mode 100644 index 0000000000..c6b0bd7a61 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/ebpf_jit/mod.rs @@ -0,0 +1,362 @@ +use alloc::{ + alloc::{Layout, alloc_zeroed, dealloc}, + collections::BTreeMap, + vec, + vec::Vec, +}; + +#[cfg(target_arch = "aarch64")] +use ax_memory_addr::VirtAddr; + +pub(crate) use super::HelperFn; +#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] +pub(crate) use super::bpf_insn; +use super::bpf_insn::{ + BPF_ALU, BPF_ALU64, BPF_EXIT, BPF_JMP, BPF_JMP32, BPF_LD, BPF_LDX, BPF_ST, BPF_STX, BpfInsn, +}; + +#[cfg(target_arch = "aarch64")] +mod jit_aarch64; +#[cfg(target_arch = "riscv64")] +mod jit_riscv64; +#[cfg(target_arch = "x86_64")] +mod jit_x86_64; + +#[cfg(target_arch = "aarch64")] +use jit_aarch64::Aarch64Backend as Backend; +#[cfg(target_arch = "riscv64")] +use jit_riscv64::Riscv64Backend as Backend; +#[cfg(target_arch = "x86_64")] +use jit_x86_64::X86_64Backend as Backend; + +pub struct JitBuffer { + ptr: *mut u8, + size: usize, + pos: usize, + /// When true, emit methods only count bytes without writing to memory. + counting: bool, +} + +// SAFETY: JitBuffer owns a single heap allocation. After finalize(), the +// buffer is read-only. The Send/Sync impls are safe because the buffer is +// never mutated concurrently. +unsafe impl Send for JitBuffer {} +unsafe impl Sync for JitBuffer {} + +impl core::fmt::Debug for JitBuffer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("JitBuffer") + .field("size", &self.size) + .field("pos", &self.pos) + .finish() + } +} + +impl JitBuffer { + pub fn new(requested_size: usize) -> Result { + let size = (requested_size + 4095) & !4095; + if size == 0 { + return Err("jit buffer size is zero"); + } + let layout = Layout::from_size_align(size, 4096).map_err(|_| "invalid layout")?; + let ptr = unsafe { alloc_zeroed(layout) }; + if ptr.is_null() { + return Err("jit buffer allocation failed"); + } + Ok(Self { + ptr, + size, + pos: 0, + counting: false, + }) + } + + /// Creates a counting-only buffer used during the sizing pass. + /// All emit calls only count bytes without writing to memory. + pub fn new_sizing() -> Self { + Self { + ptr: core::ptr::null_mut(), + size: usize::MAX, + pos: 0, + counting: true, + } + } + + pub fn emit_u8(&mut self, val: u8) { + if !self.counting { + assert!( + self.pos < self.size, + "JitBuffer overflow at offset {} (size {})", + self.pos, + self.size + ); + unsafe { + let dst = self.ptr.add(self.pos); + *dst = val; + } + } + self.pos += 1; + } + + pub fn emit_u32(&mut self, val: u32) { + if !self.counting { + assert!( + self.pos + 4 <= self.size, + "JitBuffer overflow at offset {} (size {}, need 4 bytes)", + self.pos, + self.size + ); + unsafe { + let dst = self.ptr.add(self.pos) as *mut u32; + *dst = val.to_le(); + } + } + self.pos += 4; + } + + /// Returns true when the buffer is in counting mode (sizing pass). + /// Backends should skip direct memory writes when this returns true. + pub fn counting(&self) -> bool { + self.counting + } + + pub fn offset(&self) -> usize { + self.pos + } + + pub fn entry(&self) -> *const u8 { + self.ptr + } + + pub fn finalize(&mut self) { + #[cfg(target_arch = "aarch64")] + { + let vaddr = VirtAddr::from_usize(self.ptr as usize); + ax_runtime::hal::cpu::asm::clean_dcache_range_to_pou(vaddr, self.pos); + } + ax_runtime::hal::cpu::asm::flush_icache_all(); + } +} + +impl Drop for JitBuffer { + fn drop(&mut self) { + if !self.ptr.is_null() && self.size > 0 { + let layout = Layout::from_size_align(self.size, 4096).unwrap(); + unsafe { + #[cfg(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" + ))] + dealloc(self.ptr, layout); + } + } + } +} + +pub(crate) trait JitBackend { + fn emit_prologue(buf: &mut JitBuffer) -> usize; + fn emit_epilogue(buf: &mut JitBuffer); + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool); + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool); + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32); + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn); +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +))] +struct JitCompiler<'a> { + insns: &'a [BpfInsn], + offsets: Vec, + helpers: &'a BTreeMap, +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +))] +impl<'a> JitCompiler<'a> { + fn new(insns: &'a [BpfInsn], helpers: &'a BTreeMap) -> Self { + let offsets = vec![0; insns.len()]; + Self { + insns, + offsets, + helpers, + } + } + + fn pass1_sizing(&mut self) -> usize { + let mut buf = JitBuffer::new_sizing(); + let num_insns = self.insns.len(); + let mut pc: usize = 0; + while pc < num_insns { + self.offsets[pc] = buf.offset(); + let insn = &self.insns[pc]; + let class = insn.class(); + + match class { + BPF_ALU | BPF_ALU64 => { + Backend::emit_alu(&mut buf, insn, class == BPF_ALU64); + pc += 1; + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + Backend::emit_epilogue(&mut buf); + pc += 1; + } else if op == 0x80 { + let helper_id = insn.imm as u32; + if let Some(&helper_fn) = self.helpers.get(&helper_id) { + Backend::emit_call(&mut buf, helper_fn); + } else { + Backend::emit_call(&mut buf, |_a1, _a2, _a3, _a4, _a5| u64::MAX); + } + pc += 1; + } else { + Backend::emit_jmp(&mut buf, insn, &self.offsets, pc, class == BPF_JMP); + pc += 1; + } + } + BPF_ST => { + Backend::emit_st(&mut buf, insn); + pc += 1; + } + BPF_STX => { + Backend::emit_stx(&mut buf, insn); + pc += 1; + } + BPF_LDX => { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + BPF_LD => { + if insn.is_ld_dw_imm() { + if pc + 1 < num_insns { + self.offsets[pc + 1] = buf.offset(); + } + Backend::emit_ld_imm64(&mut buf, insn, 0); + pc += 2; + } else { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + } + _ => return 0, + } + } + buf.offset() + } + + fn compile(&mut self) -> Result { + if self.insns.is_empty() { + return Err("no instructions to compile"); + } + + let insn_size_total = self.pass1_sizing(); + + let estimated = 128 + insn_size_total + 128 + 256; + let mut buf = JitBuffer::new(estimated)?; + + let prologue_size = Backend::emit_prologue(&mut buf); + + for i in 0..self.offsets.len() { + self.offsets[i] += prologue_size; + } + + let num_insns = self.insns.len(); + let mut pc: usize = 0; + while pc < num_insns { + let insn = &self.insns[pc]; + let class = insn.class(); + + match class { + BPF_ALU | BPF_ALU64 => { + let is_64 = class == BPF_ALU64; + Backend::emit_alu(&mut buf, insn, is_64); + pc += 1; + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + Backend::emit_epilogue(&mut buf); + pc += 1; + } else if op == 0x80 { + let helper_id = insn.imm as u32; + if let Some(&helper_fn) = self.helpers.get(&helper_id) { + Backend::emit_call(&mut buf, helper_fn); + } else { + Backend::emit_call(&mut buf, |_a1, _a2, _a3, _a4, _a5| u64::MAX); + } + pc += 1; + } else { + let is_64 = class == BPF_JMP; + Backend::emit_jmp(&mut buf, insn, &self.offsets, pc, is_64); + pc += 1; + } + } + BPF_ST => { + Backend::emit_st(&mut buf, insn); + pc += 1; + } + BPF_STX => { + Backend::emit_stx(&mut buf, insn); + pc += 1; + } + BPF_LDX => { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + BPF_LD => { + if insn.is_ld_dw_imm() { + let next_imm = if pc + 1 < num_insns { + self.insns[pc + 1].imm + } else { + 0 + }; + Backend::emit_ld_imm64(&mut buf, insn, next_imm); + pc += 2; + } else { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + } + _ => { + return Err("unsupported instruction class"); + } + } + } + + Backend::emit_epilogue(&mut buf); + buf.finalize(); + Ok(buf) + } +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +))] +pub fn try_jit_compile(insns: &[BpfInsn], helpers: &BTreeMap) -> Option { + let mut compiler = JitCompiler::new(insns, helpers); + compiler.compile().ok() +} + +#[cfg(not(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +)))] +pub fn try_jit_compile( + _insns: &[BpfInsn], + _helpers: &BTreeMap, +) -> Option { + None +} diff --git a/os/StarryOS/kernel/src/ebpf/mod.rs b/os/StarryOS/kernel/src/ebpf/mod.rs index e861046945..09e48a598a 100644 --- a/os/StarryOS/kernel/src/ebpf/mod.rs +++ b/os/StarryOS/kernel/src/ebpf/mod.rs @@ -31,11 +31,17 @@ use kbpf_basic::{ raw_tracepoint::BpfRawTracePointArg, }; +#[allow(dead_code)] // wired in by follow-up PRs (ebpf-jit-2/3) +pub(crate) mod bpf_insn; +#[allow(dead_code)] // wired in by follow-up PRs (ebpf-jit-2/3) +pub mod ebpf_jit; pub(crate) mod error; pub mod map; pub mod prog; pub mod transform; +pub(crate) type HelperFn = fn(u64, u64, u64, u64, u64) -> u64; + pub use transform::EbpfKernelAuxiliary; use crate::{