diff --git a/os/StarryOS/kernel/src/ebpf.rs b/os/StarryOS/kernel/src/ebpf.rs new file mode 100644 index 0000000000..944e8e892c --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf.rs @@ -0,0 +1,2429 @@ +//! eBPF (Extended Berkeley Packet Filter) subsystem for StarryOS. +//! +//! This module provides a complete in-kernel eBPF implementation including: +//! +//! - **Map management**: Array and Hash maps with fd-based lifecycle +//! - **Program loader**: Parses `bpf_attr` with correct mixed u32/u64 byte-offset layout +//! - **Instruction interpreter**: Supports ALU/JMP/MEM instruction classes with +//! BPF_EXIT and BPF_CALL handling +//! - **Helper functions**: 11 helpers including map operations, probe_read, ktime, +//! PID/TGID, UID/GID, and perf_event_output +//! - **fd table**: BpfFdTable with close/remove operations and free-fd reuse +//! +//! # Syscall interface +//! +//! Implements `bpf()` syscall commands: MAP_CREATE, LOOKUP, UPDATE, DELETE, +//! GET_NEXT_KEY, PROG_LOAD, PROG_ATTACH, LINK_CREATE, OBJ_CLOSE. + +use alloc::vec::Vec; + +use ax_errno::{AxError, AxResult}; +use ax_sync::spin::SpinNoIrq; + +use crate::task::AsThread; + +mod ebpf_jit; + +#[allow(dead_code)] +mod bpf_insn { + pub const BPF_LD: u8 = 0x00; + pub const BPF_LDX: u8 = 0x01; + pub const BPF_ST: u8 = 0x02; + pub const BPF_STX: u8 = 0x03; + pub const BPF_ALU: u8 = 0x04; + pub const BPF_JMP: u8 = 0x05; + pub const BPF_JMP32: u8 = 0x06; + pub const BPF_ALU64: u8 = 0x07; + + pub const BPF_W: u8 = 0x00; + pub const BPF_H: u8 = 0x08; + pub const BPF_B: u8 = 0x10; + pub const BPF_DW: u8 = 0x18; + + pub const BPF_IMM: u8 = 0x00; + pub const BPF_ABS: u8 = 0x20; + pub const BPF_IND: u8 = 0x40; + pub const BPF_MEM: u8 = 0x60; + pub const BPF_LEN: u8 = 0x80; + pub const BPF_MSH: u8 = 0xa0; + + pub const BPF_ADD: u8 = 0x00; + pub const BPF_SUB: u8 = 0x10; + pub const BPF_MUL: u8 = 0x20; + pub const BPF_DIV: u8 = 0x30; + pub const BPF_OR: u8 = 0x40; + pub const BPF_AND: u8 = 0x50; + pub const BPF_LSH: u8 = 0x60; + pub const BPF_RSH: u8 = 0x70; + pub const BPF_NEG: u8 = 0x80; + pub const BPF_MOD: u8 = 0x90; + pub const BPF_XOR: u8 = 0xa0; + pub const BPF_MOV: u8 = 0xb0; + pub const BPF_ARSH: u8 = 0xc0; + pub const BPF_END: u8 = 0xd0; + + pub const BPF_JA: u8 = 0x00; + pub const BPF_EXIT: u8 = 0x90; + pub const BPF_JEQ: u8 = 0x10; + pub const BPF_JGT: u8 = 0x20; + pub const BPF_JGE: u8 = 0x30; + pub const BPF_JSET: u8 = 0x40; + pub const BPF_JNE: u8 = 0x50; + pub const BPF_JSGT: u8 = 0x60; + pub const BPF_JSGE: u8 = 0x70; + pub const BPF_JLT: u8 = 0xa0; + pub const BPF_JLE: u8 = 0xb0; + pub const BPF_JSLT: u8 = 0xc0; + pub const BPF_JSLE: u8 = 0xd0; + + pub const BPF_K: u8 = 0x00; + pub const BPF_X: u8 = 0x08; + + pub const BPF_PSEUDO_MAP_FD: u8 = 1; + pub const BPF_PSEUDO_MAP_VALUE: u8 = 2; + + #[repr(C)] + #[derive(Clone, Copy, Debug, Default)] + pub struct BpfInsn { + pub code: u8, + pub dst_src_reg: u8, + pub off: i16, + pub imm: i32, + } + + impl BpfInsn { + pub const fn new(code: u8, dst: u8, src: u8, off: i16, imm: i32) -> Self { + Self { + code, + dst_src_reg: (dst & 0xf) | ((src & 0xf) << 4), + off, + imm, + } + } + + pub fn dst_reg(&self) -> u8 { + self.dst_src_reg & 0xf + } + + pub fn src_reg(&self) -> u8 { + (self.dst_src_reg >> 4) & 0xf + } + + pub fn class(&self) -> u8 { + self.code & 0x07 + } + + pub fn size(&self) -> u8 { + self.code & 0x18 + } + + pub fn mode(&self) -> u8 { + self.code & 0xe0 + } + + pub fn alu_op(&self) -> u8 { + self.code & 0xf0 + } + + pub fn is_ld_dw_imm(&self) -> bool { + self.code == (BPF_LD | BPF_IMM | BPF_DW) + } + + pub fn to_bytes(self) -> [u8; 8] { + let mut buf = [0u8; 8]; + buf[0] = self.code; + buf[1] = self.dst_src_reg; + buf[2..4].copy_from_slice(&self.off.to_le_bytes()); + buf[4..8].copy_from_slice(&self.imm.to_le_bytes()); + buf + } + + pub fn from_bytes(bytes: &[u8; 8]) -> Self { + Self { + code: bytes[0], + dst_src_reg: bytes[1], + off: i16::from_le_bytes([bytes[2], bytes[3]]), + imm: i32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), + } + } + } +} + +#[allow(dead_code)] +mod map_type { + pub const UNSPEC: u32 = 0; + pub const HASH: u32 = 1; + pub const ARRAY: u32 = 2; + pub const PROG_ARRAY: u32 = 3; + pub const PERF_EVENT_ARRAY: u32 = 4; + pub const PERCPU_HASH: u32 = 5; + pub const PERCPU_ARRAY: u32 = 6; + pub const STACK_TRACE: u32 = 7; + pub const LRU_HASH: u32 = 9; + pub const LRU_PERCPU_HASH: u32 = 10; + pub const LPM_TRIE: u32 = 11; + pub const QUEUE: u32 = 22; + pub const STACK: u32 = 23; + pub const RINGBUF: u32 = 27; +} + +#[allow(dead_code)] +mod prog_type { + pub const UNSPEC: u32 = 0; + pub const SOCKET_FILTER: u32 = 1; + pub const KPROBE: u32 = 2; + pub const SCHED_CLS: u32 = 3; + pub const TRACEPOINT: u32 = 5; + pub const XDP: u32 = 6; + pub const PERF_EVENT: u32 = 7; + pub const CGROUP_SKB: u32 = 8; + pub const RAW_TRACEPOINT: u32 = 17; + pub const LSM: u32 = 29; + pub const SYSCALL: u32 = 31; +} + +#[allow(dead_code)] +mod cmd { + pub const MAP_CREATE: u64 = 0; + pub const MAP_LOOKUP_ELEM: u64 = 1; + pub const MAP_UPDATE_ELEM: u64 = 2; + pub const MAP_DELETE_ELEM: u64 = 3; + pub const MAP_GET_NEXT_KEY: u64 = 4; + pub const PROG_LOAD: u64 = 5; + pub const OBJ_PIN: u64 = 6; + pub const OBJ_GET: u64 = 7; + pub const PROG_ATTACH: u64 = 8; + pub const PROG_DETACH: u64 = 9; + pub const OBJ_CLOSE: u64 = 11; + pub const RAW_TRACEPOINT_OPEN: u64 = 17; + pub const LINK_CREATE: u64 = 28; + pub const ENABLE_STATS: u64 = 32; +} + +#[allow(dead_code)] +mod bpf_error { + use ax_errno::AxError; + + pub const EPERM: AxError = AxError::PermissionDenied; + pub const ENOENT: AxError = AxError::NotFound; + pub const ENOMEM: AxError = AxError::NoMemory; + pub const EINVAL: AxError = AxError::InvalidInput; + pub const ENOSPC: AxError = AxError::StorageFull; + + pub fn from_linux_errno(code: i32) -> AxError { + match code { + 1 => AxError::PermissionDenied, + 2 => AxError::NotFound, + 12 => AxError::NoMemory, + 22 => AxError::InvalidInput, + 28 => AxError::StorageFull, + _ => AxError::Io, + } + } +} + +#[derive(Clone, Debug)] +#[allow(dead_code)] +struct BpfMapMeta { + map_type: u32, + key_size: u32, + value_size: u32, + max_entries: u32, + map_flags: u32, + id: u32, +} + +trait BpfMapOps: Send + Sync { + fn meta(&self) -> &BpfMapMeta; + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>>; + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> AxResult<()>; + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()>; + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>>; + fn as_any_mut(&mut self) -> &mut dyn core::any::Any; +} + +struct ArrayMap { + meta: BpfMapMeta, + data: Vec, + elem_size: usize, +} + +impl ArrayMap { + fn new(meta: BpfMapMeta) -> Self { + let elem_size = meta.value_size as usize; + let total = elem_size * meta.max_entries as usize; + Self { + meta, + data: alloc::vec![0u8; total], + elem_size, + } + } + + fn index_valid(&self, idx: u32) -> bool { + (idx as usize) < self.meta.max_entries as usize + } + + fn value_offset(&self, idx: u32) -> usize { + idx as usize * self.elem_size + } +} + +impl BpfMapOps for ArrayMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + if !self.index_valid(idx) { + return Err(bpf_error::ENOENT); + } + let start = self.value_offset(idx); + let end = start + self.elem_size; + Ok(Some(self.data[start..end].to_vec())) + } + + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + if value.len() != self.elem_size { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + if !self.index_valid(idx) { + return Err(bpf_error::ENOENT); + } + let start = self.value_offset(idx); + self.data[start..start + self.elem_size].copy_from_slice(value); + Ok(()) + } + + fn delete_elem(&mut self, _key: &[u8]) -> AxResult<()> { + Err(bpf_error::EPERM) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let next_idx = match key { + None => 0u32, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([k[0], k[1], k[2], k[3]]); + idx + 1 + } + }; + if next_idx >= self.meta.max_entries { + return Ok(None); + } + Ok(Some(next_idx.to_ne_bytes().to_vec())) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +struct HashMapInner { + meta: BpfMapMeta, + entries: hashbrown::HashMap, Vec>, +} + +impl HashMapInner { + fn new(meta: BpfMapMeta) -> Self { + Self { + meta, + entries: hashbrown::HashMap::new(), + } + } +} + +impl BpfMapOps for HashMapInner { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != self.meta.key_size as usize { + return Err(bpf_error::EINVAL); + } + Ok(self.entries.get(key).cloned()) + } + + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> AxResult<()> { + if key.len() != self.meta.key_size as usize || value.len() != self.meta.value_size as usize + { + return Err(bpf_error::EINVAL); + } + let exists = self.entries.contains_key(key); + const BPF_NOEXIST: u64 = 1; + const BPF_EXISTS: u64 = 2; + if flags & BPF_NOEXIST != 0 && exists { + return Err(bpf_error::EPERM); + } + if flags & BPF_EXISTS != 0 && !exists { + return Err(bpf_error::ENOENT); + } + if !exists && self.entries.len() >= self.meta.max_entries as usize { + return Err(bpf_error::ENOMEM); + } + self.entries.insert(key.to_vec(), value.to_vec()); + Ok(()) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != self.meta.key_size as usize { + return Err(bpf_error::EINVAL); + } + if self.entries.remove(key).is_none() { + return Err(bpf_error::ENOENT); + } + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + match key { + None => Ok(self.entries.keys().next().cloned()), + Some(k) => { + if k.len() != self.meta.key_size as usize { + return Err(bpf_error::EINVAL); + } + let mut found = false; + for existing_key in self.entries.keys() { + if found { + return Ok(Some(existing_key.clone())); + } + if existing_key.as_slice() == k { + found = true; + } + } + Ok(None) + } + } + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +struct PerfEventArrayMap { + meta: BpfMapMeta, + fds: alloc::vec::Vec, + max_entries: u32, +} + +impl PerfEventArrayMap { + fn new(meta: BpfMapMeta) -> Self { + let max_entries = meta.max_entries; + let fds = alloc::vec![0u32; max_entries as usize]; + Self { + meta, + fds, + max_entries, + } + } +} + +impl BpfMapOps for PerfEventArrayMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + if idx as usize >= self.max_entries as usize { + return Err(bpf_error::ENOENT); + } + Ok(Some(self.fds[idx as usize].to_ne_bytes().to_vec())) + } + + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> AxResult<()> { + if key.len() != 4 || value.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + if idx as usize >= self.max_entries as usize { + return Err(bpf_error::ENOENT); + } + let fd = u32::from_ne_bytes([value[0], value[1], value[2], value[3]]); + self.fds[idx as usize] = fd; + Ok(()) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + if idx as usize >= self.max_entries as usize { + return Err(bpf_error::ENOENT); + } + self.fds[idx as usize] = 0; + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let next_idx = match key { + None => 0u32, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([k[0], k[1], k[2], k[3]]); + idx + 1 + } + }; + if next_idx >= self.max_entries { + return Ok(None); + } + Ok(Some(next_idx.to_ne_bytes().to_vec())) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +#[repr(C)] +struct RingBufHdr { + len: u32, + start_offset: u32, +} + +const RINGBUF_HDR_SIZE: usize = 8; +const RINGBUF_ALIGN: usize = 8; + +struct RingBufferMap { + meta: BpfMapMeta, + buf: alloc::vec::Vec, + capacity: usize, + head: u64, + tail: u64, + mask: usize, + pending_reserve: Option<(usize, usize)>, +} + +impl RingBufferMap { + fn new(meta: BpfMapMeta) -> Self { + let capacity = (meta.max_entries as usize).next_power_of_two(); + let mask = capacity - 1; + Self { + meta, + buf: alloc::vec![0u8; capacity], + capacity, + head: 0, + tail: 0, + mask, + pending_reserve: None, + } + } + + fn write_at(&mut self, offset: usize, data: &[u8]) { + if data.is_empty() { + return; + } + let start = offset & self.mask; + if start + data.len() <= self.capacity { + unsafe { + core::ptr::copy_nonoverlapping( + data.as_ptr(), + self.buf.as_mut_ptr().add(start), + data.len(), + ); + } + } else { + let first = self.capacity - start; + unsafe { + core::ptr::copy_nonoverlapping( + data.as_ptr(), + self.buf.as_mut_ptr().add(start), + first, + ); + core::ptr::copy_nonoverlapping( + data.as_ptr().add(first), + self.buf.as_mut_ptr(), + data.len() - first, + ); + } + } + } + + fn available(&self) -> usize { + self.capacity - (self.head - self.tail) as usize + } + + fn reserve(&mut self, size: usize) -> Option<*mut u8> { + let aligned_size = (size + RINGBUF_ALIGN - 1) & !(RINGBUF_ALIGN - 1); + let total = RINGBUF_HDR_SIZE + aligned_size; + if total > self.available() { + return None; + } + if self.pending_reserve.is_some() { + return None; + } + let offset = self.head as usize; + let hdr = RingBufHdr { + len: size as u32, + start_offset: offset as u32, + }; + let hdr_bytes = unsafe { + core::slice::from_raw_parts(&hdr as *const RingBufHdr as *const u8, RINGBUF_HDR_SIZE) + }; + self.write_at(offset, hdr_bytes); + let data_offset = offset + RINGBUF_HDR_SIZE; + let data_ptr = unsafe { self.buf.as_mut_ptr().add(data_offset & self.mask) }; + self.pending_reserve = Some((aligned_size, data_offset)); + Some(data_ptr) + } + + fn submit(&mut self, _flags: u64) { + if let Some((aligned_size, _data_offset)) = self.pending_reserve.take() { + let total = RINGBUF_HDR_SIZE + aligned_size; + self.head += total as u64; + } + } + + fn discard(&mut self, _flags: u64) { + if let Some((_, data_offset)) = self.pending_reserve.take() { + let hdr_bytes_to_clear = self.head as usize + RINGBUF_HDR_SIZE; + let zero = alloc::vec![0u8; RINGBUF_HDR_SIZE]; + self.write_at(data_offset - RINGBUF_HDR_SIZE, &zero); + let _ = (hdr_bytes_to_clear, data_offset); + } + } + + fn output(&mut self, data: &[u8]) -> bool { + let aligned_size = (data.len() + RINGBUF_ALIGN - 1) & !(RINGBUF_ALIGN - 1); + let total = RINGBUF_HDR_SIZE + aligned_size; + if total > self.available() { + return false; + } + let offset = self.head as usize; + let hdr = RingBufHdr { + len: data.len() as u32, + start_offset: offset as u32, + }; + let hdr_bytes = unsafe { + core::slice::from_raw_parts(&hdr as *const RingBufHdr as *const u8, RINGBUF_HDR_SIZE) + }; + self.write_at(offset, hdr_bytes); + self.write_at(offset + RINGBUF_HDR_SIZE, data); + if aligned_size > data.len() { + let pad = alloc::vec![0u8; aligned_size - data.len()]; + self.write_at(offset + RINGBUF_HDR_SIZE + data.len(), &pad); + } + self.head += total as u64; + true + } +} + +impl BpfMapOps for RingBufferMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, _key: &[u8]) -> AxResult>> { + Err(bpf_error::EINVAL) + } + + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> AxResult<()> { + Err(bpf_error::EINVAL) + } + + fn delete_elem(&mut self, _key: &[u8]) -> AxResult<()> { + Err(bpf_error::EINVAL) + } + + fn get_next_key(&mut self, _key: Option<&[u8]>) -> AxResult>> { + Err(bpf_error::EINVAL) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +struct ProgArrayMap { + meta: BpfMapMeta, + prog_fds: alloc::vec::Vec>, +} + +impl ProgArrayMap { + fn new(meta: BpfMapMeta) -> Self { + let prog_fds = alloc::vec![None; meta.max_entries as usize]; + Self { meta, prog_fds } + } +} + +impl BpfMapOps for ProgArrayMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + let i = idx as usize; + if i >= self.prog_fds.len() { + return Ok(None); + } + match self.prog_fds[i] { + Some(fd) => Ok(Some(fd.to_ne_bytes().to_vec())), + None => Ok(None), + } + } + + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> AxResult<()> { + if key.len() != 4 || value.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + let prog_fd = u32::from_ne_bytes([value[0], value[1], value[2], value[3]]); + let i = idx as usize; + if i >= self.prog_fds.len() { + return Err(bpf_error::EINVAL); + } + { + let guard = BPF_GLOBAL.lock(); + if guard.progs.contains_key(&prog_fd) { + } else { + return Err(bpf_error::EINVAL); + } + } + self.prog_fds[i] = Some(prog_fd); + Ok(()) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + let i = idx as usize; + if i >= self.prog_fds.len() { + return Err(bpf_error::EINVAL); + } + self.prog_fds[i] = None; + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let start = match key { + None => 0, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([k[0], k[1], k[2], k[3]]); + (idx as usize).saturating_add(1) + } + }; + for i in start..self.prog_fds.len() { + if self.prog_fds[i].is_some() { + return Ok(Some((i as u32).to_ne_bytes().to_vec())); + } + } + Ok(None) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +const MAX_STACK_DEPTH: usize = 127; + +struct StackTraceMap { + meta: BpfMapMeta, + traces: hashbrown::HashMap>, + next_id: u32, +} + +impl StackTraceMap { + fn new(meta: BpfMapMeta) -> Self { + Self { + meta, + traces: hashbrown::HashMap::new(), + next_id: 0, + } + } + + fn store_trace(&mut self, ips: &[u64]) -> u32 { + let id = self.next_id; + self.next_id = self.next_id.wrapping_add(1); + if self.traces.len() >= self.meta.max_entries as usize + && let Some(oldest) = self.traces.keys().min().copied() + { + self.traces.remove(&oldest); + } + self.traces.insert(id, ips.to_vec()); + id + } +} + +impl BpfMapOps for StackTraceMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let id = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + match self.traces.get(&id) { + Some(trace) => { + let value_size = self.meta.value_size as usize; + let count = value_size / 8; + let mut buf = alloc::vec![0u8; value_size]; + for (i, ip) in trace.iter().enumerate().take(count) { + let start = i * 8; + buf[start..start + 8].copy_from_slice(&ip.to_ne_bytes()); + } + Ok(Some(buf)) + } + None => Ok(None), + } + } + + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> AxResult<()> { + Err(bpf_error::EINVAL) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let id = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + self.traces.remove(&id); + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let start = match key { + None => None, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + Some(u32::from_ne_bytes([k[0], k[1], k[2], k[3]])) + } + }; + let mut keys: alloc::vec::Vec = self.traces.keys().copied().collect(); + keys.sort(); + match start { + None => match keys.first() { + Some(&k) => Ok(Some(k.to_ne_bytes().to_vec())), + None => Ok(None), + }, + Some(sk) => match keys.iter().find(|&&k| k > sk) { + Some(&k) => Ok(Some(k.to_ne_bytes().to_vec())), + None => Ok(None), + }, + } + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +struct PerCpuArrayMap { + meta: BpfMapMeta, + per_cpu_data: alloc::vec::Vec>, + elem_size: usize, + cpu_count: usize, +} + +impl PerCpuArrayMap { + fn new(meta: BpfMapMeta, cpu_count: usize) -> Self { + let elem_size = meta.value_size as usize; + let total = elem_size * meta.max_entries as usize; + let per_cpu_data = alloc::vec![alloc::vec![0u8; total]; cpu_count]; + Self { + meta, + per_cpu_data, + elem_size, + cpu_count, + } + } + + fn current_cpu(&self) -> usize { + let cpu = ax_hal::percpu::this_cpu_id(); + if cpu < self.cpu_count { cpu } else { 0 } + } +} + +impl BpfMapOps for PerCpuArrayMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]) as usize; + if idx >= self.meta.max_entries as usize { + return Ok(None); + } + let cpu = self.current_cpu(); + let start = idx * self.elem_size; + let end = start + self.elem_size; + Ok(Some(self.per_cpu_data[cpu][start..end].to_vec())) + } + + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]) as usize; + if idx >= self.meta.max_entries as usize { + return Err(bpf_error::EINVAL); + } + let cpu = self.current_cpu(); + let start = idx * self.elem_size; + let end = start + self.elem_size.min(value.len()); + self.per_cpu_data[cpu][start..end].copy_from_slice(&value[..end - start]); + Ok(()) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]) as usize; + if idx >= self.meta.max_entries as usize { + return Err(bpf_error::EINVAL); + } + let cpu = self.current_cpu(); + let start = idx * self.elem_size; + let end = start + self.elem_size; + self.per_cpu_data[cpu][start..end].fill(0); + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let next_idx = match key { + None => 0, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([k[0], k[1], k[2], k[3]]); + idx + 1 + } + }; + if next_idx >= self.meta.max_entries { + return Ok(None); + } + Ok(Some(next_idx.to_ne_bytes().to_vec())) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +struct UnifiedMap { + inner: alloc::boxed::Box, +} + +impl core::fmt::Debug for UnifiedMap { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("UnifiedMap").finish_non_exhaustive() + } +} + +impl UnifiedMap { + fn new(map_type: u32, meta: BpfMapMeta) -> AxResult { + let inner: alloc::boxed::Box = match map_type { + map_type::ARRAY => alloc::boxed::Box::new(ArrayMap::new(meta.clone())), + map_type::HASH => alloc::boxed::Box::new(HashMapInner::new(meta.clone())), + map_type::PERF_EVENT_ARRAY => { + alloc::boxed::Box::new(PerfEventArrayMap::new(meta.clone())) + } + map_type::RINGBUF => alloc::boxed::Box::new(RingBufferMap::new(meta.clone())), + map_type::PROG_ARRAY => alloc::boxed::Box::new(ProgArrayMap::new(meta.clone())), + map_type::STACK_TRACE => alloc::boxed::Box::new(StackTraceMap::new(meta.clone())), + map_type::PERCPU_ARRAY => { + let cpu_count = Self::detect_cpu_count(); + alloc::boxed::Box::new(PerCpuArrayMap::new(meta.clone(), cpu_count)) + } + _ => { + warn!("bpf: unsupported map type {map_type}"); + return Err(bpf_error::EINVAL); + } + }; + Ok(Self { inner }) + } + + fn lookup(&mut self, key: &[u8]) -> AxResult>> { + self.inner.lookup_elem(key) + } + + fn update(&mut self, key: &[u8], value: &[u8], flags: u64) -> AxResult<()> { + self.inner.update_elem(key, value, flags) + } + + fn delete(&mut self, key: &[u8]) -> AxResult<()> { + self.inner.delete_elem(key) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + self.inner.get_next_key(key) + } + + fn meta(&self) -> &BpfMapMeta { + self.inner.meta() + } + + fn detect_cpu_count() -> usize { + let id = ax_hal::percpu::this_cpu_id(); + (id + 1).clamp(2, 256) + } +} + +#[allow(dead_code)] +struct BpfProg { + prog_type: u32, + insns: Vec, + meta: BpfProgMeta, + id: u32, + jitted: Option, +} + +#[derive(Clone, Debug)] +#[allow(dead_code)] +struct BpfProgMeta { + prog_type: u32, + name: alloc::string::String, + license: alloc::string::String, + kern_version: u32, + prog_flags: u32, + expected_attach_type: u32, +} + +#[derive(Debug)] +struct BpfFdTable { + maps: alloc::collections::BTreeMap, + progs: alloc::collections::BTreeMap, + links: alloc::collections::BTreeMap, + next_fd: u32, + free_fds: alloc::vec::Vec, +} + +impl BpfFdTable { + const fn new() -> Self { + Self { + maps: alloc::collections::BTreeMap::new(), + progs: alloc::collections::BTreeMap::new(), + links: alloc::collections::BTreeMap::new(), + next_fd: 3, + free_fds: alloc::vec::Vec::new(), + } + } + + fn alloc_fd(&mut self) -> u32 { + if let Some(fd) = self.free_fds.pop() { + fd + } else { + let fd = self.next_fd; + self.next_fd += 1; + fd + } + } + + fn insert_map(&mut self, map: UnifiedMap) -> u32 { + let fd = self.alloc_fd(); + self.maps.insert(fd, map); + fd + } + + fn get_map(&mut self, fd: u32) -> AxResult<&mut UnifiedMap> { + self.maps.get_mut(&fd).ok_or(AxError::BadFileDescriptor) + } + + fn insert_prog(&mut self, prog: BpfProg) -> u32 { + let fd = self.alloc_fd(); + self.progs.insert(fd, prog); + fd + } + + #[allow(dead_code)] + fn get_prog(&mut self, fd: u32) -> AxResult<&mut BpfProg> { + self.progs.get_mut(&fd).ok_or(AxError::BadFileDescriptor) + } + + fn remove_map(&mut self, fd: u32) -> AxResult<()> { + self.maps + .remove(&fd) + .map(|_| ()) + .ok_or(AxError::BadFileDescriptor) + } + + fn remove_prog(&mut self, fd: u32) -> AxResult<()> { + self.progs + .remove(&fd) + .map(|_| ()) + .ok_or(AxError::BadFileDescriptor) + } + + fn close_fd(&mut self, fd: u32) -> AxResult<()> { + let result = if self.maps.contains_key(&fd) { + self.remove_map(fd) + } else if self.progs.contains_key(&fd) { + self.remove_prog(fd) + } else if self.links.contains_key(&fd) { + self.links + .remove(&fd) + .map(|_| ()) + .ok_or(AxError::BadFileDescriptor) + } else { + Err(AxError::BadFileDescriptor) + }; + if result.is_ok() { + self.free_fds.push(fd); + } + result + } + + #[allow(dead_code)] + fn fd_exists(&self, fd: u32) -> bool { + self.maps.contains_key(&fd) || self.progs.contains_key(&fd) || self.links.contains_key(&fd) + } +} + +static BPF_GLOBAL: SpinNoIrq = SpinNoIrq::new(BpfFdTable::new()); +// SAFETY: This cache stores the value returned by bpf_map_lookup_elem. +// The returned pointer is valid only until the next call to the same helper. +// BPF programs must copy the value before making another map lookup. +static BPF_LOOKUP_CACHE: SpinNoIrq> = SpinNoIrq::new(alloc::vec::Vec::new()); +static BPF_TAIL_CALL_TARGET: SpinNoIrq> = SpinNoIrq::new(None); + +fn handle_map_create(uattr: usize, size: u32) -> AxResult { + if size < 24 { + return Err(bpf_error::EINVAL); + } + let map_type; + let key_size; + let value_size; + let max_entries; + let map_flags; + unsafe { + let ptr = uattr as *const u32; + map_type = core::ptr::read(ptr); + key_size = core::ptr::read(ptr.add(1)); + value_size = core::ptr::read(ptr.add(2)); + max_entries = core::ptr::read(ptr.add(3)); + map_flags = core::ptr::read(ptr.add(4)); + } + if max_entries == 0 { + return Err(bpf_error::EINVAL); + } + if map_type != map_type::RINGBUF && (key_size == 0 || value_size == 0) { + return Err(bpf_error::EINVAL); + } + let mut guard = BPF_GLOBAL.lock(); + let id = guard.maps.len() as u32; + let meta = BpfMapMeta { + map_type, + key_size, + value_size, + max_entries, + map_flags, + id, + }; + let map = UnifiedMap::new(map_type, meta)?; + let fd = guard.insert_map(map); + info!( + "bpf: created map type={map_type} key={key_size} val={value_size} max={max_entries} \ + fd={fd}" + ); + Ok(fd as isize) +} + +fn handle_map_lookup_elem(uattr: usize, size: u32) -> AxResult { + if size < 20 { + return Err(bpf_error::EINVAL); + } + let (map_fd, key_ptr, value_ptr) = unsafe { + let ptr = uattr as *const u64; + let map_fd = core::ptr::read(ptr) as u32; + let key_ptr = core::ptr::read(ptr.add(1)) as usize; + let value_ptr = core::ptr::read(ptr.add(2)) as usize; + (map_fd, key_ptr, value_ptr) + }; + let mut guard = BPF_GLOBAL.lock(); + let map = guard.get_map(map_fd)?; + let key_size = map.meta().key_size as usize; + let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; + match map.lookup(key)? { + Some(val) => { + unsafe { + core::ptr::copy_nonoverlapping(val.as_ptr(), value_ptr as *mut u8, val.len()); + } + Ok(0) + } + None => Err(bpf_error::ENOENT), + } +} + +fn handle_map_update_elem(uattr: usize, size: u32) -> AxResult { + if size < 28 { + return Err(bpf_error::EINVAL); + } + let (map_fd, key_ptr, value_ptr, flags) = unsafe { + let ptr = uattr as *const u64; + let map_fd = core::ptr::read(ptr) as u32; + let key_ptr = core::ptr::read(ptr.add(1)) as usize; + let value_ptr = core::ptr::read(ptr.add(2)) as usize; + let flags = core::ptr::read(ptr.add(3)); + (map_fd, key_ptr, value_ptr, flags) + }; + let mut guard = BPF_GLOBAL.lock(); + let map = guard.get_map(map_fd)?; + let key_size = map.meta().key_size as usize; + let value_size = map.meta().value_size as usize; + let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; + let value = unsafe { core::slice::from_raw_parts(value_ptr as *const u8, value_size) }; + map.update(key, value, flags)?; + Ok(0) +} + +fn handle_map_delete_elem(uattr: usize, size: u32) -> AxResult { + if size < 12 { + return Err(bpf_error::EINVAL); + } + let (map_fd, key_ptr) = unsafe { + let ptr = uattr as *const u64; + let map_fd = core::ptr::read(ptr) as u32; + let key_ptr = core::ptr::read(ptr.add(1)) as usize; + (map_fd, key_ptr) + }; + let mut guard = BPF_GLOBAL.lock(); + let map = guard.get_map(map_fd)?; + let key_size = map.meta().key_size as usize; + let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; + map.delete(key)?; + Ok(0) +} + +fn handle_map_get_next_key(uattr: usize, size: u32) -> AxResult { + if size < 28 { + return Err(bpf_error::EINVAL); + } + let (map_fd, key_ptr, next_key_ptr) = unsafe { + let ptr = uattr as *const u64; + let map_fd = core::ptr::read(ptr) as u32; + let key_ptr = core::ptr::read(ptr.add(1)) as usize; + let next_key_ptr = core::ptr::read(ptr.add(2)) as usize; + (map_fd, key_ptr, next_key_ptr) + }; + let mut guard = BPF_GLOBAL.lock(); + let map = guard.get_map(map_fd)?; + let key_size = map.meta().key_size as usize; + let key_opt = if key_ptr != 0 { + Some(unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }) + } else { + None + }; + match map.get_next_key(key_opt)? { + Some(next_key) => { + unsafe { + core::ptr::copy_nonoverlapping( + next_key.as_ptr(), + next_key_ptr as *mut u8, + key_size, + ); + } + Ok(0) + } + None => Err(bpf_error::ENOENT), + } +} + +fn handle_prog_load(uattr: usize, size: u32) -> AxResult { + if size < 48 { + return Err(bpf_error::EINVAL); + } + let ( + prog_type, + insn_cnt, + insns_ptr, + _license_ptr, + log_level, + _log_size, + _log_buf, + kern_version, + prog_flags, + ) = unsafe { + let read_u32 = |off: usize| core::ptr::read((uattr + off) as *const u32); + let read_u64 = |off: usize| core::ptr::read((uattr + off) as *const u64); + let prog_type = read_u32(0); + let insn_cnt = read_u32(4); + let insns_ptr = read_u64(8); + let license_ptr = read_u64(16); + let log_level = read_u32(24); + let log_size = read_u32(28); + let log_buf = read_u64(32); + let kern_version = read_u32(40); + let prog_flags = read_u32(44); + ( + prog_type, + insn_cnt, + insns_ptr, + license_ptr, + log_level, + log_size, + log_buf, + kern_version, + prog_flags, + ) + }; + if log_level > 0 { + warn!("bpf: BPF_PROG_LOAD verifier log requested but not implemented"); + } + match prog_type { + prog_type::KPROBE + | prog_type::TRACEPOINT + | prog_type::RAW_TRACEPOINT + | prog_type::PERF_EVENT + | prog_type::UNSPEC => {} + _ => { + warn!("bpf: unsupported prog type {prog_type}"); + return Err(bpf_error::EINVAL); + } + } + let insn_bytes = insn_cnt as usize * 8; + let raw_insns = unsafe { core::slice::from_raw_parts(insns_ptr as *const u8, insn_bytes) }; + let mut insns = Vec::new(); + for chunk in raw_insns.chunks_exact(8) { + let arr: [u8; 8] = chunk.try_into().unwrap(); + insns.push(bpf_insn::BpfInsn::from_bytes(&arr)); + } + if let Err(e) = BpfVm::verify_program(&insns) { + warn!("bpf: program verification failed: {e}"); + return Err(bpf_error::EINVAL); + } + let mut guard = BPF_GLOBAL.lock(); + let id = guard.progs.len() as u32; + let jitted = { + let helpers = init_helper_functions(); + ebpf_jit::try_jit_compile(&insns, &helpers) + }; + if jitted.is_some() { + info!("bpf: JIT compilation successful for prog_{id}"); + } else { + warn!("bpf: JIT compilation failed, will use interpreter for prog_{id}"); + } + let prog = BpfProg { + prog_type, + insns, + meta: BpfProgMeta { + prog_type, + name: alloc::format!("prog_{id}"), + license: alloc::string::String::new(), + kern_version, + prog_flags, + expected_attach_type: 0, + }, + id, + jitted, + }; + let fd = guard.insert_prog(prog); + info!("bpf: loaded prog type={prog_type} insns={insn_cnt} fd={fd}"); + Ok(fd as isize) +} + +fn handle_raw_tracepoint_open(_uattr: usize, _size: u32) -> AxResult { + warn!("bpf: BPF_RAW_TRACEPOINT_OPEN not yet implemented"); + Err(bpf_error::EINVAL) +} + +#[allow(dead_code)] +mod helper_id { + pub const MAP_LOOKUP_ELEM: u32 = 1; + pub const MAP_UPDATE_ELEM: u32 = 2; + pub const MAP_DELETE_ELEM: u32 = 3; + pub const PROBE_READ: u32 = 4; + pub const KTIME_GET_NS: u32 = 5; + pub const TRACE_PRINTK: u32 = 6; + pub const GET_PRANDOM_U32: u32 = 7; + pub const GET_SMP_PROCESSOR_ID: u32 = 8; + pub const SKB_STORE_BYTES: u32 = 9; + pub const CSUM_DIFF: u32 = 10; + pub const TAIL_CALL: u32 = 12; + pub const GET_CURRENT_PID_TGID: u32 = 14; + pub const GET_CURRENT_UID_GID: u32 = 15; + pub const GET_CURRENT_COMM: u32 = 16; + pub const PERF_EVENT_OUTPUT: u32 = 25; + pub const GET_STACK_ID: u32 = 27; + pub const GET_CURRENT_CGROUP_ID: u32 = 43; + pub const PROBE_READ_USER: u32 = 112; + pub const PROBE_READ_KERNEL: u32 = 113; + pub const PROBE_READ_USER_STR: u32 = 114; + pub const PROBE_READ_KERNEL_STR: u32 = 115; + pub const MAP_PUSH_ELEM: u32 = 87; + pub const MAP_POP_ELEM: u32 = 88; + pub const MAP_PEEK_ELEM: u32 = 89; + pub const RINGBUF_OUTPUT: u32 = 130; + pub const RINGBUF_RESERVE: u32 = 131; + pub const RINGBUF_SUBMIT: u32 = 132; + pub const RINGBUF_DISCARD: u32 = 133; + pub const GET_CURRENT_TASK: u32 = 35; + pub const MAP_FOR_EACH_ELEM: u32 = 164; + pub const GET_ATTACHED_FUNC_ARGS: u32 = 186; +} + +type HelperFn = fn(u64, u64, u64, u64, u64) -> u64; + +fn init_helper_functions() -> alloc::collections::BTreeMap { + let mut m: alloc::collections::BTreeMap = alloc::collections::BTreeMap::new(); + m.insert(helper_id::MAP_LOOKUP_ELEM, helper_map_lookup_elem); + m.insert(helper_id::MAP_UPDATE_ELEM, helper_map_update_elem); + m.insert(helper_id::MAP_DELETE_ELEM, helper_map_delete_elem); + m.insert(helper_id::TAIL_CALL, helper_tail_call); + m.insert(helper_id::GET_STACK_ID, helper_get_stackid); + m.insert(helper_id::PROBE_READ, helper_probe_read); + m.insert(helper_id::PROBE_READ_KERNEL, helper_probe_read); + m.insert(helper_id::PROBE_READ_USER, helper_probe_read_user); + m.insert(helper_id::PROBE_READ_USER_STR, helper_probe_read_user_str); + m.insert( + helper_id::PROBE_READ_KERNEL_STR, + helper_probe_read_kernel_str, + ); + m.insert(helper_id::KTIME_GET_NS, helper_ktime_get_ns); + m.insert(helper_id::GET_SMP_PROCESSOR_ID, helper_get_smp_processor_id); + m.insert(helper_id::GET_CURRENT_PID_TGID, helper_get_current_pid_tgid); + m.insert(helper_id::GET_CURRENT_UID_GID, helper_get_current_uid_gid); + m.insert(helper_id::GET_PRANDOM_U32, helper_get_prandom_u32); + m.insert(helper_id::PERF_EVENT_OUTPUT, helper_perf_event_output); + m.insert(helper_id::RINGBUF_OUTPUT, helper_ringbuf_output); + m.insert(helper_id::RINGBUF_RESERVE, helper_ringbuf_reserve); + m.insert(helper_id::RINGBUF_SUBMIT, helper_ringbuf_submit); + m.insert(helper_id::RINGBUF_DISCARD, helper_ringbuf_discard); + m.insert(helper_id::TRACE_PRINTK, helper_trace_printk); + m.insert(helper_id::GET_CURRENT_TASK, helper_get_current_task); + m.insert(helper_id::GET_CURRENT_COMM, helper_get_current_comm); + m +} + +fn helper_map_lookup_elem(map_ptr: u64, key_ptr: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if map_ptr == 0 || key_ptr == 0 { + return 0; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_ptr as u32) { + Ok(m) => m, + Err(_) => return 0, + }; + let key_size = map.meta().key_size as usize; + let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; + match map.lookup(key) { + Ok(Some(value)) => { + let mut cache = BPF_LOOKUP_CACHE.lock(); + cache.clear(); + cache.extend_from_slice(&value); + cache.as_ptr() as u64 + } + _ => 0, + } +} + +fn helper_map_update_elem(map_ptr: u64, key_ptr: u64, value_ptr: u64, flags: u64, _a5: u64) -> u64 { + if map_ptr == 0 || key_ptr == 0 || value_ptr == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_ptr as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + let key_size = map.meta().key_size as usize; + let value_size = map.meta().value_size as usize; + let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; + let value = unsafe { core::slice::from_raw_parts(value_ptr as *const u8, value_size) }; + match map.update(key, value, flags) { + Ok(()) => 0, + Err(_) => u64::MAX, + } +} + +fn helper_map_delete_elem(map_ptr: u64, key_ptr: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if map_ptr == 0 || key_ptr == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_ptr as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + let key_size = map.meta().key_size as usize; + let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; + match map.delete(key) { + Ok(()) => 0, + Err(_) => u64::MAX, + } +} + +fn helper_tail_call(_ctx: u64, map_fd: u64, index: u64, _a4: u64, _a5: u64) -> u64 { + if map_fd == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + if map.meta().map_type != map_type::PROG_ARRAY { + return u64::MAX; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(p) => p, + None => return u64::MAX, + }; + let i = index as usize; + if i >= inner.prog_fds.len() { + return u64::MAX; + } + let target_fd = match inner.prog_fds[i] { + Some(fd) => fd, + None => return u64::MAX, + }; + if !guard.progs.contains_key(&target_fd) { + return u64::MAX; + } + drop(guard); + let mut tail_target = BPF_TAIL_CALL_TARGET.lock(); + *tail_target = Some(target_fd); + 0 +} + +fn helper_get_stackid(_ctx: u64, map_fd: u64, _flags: u64, _a4: u64, _a5: u64) -> u64 { + if map_fd == 0 { + return u64::MAX; + } + let fp: usize; + #[cfg(target_arch = "x86_64")] + unsafe { + core::arch::asm!("mov {}, rbp", out(reg) fp) + } + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] + unsafe { + core::arch::asm!("addi {0}, s0, 0", out(reg) fp) + } + #[cfg(target_arch = "aarch64")] + unsafe { + core::arch::asm!("mov {0}, x29", out(reg) fp) + } + #[cfg(target_arch = "loongarch64")] + unsafe { + core::arch::asm!("move {0}, $fp", out(reg) fp) + } + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "aarch64", + target_arch = "loongarch64" + )))] + { + fp = 0; + } + let mut ips = alloc::vec::Vec::new(); + let mut current_fp = fp; + for _ in 0..MAX_STACK_DEPTH { + if current_fp == 0 { + break; + } + unsafe { + let ip_ptr = current_fp as *const usize; + if ip_ptr.is_null() { + break; + } + let next_fp_ptr = ip_ptr.add(1); + if core::ptr::read(next_fp_ptr) == 0 { + break; + } + ips.push(core::ptr::read(ip_ptr) as u64); + current_fp = core::ptr::read(next_fp_ptr); + } + } + if ips.is_empty() { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + if map.meta().map_type != map_type::STACK_TRACE { + return u64::MAX; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(s) => s, + None => return u64::MAX, + }; + inner.store_trace(&ips) as u64 +} + +fn helper_probe_read(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { + if dst == 0 || size == 0 { + return u64::MAX; + } + let len = size as usize; + if len > 4096 { + return u64::MAX; + } + if src == 0 { + unsafe { core::ptr::write_bytes(dst as *mut u8, 0, len) }; + return 0; + } + let src_slice = unsafe { core::slice::from_raw_parts(src as *const u8, len) }; + let dst_slice = unsafe { core::slice::from_raw_parts_mut(dst as *mut u8, len) }; + let copied = { + let buf = unsafe { + core::slice::from_raw_parts_mut( + dst_slice.as_mut_ptr() as *mut core::mem::MaybeUninit, + len, + ) + }; + match starry_vm::vm_read_slice(src as *const u8, buf) { + Ok(()) => len, + Err(_) => { + unsafe { + core::ptr::copy_nonoverlapping(src_slice.as_ptr(), dst_slice.as_mut_ptr(), len) + }; + len + } + } + }; + let _ = copied; + 0 +} + +fn helper_probe_read_user(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { + if dst == 0 || size == 0 { + return u64::MAX; + } + let len = size as usize; + if len > 4096 { + return u64::MAX; + } + if src == 0 { + unsafe { core::ptr::write_bytes(dst as *mut u8, 0, len) }; + return 0; + } + let buf = + unsafe { core::slice::from_raw_parts_mut(dst as *mut core::mem::MaybeUninit, len) }; + match starry_vm::vm_read_slice(src as *const u8, buf) { + Ok(()) => 0, + Err(_) => { + unsafe { core::ptr::write_bytes(dst as *mut u8, 0, len) }; + u64::MAX + } + } +} + +unsafe fn probe_read_str_kernel(dst: *mut u8, size: usize, src: *const u8) -> usize { + let mut i = 0; + while i < size { + unsafe { + let byte = *src.add(i); + *dst.add(i) = byte; + if byte == 0 { + return i; + } + } + i += 1; + } + if size > 0 { + unsafe { *dst.add(size - 1) = 0 }; + } + size +} + +unsafe fn probe_read_str_user(dst: *mut u8, size: usize, src: *const u8) -> usize { + let mut i = 0; + while i < size { + let mut one_buf = core::mem::MaybeUninit::::uninit(); + unsafe { + match starry_vm::vm_read_slice(src.add(i), core::slice::from_mut(&mut one_buf)) { + Ok(()) => { + let byte = one_buf.assume_init(); + *dst.add(i) = byte; + if byte == 0 { + return i; + } + } + Err(_) => { + if i < size { + *dst.add(i) = 0; + } + return i; + } + } + } + i += 1; + } + if size > 0 { + unsafe { *dst.add(size - 1) = 0 }; + } + size +} + +fn helper_probe_read_user_str(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { + if dst == 0 || size == 0 { + return u64::MAX; + } + let len = size as usize; + if len > 4096 { + return u64::MAX; + } + if src == 0 { + unsafe { *(dst as *mut u8) = 0 }; + return u64::MAX; + } + unsafe { probe_read_str_user(dst as *mut u8, len, src as *const u8) as u64 } +} + +fn helper_probe_read_kernel_str(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { + if dst == 0 || size == 0 { + return u64::MAX; + } + let len = size as usize; + if len > 4096 { + return u64::MAX; + } + if src == 0 { + unsafe { *(dst as *mut u8) = 0 }; + return u64::MAX; + } + unsafe { probe_read_str_kernel(dst as *mut u8, len, src as *const u8) as u64 } +} + +fn helper_ktime_get_ns(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + ax_hal::time::monotonic_time_nanos() +} + +fn helper_get_smp_processor_id(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + ax_hal::percpu::this_cpu_id() as u64 +} + +fn helper_get_current_pid_tgid(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + let curr = ax_task::current(); + let pid = curr.id().as_u64(); + let tgid = pid; + (tgid << 32) | pid +} + +fn helper_get_current_uid_gid(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + let curr = ax_task::current(); + let cred = curr.as_thread().cred(); + let uid = cred.uid as u64; + let gid = cred.gid as u64; + (gid << 32) | uid +} + +fn helper_get_prandom_u32(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + use core::sync::atomic::{AtomicU32, Ordering}; + static SEED: AtomicU32 = AtomicU32::new(12345); + let prev = SEED.load(Ordering::Relaxed); + let next = prev.wrapping_mul(1103515245).wrapping_add(12345); + SEED.store(next, Ordering::Relaxed); + next as u64 +} + +fn helper_perf_event_output( + _ctx: u64, + map_fd: u64, + _flags: u64, + data_ptr: u64, + data_size: u64, +) -> u64 { + if data_ptr == 0 || data_size == 0 { + return u64::MAX; + } + let data = unsafe { core::slice::from_raw_parts(data_ptr as *const u8, data_size as usize) }; + match crate::perf_event::perf_event_write(map_fd as u32, data) { + Ok(()) => 0, + Err(_) => u64::MAX, + } +} + +fn helper_ringbuf_output(map_fd: u64, data_ptr: u64, data_size: u64, _flags: u64, _a5: u64) -> u64 { + if map_fd == 0 || data_ptr == 0 || data_size == 0 { + return u64::MAX; + } + let data = unsafe { core::slice::from_raw_parts(data_ptr as *const u8, data_size as usize) }; + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + if map.meta().map_type != map_type::RINGBUF { + return u64::MAX; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => return u64::MAX, + }; + if inner.output(data) { 0 } else { u64::MAX } +} + +fn helper_ringbuf_reserve(map_fd: u64, size: u64, _flags: u64, _a4: u64, _a5: u64) -> u64 { + if map_fd == 0 || size == 0 || size > 4096 { + return 0; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return 0, + }; + if map.meta().map_type != map_type::RINGBUF { + return 0; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => return 0, + }; + match inner.reserve(size as usize) { + Some(ptr) => ptr as u64, + None => 0, + } +} + +fn helper_ringbuf_submit(sample_ptr: u64, flags: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if sample_ptr == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + for (_, map) in guard.maps.iter_mut() { + if map.meta().map_type == map_type::RINGBUF { + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => continue, + }; + if inner.pending_reserve.is_some() { + inner.submit(flags); + return 0; + } + } + } + u64::MAX +} + +fn helper_ringbuf_discard(sample_ptr: u64, flags: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if sample_ptr == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + for (_, map) in guard.maps.iter_mut() { + if map.meta().map_type == map_type::RINGBUF { + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => continue, + }; + if inner.pending_reserve.is_some() { + inner.discard(flags); + return 0; + } + } + } + u64::MAX +} + +fn helper_trace_printk(fmt_ptr: u64, fmt_size: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if fmt_ptr == 0 || fmt_size == 0 || fmt_size > 128 { + return u64::MAX; + } + let len = fmt_size as usize; + let bytes = unsafe { core::slice::from_raw_parts(fmt_ptr as *const u8, len) }; + let s = core::str::from_utf8(bytes).unwrap_or(""); + let trimmed = s.trim_end_matches('\0'); + if !trimmed.is_empty() { + warn!("bpf trace_printk: {trimmed}"); + } + len as u64 +} + +fn helper_get_current_task(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + let curr = ax_task::current(); + curr.as_ref() as *const _ as u64 +} + +fn helper_get_current_comm(buf: u64, size: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if buf == 0 || size == 0 { + return u64::MAX; + } + let curr = ax_task::current(); + let name = curr.name(); + let copy_len = core::cmp::min(name.len(), size as usize - 1); + unsafe { + let dst = core::slice::from_raw_parts_mut(buf as *mut u8, copy_len); + dst.copy_from_slice(name.as_bytes()); + if copy_len < size as usize { + *dst.get_unchecked_mut(copy_len) = 0; + } + } + 0 +} + +const BPF_MAX_INSN: usize = 1000000; +const BPF_MAX_STACK: usize = 512; + +struct BpfVm { + #[allow(dead_code)] + helpers: alloc::collections::BTreeMap, +} + +impl BpfVm { + fn new() -> Self { + Self { + helpers: init_helper_functions(), + } + } + + fn verify_program(insns: &[bpf_insn::BpfInsn]) -> Result<(), &'static str> { + if insns.len() > BPF_MAX_INSN { + warn!("bpf verifier: program too large: {} insns", insns.len()); + return Err("program too large"); + } + if insns.is_empty() { + return Err("empty program"); + } + + let max_pc = insns.len(); + + for (pc, insn) in insns.iter().enumerate() { + let dst = insn.dst_reg() as usize; + let src = insn.src_reg() as usize; + if dst > 10 { + warn!("bpf verifier: invalid dst_reg {dst} at pc={pc}"); + return Err("invalid destination register"); + } + if src > 10 { + warn!("bpf verifier: invalid src_reg {src} at pc={pc}"); + return Err("invalid source register"); + } + } + + let mut visited = alloc::vec![false; max_pc]; + let mut stack = alloc::vec![0usize]; + visited[0] = true; + while let Some(pc) = stack.pop() { + let insn = &insns[pc]; + let class = insn.class(); + + let successors = Self::insn_successors(insn, pc, max_pc)?; + if successors.is_empty() { + let op = insn.code & 0xf0; + if class == bpf_insn::BPF_JMP && op == bpf_insn::BPF_EXIT { + // terminal + } else { + warn!("bpf verifier: unreachable termination at pc={pc}"); + return Err("instruction has no valid successor"); + } + } + for s in successors { + if !visited[s] { + visited[s] = true; + stack.push(s); + } + } + } + + let mut reachable_with_exit = false; + for (pc, &v) in visited.iter().enumerate() { + if !v { + continue; + } + let insn = &insns[pc]; + let class = insn.class(); + let op = insn.code & 0xf0; + if class == bpf_insn::BPF_JMP && op == bpf_insn::BPF_EXIT { + reachable_with_exit = true; + } + + if class == bpf_insn::BPF_ST || class == bpf_insn::BPF_STX || class == bpf_insn::BPF_LDX + { + let off = insn.off as isize; + let abs_off = if off >= 0 { off } else { -off }; + if abs_off as usize >= BPF_MAX_STACK { + warn!("bpf verifier: stack access out of bounds at pc={pc} off={off}"); + return Err("stack access out of bounds"); + } + } + + if class == bpf_insn::BPF_ALU || class == bpf_insn::BPF_ALU64 { + let alu_op = insn.alu_op(); + if (alu_op == 0x30 || alu_op == 0x40) + && (insn.code & bpf_insn::BPF_X) == 0 + && insn.imm == 0 + { + warn!("bpf verifier: division by zero at pc={pc}"); + return Err("division by zero"); + } + } + } + if !reachable_with_exit { + warn!("bpf verifier: no reachable BPF_EXIT instruction"); + return Err("no reachable BPF_EXIT instruction"); + } + + Ok(()) + } + + fn insn_successors( + insn: &bpf_insn::BpfInsn, + pc: usize, + max_pc: usize, + ) -> Result, &'static str> { + let class = insn.class(); + let op = insn.code & 0xf0; + match class { + bpf_insn::BPF_ALU + | bpf_insn::BPF_ALU64 + | bpf_insn::BPF_ST + | bpf_insn::BPF_STX + | bpf_insn::BPF_LDX => { + let next = pc + 1; + if next >= max_pc { + return Ok(alloc::vec![]); + } + Ok(alloc::vec![next]) + } + bpf_insn::BPF_LD => { + if insn.is_ld_dw_imm() { + let next = pc + 2; + if next > max_pc { + return Ok(alloc::vec![]); + } + Ok(alloc::vec![next]) + } else { + Ok(alloc::vec![]) + } + } + bpf_insn::BPF_JMP | bpf_insn::BPF_JMP32 => { + if op == bpf_insn::BPF_EXIT { + return Ok(alloc::vec![]); + } + if op == 0x80 { + let next = pc + 1; + if next >= max_pc { + return Ok(alloc::vec![]); + } + return Ok(alloc::vec![next]); + } + let fallthrough = pc + 1; + let target = (pc as isize + 1 + insn.off as isize) as usize; + let mut succs = alloc::vec![]; + if fallthrough < max_pc { + succs.push(fallthrough); + } + if target < max_pc { + succs.push(target); + } else { + warn!("bpf verifier: jump out of bounds at pc={pc} target={target}"); + return Err("jump out of bounds"); + } + Ok(succs) + } + _ => Ok(alloc::vec![]), + } + } + + fn execute(&self, insns: &[bpf_insn::BpfInsn], ctx: u64) -> Result { + if insns.is_empty() { + return Err("empty program"); + } + let mut regs = [0u64; 11]; + regs[1] = ctx; + regs[10] = 0; + let mut stack = [0u8; BPF_MAX_STACK]; + regs[10] = stack.as_mut_ptr() as u64 + BPF_MAX_STACK as u64; + let mut pc: usize = 0; + let max_pc = insns.len(); + for _ in 0..BPF_MAX_INSN { + if pc >= max_pc { + return Err("PC out of bounds"); + } + let insn = &insns[pc]; + let class = insn.class(); + match class { + bpf_insn::BPF_ALU | bpf_insn::BPF_ALU64 => { + let is_64 = class == bpf_insn::BPF_ALU64; + let dst = insn.dst_reg() as usize; + let src_val = if insn.code & bpf_insn::BPF_X != 0 { + regs[insn.src_reg() as usize] + } else { + insn.imm as u64 + }; + let result = Self::exec_alu(insn.alu_op(), regs[dst], src_val, is_64); + regs[dst] = result; + pc += 1; + } + bpf_insn::BPF_JMP | bpf_insn::BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == bpf_insn::BPF_EXIT { + return Ok(regs[0]); + } + if op == 0x80 { + let helper_id = insn.imm as u32; + if let Some(helper_fn) = self.helpers.get(&helper_id) { + regs[0] = helper_fn(regs[1], regs[2], regs[3], regs[4], regs[5]); + } else { + warn!("bpf: unknown helper {}", helper_id); + regs[0] = u64::MAX; + } + if helper_id == helper_id::TAIL_CALL && regs[0] == 0 { + let target_fd = { + let mut tail_target = BPF_TAIL_CALL_TARGET.lock(); + tail_target.take() + }; + if let Some(fd) = target_fd { + let guard = BPF_GLOBAL.lock(); + if let Some(target_prog) = guard.progs.get(&fd) { + let target_insns = target_prog.insns.clone(); + drop(guard); + return self.execute(&target_insns, regs[1]); + } + } + } + pc += 1; + continue; + } + let is_64 = class == bpf_insn::BPF_JMP; + let dst = insn.dst_reg() as usize; + let src_val = if insn.code & bpf_insn::BPF_X != 0 { + regs[insn.src_reg() as usize] + } else { + insn.imm as u64 + }; + let dst_val = regs[dst]; + let off = insn.off as isize; + if insn.code == (bpf_insn::BPF_JMP | bpf_insn::BPF_JA) { + pc = (pc as isize + 1 + off) as usize; + continue; + } + if insn.code == (bpf_insn::BPF_JMP32 | bpf_insn::BPF_JA) { + pc = (pc as isize + 1 + off) as usize; + continue; + } + if Self::eval_jmp(insn.code, dst_val, src_val, is_64) { + pc = (pc as isize + 1 + off) as usize; + } else { + pc += 1; + } + } + bpf_insn::BPF_ST | bpf_insn::BPF_STX => { + Self::exec_store(insn, &mut regs, &mut stack); + pc += 1; + } + bpf_insn::BPF_LDX => { + Self::exec_load(insn, &mut regs, &stack); + pc += 1; + } + bpf_insn::BPF_LD => { + if insn.is_ld_dw_imm() && pc + 1 < max_pc { + let next = &insns[pc + 1]; + let imm_lo = insn.imm as u64; + let imm_hi = next.imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + regs[insn.dst_reg() as usize] = val; + pc += 2; + } else { + return Err("unsupported LD instruction"); + } + } + _ => return Err("unsupported instruction class"), + } + } + Err("max instructions exceeded") + } + + fn exec_alu(op: u8, dst: u64, src: u64, is_64: bool) -> u64 { + let (result, mask) = match op { + bpf_insn::BPF_ADD => (dst.wrapping_add(src), !0), + bpf_insn::BPF_SUB => (dst.wrapping_sub(src), !0), + bpf_insn::BPF_MUL => (dst.wrapping_mul(src), !0), + bpf_insn::BPF_DIV => { + if src == 0 { + return 0; + } + (dst / src, !0) + } + bpf_insn::BPF_OR => (dst | src, !0), + bpf_insn::BPF_AND => (dst & src, !0), + bpf_insn::BPF_LSH => (dst.wrapping_shl(src as u32), !0), + bpf_insn::BPF_RSH => { + if is_64 { + (dst >> src, !0) + } else { + ((dst as u32 >> src as u32) as u64, 0xffffffff) + } + } + bpf_insn::BPF_NEG => ((-(dst as i64)) as u64, !0), + bpf_insn::BPF_MOD => { + if src == 0 { + return dst; + } + (dst % src, !0) + } + bpf_insn::BPF_XOR => (dst ^ src, !0), + bpf_insn::BPF_MOV => (src, !0), + bpf_insn::BPF_ARSH => { + if is_64 { + (((dst as i64) >> src) as u64, !0) + } else { + ((((dst as i32) as i64) >> src) as u64, 0xffffffff) + } + } + _ => return dst, + }; + result & mask + } + + fn eval_jmp(code: u8, dst: u64, src: u64, is_64: bool) -> bool { + let op = code & 0xf0; + let (d, s) = if is_64 { + (dst, src) + } else { + (dst as u32 as u64, src as u32 as u64) + }; + match op { + bpf_insn::BPF_JEQ => d == s, + bpf_insn::BPF_JGT => d > s, + bpf_insn::BPF_JGE => d >= s, + bpf_insn::BPF_JSET => (d & s) != 0, + bpf_insn::BPF_JNE => d != s, + bpf_insn::BPF_JSGT => (d as i64) > (s as i64), + bpf_insn::BPF_JSGE => (d as i64) >= (s as i64), + bpf_insn::BPF_JLT => d < s, + bpf_insn::BPF_JLE => d <= s, + bpf_insn::BPF_JSLT => (d as i64) < (s as i64), + bpf_insn::BPF_JSLE => (d as i64) <= (s as i64), + _ => false, + } + } + + #[allow(clippy::comparison_chain)] + fn exec_store(insn: &bpf_insn::BpfInsn, regs: &mut [u64; 11], stack: &mut [u8; BPF_MAX_STACK]) { + let dst_base = regs[10]; + let off = insn.off as i32 as isize; + let mem = insn.mode(); + if mem == bpf_insn::BPF_MEM { + let addr = (dst_base as isize + off) as usize; + let stack_base = stack.as_mut_ptr() as usize; + if addr < stack_base || addr + 8 > stack_base + BPF_MAX_STACK { + return; + } + let val = if insn.class() == bpf_insn::BPF_ST { + insn.imm as u64 + } else { + regs[insn.src_reg() as usize] + }; + match insn.size() { + bpf_insn::BPF_W => unsafe { + let p = (addr - stack_base) as *mut u32; + *p = val as u32; + }, + bpf_insn::BPF_H => unsafe { + let p = (addr - stack_base) as *mut u16; + *p = val as u16; + }, + bpf_insn::BPF_B => unsafe { + let p = (addr - stack_base) as *mut u8; + *p = val as u8; + }, + bpf_insn::BPF_DW => unsafe { + let p = (addr - stack_base) as *mut u64; + *p = val; + }, + _ => {} + } + } + } + + fn exec_load(insn: &bpf_insn::BpfInsn, regs: &mut [u64; 11], stack: &[u8; BPF_MAX_STACK]) { + let src_base = regs[insn.src_reg() as usize]; + let off = insn.off as i32 as isize; + let mem = insn.mode(); + if mem == bpf_insn::BPF_MEM { + let addr = (src_base as isize + off) as usize; + let stack_base = stack.as_ptr() as usize; + if addr < stack_base || addr + 8 > stack_base + BPF_MAX_STACK { + return; + } + let val: u64 = match insn.size() { + bpf_insn::BPF_W => unsafe { + let p = (addr - stack_base) as *const u32; + (*p) as u64 + }, + bpf_insn::BPF_H => unsafe { + let p = (addr - stack_base) as *const u16; + (*p) as u64 + }, + bpf_insn::BPF_B => unsafe { + let p = (addr - stack_base) as *const u8; + (*p) as u64 + }, + bpf_insn::BPF_DW => unsafe { + let p = (addr - stack_base) as *const u64; + *p + }, + _ => 0, + }; + regs[insn.dst_reg() as usize] = val; + } + } +} + +#[allow(dead_code)] +pub fn run_bpf_prog(fd: u32, ctx: u64) -> AxResult { + let (insns, prog_type, has_jit, jit_entry) = { + let guard = BPF_GLOBAL.lock(); + let prog = guard.progs.get(&fd).ok_or(AxError::BadFileDescriptor)?; + let entry = prog.jitted.as_ref().map(|j| j.entry()); + ( + prog.insns.clone(), + prog.prog_type, + prog.jitted.is_some(), + entry, + ) + }; + let _ = prog_type; + + if has_jit { + if let Some(entry) = jit_entry { + let result: u64; + unsafe { + let jit_fn: extern "C" fn(u64) -> u64 = core::mem::transmute(entry); + result = jit_fn(ctx); + } + return Ok(result); + } + } + + let vm = BpfVm::new(); + vm.execute(&insns, ctx).map_err(|e| { + warn!("bpf: program execution failed: {e}"); + AxError::Io + }) +} + +fn handle_link_create(uattr: usize, size: u32) -> AxResult { + if size < 20 { + return Err(bpf_error::EINVAL); + } + let (prog_fd, target_fd) = unsafe { + let ptr = uattr as *const u32; + let prog_fd = core::ptr::read(ptr) as u32; + let target_fd = core::ptr::read(ptr.add(1)) as u32; + let _attach_type = core::ptr::read(ptr.add(2)); + (prog_fd, target_fd) + }; + crate::perf_event::perf_event_attach_prog(target_fd, prog_fd)?; + crate::perf_event::perf_event_enable(target_fd)?; + let link_fd = { + let mut guard = BPF_GLOBAL.lock(); + let link_fd = guard.alloc_fd(); + guard.links.insert(link_fd, (prog_fd, target_fd)); + link_fd + }; + info!("bpf: LINK_CREATE prog_fd={prog_fd} target_fd={target_fd} link_fd={link_fd}"); + Ok(link_fd as isize) +} + +fn handle_obj_close(uattr: usize, size: u32) -> AxResult { + if size < 4 { + return Err(bpf_error::EINVAL); + } + let fd = unsafe { core::ptr::read(uattr as *const u32) }; + let mut guard = BPF_GLOBAL.lock(); + guard.close_fd(fd)?; + info!("bpf: OBJ_CLOSE fd={fd}"); + Ok(0) +} + +fn handle_prog_attach(cmd: u64, uattr: usize, size: u32) -> AxResult { + if size < 16 { + return Err(bpf_error::EINVAL); + } + let (target_fd, attach_prog_fd) = unsafe { + let ptr = uattr as *const u32; + let target_fd = core::ptr::read(ptr) as u32; + let attach_prog_fd = core::ptr::read(ptr.add(1)) as u32; + let _attach_type = core::ptr::read(ptr.add(2)); + (target_fd, attach_prog_fd) + }; + if cmd == cmd::PROG_ATTACH { + crate::perf_event::perf_event_attach_prog(target_fd, attach_prog_fd)?; + crate::perf_event::perf_event_enable(target_fd)?; + info!("bpf: PROG_ATTACH attach_prog_fd={attach_prog_fd} target_fd={target_fd}"); + } else { + crate::perf_event::perf_event_disable(target_fd)?; + info!("bpf: PROG_DETACH target_fd={target_fd}"); + } + Ok(0) +} + +pub fn sys_bpf(cmd: u64, uattr: usize, size: u32) -> AxResult { + match cmd { + cmd::MAP_CREATE => handle_map_create(uattr, size), + cmd::PROG_LOAD => handle_prog_load(uattr, size), + cmd::MAP_LOOKUP_ELEM => handle_map_lookup_elem(uattr, size), + cmd::MAP_UPDATE_ELEM => handle_map_update_elem(uattr, size), + cmd::MAP_DELETE_ELEM => handle_map_delete_elem(uattr, size), + cmd::MAP_GET_NEXT_KEY => handle_map_get_next_key(uattr, size), + cmd::RAW_TRACEPOINT_OPEN => handle_raw_tracepoint_open(uattr, size), + cmd::OBJ_CLOSE => handle_obj_close(uattr, size), + cmd::OBJ_PIN | cmd::OBJ_GET => { + warn!("bpf: obj pin/get not yet implemented"); + Err(bpf_error::EINVAL) + } + cmd::PROG_ATTACH | cmd::PROG_DETACH => handle_prog_attach(cmd, uattr, size), + cmd::LINK_CREATE => handle_link_create(uattr, size), + cmd::ENABLE_STATS => { + warn!("bpf: ENABLE_STATS not yet implemented"); + Err(bpf_error::EINVAL) + } + _ => { + warn!("bpf: unknown command {cmd}"); + Err(bpf_error::EINVAL) + } + } +} + +pub fn sys_perf_event_open( + attr_uptr: usize, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u64, +) -> AxResult { + crate::perf_event::sys_perf_event_open_impl(attr_uptr, pid, cpu, group_fd, flags) +} + +#[allow(dead_code)] +pub fn bpf_close_fd(fd: u32) -> AxResult<()> { + let mut guard = BPF_GLOBAL.lock(); + guard.close_fd(fd) +} + +#[allow(dead_code)] +pub fn bpf_fd_exists(fd: u32) -> bool { + let guard = BPF_GLOBAL.lock(); + guard.fd_exists(fd) +} diff --git a/os/StarryOS/kernel/src/ebpf_jit/jit_aarch64.rs b/os/StarryOS/kernel/src/ebpf_jit/jit_aarch64.rs new file mode 100644 index 0000000000..2398fac858 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf_jit/jit_aarch64.rs @@ -0,0 +1,662 @@ +use alloc::vec::Vec; + +use super::JitBackend; +use super::super::bpf_insn::{ + BpfInsn, BPF_ADD, BPF_ALU, BPF_ALU64, BPF_AND, BPF_ARSH, BPF_B, BPF_DW, BPF_DIV, BPF_EXIT, + BPF_H, BPF_JA, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, BPF_JNE, + BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_JSET, BPF_K, BPF_LD, BPF_LDX, BPF_LSH, BPF_MEM, + BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, BPF_OR, BPF_RSH, BPF_ST, BPF_STX, BPF_SUB, BPF_W, + BPF_X, BPF_XOR, +}; +use super::super::HelperFn; +use super::JitBuffer; + +const AA_X0: u32 = 0; +const AA_X1: u32 = 1; +const AA_X2: u32 = 2; +const AA_X3: u32 = 3; +const AA_X4: u32 = 4; +const AA_X5: u32 = 5; +const AA_X7: u32 = 7; +const AA_X9: u32 = 9; +const AA_X15: u32 = 15; +const AA_X16: u32 = 16; +const AA_X17: u32 = 17; +const AA_SP: u32 = 31; +const AA_X29: u32 = 29; +const AA_LR: u32 = 30; + +fn bpf_to_aa(r: u8) -> u32 { + match r { + 0 => AA_X0, + 1 => AA_X1, + 2 => AA_X2, + 3 => AA_X3, + 4 => AA_X4, + 5 => AA_X5, + 6 => AA_X7, + 7 => AA_X9, + 8 => AA_X15, + 9 => AA_X16, + 10 => AA_X29, + _ => AA_X0, + } +} + +fn emit_add(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x8B000000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_addw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x0B000000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_sub(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0xCB000000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_subw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x4B000000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_addi(buf: &mut JitBuffer, rd: u32, rn: u32, imm: i32) { + let sh = 0u32; + let imm12 = (imm as u32) & 0xFFF; + buf.emit_u32(0x91000000 | (sh << 22) | (imm12 << 10) | (rn << 5) | rd); +} + +fn emit_addiw(buf: &mut JitBuffer, rd: u32, rn: u32, imm: i32) { + let imm12 = (imm as u32) & 0xFFF; + buf.emit_u32(0x11000000 | (imm12 << 10) | (rn << 5) | rd); +} + +fn emit_subi(buf: &mut JitBuffer, rd: u32, rn: u32, imm: i32) { + let imm12 = (imm as u32) & 0xFFF; + buf.emit_u32(0xD1000000 | (imm12 << 10) | (rn << 5) | rd); +} + +fn emit_and(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x8A000000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_andw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x0A000000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_or(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x8A200000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_orw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x0A200000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_xor(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x8A400000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_xorw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x0A400000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_mul(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x9B007C00 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_mulw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x1B007C00 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_udiv(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x9AC00C00 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_udivw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x1AC00C00 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_msub(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32, ra: u32) { + buf.emit_u32(0x9B00FC00 | (rm << 16) | (ra << 10) | (rn << 5) | rd); +} + +fn emit_msubw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32, ra: u32) { + buf.emit_u32(0x1B00FC00 | (rm << 16) | (ra << 10) | (rn << 5) | rd); +} + +fn emit_lsl(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x9AC02000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_lslw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x1AC02000 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_lsr(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x9AC02400 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_lsrw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x1AC02400 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_asr(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x9AC02800 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_asrw(buf: &mut JitBuffer, rd: u32, rn: u32, rm: u32) { + buf.emit_u32(0x1AC02800 | (rm << 16) | (rn << 5) | rd); +} + +fn emit_lsl_imm(buf: &mut JitBuffer, rd: u32, rn: u32, imm: u32) { + buf.emit_u32(0xD3400000 | (((64 - imm) & 63) << 16) | (rn << 5) | rd); +} + +fn emit_lsl_immw(buf: &mut JitBuffer, rd: u32, rn: u32, imm: u32) { + buf.emit_u32(0x53000000 | (((32 - imm) & 31) << 16) | (rn << 5) | rd); +} + +fn emit_lsr_imm(buf: &mut JitBuffer, rd: u32, rn: u32, imm: u32) { + buf.emit_u32(0xD340FC00 | ((imm & 63) << 16) | (rn << 5) | rd); +} + +fn emit_lsr_immw(buf: &mut JitBuffer, rd: u32, rn: u32, imm: u32) { + buf.emit_u32(0x53007C00 | ((imm & 31) << 16) | (rn << 5) | rd); +} + +fn emit_asr_imm(buf: &mut JitBuffer, rd: u32, rn: u32, imm: u32) { + buf.emit_u32(0x9340FC00 | ((imm & 63) << 16) | (rn << 5) | rd); +} + +fn emit_asr_immw(buf: &mut JitBuffer, rd: u32, rn: u32, imm: u32) { + buf.emit_u32(0x13007C00 | ((imm & 31) << 16) | (rn << 5) | rd); +} + +fn emit_neg(buf: &mut JitBuffer, rd: u32, rn: u32) { + emit_sub(buf, rd, 31, rn); +} + +fn emit_negw(buf: &mut JitBuffer, rd: u32, rn: u32) { + emit_subw(buf, rd, 31, rn); +} + +fn emit_mov(buf: &mut JitBuffer, rd: u32, rn: u32) { + buf.emit_u32(0xAA0003E0 | (rn << 16) | rd); +} + +fn emit_movz16(buf: &mut JitBuffer, rd: u32, imm: u16, shift: u32) { + buf.emit_u32(0x52800000 | (shift << 21) | ((imm as u32) << 5) | rd); +} + +fn emit_movk16(buf: &mut JitBuffer, rd: u32, imm: u16, shift: u32) { + buf.emit_u32(0x72800000 | (shift << 21) | ((imm as u32) << 5) | rd); +} + +fn emit_load_imm64(buf: &mut JitBuffer, rd: u32, val: u64) { + emit_movz16(buf, rd, (val & 0xFFFF) as u16, 0); + emit_movk16(buf, rd, ((val >> 16) & 0xFFFF) as u16, 1); + emit_movk16(buf, rd, ((val >> 32) & 0xFFFF) as u16, 2); + emit_movk16(buf, rd, ((val >> 48) & 0xFFFF) as u16, 3); +} + +fn emit_load_imm32(buf: &mut JitBuffer, rd: u32, val: i32) { + let v = val as u32; + emit_movz16(buf, rd, (v & 0xFFFF) as u16, 0); + emit_movk16(buf, rd, ((v >> 16) & 0xFFFF) as u16, 1); +} + +fn emit_str(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off << 2) as u32 & 0x1FFC; + buf.emit_u32(0xF9000000 | imm9 | (rn << 5) | rt); +} + +fn emit_strw(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off << 2) as u32 & 0x1FFC; + buf.emit_u32(0xB9000000 | imm9 | (rn << 5) | rt); +} + +fn emit_strh(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off << 1) as u32 & 0x1FFE; + buf.emit_u32(0x79000000 | imm9 | (rn << 5) | rt); +} + +fn emit_strb(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off) as u32 & 0xFFF; + buf.emit_u32(0x39000000 | imm9 | (rn << 5) | rt); +} + +fn emit_ldr(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off << 2) as u32 & 0x1FFC; + buf.emit_u32(0xF9400000 | imm9 | (rn << 5) | rt); +} + +fn emit_ldrw(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off << 2) as u32 & 0x1FFC; + buf.emit_u32(0xB9400000 | imm9 | (rn << 5) | rt); +} + +fn emit_ldrh(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off << 1) as u32 & 0x1FFE; + buf.emit_u32(0x79400000 | imm9 | (rn << 5) | rt); +} + +fn emit_ldrb(buf: &mut JitBuffer, rt: u32, rn: u32, off: i32) { + let imm9 = (off) as u32 & 0xFFF; + buf.emit_u32(0x39400000 | imm9 | (rn << 5) | rt); +} + +fn emit_stp(buf: &mut JitBuffer, rt1: u32, rt2: u32, rn: u32, off: i32) { + let imm7 = ((off as i64) << 3) as u32 & 0x1FF8; + buf.emit_u32(0xA9000000 | imm7 | (rt2 << 10) | (rn << 5) | rt1); +} + +fn emit_ldp(buf: &mut JitBuffer, rt1: u32, rt2: u32, rn: u32, off: i32) { + let imm7 = ((off as i64) << 3) as u32 & 0x1FF8; + buf.emit_u32(0xA9400000 | imm7 | (rt2 << 10) | (rn << 5) | rt1); +} + +fn emit_cmp(buf: &mut JitBuffer, rn: u32, rm: u32) { + emit_sub(buf, 31, rn, rm); +} + +fn emit_cmpw(buf: &mut JitBuffer, rn: u32, rm: u32) { + emit_subw(buf, 31, rn, rm); +} + +fn emit_cbz(buf: &mut JitBuffer, rt: u32, off: i32) { + let imm19 = ((off as i64) >> 2) as u32 & 0x7FFFF; + buf.emit_u32(0x34000000 | (imm19 << 5) | rt); +} + +fn emit_cbnz(buf: &mut JitBuffer, rt: u32, off: i32) { + let imm19 = ((off as i64) >> 2) as u32 & 0x7FFFF; + buf.emit_u32(0x35000000 | (imm19 << 5) | rt); +} + +fn emit_bcond(buf: &mut JitBuffer, cond: u32, off: i32) { + let imm19 = ((off as i64) >> 2) as u32 & 0x7FFFF; + buf.emit_u32(0x54000000 | (imm19 << 5) | cond); +} + +fn emit_b(buf: &mut JitBuffer, off: i32) { + let imm26 = ((off as i64) >> 2) as u32 & 0x3FFFFFF; + buf.emit_u32(0x14000000 | imm26); +} + +fn emit_blr(buf: &mut JitBuffer, rn: u32) { + buf.emit_u32(0xD63F0000 | (rn << 5)); +} + +fn emit_ret(buf: &mut JitBuffer) { + buf.emit_u32(0xD65F03C0); +} + +fn emit_zext32(buf: &mut JitBuffer, rd: u32, rn: u32) { + buf.emit_u32(0x5300001F | (rn << 5) | rd); +} + +pub(crate) struct Aarch64Backend; + +const BPF_STACK_SIZE: usize = 512; +const FRAME_SIZE: usize = BPF_STACK_SIZE + 8 * 5; + +impl JitBackend for Aarch64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + emit_subi(buf, AA_SP, AA_SP, FRAME_SIZE as i32); + emit_stp(buf, AA_X29, AA_X7, AA_SP, BPF_STACK_SIZE as i32); + emit_stp(buf, AA_X9, AA_X15, AA_SP, (BPF_STACK_SIZE + 16) as i32); + emit_str(buf, AA_X16, AA_SP, (BPF_STACK_SIZE + 32) as i32); + emit_addi(buf, AA_X29, AA_SP, FRAME_SIZE as i32); + emit_mov(buf, AA_X1, AA_X0); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + emit_ldp(buf, AA_X29, AA_X7, AA_SP, BPF_STACK_SIZE as i32); + emit_ldp(buf, AA_X9, AA_X15, AA_SP, (BPF_STACK_SIZE + 16) as i32); + emit_ldr(buf, AA_X16, AA_SP, (BPF_STACK_SIZE + 32) as i32); + emit_addi(buf, AA_SP, AA_SP, FRAME_SIZE as i32); + emit_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_aa(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { AA_X17 } else { bpf_to_aa(insn.src_reg()) }; + + if use_imm { + if is_64 { + emit_load_imm64(buf, AA_X17, insn.imm as u64); + } else { + emit_load_imm32(buf, AA_X17, insn.imm); + } + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + emit_add(buf, dst, dst, src); + } else { + emit_addw(buf, dst, dst, src); + } + } + BPF_SUB => { + if is_64 { + emit_sub(buf, dst, dst, src); + } else { + emit_subw(buf, dst, dst, src); + } + } + BPF_MUL => { + if is_64 { + emit_mul(buf, dst, dst, src); + } else { + emit_mulw(buf, dst, dst, src); + } + } + BPF_DIV => { + if is_64 { + emit_cbz(buf, src, 8); + emit_udiv(buf, dst, dst, src); + emit_b(buf, 8); + emit_movz16(buf, dst, 0, 0); + } else { + emit_cbz(buf, src, 8); + emit_udivw(buf, dst, dst, src); + emit_b(buf, 8); + emit_movz16(buf, dst, 0, 0); + } + } + BPF_OR => { + if is_64 { + emit_or(buf, dst, dst, src); + } else { + emit_orw(buf, dst, dst, src); + } + } + BPF_AND => { + if is_64 { + emit_and(buf, dst, dst, src); + } else { + emit_andw(buf, dst, dst, src); + } + } + BPF_LSH => { + if use_imm { + let shamt = (insn.imm as u32) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_lsl_imm(buf, dst, dst, shamt); + } else { + emit_lsl_immw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_lsl(buf, dst, dst, src); + } else { + emit_lslw(buf, dst, dst, src); + } + } + BPF_RSH => { + if use_imm { + let shamt = (insn.imm as u32) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_lsr_imm(buf, dst, dst, shamt); + } else { + emit_lsr_immw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_lsr(buf, dst, dst, src); + } else { + emit_lsrw(buf, dst, dst, src); + } + } + BPF_NEG => { + if is_64 { + emit_neg(buf, dst, dst); + } else { + emit_negw(buf, dst, dst); + } + } + BPF_MOD => { + if is_64 { + emit_cbz(buf, src, 12); + emit_udiv(buf, AA_X17, dst, src); + emit_msub(buf, dst, AA_X17, src, dst); + emit_b(buf, 8); + emit_movz16(buf, dst, 0, 0); + } else { + emit_cbz(buf, src, 12); + emit_udivw(buf, AA_X17, dst, src); + emit_msubw(buf, dst, AA_X17, src, dst); + emit_b(buf, 8); + emit_movz16(buf, dst, 0, 0); + } + } + BPF_XOR => { + if is_64 { + emit_xor(buf, dst, dst, src); + } else { + emit_xorw(buf, dst, dst, src); + } + } + BPF_MOV => { + if is_64 { + if use_imm { + emit_load_imm64(buf, dst, insn.imm as u64); + } else { + emit_mov(buf, dst, src); + } + } else { + if use_imm { + emit_load_imm32(buf, dst, insn.imm); + } else { + emit_mov(buf, dst, src); + } + } + } + BPF_ARSH => { + if use_imm { + let shamt = (insn.imm as u32) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_asr_imm(buf, dst, dst, shamt); + } else { + emit_asr_immw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_asr(buf, dst, dst, src); + } else { + emit_asrw(buf, dst, dst, src); + } + } + _ => {} + } + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let off = offsets[target_pc] as isize - buf.offset() as isize; + emit_b(buf, off as i32); + } + return; + } + + if op == BPF_EXIT { + let off = 0isize; + emit_b(buf, off as i32); + return; + } + + let dst = bpf_to_aa(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { AA_X17 } else { bpf_to_aa(insn.src_reg()) }; + + if use_imm { + if is_64 { + emit_load_imm64(buf, AA_X17, insn.imm as u64); + } else { + emit_load_imm32(buf, AA_X17, insn.imm); + } + } + + if is_64 { + emit_cmp(buf, dst, src); + } else { + emit_cmpw(buf, dst, src); + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + let target_off = if target_pc < offsets.len() { + (offsets[target_pc] as isize - buf.offset() as isize) as i32 + } else { + 0 + }; + + match op { + BPF_JEQ => emit_bcond(buf, 0, target_off), + BPF_JGT => emit_bcond(buf, 2, target_off), + BPF_JGE => emit_bcond(buf, 2, target_off), + BPF_JSET => { + emit_and(buf, AA_X17, dst, src); + emit_cbnz(buf, AA_X17, target_off); + } + BPF_JNE => emit_bcond(buf, 1, target_off), + BPF_JSGT => emit_bcond(buf, 0xC, target_off), + BPF_JSGE => emit_bcond(buf, 0xA, target_off), + BPF_JLT => emit_bcond(buf, 3, target_off), + BPF_JLE => emit_bcond(buf, 1, target_off), + BPF_JSLT => emit_bcond(buf, 0xB, target_off), + BPF_JSLE => emit_bcond(buf, 0xD, target_off), + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + if insn.size() == BPF_DW { + emit_load_imm64(buf, AA_X17, insn.imm as u64); + emit_str(buf, AA_X17, AA_X29, off); + } else { + emit_load_imm32(buf, AA_X17, insn.imm); + match insn.size() { + BPF_B => emit_strb(buf, AA_X17, AA_X29, off), + BPF_H => emit_strh(buf, AA_X17, AA_X29, off), + BPF_W => emit_strw(buf, AA_X17, AA_X29, off), + _ => {} + } + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_aa(insn.src_reg()); + match insn.size() { + BPF_B => emit_strb(buf, src, AA_X29, off), + BPF_H => emit_strh(buf, src, AA_X29, off), + BPF_W => emit_strw(buf, src, AA_X29, off), + BPF_DW => emit_str(buf, src, AA_X29, off), + _ => {} + } + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_aa(insn.src_reg()); + let dst = bpf_to_aa(insn.dst_reg()); + match insn.size() { + BPF_B => emit_ldrb(buf, dst, base, off), + BPF_H => emit_ldrh(buf, dst, base, off), + BPF_W => { + emit_ldrw(buf, dst, base, off); + } + BPF_DW => emit_ldr(buf, dst, base, off), + _ => {} + } + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_aa(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + emit_load_imm64(buf, dst, val); + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + emit_mov(buf, AA_X17, AA_X5); + emit_mov(buf, AA_X5, AA_X4); + emit_mov(buf, AA_X4, AA_X3); + emit_mov(buf, AA_X3, AA_X2); + emit_mov(buf, AA_X2, AA_X1); + emit_mov(buf, AA_X1, AA_X0); + emit_load_imm64(buf, AA_X17, helper_fn as u64); + emit_blr(buf, AA_X17); + } + + fn insn_size(insn: &BpfInsn) -> usize { + let class = insn.class(); + let use_imm = (insn.code & BPF_X) == 0; + + match class { + BPF_ALU | BPF_ALU64 => { + let alu_op = insn.alu_op(); + let imm_size = if use_imm { + if alu_op == BPF_MOV { + 16 + } else { + 16 + } + } else { + 4 + }; + match alu_op { + BPF_DIV => imm_size + 12, + BPF_MOD => imm_size + 16, + _ => imm_size, + } + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + 4 + } else if op == 0x80 { + 8 + 16 + 4 + } else if insn.code == (BPF_JMP | BPF_JA) + || insn.code == (BPF_JMP32 | BPF_JA) + { + 4 + } else { + let cmp_size = if use_imm { 16 } else { 4 }; + cmp_size + 4 + } + } + BPF_ST => { + if insn.size() == BPF_DW { + 24 + } else { + 20 + } + } + BPF_STX => 4, + BPF_LDX => 4, + BPF_LD => { + if insn.is_ld_dw_imm() { + 16 + } else { + 4 + } + } + _ => 4, + } + } +} diff --git a/os/StarryOS/kernel/src/ebpf_jit/jit_riscv64.rs b/os/StarryOS/kernel/src/ebpf_jit/jit_riscv64.rs new file mode 100644 index 0000000000..a664331279 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf_jit/jit_riscv64.rs @@ -0,0 +1,824 @@ +use alloc::vec::Vec; + +use super::JitBackend; +use super::super::bpf_insn::{ + BpfInsn, BPF_ADD, BPF_ALU, BPF_ALU64, BPF_AND, BPF_ARSH, BPF_B, BPF_DW, BPF_DIV, BPF_END, + BPF_EXIT, BPF_H, BPF_JA, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, + BPF_JNE, BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_JSET, BPF_K, BPF_LD, BPF_LDX, BPF_LSH, + BPF_MEM, BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, BPF_OR, BPF_RSH, BPF_ST, BPF_STX, BPF_SUB, + BPF_W, BPF_X, BPF_XOR, +}; +use super::super::HelperFn; +use super::JitBuffer; + +const RV_ZERO: u32 = 0; +const RV_RA: u32 = 1; +const RV_SP: u32 = 2; +const RV_T1: u32 = 6; +const RV_T2: u32 = 7; +const RV_S1: u32 = 9; +const RV_A0: u32 = 10; +const RV_A1: u32 = 11; +const RV_A2: u32 = 12; +const RV_A3: u32 = 13; +const RV_A4: u32 = 14; +const RV_A5: u32 = 15; +const RV_S2: u32 = 18; +const RV_S3: u32 = 19; +const RV_S4: u32 = 20; +const RV_S5: u32 = 21; +const RV_T6: u32 = 31; + +const BPF_STACK_SIZE: usize = 512; +const CALLEE_SAVED_COUNT: usize = 5; +const FRAME_SIZE: usize = BPF_STACK_SIZE + CALLEE_SAVED_COUNT * 8 + 8; + +fn bpf_to_rv(r: u8) -> u32 { + match r { + 0 => RV_A0, + 1 => RV_A1, + 2 => RV_A2, + 3 => RV_A3, + 4 => RV_A4, + 5 => RV_A5, + 6 => RV_S1, + 7 => RV_S2, + 8 => RV_S3, + 9 => RV_S4, + 10 => RV_S5, + _ => RV_ZERO, + } +} + +fn rv_r(funct7: u32, rs2: u32, rs1: u32, funct3: u32, rd: u32) -> u32 { + (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 0x33 +} + +fn rv_i(imm: u32, rs1: u32, funct3: u32, rd: u32, opcode: u32) -> u32 { + (imm << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | opcode +} + +fn rv_s(imm: u32, rs2: u32, rs1: u32, funct3: u32) -> u32 { + ((imm >> 5) << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | ((imm & 0x1f) << 7) | 0x23 +} + +fn rv_b(imm: u32, rs2: u32, rs1: u32, funct3: u32) -> u32 { + let bit12 = (imm >> 12) & 1; + let bits10_5 = (imm >> 5) & 0x3f; + let bits4_1 = (imm >> 1) & 0xf; + let bit11 = (imm >> 11) & 1; + (bit12 << 31) + | (bits10_5 << 25) + | (rs2 << 20) + | (rs1 << 15) + | (funct3 << 12) + | (bits4_1 << 8) + | (bit11 << 7) + | 0x63 +} + +fn rv_u(imm: u32, rd: u32, opcode: u32) -> u32 { + (imm << 12) | (rd << 7) | opcode +} + +fn emit_add(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 0, rd)); +} + +fn emit_addw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 0, rd) | (0x3b ^ 0x33)); +} + +fn emit_sub(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 0, rd)); +} + +fn emit_subw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 0, rd) | (0x3b ^ 0x33)); +} + +fn emit_addi(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 0, rd, 0x13)); +} + +fn emit_addiw(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 0, rd, 0x1b)); +} + +fn emit_and(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 7, rd)); +} + +fn emit_andw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 7, rd) | (0x3b ^ 0x33)); +} + +fn emit_or(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 6, rd)); +} + +fn emit_orw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 6, rd) | (0x3b ^ 0x33)); +} + +fn emit_xor(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 4, rd)); +} + +fn emit_xorw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 4, rd) | (0x3b ^ 0x33)); +} + +fn emit_sll(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 1, rd)); +} + +fn emit_sllw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 1, rd) | (0x3b ^ 0x33)); +} + +fn emit_srl(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 5, rd)); +} + +fn emit_srlw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0, rs2, rs1, 5, rd) | (0x3b ^ 0x33)); +} + +fn emit_sra(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 5, rd)); +} + +fn emit_sraw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(0x20, rs2, rs1, 5, rd) | (0x3b ^ 0x33)); +} + +fn emit_mul(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 0, rd)); +} + +fn emit_mulw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 0, rd) | (0x3b ^ 0x33)); +} + +fn emit_divu(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 5, rd)); +} + +fn emit_divuw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 5, rd) | (0x3b ^ 0x33)); +} + +fn emit_remu(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 7, rd)); +} + +fn emit_remuw(buf: &mut JitBuffer, rd: u32, rs1: u32, rs2: u32) { + buf.emit_u32(rv_r(1, rs2, rs1, 7, rd) | (0x3b ^ 0x33)); +} + +fn emit_andi(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 7, rd, 0x13)); +} + +fn emit_ori(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 6, rd, 0x13)); +} + +fn emit_xori(buf: &mut JitBuffer, rd: u32, rs1: u32, imm: i32) { + buf.emit_u32(rv_i(imm as u32, rs1, 4, rd, 0x13)); +} + +fn emit_slli(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 1, rd, 0x13)); +} + +fn emit_srli(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 5, rd, 0x13)); +} + +fn emit_srai(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(0x400 | shamt, rs1, 5, rd, 0x13)); +} + +fn emit_slliw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 1, rd, 0x1b)); +} + +fn emit_srliw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(shamt, rs1, 5, rd, 0x1b)); +} + +fn emit_sraiw(buf: &mut JitBuffer, rd: u32, rs1: u32, shamt: u32) { + buf.emit_u32(rv_i(0x400 | shamt, rs1, 5, rd, 0x1b)); +} + +fn emit_lui(buf: &mut JitBuffer, rd: u32, imm: u32) { + buf.emit_u32(rv_u(imm, rd, 0x37)); +} + +fn emit_ld(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 3, rd, 0x03)); +} + +fn emit_lwu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 6, rd, 0x03)); +} + +fn emit_lw(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 2, rd, 0x03)); +} + +fn emit_lhu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 5, rd, 0x03)); +} + +fn emit_lh(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 1, rd, 0x03)); +} + +fn emit_lbu(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 4, rd, 0x03)); +} + +fn emit_lb(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 0, rd, 0x03)); +} + +fn emit_sd(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 3)); +} + +fn emit_sw(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 2)); +} + +fn emit_sh(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 1)); +} + +fn emit_sb(buf: &mut JitBuffer, rs2: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_s(off as u32, rs2, rs1, 0)); +} + +fn emit_beq(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 0)); +} + +fn emit_bne(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 1)); +} + +fn emit_blt(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 4)); +} + +fn emit_bge(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 5)); +} + +fn emit_bltu(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 6)); +} + +fn emit_bgeu(buf: &mut JitBuffer, rs1: u32, rs2: u32, off: i32) { + buf.emit_u32(rv_b(off as u32, rs2, rs1, 7)); +} + +fn emit_jalr(buf: &mut JitBuffer, rd: u32, rs1: u32, off: i32) { + buf.emit_u32(rv_i(off as u32, rs1, 0, rd, 0x67)); +} + +fn emit_ret(buf: &mut JitBuffer) { + emit_jalr(buf, RV_ZERO, RV_RA, 0); +} + +fn emit_mv(buf: &mut JitBuffer, rd: u32, rs: u32) { + emit_addi(buf, rd, rs, 0); +} + +fn emit_load_imm64(buf: &mut JitBuffer, rd: u32, val: u64) { + let lo12 = (val as i16) as i32; + let hi52 = ((val as i64) - lo12 as i64) as u64; + let hi20 = (hi52 >> 12) as u32; + if hi52 >> 32 == 0 { + emit_lui(buf, rd, hi20); + if lo12 != 0 { + emit_addi(buf, rd, rd, lo12); + } + } else { + emit_lui(buf, rd, hi20 & 0xfffff); + emit_addi(buf, rd, rd, lo12); + emit_slli(buf, rd, rd, 12); + let mid20 = ((hi52 >> 32) & 0xfffff) as u32; + emit_addi(buf, rd, rd, (mid20 & 0xfff) as i32); + emit_slli(buf, rd, rd, 12); + emit_addi(buf, rd, rd, ((mid20 >> 12) & 0xfff) as i32); + emit_slli(buf, rd, rd, 8); + } +} + +fn emit_load_imm32(buf: &mut JitBuffer, rd: u32, val: i32) { + let needs_upper = (val as i32) < -2048 || (val as i32) >= 2048; + if !needs_upper { + emit_addi(buf, rd, RV_ZERO, val); + } else { + let hi20 = ((val as i32 as i64 + 2048) >> 12) as u32; + let lo12 = val.wrapping_sub((hi20 as i32) << 12); + emit_lui(buf, rd, hi20); + if lo12 != 0 { + emit_addi(buf, rd, rd, lo12); + } + } +} + +fn emit_zext32(buf: &mut JitBuffer, rd: u32) { + emit_slli(buf, rd, rd, 32); + emit_srli(buf, rd, rd, 32); +} + +pub(crate) struct Riscv64Backend; + +impl JitBackend for Riscv64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + emit_addi(buf, RV_SP, RV_SP, -(FRAME_SIZE as i32)); + emit_sd(buf, RV_S1, RV_SP, (BPF_STACK_SIZE) as i32); + emit_sd(buf, RV_S2, RV_SP, (BPF_STACK_SIZE + 8) as i32); + emit_sd(buf, RV_S3, RV_SP, (BPF_STACK_SIZE + 16) as i32); + emit_sd(buf, RV_S4, RV_SP, (BPF_STACK_SIZE + 24) as i32); + emit_sd(buf, RV_S5, RV_SP, (BPF_STACK_SIZE + 32) as i32); + emit_addi(buf, RV_S5, RV_SP, FRAME_SIZE as i32); + emit_mv(buf, RV_A1, RV_A0); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + emit_ld(buf, RV_S1, RV_SP, BPF_STACK_SIZE as i32); + emit_ld(buf, RV_S2, RV_SP, (BPF_STACK_SIZE + 8) as i32); + emit_ld(buf, RV_S3, RV_SP, (BPF_STACK_SIZE + 16) as i32); + emit_ld(buf, RV_S4, RV_SP, (BPF_STACK_SIZE + 24) as i32); + emit_ld(buf, RV_S5, RV_SP, (BPF_STACK_SIZE + 32) as i32); + emit_addi(buf, RV_SP, RV_SP, FRAME_SIZE as i32); + emit_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_rv(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { + RV_T1 + } else { + bpf_to_rv(insn.src_reg()) + }; + if use_imm { + emit_load_imm64(buf, RV_T1, insn.imm as u64); + } + + if !is_64 && use_imm { + emit_zext32(buf, dst); + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + emit_add(buf, dst, dst, src); + } else { + emit_addw(buf, dst, dst, src); + } + } + BPF_SUB => { + if is_64 { + emit_sub(buf, dst, dst, src); + } else { + emit_subw(buf, dst, dst, src); + } + } + BPF_MUL => { + if is_64 { + emit_mul(buf, dst, dst, src); + } else { + emit_mulw(buf, dst, dst, src); + } + } + BPF_DIV => { + let skip = buf.offset(); + if is_64 { + emit_beq(buf, src, RV_ZERO, 0); + emit_divu(buf, dst, dst, src); + } else { + emit_beq(buf, src, RV_ZERO, 0); + emit_divuw(buf, dst, dst, src); + } + let after = buf.offset(); + let skip_off = (after - skip) as i32; + unsafe { + let ptr = buf.entry().add(skip) as *mut u32; + *ptr = rv_b(skip_off as u32 * 2, RV_ZERO, src, 0); + } + } + BPF_OR => { + if is_64 { + emit_or(buf, dst, dst, src); + } else { + emit_orw(buf, dst, dst, src); + } + } + BPF_AND => { + if is_64 { + emit_and(buf, dst, dst, src); + } else { + emit_andw(buf, dst, dst, src); + } + } + BPF_LSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_slli(buf, dst, dst, shamt); + } else { + emit_slliw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_sll(buf, dst, dst, src); + emit_andi(buf, src, src, 63); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_sllw(buf, dst, dst, RV_T2); + } + } + BPF_RSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_srli(buf, dst, dst, shamt); + } else { + emit_srliw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_srl(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_srlw(buf, dst, dst, RV_T2); + } + } + BPF_NEG => { + if is_64 { + emit_sub(buf, dst, RV_ZERO, dst); + } else { + emit_subw(buf, dst, RV_ZERO, dst); + } + } + BPF_MOD => { + let skip = buf.offset(); + if is_64 { + emit_beq(buf, src, RV_ZERO, 0); + emit_remu(buf, dst, dst, src); + } else { + emit_beq(buf, src, RV_ZERO, 0); + emit_remuw(buf, dst, dst, src); + } + let after = buf.offset(); + let skip_off = (after - skip) as i32; + unsafe { + let ptr = buf.entry().add(skip) as *mut u32; + *ptr = rv_b(skip_off as u32 * 2, RV_ZERO, src, 0); + } + } + BPF_XOR => { + if is_64 { + emit_xor(buf, dst, dst, src); + } else { + emit_xorw(buf, dst, dst, src); + } + } + BPF_MOV => { + if is_64 { + emit_mv(buf, dst, src); + } else { + emit_addiw(buf, dst, src, 0); + } + } + BPF_ARSH => { + if use_imm { + let shamt = if is_64 { + (insn.imm as u32) & 63 + } else { + (insn.imm as u32) & 31 + }; + if is_64 { + emit_srai(buf, dst, dst, shamt); + } else { + emit_sraiw(buf, dst, dst, shamt); + } + } else if is_64 { + emit_andi(buf, RV_T2, src, 63); + emit_sra(buf, dst, dst, RV_T2); + } else { + emit_andi(buf, RV_T2, src, 31); + emit_sraw(buf, dst, dst, RV_T2); + } + } + BPF_END => {} + _ => {} + } + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let off = offsets[target_pc] as isize - buf.offset() as isize; + let off_words = (off / 4) as i32; + if off_words >= -1048576 && off_words <= 1048575 { + let jal_off = (off as i32) & !3; + emit_jalr(buf, RV_ZERO, RV_ZERO, jal_off); + } else { + emit_load_imm64(buf, RV_T6, off as u64); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + } + } + return; + } + + if op == BPF_EXIT { + let off = buf.offset(); + let target = offsets[0] as isize; + if target == 0 {} + emit_load_imm64(buf, RV_T6, 0); + emit_jalr(buf, RV_ZERO, RV_T6, 0); + return; + } + + if op == 0x80 { + return; + } + + let dst = bpf_to_rv(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { RV_T1 } else { bpf_to_rv(insn.src_reg()) }; + + if use_imm { + if is_64 { + emit_load_imm64(buf, RV_T1, insn.imm as u64); + } else { + emit_load_imm32(buf, RV_T1, insn.imm); + } + } + + if !is_64 { + emit_zext32(buf, dst); + emit_zext32(buf, src); + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + let branch_off = if target_pc < offsets.len() { + (offsets[target_pc] as isize - buf.offset() as isize) as i32 + } else { + 0 + }; + + match op { + BPF_JEQ => { + emit_bne(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JGT => { + emit_bgeu(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JGE => { + emit_bltu(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JSET => { + emit_and(buf, RV_T2, dst, src); + emit_beq(buf, RV_T2, RV_ZERO, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JNE => { + emit_beq(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JSGT => { + emit_bge(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JSGE => { + emit_blt(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JLT => { + emit_bgeu(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JLE => { + emit_bltu(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JSLT => { + emit_bge(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + BPF_JSLE => { + emit_blt(buf, dst, src, 8); + emit_jalr(buf, RV_ZERO, RV_ZERO, branch_off); + } + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + emit_addi(buf, RV_T1, RV_S5, off); + let val = insn.imm as u64; + match insn.size() { + BPF_B => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sb(buf, RV_T2, RV_T1, 0); + } + BPF_H => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sh(buf, RV_T2, RV_T1, 0); + } + BPF_W => { + emit_load_imm32(buf, RV_T2, val as i32); + emit_sw(buf, RV_T2, RV_T1, 0); + } + BPF_DW => { + emit_load_imm64(buf, RV_T2, val); + emit_sd(buf, RV_T2, RV_T1, 0); + } + _ => {} + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_rv(insn.src_reg()); + emit_addi(buf, RV_T1, RV_S5, off); + match insn.size() { + BPF_B => emit_sb(buf, src, RV_T1, 0), + BPF_H => emit_sh(buf, src, RV_T1, 0), + BPF_W => emit_sw(buf, src, RV_T1, 0), + BPF_DW => emit_sd(buf, src, RV_T1, 0), + _ => {} + } + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_rv(insn.src_reg()); + let dst = bpf_to_rv(insn.dst_reg()); + match insn.size() { + BPF_B => { + emit_addi(buf, RV_T1, src, off); + emit_lbu(buf, dst, RV_T1, 0); + } + BPF_H => { + emit_addi(buf, RV_T1, src, off); + emit_lhu(buf, dst, RV_T1, 0); + } + BPF_W => { + emit_addi(buf, RV_T1, src, off); + emit_lwu(buf, dst, RV_T1, 0); + } + BPF_DW => { + emit_addi(buf, RV_T1, src, off); + emit_ld(buf, dst, RV_T1, 0); + } + _ => {} + } + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_rv(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + emit_load_imm64(buf, dst, val); + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + emit_mv(buf, RV_T2, RV_A5); + emit_mv(buf, RV_A5, RV_A4); + emit_mv(buf, RV_A4, RV_A3); + emit_mv(buf, RV_A3, RV_A2); + emit_mv(buf, RV_A2, RV_A1); + emit_mv(buf, RV_A1, RV_A0); + + emit_load_imm64(buf, RV_T1, helper_fn as u64); + emit_jalr(buf, RV_RA, RV_T1, 0); + + emit_mv(buf, RV_A0, RV_A0); + } + + fn insn_size(insn: &BpfInsn) -> usize { + let class = insn.class(); + let use_imm = (insn.code & BPF_X) == 0; + + match class { + BPF_ALU | BPF_ALU64 => { + let alu_op = insn.alu_op(); + let base = if use_imm { + if insn.imm >= -2048 && insn.imm < 2048 { + 4 + } else if insn.imm as i32 >= -2048 && insn.imm as i32 < 2048 { + 8 + } else { + 24 + } + } else { + 4 + }; + match alu_op { + BPF_DIV | BPF_MOD => base + 8, + BPF_LSH | BPF_RSH | BPF_ARSH => { + if use_imm { + 4 + } else { + 8 + } + } + _ => base, + } + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + 32 + } else if op == 0x80 { + 8 + 24 + 4 + 4 + } else if insn.code == (BPF_JMP | BPF_JA) + || insn.code == (BPF_JMP32 | BPF_JA) + { + 24 + } else { + let imm_size = if use_imm { + if insn.imm >= -2048 && insn.imm < 2048 { + 8 + } else { + 28 + } + } else { + 0 + }; + let cmp_size = if class == BPF_JMP32 { 16 } else { 0 }; + 4 + cmp_size + imm_size + 8 + } + } + BPF_ST | BPF_STX => { + let off_size = if insn.off >= -2048 && insn.off < 2048 { + 0 + } else { + 8 + }; + let imm_size = if class == BPF_ST { + if insn.imm >= -2048 && insn.imm < 2048 { + 8 + } else { + 24 + } + } else { + 0 + }; + 4 + off_size + imm_size + 4 + } + BPF_LDX => { + let off_size = if insn.off >= -2048 && insn.off < 2048 { + 0 + } else { + 8 + }; + 4 + off_size + 4 + } + BPF_LD => { + if insn.is_ld_dw_imm() { + 24 + } else { + 4 + } + } + _ => 4, + } + } +} diff --git a/os/StarryOS/kernel/src/ebpf_jit/jit_x86_64.rs b/os/StarryOS/kernel/src/ebpf_jit/jit_x86_64.rs new file mode 100644 index 0000000000..abe80f314e --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf_jit/jit_x86_64.rs @@ -0,0 +1,895 @@ +use alloc::vec::Vec; + +use super::JitBackend; +use super::super::bpf_insn::{ + BpfInsn, BPF_ADD, BPF_ALU, BPF_ALU64, BPF_AND, BPF_ARSH, BPF_B, BPF_DW, BPF_DIV, BPF_EXIT, + BPF_H, BPF_JA, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP, BPF_JMP32, BPF_JNE, + BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_JSET, BPF_K, BPF_LD, BPF_LDX, BPF_LSH, BPF_MEM, + BPF_MOD, BPF_MOV, BPF_MUL, BPF_NEG, BPF_OR, BPF_RSH, BPF_ST, BPF_STX, BPF_SUB, BPF_W, + BPF_X, BPF_XOR, +}; +use super::super::HelperFn; +use super::JitBuffer; + +const X86_RAX: u8 = 0; +const X86_RCX: u8 = 1; +const X86_RDX: u8 = 2; +const X86_RBX: u8 = 3; +const X86_RSP: u8 = 4; +const X86_RBP: u8 = 5; +const X86_RSI: u8 = 6; +const X86_RDI: u8 = 7; +const X86_R8: u8 = 8; +const X86_R9: u8 = 9; +const X86_R10: u8 = 10; +const X86_R11: u8 = 11; +const X86_R12: u8 = 12; +const X86_R13: u8 = 13; +const X86_R14: u8 = 14; +const X86_R15: u8 = 15; + +fn bpf_to_x86(r: u8) -> u8 { + match r { + 0 => X86_RAX, + 1 => X86_RDI, + 2 => X86_RSI, + 3 => X86_RDX, + 4 => X86_RCX, + 5 => X86_R8, + 6 => X86_RBX, + 7 => X86_R13, + 8 => X86_R14, + 9 => X86_R15, + 10 => X86_RBP, + _ => X86_RAX, + } +} + +fn need_rex(r: u8) -> bool { + r >= 8 +} + +fn emit_rex(buf: &mut JitBuffer, w: bool, r: u8, x: bool, b: u8) { + let mut rex: u8 = 0x40; + if w { + rex |= 0x08; + } + if need_rex(r) { + rex |= 0x04; + } + if x { + rex |= 0x02; + } + if need_rex(b) { + rex |= 0x01; + } + buf.emit_u8(rex); +} + +fn emit_modrm(buf: &mut JitBuffer, mod_bits: u8, reg: u8, rm: u8) { + buf.emit_u8((mod_bits << 6) | ((reg & 7) << 3) | (rm & 7)); +} + +fn emit_rex_if(buf: &mut JitBuffer, r: u8, b: u8) { + if need_rex(r) || need_rex(b) { + emit_rex(buf, false, r, false, b); + } +} + +fn emit_rex_w(buf: &mut JitBuffer, r: u8, b: u8) { + emit_rex(buf, true, r, false, b); +} + +fn emit_modrm_disp(buf: &mut JitBuffer, reg: u8, rm: u8, disp: i32) { + if disp == 0 && (rm & 7) != X86_RBP { + emit_modrm(buf, 0, reg, rm); + } else if disp >= -128 && disp <= 127 { + emit_modrm(buf, 1, reg, rm); + buf.emit_u8(disp as u8); + } else { + emit_modrm(buf, 2, reg, rm); + buf.emit_u32(disp as u32); + } +} + +fn emit_push(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0x50 | (r & 7)); +} + +fn emit_pop(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0x58 | (r & 7)); +} + +fn emit_mov_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x89); + emit_modrm(buf, 3, src, dst); +} + +fn emit_mov_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x89); + emit_modrm(buf, 3, src, dst); +} + +fn emit_mov_imm64(buf: &mut JitBuffer, dst: u8, imm: u64) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xB8 | (dst & 7)); + buf.emit_u32(imm as u32); + buf.emit_u32((imm >> 32) as u32); +} + +fn emit_mov_imm32(buf: &mut JitBuffer, dst: u8, imm: i32) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC7); + emit_modrm(buf, 3, 0, dst); + buf.emit_u32(imm as u32); +} + +fn emit_add_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x01); + emit_modrm(buf, 3, src, dst); +} + +fn emit_sub_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x29); + emit_modrm(buf, 3, src, dst); +} + +fn emit_add_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x01); + emit_modrm(buf, 3, src, dst); +} + +fn emit_sub_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x29); + emit_modrm(buf, 3, src, dst); +} + +fn emit_imul_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, dst, src); + buf.emit_u8(0x0F); + buf.emit_u8(0xAF); + emit_modrm(buf, 3, dst, src); +} + +fn emit_imul_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, dst, src); + buf.emit_u8(0x0F); + buf.emit_u8(0xAF); + emit_modrm(buf, 3, dst, src); +} + +fn emit_xor_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x31); + emit_modrm(buf, 3, src, dst); +} + +fn emit_xor_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x31); + emit_modrm(buf, 3, src, dst); +} + +fn emit_or_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x09); + emit_modrm(buf, 3, src, dst); +} + +fn emit_or_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x09); + emit_modrm(buf, 3, src, dst); +} + +fn emit_and_reg64(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_w(buf, src, dst); + buf.emit_u8(0x21); + emit_modrm(buf, 3, src, dst); +} + +fn emit_and_reg32(buf: &mut JitBuffer, dst: u8, src: u8) { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x21); + emit_modrm(buf, 3, src, dst); +} + +fn emit_neg_reg64(buf: &mut JitBuffer, r: u8) { + emit_rex_w(buf, 0, r); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 3, r); +} + +fn emit_neg_reg32(buf: &mut JitBuffer, r: u8) { + emit_rex_if(buf, 0, r); + buf.emit_u8(0xF7); + emit_modrm(buf, 3, 3, r); +} + +fn emit_shl_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 4, dst); +} + +fn emit_shr_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 5, dst); +} + +fn emit_sar_reg64(buf: &mut JitBuffer, dst: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 7, dst); +} + +fn emit_shl_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 4, dst); +} + +fn emit_shr_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 5, dst); +} + +fn emit_sar_reg32(buf: &mut JitBuffer, dst: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xD3); + emit_modrm(buf, 3, 7, dst); +} + +fn emit_shl_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 4, dst); + buf.emit_u8(imm); +} + +fn emit_shr_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 5, dst); + buf.emit_u8(imm); +} + +fn emit_sar_imm64(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_w(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 7, dst); + buf.emit_u8(imm); +} + +fn emit_shl_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 4, dst); + buf.emit_u8(imm); +} + +fn emit_shr_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 5, dst); + buf.emit_u8(imm); +} + +fn emit_sar_imm32(buf: &mut JitBuffer, dst: u8, imm: u8) { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0xC1); + emit_modrm(buf, 3, 7, dst); + buf.emit_u8(imm); +} + +fn emit_test_reg64(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_w(buf, r2, r1); + buf.emit_u8(0x85); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_cmp_reg64(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_w(buf, r2, r1); + buf.emit_u8(0x39); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_cmp_reg32(buf: &mut JitBuffer, r1: u8, r2: u8) { + emit_rex_if(buf, r2, r1); + buf.emit_u8(0x39); + emit_modrm(buf, 3, r2, r1); +} + +fn emit_je(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x84); + buf.emit_u32(off as u32); +} + +fn emit_jne(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x85); + buf.emit_u32(off as u32); +} + +fn emit_ja(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x87); + buf.emit_u32(off as u32); +} + +fn emit_jae(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x83); + buf.emit_u32(off as u32); +} + +fn emit_jb(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x82); + buf.emit_u32(off as u32); +} + +fn emit_jbe(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x86); + buf.emit_u32(off as u32); +} + +fn emit_jg(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8F); + buf.emit_u32(off as u32); +} + +fn emit_jge(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8D); + buf.emit_u32(off as u32); +} + +fn emit_jl(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8C); + buf.emit_u32(off as u32); +} + +fn emit_jle(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0x0F); + buf.emit_u8(0x8E); + buf.emit_u32(off as u32); +} + +fn emit_jmp_rel32(buf: &mut JitBuffer, off: i32) { + buf.emit_u8(0xE9); + buf.emit_u32(off as u32); +} + +fn emit_call_reg(buf: &mut JitBuffer, r: u8) { + if need_rex(r) { + buf.emit_u8(0x41); + } + buf.emit_u8(0xFF); + emit_modrm(buf, 3, 2, r); +} + +fn emit_ret(buf: &mut JitBuffer) { + buf.emit_u8(0xC3); +} + +fn emit_store_mem(buf: &mut JitBuffer, base: u8, off: i32, src: u8, size: u8) { + match size { + BPF_B => { + emit_rex_if(buf, 0, src); + buf.emit_u8(0x88); + emit_modrm_disp(buf, src, base, off); + } + BPF_H => { + buf.emit_u8(0x66); + emit_rex_if(buf, 0, src); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + BPF_W => { + emit_rex_if(buf, 0, src); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + BPF_DW => { + emit_rex_w(buf, src, base); + buf.emit_u8(0x89); + emit_modrm_disp(buf, src, base, off); + } + _ => {} + } +} + +fn emit_load_mem(buf: &mut JitBuffer, dst: u8, base: u8, off: i32, size: u8) { + match size { + BPF_B => { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0x0F); + buf.emit_u8(0xB6); + emit_modrm_disp(buf, dst, base, off); + } + BPF_H => { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0x0F); + buf.emit_u8(0xB7); + emit_modrm_disp(buf, dst, base, off); + } + BPF_W => { + emit_rex_if(buf, 0, dst); + buf.emit_u8(0x8B); + emit_modrm_disp(buf, dst, base, off); + emit_rex_if(buf, 0, dst); + buf.emit_u8(0x23); + buf.emit_u8(0xC0 | ((dst & 7) << 3) | (dst & 7)); + } + BPF_DW => { + emit_rex_w(buf, dst, base); + buf.emit_u8(0x8B); + emit_modrm_disp(buf, dst, base, off); + } + _ => {} + } +} + +fn emit_zext32(buf: &mut JitBuffer, r: u8) { + emit_rex_if(buf, 0, r); + buf.emit_u8(0x23); + buf.emit_u8(0xC0 | ((r & 7) << 3) | (r & 7)); +} + +fn emit_divmod(buf: &mut JitBuffer, dst: u8, src: u8, is_div: bool, is_64: bool) { + let saved_rdx = if is_64 { X86_RCX } else { X86_RCX }; + if is_64 { + emit_push(buf, saved_rdx); + } else { + emit_push(buf, X86_RCX); + } + + if is_64 { + emit_mov_reg64(buf, X86_RCX, X86_RAX); + emit_mov_reg64(buf, X86_RAX, src); + emit_test_reg64(buf, X86_RAX, X86_RAX); + let skip = buf.offset(); + emit_je(buf, 0); + emit_mov_reg64(buf, X86_RAX, X86_RCX); + emit_xor_reg64(buf, X86_RDX, X86_RDX); + if is_div { + buf.emit_u8(0x48); + buf.emit_u8(0xF7); + buf.emit_u8(0xF1); + } else { + buf.emit_u8(0x48); + buf.emit_u8(0xF7); + buf.emit_u8(0xF1); + } + let after = buf.offset(); + unsafe { + let ptr = buf.entry().add(skip) as *mut u8; + let off = (after - skip - 6) as i32; + core::ptr::copy_nonoverlapping(off.to_le_bytes().as_ptr(), ptr.add(2), 4); + } + emit_mov_reg64(buf, dst, if is_div { X86_RAX } else { X86_RDX }); + } else { + emit_mov_reg32(buf, X86_RCX, X86_RAX); + emit_mov_reg32(buf, X86_RAX, src); + emit_test_reg64(buf, X86_RAX, X86_RAX); + let skip = buf.offset(); + emit_je(buf, 0); + emit_mov_reg32(buf, X86_RAX, X86_RCX); + emit_zext32(buf, X86_RAX); + emit_xor_reg32(buf, X86_RDX, X86_RDX); + if is_div { + buf.emit_u8(0xF7); + buf.emit_u8(0xF1); + } else { + buf.emit_u8(0xF7); + buf.emit_u8(0xF1); + } + emit_zext32(buf, if is_div { X86_RAX } else { X86_RDX }); + let after = buf.offset(); + unsafe { + let ptr = buf.entry().add(skip) as *mut u8; + let off = (after - skip - 6) as i32; + core::ptr::copy_nonoverlapping(off.to_le_bytes().as_ptr(), ptr.add(2), 4); + } + emit_mov_reg32(buf, dst, if is_div { X86_RAX } else { X86_RDX }); + } + + if is_64 { + emit_pop(buf, saved_rdx); + } else { + emit_pop(buf, X86_RCX); + } +} + +pub(crate) struct X86_64Backend; + +impl JitBackend for X86_64Backend { + fn emit_prologue(buf: &mut JitBuffer) -> usize { + emit_push(buf, X86_RBP); + emit_mov_reg64(buf, X86_RSP, X86_RBP); + emit_push(buf, X86_RBX); + emit_push(buf, X86_R13); + emit_push(buf, X86_R14); + emit_push(buf, X86_R15); + buf.emit_u8(0x48); + buf.emit_u8(0x81); + buf.emit_u8(0xEC); + buf.emit_u32(512); + buf.emit_u8(0x48); + buf.emit_u8(0x8D); + buf.emit_u8(0x65); + buf.emit_u8(0x00); + emit_mov_reg64(buf, X86_RDI, X86_RBP); + buf.offset() + } + + fn emit_epilogue(buf: &mut JitBuffer) { + buf.emit_u8(0x48); + buf.emit_u8(0x81); + buf.emit_u8(0xC4); + buf.emit_u32(512); + emit_pop(buf, X86_R15); + emit_pop(buf, X86_R14); + emit_pop(buf, X86_R13); + emit_pop(buf, X86_RBX); + emit_pop(buf, X86_RBP); + emit_ret(buf); + } + + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool) { + let dst = bpf_to_x86(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { + X86_RCX + } else { + bpf_to_x86(insn.src_reg()) + }; + + if use_imm { + let imm = insn.imm as i32; + if is_64 { + if imm >= 0 && imm < 256 { + emit_mov_imm32(buf, dst, imm); + } else { + emit_mov_imm64(buf, X86_RCX, insn.imm as u64); + } + } else { + emit_mov_imm32(buf, X86_RCX, imm); + } + } + + match insn.alu_op() { + BPF_ADD => { + if is_64 { + emit_add_reg64(buf, dst, src); + } else { + emit_add_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_SUB => { + if is_64 { + emit_sub_reg64(buf, dst, src); + } else { + emit_sub_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_MUL => { + if is_64 { + emit_imul_reg64(buf, dst, src); + } else { + emit_imul_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_DIV => { + emit_divmod(buf, dst, src, true, is_64); + } + BPF_OR => { + if is_64 { + emit_or_reg64(buf, dst, src); + } else { + emit_or_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_AND => { + if is_64 { + emit_and_reg64(buf, dst, src); + } else { + emit_and_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_LSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_shl_imm64(buf, dst, shamt); + } else { + emit_shl_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_shl_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_shl_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_RSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_shr_imm64(buf, dst, shamt); + } else { + emit_shr_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_shr_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_shr_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_NEG => { + if is_64 { + emit_neg_reg64(buf, dst); + } else { + emit_neg_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + BPF_MOD => { + emit_divmod(buf, dst, src, false, is_64); + } + BPF_XOR => { + if is_64 { + emit_xor_reg64(buf, dst, src); + } else { + emit_xor_reg32(buf, dst, src); + emit_zext32(buf, dst); + } + } + BPF_MOV => { + if is_64 { + if use_imm { + emit_mov_imm64(buf, dst, insn.imm as u64); + } else { + emit_mov_reg64(buf, dst, src); + } + } else { + if use_imm { + emit_mov_imm32(buf, dst, insn.imm); + } else { + emit_mov_reg32(buf, dst, src); + } + emit_zext32(buf, dst); + } + } + BPF_ARSH => { + if use_imm { + let shamt = (insn.imm as u8) & (if is_64 { 63 } else { 31 }); + if is_64 { + emit_sar_imm64(buf, dst, shamt); + } else { + emit_sar_imm32(buf, dst, shamt); + emit_zext32(buf, dst); + } + } else if is_64 { + emit_mov_reg64(buf, X86_RCX, src); + emit_sar_reg64(buf, dst); + } else { + emit_mov_reg32(buf, X86_RCX, src); + emit_sar_reg32(buf, dst); + emit_zext32(buf, dst); + } + } + _ => {} + } + } + + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool) { + let op = insn.code & 0xf0; + + if insn.code == (BPF_JMP | BPF_JA) || insn.code == (BPF_JMP32 | BPF_JA) { + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + if target_pc < offsets.len() { + let off = offsets[target_pc] as isize - buf.offset() as isize - 5; + emit_jmp_rel32(buf, off as i32); + } + return; + } + + if op == BPF_EXIT { + let epilogue_offset = 0isize; + emit_jmp_rel32(buf, epilogue_offset as i32); + return; + } + + let dst = bpf_to_x86(insn.dst_reg()); + let use_imm = (insn.code & BPF_X) == 0; + let src = if use_imm { X86_RCX } else { bpf_to_x86(insn.src_reg()) }; + + if use_imm { + if is_64 { + emit_mov_imm64(buf, X86_RCX, insn.imm as u64); + } else { + emit_mov_imm32(buf, X86_RCX, insn.imm); + } + } + + if is_64 { + emit_cmp_reg64(buf, dst, src); + } else { + emit_cmp_reg32(buf, dst, src); + } + + let target_pc = (pc as isize + 1 + insn.off as isize) as usize; + let target_off = if target_pc < offsets.len() { + (offsets[target_pc] as isize - buf.offset() as isize - 6) as i32 + } else { + 0 + }; + + match op { + BPF_JEQ => emit_je(buf, target_off), + BPF_JGT => emit_ja(buf, target_off), + BPF_JGE => emit_jae(buf, target_off), + BPF_JSET => { + if is_64 { + emit_test_reg64(buf, dst, src); + } else { + emit_rex_if(buf, src, dst); + buf.emit_u8(0x85); + emit_modrm(buf, 3, src, dst); + } + emit_jne(buf, target_off); + } + BPF_JNE => emit_jne(buf, target_off), + BPF_JSGT => emit_jg(buf, target_off), + BPF_JSGE => emit_jge(buf, target_off), + BPF_JLT => emit_jb(buf, target_off), + BPF_JLE => emit_jbe(buf, target_off), + BPF_JSLT => emit_jl(buf, target_off), + BPF_JSLE => emit_jle(buf, target_off), + _ => {} + } + } + + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let imm = insn.imm as i64; + if insn.size() == BPF_DW { + emit_mov_imm64(buf, X86_RCX, imm as u64); + emit_store_mem(buf, X86_RBP, off, X86_RCX, BPF_DW); + } else { + emit_mov_imm32(buf, X86_RCX, imm as i32); + emit_store_mem(buf, X86_RBP, off, X86_RCX, insn.size()); + } + } + + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let src = bpf_to_x86(insn.src_reg()); + let sz = if insn.size() == BPF_DW { BPF_DW } else { insn.size() }; + emit_store_mem(buf, X86_RBP, off, src, sz); + } + + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn) { + if insn.mode() != BPF_MEM { + return; + } + let off = insn.off as i32; + let base = bpf_to_x86(insn.src_reg()); + let dst = bpf_to_x86(insn.dst_reg()); + emit_load_mem(buf, dst, base, off, insn.size()); + } + + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32) { + let dst = bpf_to_x86(insn.dst_reg()); + let imm_lo = insn.imm as u64; + let imm_hi = next_imm as u64; + let val = (imm_hi << 32) | (imm_lo & 0xffffffff); + emit_mov_imm64(buf, dst, val); + } + + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn) { + emit_mov_imm64(buf, X86_RAX, helper_fn as u64); + emit_call_reg(buf, X86_RAX); + } + + fn insn_size(insn: &BpfInsn) -> usize { + let class = insn.class(); + let use_imm = (insn.code & BPF_X) == 0; + + match class { + BPF_ALU | BPF_ALU64 => { + let alu_op = insn.alu_op(); + let imm_size = if use_imm { + if alu_op == BPF_MOV && insn.imm != 0 { + 10 + } else { + 6 + } + } else { + 3 + }; + match alu_op { + BPF_DIV | BPF_MOD => imm_size + 40, + BPF_MOV => { + if use_imm { + 10 + } else { + 3 + } + } + _ => imm_size + 3, + } + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + 5 + } else if op == 0x80 { + 16 + } else if insn.code == (BPF_JMP | BPF_JA) + || insn.code == (BPF_JMP32 | BPF_JA) + { + 5 + } else { + let cmp_size = if use_imm { 10 } else { 3 }; + cmp_size + 6 + } + } + BPF_ST => { + if insn.size() == BPF_DW { + 20 + } else { + 12 + } + } + BPF_STX => 8, + BPF_LDX => 8, + BPF_LD => { + if insn.is_ld_dw_imm() { + 10 + } else { + 4 + } + } + _ => 4, + } + } +} diff --git a/os/StarryOS/kernel/src/ebpf_jit/mod.rs b/os/StarryOS/kernel/src/ebpf_jit/mod.rs new file mode 100644 index 0000000000..6e6ab91799 --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf_jit/mod.rs @@ -0,0 +1,234 @@ +use alloc::alloc::{alloc_zeroed, dealloc, Layout}; +use alloc::collections::BTreeMap; +use alloc::vec; +use alloc::vec::Vec; + +use ax_memory_addr::VirtAddr; + +use super::bpf_insn::{BpfInsn, BPF_ALU, BPF_ALU64, BPF_EXIT, BPF_JMP, BPF_JMP32, BPF_LD, BPF_LDX, BPF_ST, BPF_STX}; +use super::HelperFn; + +#[cfg(target_arch = "riscv64")] +mod jit_riscv64; +#[cfg(target_arch = "x86_64")] +mod jit_x86_64; +#[cfg(target_arch = "aarch64")] +mod jit_aarch64; + +#[cfg(target_arch = "riscv64")] +use jit_riscv64::Riscv64Backend as Backend; +#[cfg(target_arch = "x86_64")] +use jit_x86_64::X86_64Backend as Backend; +#[cfg(target_arch = "aarch64")] +use jit_aarch64::Aarch64Backend as Backend; + +pub struct JitBuffer { + ptr: *mut u8, + size: usize, + pos: usize, +} + +impl JitBuffer { + pub fn new(requested_size: usize) -> Result { + let size = (requested_size + 4095) & !4095; + if size == 0 { + return Err("jit buffer size is zero"); + } + let layout = Layout::from_size_align(size, 4096).map_err(|_| "invalid layout")?; + let ptr = unsafe { alloc_zeroed(layout) }; + if ptr.is_null() { + return Err("jit buffer allocation failed"); + } + Ok(Self { ptr, size, pos: 0 }) + } + + pub fn emit_u8(&mut self, val: u8) { + if self.pos >= self.size { + return; + } + unsafe { + *self.ptr.add(self.pos) = val; + } + self.pos += 1; + } + + pub fn emit_u32(&mut self, val: u32) { + if self.pos + 4 > self.size { + return; + } + unsafe { + let dst = self.ptr.add(self.pos) as *mut u32; + *dst = val.to_le(); + } + self.pos += 4; + } + + pub fn offset(&self) -> usize { + self.pos + } + + pub fn entry(&self) -> *const u8 { + self.ptr + } + + pub fn finalize(&mut self) { + #[cfg(target_arch = "aarch64")] + { + let vaddr = VirtAddr::from_usize(self.ptr as usize); + ax_hal::asm::clean_dcache_range_to_pou(vaddr, self.pos); + } + ax_hal::asm::flush_icache_all(); + } +} + +impl Drop for JitBuffer { + fn drop(&mut self) { + if !self.ptr.is_null() && self.size > 0 { + let layout = Layout::from_size_align(self.size, 4096).unwrap(); + unsafe { + dealloc(self.ptr, layout); + } + } + } +} + +pub(crate) trait JitBackend { + fn emit_prologue(buf: &mut JitBuffer) -> usize; + fn emit_epilogue(buf: &mut JitBuffer); + fn emit_alu(buf: &mut JitBuffer, insn: &BpfInsn, is_64: bool); + fn emit_jmp(buf: &mut JitBuffer, insn: &BpfInsn, offsets: &[usize], pc: usize, is_64: bool); + fn emit_st(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_stx(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_ldx(buf: &mut JitBuffer, insn: &BpfInsn); + fn emit_ld_imm64(buf: &mut JitBuffer, insn: &BpfInsn, next_imm: i32); + fn emit_call(buf: &mut JitBuffer, helper_fn: HelperFn); + fn insn_size(insn: &BpfInsn) -> usize; +} + +struct JitCompiler<'a> { + insns: &'a [BpfInsn], + offsets: Vec, + helpers: &'a BTreeMap, +} + +impl<'a> JitCompiler<'a> { + fn new(insns: &'a [BpfInsn], helpers: &'a BTreeMap) -> Self { + let offsets = vec![0; insns.len()]; + Self { + insns, + offsets, + helpers, + } + } + + fn pass1_sizing(&mut self) -> usize { + let mut insn_size_total: usize = 0; + let num_insns = self.insns.len(); + let mut pc: usize = 0; + while pc < num_insns { + self.offsets[pc] = insn_size_total; + let insn = &self.insns[pc]; + if insn.is_ld_dw_imm() { + let sz = Backend::insn_size(insn); + insn_size_total += sz; + if pc + 1 < num_insns { + self.offsets[pc + 1] = insn_size_total; + } + pc += 2; + } else { + insn_size_total += Backend::insn_size(insn); + pc += 1; + } + } + insn_size_total + } + + fn compile(&mut self) -> Result { + if self.insns.is_empty() { + return Err("no instructions to compile"); + } + + let insn_size_total = self.pass1_sizing(); + + let estimated = 128 + insn_size_total + 128 + 256; + let mut buf = JitBuffer::new(estimated)?; + + let prologue_size = Backend::emit_prologue(&mut buf); + + for i in 0..self.offsets.len() { + self.offsets[i] += prologue_size; + } + + let num_insns = self.insns.len(); + let mut pc: usize = 0; + while pc < num_insns { + let insn = &self.insns[pc]; + let class = insn.class(); + + match class { + BPF_ALU | BPF_ALU64 => { + let is_64 = class == BPF_ALU64; + Backend::emit_alu(&mut buf, insn, is_64); + pc += 1; + } + BPF_JMP | BPF_JMP32 => { + let op = insn.code & 0xf0; + if op == BPF_EXIT { + Backend::emit_epilogue(&mut buf); + pc += 1; + } else if op == 0x80 { + let helper_id = insn.imm as u32; + if let Some(&helper_fn) = self.helpers.get(&helper_id) { + Backend::emit_call(&mut buf, helper_fn); + } else { + Backend::emit_call(&mut buf, |_a1, _a2, _a3, _a4, _a5| u64::MAX); + } + pc += 1; + } else { + let is_64 = class == BPF_JMP; + Backend::emit_jmp(&mut buf, insn, &self.offsets, pc, is_64); + pc += 1; + } + } + BPF_ST => { + Backend::emit_st(&mut buf, insn); + pc += 1; + } + BPF_STX => { + Backend::emit_stx(&mut buf, insn); + pc += 1; + } + BPF_LDX => { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + BPF_LD => { + if insn.is_ld_dw_imm() { + let next_imm = if pc + 1 < num_insns { + self.insns[pc + 1].imm + } else { + 0 + }; + Backend::emit_ld_imm64(&mut buf, insn, next_imm); + pc += 2; + } else { + Backend::emit_ldx(&mut buf, insn); + pc += 1; + } + } + _ => { + return Err("unsupported instruction class"); + } + } + } + + Backend::emit_epilogue(&mut buf); + buf.finalize(); + Ok(buf) + } +} + +pub fn try_jit_compile(insns: &[BpfInsn], helpers: &BTreeMap) -> Option { + let mut compiler = JitCompiler::new(insns, helpers); + compiler.compile().ok() +} diff --git a/os/StarryOS/kernel/src/perf_event.rs b/os/StarryOS/kernel/src/perf_event.rs new file mode 100644 index 0000000000..aa297752c3 --- /dev/null +++ b/os/StarryOS/kernel/src/perf_event.rs @@ -0,0 +1,430 @@ +//! Performance event ring buffer infrastructure for StarryOS. +//! +//! Provides a Linux-compatible perf event interface for eBPF data output +//! and kernel event sampling. Key components: +//! +//! - **PerfEventMmapPage**: mmap header compatible with Linux's `perf_event_mmap_page` +//! - **RingBuffer**: Wrap-around circular buffer for PERF_RECORD_SAMPLE/LOST records +//! - **perf_event_open syscall**: Supports KPROBE, TRACEPOINT, SOFTWARE, and RAW event types +//! +//! # Public API +//! +//! - `perf_event_write`: Write sample data to a specific perf event fd +//! - `perf_event_close` / `perf_event_fd_exists`: fd lifecycle management + +use alloc::vec::Vec; + +use ax_errno::{AxError, AxResult}; +use ax_sync::spin::SpinNoIrq; + +const PAGE_SIZE: usize = 4096; +const PERF_RECORD_SAMPLE: u32 = 9; +const PERF_RECORD_LOST: u32 = 2; + +#[allow(dead_code)] +const PERF_TYPE_HARDWARE: u32 = 0; +const PERF_TYPE_SOFTWARE: u32 = 1; +const PERF_TYPE_TRACEPOINT: u32 = 2; +const PERF_TYPE_RAW: u32 = 4; +const PERF_TYPE_KPROBE: u32 = 6; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct PerfEventHeader { + type_: u32, + misc: u16, + size: u16, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct SampleHeader { + header: PerfEventHeader, + size: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct LostSamples { + header: PerfEventHeader, + id: u64, + count: u64, +} + +#[repr(C)] +struct PerfEventMmapPage { + version: u32, + compat_version: u32, + lock: u32, + index: u32, + offset: i64, + time_enabled: u64, + time_running: u64, + capabilities: u64, + pmc_width: u16, + time_shift: u16, + time_mult: u32, + time_offset: u64, + time_zero: u64, + size: u32, + _reserved_1: u32, + time_cycles: u64, + time_mask: u64, + _reserved: [u8; 928], + data_head: u64, + data_tail: u64, + data_offset: u64, + data_size: u64, + aux_head: u64, + aux_tail: u64, + aux_offset: u64, + aux_size: u64, +} + +const MMAP_PAGE_SIZE: usize = core::mem::size_of::(); + +struct RingBuffer { + pages: Vec, + data_region_size: usize, + lost: u64, +} + +impl RingBuffer { + fn new(page_count: usize) -> AxResult { + if page_count < 2 { + return Err(AxError::InvalidInput); + } + let total_size = page_count * PAGE_SIZE; + let data_region_size = total_size - PAGE_SIZE; + let mut pages = alloc::vec![0u8; total_size]; + let mmap_page = unsafe { &mut *(pages.as_mut_ptr() as *mut PerfEventMmapPage) }; + mmap_page.version = 1; + mmap_page.compat_version = 1; + mmap_page.size = MMAP_PAGE_SIZE as u32; + mmap_page.data_offset = PAGE_SIZE as u64; + mmap_page.data_size = data_region_size as u64; + mmap_page.data_head = 0; + mmap_page.data_tail = 0; + Ok(Self { + pages, + data_region_size, + lost: 0, + }) + } + + fn data_head(&self) -> u64 { + let mmap_page = unsafe { &*(self.pages.as_ptr() as *const PerfEventMmapPage) }; + mmap_page.data_head + } + + fn set_data_head(&mut self, val: u64) { + let mmap_page = unsafe { &mut *(self.pages.as_mut_ptr() as *mut PerfEventMmapPage) }; + mmap_page.data_head = val; + } + + fn data_tail(&self) -> u64 { + let mmap_page = unsafe { &*(self.pages.as_ptr() as *const PerfEventMmapPage) }; + mmap_page.data_tail + } + + fn can_write(&self, needed: usize, tail: u64, head: u64) -> bool { + let capacity = self.data_region_size as u64; + (capacity - (head - tail)) as usize >= needed + } + + fn write_bytes(&mut self, data: &[u8], offset_in_data_region: usize) { + if data.is_empty() { + return; + } + let start = offset_in_data_region % self.data_region_size; + let data_ptr = unsafe { self.pages.as_mut_ptr().add(PAGE_SIZE) }; + if start + data.len() <= self.data_region_size { + unsafe { + core::ptr::copy_nonoverlapping(data.as_ptr(), data_ptr.add(start), data.len()); + } + } else { + let first_len = self.data_region_size - start; + unsafe { + core::ptr::copy_nonoverlapping(data.as_ptr(), data_ptr.add(start), first_len); + core::ptr::copy_nonoverlapping( + data.as_ptr().add(first_len), + data_ptr, + data.len() - first_len, + ); + } + } + } + + fn fill_size(&self, head_mod: usize) -> usize { + let remaining = self.data_region_size - head_mod; + if remaining > 0 && remaining < core::mem::size_of::() { + remaining + } else { + 0 + } + } + + fn write_sample(&mut self, data: &[u8], head: u64) -> AxResult { + let head_mod = (head as usize) % self.data_region_size; + let fill = self.fill_size(head_mod); + let total_size = core::mem::size_of::() + data.len() + fill; + let hdr = SampleHeader { + header: PerfEventHeader { + type_: PERF_RECORD_SAMPLE, + misc: 0, + size: total_size as u16, + }, + size: data.len() as u32, + }; + let hdr_bytes = unsafe { + core::slice::from_raw_parts( + &hdr as *const SampleHeader as *const u8, + core::mem::size_of::(), + ) + }; + self.write_bytes(hdr_bytes, head_mod); + let data_offset = (head_mod + core::mem::size_of::()) % self.data_region_size; + self.write_bytes(data, data_offset); + if fill > 0 { + let fill_offset = (head_mod + total_size - fill) % self.data_region_size; + let zeros = alloc::vec![0u8; fill]; + self.write_bytes(&zeros, fill_offset); + } + Ok(head + total_size as u64) + } + + fn write_lost(&mut self, head: u64, count: u64) -> AxResult { + let head_mod = (head as usize) % self.data_region_size; + let lost = LostSamples { + header: PerfEventHeader { + type_: PERF_RECORD_LOST, + misc: 0, + size: core::mem::size_of::() as u16, + }, + id: 0, + count, + }; + let lost_bytes = unsafe { + core::slice::from_raw_parts( + &lost as *const LostSamples as *const u8, + core::mem::size_of::(), + ) + }; + self.write_bytes(lost_bytes, head_mod); + Ok(head + core::mem::size_of::() as u64) + } + + fn write_event(&mut self, data: &[u8]) -> AxResult<()> { + let tail = self.data_tail(); + let mut head = self.data_head(); + let hdr_size = core::mem::size_of::(); + if !self.can_write(hdr_size, tail, head) { + self.lost += 1; + return Ok(()); + } + if self.lost > 0 { + let lost_size = core::mem::size_of::(); + if self.can_write(lost_size, tail, head) { + head = self.write_lost(head, self.lost)?; + self.lost = 0; + } + } + let sample_size = core::mem::size_of::() + data.len(); + let head_mod = (head as usize) % self.data_region_size; + let fill = self.fill_size(head_mod); + let total = sample_size + fill; + if self.can_write(total, tail, head) { + head = self.write_sample(data, head)?; + } else { + self.lost += 1; + } + self.set_data_head(head); + Ok(()) + } + + #[allow(dead_code)] + fn readable(&self) -> bool { + self.data_head() != self.data_tail() + } +} + +struct PerfEvent { + ring: RingBuffer, + enabled: bool, + prog_fd: Option, +} + +impl PerfEvent { + fn new(page_count: usize) -> AxResult { + Ok(Self { + ring: RingBuffer::new(page_count)?, + enabled: false, + prog_fd: None, + }) + } + + fn enable(&mut self) { + self.enabled = true; + } + + fn disable(&mut self) { + self.enabled = false; + } + + fn write_event(&mut self, data: &[u8]) -> AxResult<()> { + if !self.enabled { + return Ok(()); + } + self.ring.write_event(data) + } + + fn attach_prog(&mut self, prog_fd: u32) { + self.prog_fd = Some(prog_fd); + } + + #[allow(dead_code)] + fn attached_prog(&self) -> Option { + self.prog_fd + } +} + +struct PerfEventEntry { + event: PerfEvent, + fd: u32, + #[allow(dead_code)] + event_type: u32, + #[allow(dead_code)] + config: u64, + #[allow(dead_code)] + pid: i32, + #[allow(dead_code)] + cpu: i32, +} + +static PERF_EVENTS: SpinNoIrq> = SpinNoIrq::new(Vec::new()); + +fn alloc_perf_fd() -> u32 { + let guard = PERF_EVENTS.lock(); + (guard.len() as u32) + 100 +} + +pub fn sys_perf_event_open_impl( + attr_uptr: usize, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u64, +) -> AxResult { + let (event_type, config) = unsafe { + if attr_uptr == 0 { + return Err(AxError::InvalidInput); + } + let ptr = attr_uptr as *const u32; + let event_type = core::ptr::read(ptr); + let size = core::ptr::read(ptr.add(1)); + if size < 16 { + return Err(AxError::InvalidInput); + } + let config = core::ptr::read((ptr as *const u64).add(1)); + (event_type, config) + }; + match event_type { + PERF_TYPE_KPROBE | PERF_TYPE_TRACEPOINT | PERF_TYPE_SOFTWARE | PERF_TYPE_RAW => {} + _ => { + warn!("perf_event_open: unsupported type {event_type}"); + return Err(AxError::Unsupported); + } + } + let page_count = 2 + 1; + let event = PerfEvent::new(page_count)?; + let fd = alloc_perf_fd(); + let entry = PerfEventEntry { + event, + fd, + event_type, + config, + pid, + cpu, + }; + if group_fd >= 0 { + let _ = group_fd; + } + let _ = flags; + PERF_EVENTS.lock().push(entry); + info!("perf_event_open: type={event_type} config={config:#x} pid={pid} cpu={cpu} fd={fd}"); + Ok(fd as isize) +} + +pub fn perf_event_write(fd: u32, data: &[u8]) -> AxResult<()> { + let mut guard = PERF_EVENTS.lock(); + for entry in guard.iter_mut() { + if entry.fd == fd { + return entry.event.write_event(data); + } + } + Err(AxError::BadFileDescriptor) +} + +#[allow(dead_code)] +pub fn perf_event_close(fd: u32) -> AxResult<()> { + let mut guard = PERF_EVENTS.lock(); + let idx = guard.iter().position(|e| e.fd == fd); + match idx { + Some(i) => { + guard.remove(i); + info!("perf_event_close: fd={fd}"); + Ok(()) + } + None => Err(AxError::BadFileDescriptor), + } +} + +#[allow(dead_code)] +pub fn perf_event_fd_exists(fd: u32) -> bool { + let guard = PERF_EVENTS.lock(); + guard.iter().any(|e| e.fd == fd) +} + +pub fn perf_event_enable(fd: u32) -> AxResult<()> { + let mut guard = PERF_EVENTS.lock(); + for entry in guard.iter_mut() { + if entry.fd == fd { + entry.event.enable(); + return Ok(()); + } + } + Err(AxError::BadFileDescriptor) +} + +pub fn perf_event_disable(fd: u32) -> AxResult<()> { + let mut guard = PERF_EVENTS.lock(); + for entry in guard.iter_mut() { + if entry.fd == fd { + entry.event.disable(); + return Ok(()); + } + } + Err(AxError::BadFileDescriptor) +} + +pub fn perf_event_attach_prog(fd: u32, prog_fd: u32) -> AxResult<()> { + let mut guard = PERF_EVENTS.lock(); + for entry in guard.iter_mut() { + if entry.fd == fd { + entry.event.attach_prog(prog_fd); + return Ok(()); + } + } + Err(AxError::BadFileDescriptor) +} + +#[allow(dead_code)] +pub fn perf_event_get_prog_fd(fd: u32) -> AxResult> { + let guard = PERF_EVENTS.lock(); + for entry in guard.iter() { + if entry.fd == fd { + return Ok(entry.event.attached_prog()); + } + } + Err(AxError::BadFileDescriptor) +}