diff --git a/Cargo.lock b/Cargo.lock index 3b4b99b53d..3ca66bcd4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -664,9 +664,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" @@ -2335,9 +2335,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.3" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bwbench-client" @@ -2650,9 +2650,9 @@ dependencies = [ [[package]] name = "compact_str" -version = "0.8.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" dependencies = [ "castaway", "cfg-if", @@ -2664,9 +2664,9 @@ dependencies = [ [[package]] name = "compact_str" -version = "0.9.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" dependencies = [ "castaway", "cfg-if", @@ -3044,9 +3044,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" dependencies = [ "hybrid-array", ] @@ -3128,7 +3128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "321ec774d27fafc66e812034d0025f8858bd7d9095304ff8fc200e0b9f9cc257" dependencies = [ "ahash 0.8.12", - "compact_str 0.8.2", + "compact_str 0.8.1", "crossbeam-channel", "cursive-macros", "enum-map", @@ -3388,14 +3388,14 @@ checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.0", "const-oid", - "crypto-common 0.2.2", + "crypto-common 0.2.1", ] [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", @@ -3512,9 +3512,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.16.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "embedded-graphics" @@ -3636,9 +3636,9 @@ dependencies = [ [[package]] name = "enumset" -version = "1.1.13" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d" +checksum = "7f96a4a12fe60ac746ae295a1a4ecb5bb02debc20856506c8635288065f142de" dependencies = [ "enumset_derive", ] @@ -3891,9 +3891,9 @@ checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" [[package]] name = "fitimage" -version = "0.1.4" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92edd7c1c55efd27b5a17dd6d505affafcfe48dfaed7423d4c4ae7361e56a7d8" +checksum = "8cb4d07a1e7a76a7ac4b6b754d45670b1a82c8a0484d80cceeb30c99ad65ce1d" dependencies = [ "anyhow", "byteorder", @@ -4070,9 +4070,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.4" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" @@ -4188,9 +4188,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "goblin" -version = "0.10.6" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d494b2004fbc8cf419a6d2115488df4e11140f6f4abd877519de1bbd90c5370" +checksum = "983a6aafb3b12d4c41ea78d39e189af4298ce747353945ff5105b54a056e5cd9" dependencies = [ "log", "plain", @@ -4332,9 +4332,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", "itoa", @@ -4794,9 +4794,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.27" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392c70591e8749fe235ddaf513e6f58b26bce3dcc16524cecc8936f75afa161e" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "log", @@ -4807,9 +4807,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.27" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b605b0c050d845fc355bb11eb3f9a8deddc218ea60c76e61aa1f2adfb2c96a" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -4841,9 +4841,9 @@ dependencies = [ [[package]] name = "jkconfig" -version = "0.2.4" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4cbaedffb63dabcbab61e23e0adc1fc7bc0d7420f2f56124e2bc0002d9274ab" +checksum = "d8746ebf553e6e9d0620a918072432b44aa9563f6f19d17432b613828df708a1" dependencies = [ "anyhow", "cargo_metadata", @@ -4929,9 +4929,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", @@ -5241,9 +5241,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.30" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "loongArch64" @@ -5380,9 +5380,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" @@ -5593,9 +5593,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-derive" @@ -5803,7 +5803,7 @@ dependencies = [ "fitimage", "futures", "indicatif", - "jkconfig 0.2.4", + "jkconfig 0.2.3", "log", "lzma-rs", "network-interface", @@ -6490,7 +6490,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ef8dea09a92caaf73bff7adb70b76162e5937524058a7e5bff37869cbbec293" dependencies = [ "bitflags 2.11.1", - "compact_str 0.9.1", + "compact_str 0.9.0", "hashbrown 0.16.1", "indoc", "itertools 0.14.0", @@ -6592,6 +6592,17 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rbpf" +version = "0.4.1" +source = "git+https://github.com/CN-TangLin/rbpf.git?branch=feat%2Fjit-multi-arch#4e8cf992d9c9f885906119efa29e8dbe87d69add" +dependencies = [ + "byteorder", + "combine", + "hashbrown 0.16.1", + "log", +] + [[package]] name = "rd-block" version = "0.1.2" @@ -6850,9 +6861,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" dependencies = [ "base64 0.22.1", "bytes", @@ -7590,9 +7601,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", @@ -8088,6 +8099,7 @@ dependencies = [ "num_enum", "ouroboros", "rand 0.10.1", + "rbpf", "rdrive", "ringbuf", "riscv 0.16.0", @@ -8331,9 +8343,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.46" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" dependencies = [ "filetime", "libc", @@ -9091,9 +9103,9 @@ dependencies = [ [[package]] name = "uboot-shell" -version = "0.2.4" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2237321472e37a34980340e0a6f8ba406167fb010209ead2f0e594bdfb718e02" +checksum = "c3b3d5d2959cc2e9a28cb2c4ad30f99a33f6831f4f159b08c696604b9494b672" dependencies = [ "colored", "futures", @@ -9504,9 +9516,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -9517,9 +9529,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -9527,9 +9539,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9537,9 +9549,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -9550,9 +9562,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -9612,9 +9624,9 @@ checksum = "dba9c05687e501b2710833fbe2cc7ff8cc24411fb0d81967498ca44598942f88" [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -10324,9 +10336,9 @@ dependencies = [ [[package]] name = "yaxpeax-x86" -version = "2.1.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a159e15f66ce40b5dfc28213366f8ff42ff8f0508e2e72e431c62573f025ac5e" +checksum = "9a9a30b7dd533c7b1a73eaf7c4ea162a7a632a2bb29b9fff47d8f2cc8513a883" dependencies = [ "cfg-if", "num-traits", diff --git a/apps/starry/ebpf/README.md b/apps/starry/ebpf/README.md new file mode 100644 index 0000000000..c40875033b --- /dev/null +++ b/apps/starry/ebpf/README.md @@ -0,0 +1,13 @@ +# Starry eBPF Programs + +These programs are comprehensive eBPF test applications migrated from the original Starry `user/musl` directory. Since they are complex and require significant execution time, they are placed in `apps/starry/ebpf` rather than `test-suit` to serve as integration/application examples. + +## Contents + +- `ebpf-basics`: Basic eBPF instruction tests +- `ebpf-advanced`: Advanced eBPF instruction tests (ALU32, JMP32, etc.) +- `ebpf-attach`: eBPF attach and perf_event_open tests + +## Build and Run + +These programs can be built as part of the rootfs and run interactively or via an app script to comprehensively test the eBPF subsystem. diff --git a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-advanced/c/CMakeLists.txt b/apps/starry/ebpf/ebpf-advanced/c/CMakeLists.txt similarity index 77% rename from test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-advanced/c/CMakeLists.txt rename to apps/starry/ebpf/ebpf-advanced/c/CMakeLists.txt index 7a7d0ded3d..ac391432ee 100644 --- a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-advanced/c/CMakeLists.txt +++ b/apps/starry/ebpf/ebpf-advanced/c/CMakeLists.txt @@ -5,4 +5,4 @@ set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_C_EXTENSIONS OFF) add_executable(test-ebpf-advanced src/main.c) target_compile_options(test-ebpf-advanced PRIVATE -Wall -Wextra -Werror) -install(TARGETS test-ebpf-advanced RUNTIME DESTINATION usr/bin/starry-test-suit) +install(TARGETS test-ebpf-advanced RUNTIME DESTINATION usr/bin) diff --git a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-advanced/c/src/main.c b/apps/starry/ebpf/ebpf-advanced/c/src/main.c similarity index 100% rename from test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-advanced/c/src/main.c rename to apps/starry/ebpf/ebpf-advanced/c/src/main.c diff --git a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-attach/c/CMakeLists.txt b/apps/starry/ebpf/ebpf-attach/c/CMakeLists.txt similarity index 77% rename from test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-attach/c/CMakeLists.txt rename to apps/starry/ebpf/ebpf-attach/c/CMakeLists.txt index 71310a8362..4dd1c72eab 100644 --- a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-attach/c/CMakeLists.txt +++ b/apps/starry/ebpf/ebpf-attach/c/CMakeLists.txt @@ -5,4 +5,4 @@ set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_C_EXTENSIONS OFF) add_executable(test-ebpf-attach src/main.c) target_compile_options(test-ebpf-attach PRIVATE -Wall -Wextra -Werror) -install(TARGETS test-ebpf-attach RUNTIME DESTINATION usr/bin/starry-test-suit) +install(TARGETS test-ebpf-attach RUNTIME DESTINATION usr/bin) diff --git a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-attach/c/src/main.c b/apps/starry/ebpf/ebpf-attach/c/src/main.c similarity index 100% rename from test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-attach/c/src/main.c rename to apps/starry/ebpf/ebpf-attach/c/src/main.c diff --git a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-basics/c/CMakeLists.txt b/apps/starry/ebpf/ebpf-basics/c/CMakeLists.txt similarity index 80% rename from test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-basics/c/CMakeLists.txt rename to apps/starry/ebpf/ebpf-basics/c/CMakeLists.txt index 61bd3360cf..49a4c043af 100644 --- a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-basics/c/CMakeLists.txt +++ b/apps/starry/ebpf/ebpf-basics/c/CMakeLists.txt @@ -6,4 +6,4 @@ set(CMAKE_C_EXTENSIONS OFF) add_executable(test-ebpf-basics src/main.c) target_include_directories(test-ebpf-basics PRIVATE src) target_compile_options(test-ebpf-basics PRIVATE -Wall -Wextra -Werror) -install(TARGETS test-ebpf-basics RUNTIME DESTINATION usr/bin/starry-test-suit) +install(TARGETS test-ebpf-basics RUNTIME DESTINATION usr/bin) diff --git a/test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-basics/c/src/main.c b/apps/starry/ebpf/ebpf-basics/c/src/main.c similarity index 100% rename from test-suit/starryos/normal/qemu-smp1/syscall/test-ebpf-basics/c/src/main.c rename to apps/starry/ebpf/ebpf-basics/c/src/main.c diff --git a/os/StarryOS/kernel/Cargo.toml b/os/StarryOS/kernel/Cargo.toml index a3ac6d926f..cd3e08a9af 100644 --- a/os/StarryOS/kernel/Cargo.toml +++ b/os/StarryOS/kernel/Cargo.toml @@ -18,7 +18,7 @@ default = ["dynamic_debug", "ebpf"] dev-log = [] ext4 = ["ax-fs/ext4"] input = ["dep:ax-input", "ax-feat/input"] -ebpf = [] +ebpf = ["dep:rbpf"] memtrack = ["ax-feat/dwarf", "ax-alloc/tracking", "dep:gimli"] rknpu = ["dep:ax-driver", "ax-driver/rknpu"] plat-dyn = [ @@ -70,6 +70,7 @@ axplat-dyn = { workspace = true, optional = true } ax-runtime.workspace = true ax-sync.workspace = true ax-task.workspace = true +rbpf = { git = "https://github.com/CN-TangLin/rbpf.git", branch = "feat/jit-multi-arch", default-features = false, optional = true } ax-dma = { workspace = true, optional = true } sg2002-tpu = { workspace = true, optional = true } crab-usb = { workspace = true, optional = true } diff --git a/os/StarryOS/kernel/src/ebpf.rs b/os/StarryOS/kernel/src/ebpf.rs index b47243b3a4..5286df1105 100644 --- a/os/StarryOS/kernel/src/ebpf.rs +++ b/os/StarryOS/kernel/src/ebpf.rs @@ -22,8 +22,15 @@ use ax_sync::spin::SpinNoIrq; use crate::task::AsThread; +#[cfg(any( + target_arch = "x86_64", + target_arch = "riscv64", + target_arch = "aarch64" +))] +mod rbpf_jit; + #[allow(dead_code)] -mod bpf_insn { +pub(crate) mod bpf_insn { pub const BPF_LD: u8 = 0x00; pub const BPF_LDX: u8 = 0x01; pub const BPF_ST: u8 = 0x02; @@ -77,8 +84,6 @@ mod bpf_insn { pub const BPF_K: u8 = 0x00; pub const BPF_X: u8 = 0x08; - pub const BPF_CALL_OP: u8 = 0x80; - pub const BPF_PSEUDO_MAP_FD: u8 = 1; pub const BPF_PSEUDO_MAP_VALUE: u8 = 2; @@ -182,7 +187,6 @@ mod prog_type { pub const SYSCALL: u32 = 31; } -#[allow(dead_code)] mod cmd { pub const MAP_CREATE: u64 = 0; pub const MAP_LOOKUP_ELEM: u64 = 1; @@ -239,6 +243,7 @@ trait BpfMapOps: Send + Sync { fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> AxResult<()>; fn delete_elem(&mut self, key: &[u8]) -> AxResult<()>; fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>>; + fn as_any_mut(&mut self) -> &mut dyn core::any::Any; } struct ArrayMap { @@ -321,18 +326,22 @@ impl BpfMapOps for ArrayMap { } Ok(Some(next_idx.to_ne_bytes().to_vec())) } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } } struct HashMapInner { meta: BpfMapMeta, - entries: alloc::collections::BTreeMap, Vec>, + entries: hashbrown::HashMap, Vec>, } impl HashMapInner { fn new(meta: BpfMapMeta) -> Self { Self { meta, - entries: alloc::collections::BTreeMap::new(), + entries: hashbrown::HashMap::new(), } } } @@ -400,6 +409,10 @@ impl BpfMapOps for HashMapInner { } } } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } } struct PerfEventArrayMap { @@ -477,6 +490,446 @@ impl BpfMapOps for PerfEventArrayMap { } Ok(Some(next_idx.to_ne_bytes().to_vec())) } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +#[repr(C)] +struct RingBufHdr { + len: u32, + start_offset: u32, +} + +const RINGBUF_HDR_SIZE: usize = 8; +const RINGBUF_ALIGN: usize = 8; + +struct RingBufferMap { + meta: BpfMapMeta, + buf: alloc::vec::Vec, + capacity: usize, + head: u64, + tail: u64, + mask: usize, + pending_reserve: Option<(usize, usize)>, +} + +impl RingBufferMap { + fn new(meta: BpfMapMeta) -> Self { + let capacity = (meta.max_entries as usize).next_power_of_two(); + let mask = capacity - 1; + Self { + meta, + buf: alloc::vec![0u8; capacity], + capacity, + head: 0, + tail: 0, + mask, + pending_reserve: None, + } + } + + fn write_at(&mut self, offset: usize, data: &[u8]) { + if data.is_empty() { + return; + } + let start = offset & self.mask; + if start + data.len() <= self.capacity { + unsafe { + core::ptr::copy_nonoverlapping( + data.as_ptr(), + self.buf.as_mut_ptr().add(start), + data.len(), + ); + } + } else { + let first = self.capacity - start; + unsafe { + core::ptr::copy_nonoverlapping( + data.as_ptr(), + self.buf.as_mut_ptr().add(start), + first, + ); + core::ptr::copy_nonoverlapping( + data.as_ptr().add(first), + self.buf.as_mut_ptr(), + data.len() - first, + ); + } + } + } + + fn available(&self) -> usize { + self.capacity - (self.head - self.tail) as usize + } + + fn reserve(&mut self, size: usize) -> Option<*mut u8> { + let aligned_size = (size + RINGBUF_ALIGN - 1) & !(RINGBUF_ALIGN - 1); + let total = RINGBUF_HDR_SIZE + aligned_size; + if total > self.available() { + return None; + } + if self.pending_reserve.is_some() { + return None; + } + let offset = self.head as usize; + let hdr = RingBufHdr { + len: size as u32, + start_offset: offset as u32, + }; + let hdr_bytes = unsafe { + core::slice::from_raw_parts(&hdr as *const RingBufHdr as *const u8, RINGBUF_HDR_SIZE) + }; + self.write_at(offset, hdr_bytes); + let data_offset = offset + RINGBUF_HDR_SIZE; + let data_ptr = unsafe { self.buf.as_mut_ptr().add(data_offset & self.mask) }; + self.pending_reserve = Some((aligned_size, data_offset)); + Some(data_ptr) + } + + fn submit(&mut self, _flags: u64) { + if let Some((aligned_size, _data_offset)) = self.pending_reserve.take() { + let total = RINGBUF_HDR_SIZE + aligned_size; + self.head += total as u64; + } + } + + fn discard(&mut self, _flags: u64) { + if let Some((_, data_offset)) = self.pending_reserve.take() { + let hdr_bytes_to_clear = self.head as usize + RINGBUF_HDR_SIZE; + let zero = alloc::vec![0u8; RINGBUF_HDR_SIZE]; + self.write_at(data_offset - RINGBUF_HDR_SIZE, &zero); + let _ = (hdr_bytes_to_clear, data_offset); + } + } + + fn output(&mut self, data: &[u8]) -> bool { + let aligned_size = (data.len() + RINGBUF_ALIGN - 1) & !(RINGBUF_ALIGN - 1); + let total = RINGBUF_HDR_SIZE + aligned_size; + if total > self.available() { + return false; + } + let offset = self.head as usize; + let hdr = RingBufHdr { + len: data.len() as u32, + start_offset: offset as u32, + }; + let hdr_bytes = unsafe { + core::slice::from_raw_parts(&hdr as *const RingBufHdr as *const u8, RINGBUF_HDR_SIZE) + }; + self.write_at(offset, hdr_bytes); + self.write_at(offset + RINGBUF_HDR_SIZE, data); + if aligned_size > data.len() { + let pad = alloc::vec![0u8; aligned_size - data.len()]; + self.write_at(offset + RINGBUF_HDR_SIZE + data.len(), &pad); + } + self.head += total as u64; + true + } +} + +impl BpfMapOps for RingBufferMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, _key: &[u8]) -> AxResult>> { + Err(bpf_error::EINVAL) + } + + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> AxResult<()> { + Err(bpf_error::EINVAL) + } + + fn delete_elem(&mut self, _key: &[u8]) -> AxResult<()> { + Err(bpf_error::EINVAL) + } + + fn get_next_key(&mut self, _key: Option<&[u8]>) -> AxResult>> { + Err(bpf_error::EINVAL) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +struct ProgArrayMap { + meta: BpfMapMeta, + prog_fds: alloc::vec::Vec>, +} + +impl ProgArrayMap { + fn new(meta: BpfMapMeta) -> Self { + let prog_fds = alloc::vec![None; meta.max_entries as usize]; + Self { meta, prog_fds } + } +} + +impl BpfMapOps for ProgArrayMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + let i = idx as usize; + if i >= self.prog_fds.len() { + return Ok(None); + } + match self.prog_fds[i] { + Some(fd) => Ok(Some(fd.to_ne_bytes().to_vec())), + None => Ok(None), + } + } + + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> AxResult<()> { + if key.len() != 4 || value.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + let prog_fd = u32::from_ne_bytes([value[0], value[1], value[2], value[3]]); + let i = idx as usize; + if i >= self.prog_fds.len() { + return Err(bpf_error::EINVAL); + } + self.prog_fds[i] = Some(prog_fd); + Ok(()) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + let i = idx as usize; + if i >= self.prog_fds.len() { + return Err(bpf_error::EINVAL); + } + self.prog_fds[i] = None; + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let start = match key { + None => 0, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([k[0], k[1], k[2], k[3]]); + (idx as usize).saturating_add(1) + } + }; + for i in start..self.prog_fds.len() { + if self.prog_fds[i].is_some() { + return Ok(Some((i as u32).to_ne_bytes().to_vec())); + } + } + Ok(None) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +const MAX_STACK_DEPTH: usize = 127; + +struct StackTraceMap { + meta: BpfMapMeta, + traces: hashbrown::HashMap>, + next_id: u32, +} + +impl StackTraceMap { + fn new(meta: BpfMapMeta) -> Self { + Self { + meta, + traces: hashbrown::HashMap::new(), + next_id: 0, + } + } + + fn store_trace(&mut self, ips: &[u64]) -> u32 { + let id = self.next_id; + self.next_id = self.next_id.wrapping_add(1); + if self.traces.len() >= self.meta.max_entries as usize + && let Some(oldest) = self.traces.keys().min().copied() + { + self.traces.remove(&oldest); + } + self.traces.insert(id, ips.to_vec()); + id + } +} + +impl BpfMapOps for StackTraceMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let id = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + match self.traces.get(&id) { + Some(trace) => { + let value_size = self.meta.value_size as usize; + let count = value_size / 8; + let mut buf = alloc::vec![0u8; value_size]; + for (i, ip) in trace.iter().enumerate().take(count) { + let start = i * 8; + buf[start..start + 8].copy_from_slice(&ip.to_ne_bytes()); + } + Ok(Some(buf)) + } + None => Ok(None), + } + } + + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> AxResult<()> { + Err(bpf_error::EINVAL) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let id = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]); + self.traces.remove(&id); + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let start = match key { + None => None, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + Some(u32::from_ne_bytes([k[0], k[1], k[2], k[3]])) + } + }; + let mut keys: alloc::vec::Vec = self.traces.keys().copied().collect(); + keys.sort(); + match start { + None => match keys.first() { + Some(&k) => Ok(Some(k.to_ne_bytes().to_vec())), + None => Ok(None), + }, + Some(sk) => match keys.iter().find(|&&k| k > sk) { + Some(&k) => Ok(Some(k.to_ne_bytes().to_vec())), + None => Ok(None), + }, + } + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } +} + +struct PerCpuArrayMap { + meta: BpfMapMeta, + per_cpu_data: alloc::vec::Vec>, + elem_size: usize, + cpu_count: usize, +} + +impl PerCpuArrayMap { + fn new(meta: BpfMapMeta, cpu_count: usize) -> Self { + let elem_size = meta.value_size as usize; + let total = elem_size * meta.max_entries as usize; + let per_cpu_data = alloc::vec![alloc::vec![0u8; total]; cpu_count]; + Self { + meta, + per_cpu_data, + elem_size, + cpu_count, + } + } + + fn current_cpu(&self) -> usize { + let cpu = ax_runtime::hal::percpu::this_cpu_id(); + if cpu < self.cpu_count { cpu } else { 0 } + } +} + +impl BpfMapOps for PerCpuArrayMap { + fn meta(&self) -> &BpfMapMeta { + &self.meta + } + + fn lookup_elem(&mut self, key: &[u8]) -> AxResult>> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]) as usize; + if idx >= self.meta.max_entries as usize { + return Ok(None); + } + let cpu = self.current_cpu(); + let start = idx * self.elem_size; + let end = start + self.elem_size; + Ok(Some(self.per_cpu_data[cpu][start..end].to_vec())) + } + + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]) as usize; + if idx >= self.meta.max_entries as usize { + return Err(bpf_error::EINVAL); + } + let cpu = self.current_cpu(); + let start = idx * self.elem_size; + let end = start + self.elem_size.min(value.len()); + self.per_cpu_data[cpu][start..end].copy_from_slice(&value[..end - start]); + Ok(()) + } + + fn delete_elem(&mut self, key: &[u8]) -> AxResult<()> { + if key.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([key[0], key[1], key[2], key[3]]) as usize; + if idx >= self.meta.max_entries as usize { + return Err(bpf_error::EINVAL); + } + let cpu = self.current_cpu(); + let start = idx * self.elem_size; + let end = start + self.elem_size; + self.per_cpu_data[cpu][start..end].fill(0); + Ok(()) + } + + fn get_next_key(&mut self, key: Option<&[u8]>) -> AxResult>> { + let next_idx = match key { + None => 0, + Some(k) => { + if k.len() != 4 { + return Err(bpf_error::EINVAL); + } + let idx = u32::from_ne_bytes([k[0], k[1], k[2], k[3]]); + idx + 1 + } + }; + if next_idx >= self.meta.max_entries { + return Ok(None); + } + Ok(Some(next_idx.to_ne_bytes().to_vec())) + } + + fn as_any_mut(&mut self) -> &mut dyn core::any::Any { + self + } } struct UnifiedMap { @@ -497,6 +950,13 @@ impl UnifiedMap { map_type::PERF_EVENT_ARRAY => { alloc::boxed::Box::new(PerfEventArrayMap::new(meta.clone())) } + map_type::RINGBUF => alloc::boxed::Box::new(RingBufferMap::new(meta.clone())), + map_type::PROG_ARRAY => alloc::boxed::Box::new(ProgArrayMap::new(meta.clone())), + map_type::STACK_TRACE => alloc::boxed::Box::new(StackTraceMap::new(meta.clone())), + map_type::PERCPU_ARRAY => { + let cpu_count = Self::detect_cpu_count(); + alloc::boxed::Box::new(PerCpuArrayMap::new(meta.clone(), cpu_count)) + } _ => { warn!("bpf: unsupported map type {map_type}"); return Err(bpf_error::EINVAL); @@ -524,15 +984,25 @@ impl UnifiedMap { fn meta(&self) -> &BpfMapMeta { self.inner.meta() } + + fn detect_cpu_count() -> usize { + ax_runtime::hal::cpu_num() + } } -#[derive(Debug)] #[allow(dead_code)] +#[derive(Debug)] struct BpfProg { prog_type: u32, insns: Vec, meta: BpfProgMeta, id: u32, + #[cfg(any( + target_arch = "x86_64", + target_arch = "riscv64", + target_arch = "aarch64" + ))] + jitted: Option, } #[derive(Clone, Debug)] @@ -636,9 +1106,21 @@ impl BpfFdTable { } } +// Lock ordering: BPF_GLOBAL -> BPF_TAIL_CALL_TARGET +// BPF_LOOKUP_CACHE is per-CPU (no lock needed). +// ProgArrayMap::update_elem does NOT acquire BPF_GLOBAL; the caller +// (handle_map_update_elem / helper_map_update_elem) validates prog_fd +// while holding BPF_GLOBAL before calling map.update(). +// BpfVm::execute() must never be called while BPF_GLOBAL is held. +// All helpers that acquire BPF_GLOBAL are called from execute(), which +// runs without any lock. Syscall handlers acquire BPF_GLOBAL independently. +// run_bpf_prog clones insns under BPF_GLOBAL then drops the lock before +// calling execute(). The tail-call path in execute() also drops BPF_GLOBAL +// before recursing into execute(). static BPF_GLOBAL: SpinNoIrq = SpinNoIrq::new(BpfFdTable::new()); #[ax_percpu::def_percpu] static BPF_LOOKUP_CACHE: alloc::vec::Vec = alloc::vec::Vec::new(); +static BPF_TAIL_CALL_TARGET: SpinNoIrq> = SpinNoIrq::new(None); fn handle_map_create(uattr: usize, size: u32) -> AxResult { if size < 24 { @@ -657,7 +1139,10 @@ fn handle_map_create(uattr: usize, size: u32) -> AxResult { max_entries = core::ptr::read(ptr.add(3)); map_flags = core::ptr::read(ptr.add(4)); } - if max_entries == 0 || key_size == 0 || value_size == 0 { + if max_entries == 0 { + return Err(bpf_error::EINVAL); + } + if map_type != map_type::RINGBUF && (key_size == 0 || value_size == 0) { return Err(bpf_error::EINVAL); } let mut guard = BPF_GLOBAL.lock(); @@ -718,9 +1203,21 @@ fn handle_map_update_elem(uattr: usize, size: u32) -> AxResult { (map_fd, key_ptr, value_ptr, flags) }; let mut guard = BPF_GLOBAL.lock(); + let meta = guard + .maps + .get(&map_fd) + .map(|m| m.meta().clone()) + .ok_or(AxError::BadFileDescriptor)?; + let key_size = meta.key_size as usize; + let value_size = meta.value_size as usize; + if meta.map_type == map_type::PROG_ARRAY && value_size == 4 { + let value = unsafe { core::slice::from_raw_parts(value_ptr as *const u8, value_size) }; + let prog_fd = u32::from_ne_bytes([value[0], value[1], value[2], value[3]]); + if !guard.progs.contains_key(&prog_fd) { + return Err(bpf_error::EINVAL); + } + } let map = guard.get_map(map_fd)?; - let key_size = map.meta().key_size as usize; - let value_size = map.meta().value_size as usize; let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; let value = unsafe { core::slice::from_raw_parts(value_ptr as *const u8, value_size) }; map.update(key, value, flags)?; @@ -844,6 +1341,25 @@ fn handle_prog_load(uattr: usize, size: u32) -> AxResult { } let mut guard = BPF_GLOBAL.lock(); let id = guard.progs.len() as u32; + #[cfg(any( + target_arch = "x86_64", + target_arch = "riscv64", + target_arch = "aarch64" + ))] + let jitted = { + let helpers = init_helper_functions(); + rbpf_jit::try_jit_compile(&insns, &helpers) + }; + #[cfg(any( + target_arch = "x86_64", + target_arch = "riscv64", + target_arch = "aarch64" + ))] + if jitted.is_some() { + info!("bpf: JIT compilation successful for prog_{id}"); + } else { + warn!("bpf: JIT compilation failed, will use interpreter for prog_{id}"); + } let prog = BpfProg { prog_type, insns, @@ -856,6 +1372,12 @@ fn handle_prog_load(uattr: usize, size: u32) -> AxResult { expected_attach_type: 0, }, id, + #[cfg(any( + target_arch = "x86_64", + target_arch = "riscv64", + target_arch = "aarch64" + ))] + jitted, }; let fd = guard.insert_prog(prog); info!("bpf: loaded prog type={prog_type} insns={insn_cnt} fd={fd}"); @@ -879,6 +1401,7 @@ mod helper_id { pub const GET_SMP_PROCESSOR_ID: u32 = 8; pub const SKB_STORE_BYTES: u32 = 9; pub const CSUM_DIFF: u32 = 10; + pub const TAIL_CALL: u32 = 12; pub const GET_CURRENT_PID_TGID: u32 = 14; pub const GET_CURRENT_UID_GID: u32 = 15; pub const GET_CURRENT_COMM: u32 = 16; @@ -901,21 +1424,33 @@ mod helper_id { pub const GET_ATTACHED_FUNC_ARGS: u32 = 186; } -type HelperFn = fn(u64, u64, u64, u64, u64) -> u64; +pub(crate) type HelperFn = fn(u64, u64, u64, u64, u64) -> u64; fn init_helper_functions() -> alloc::collections::BTreeMap { let mut m: alloc::collections::BTreeMap = alloc::collections::BTreeMap::new(); m.insert(helper_id::MAP_LOOKUP_ELEM, helper_map_lookup_elem); m.insert(helper_id::MAP_UPDATE_ELEM, helper_map_update_elem); m.insert(helper_id::MAP_DELETE_ELEM, helper_map_delete_elem); + m.insert(helper_id::TAIL_CALL, helper_tail_call); + m.insert(helper_id::GET_STACK_ID, helper_get_stackid); m.insert(helper_id::PROBE_READ, helper_probe_read); m.insert(helper_id::PROBE_READ_KERNEL, helper_probe_read); + m.insert(helper_id::PROBE_READ_USER, helper_probe_read_user); + m.insert(helper_id::PROBE_READ_USER_STR, helper_probe_read_user_str); + m.insert( + helper_id::PROBE_READ_KERNEL_STR, + helper_probe_read_kernel_str, + ); m.insert(helper_id::KTIME_GET_NS, helper_ktime_get_ns); m.insert(helper_id::GET_SMP_PROCESSOR_ID, helper_get_smp_processor_id); m.insert(helper_id::GET_CURRENT_PID_TGID, helper_get_current_pid_tgid); m.insert(helper_id::GET_CURRENT_UID_GID, helper_get_current_uid_gid); m.insert(helper_id::GET_PRANDOM_U32, helper_get_prandom_u32); m.insert(helper_id::PERF_EVENT_OUTPUT, helper_perf_event_output); + m.insert(helper_id::RINGBUF_OUTPUT, helper_ringbuf_output); + m.insert(helper_id::RINGBUF_RESERVE, helper_ringbuf_reserve); + m.insert(helper_id::RINGBUF_SUBMIT, helper_ringbuf_submit); + m.insert(helper_id::RINGBUF_DISCARD, helper_ringbuf_discard); m.insert(helper_id::TRACE_PRINTK, helper_trace_printk); m.insert(helper_id::GET_CURRENT_TASK, helper_get_current_task); m.insert(helper_id::GET_CURRENT_COMM, helper_get_current_comm); @@ -934,11 +1469,14 @@ fn helper_map_lookup_elem(map_ptr: u64, key_ptr: u64, _a3: u64, _a4: u64, _a5: u let key_size = map.meta().key_size as usize; let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; match map.lookup(key) { - Ok(Some(value)) => BPF_LOOKUP_CACHE.with_current(|cache| { - cache.clear(); - cache.extend_from_slice(&value); - cache.as_ptr() as u64 - }), + Ok(Some(value)) => { + drop(guard); + BPF_LOOKUP_CACHE.with_current(|cache| { + cache.clear(); + cache.extend_from_slice(&value); + cache.as_ptr() as u64 + }) + } _ => 0, } } @@ -948,14 +1486,29 @@ fn helper_map_update_elem(map_ptr: u64, key_ptr: u64, value_ptr: u64, flags: u64 return u64::MAX; } let mut guard = BPF_GLOBAL.lock(); + let meta = guard + .maps + .get(&(map_ptr as u32)) + .map(|m| m.meta().clone()) + .ok_or(u64::MAX); + let meta = match meta { + Ok(m) => m, + Err(e) => return e, + }; + if meta.map_type == map_type::PROG_ARRAY && meta.value_size == 4 { + let value = unsafe { core::slice::from_raw_parts(value_ptr as *const u8, 4) }; + let prog_fd = u32::from_ne_bytes([value[0], value[1], value[2], value[3]]); + if !guard.progs.contains_key(&prog_fd) { + return u64::MAX; + } + } let map = match guard.get_map(map_ptr as u32) { Ok(m) => m, Err(_) => return u64::MAX, }; - let key_size = map.meta().key_size as usize; - let value_size = map.meta().value_size as usize; - let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, key_size) }; - let value = unsafe { core::slice::from_raw_parts(value_ptr as *const u8, value_size) }; + let key = unsafe { core::slice::from_raw_parts(key_ptr as *const u8, meta.key_size as usize) }; + let value = + unsafe { core::slice::from_raw_parts(value_ptr as *const u8, meta.value_size as usize) }; match map.update(key, value, flags) { Ok(()) => 0, Err(_) => u64::MAX, @@ -979,6 +1532,107 @@ fn helper_map_delete_elem(map_ptr: u64, key_ptr: u64, _a3: u64, _a4: u64, _a5: u } } +fn helper_tail_call(_ctx: u64, map_fd: u64, index: u64, _a4: u64, _a5: u64) -> u64 { + if map_fd == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + if map.meta().map_type != map_type::PROG_ARRAY { + return u64::MAX; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(p) => p, + None => return u64::MAX, + }; + let i = index as usize; + if i >= inner.prog_fds.len() { + return u64::MAX; + } + let target_fd = match inner.prog_fds[i] { + Some(fd) => fd, + None => return u64::MAX, + }; + if !guard.progs.contains_key(&target_fd) { + return u64::MAX; + } + drop(guard); + let mut tail_target = BPF_TAIL_CALL_TARGET.lock(); + *tail_target = Some(target_fd); + 0 +} + +fn helper_get_stackid(_ctx: u64, map_fd: u64, _flags: u64, _a4: u64, _a5: u64) -> u64 { + if map_fd == 0 { + return u64::MAX; + } + let fp: usize; + #[cfg(target_arch = "x86_64")] + unsafe { + core::arch::asm!("mov {}, rbp", out(reg) fp) + } + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] + unsafe { + core::arch::asm!("addi {0}, s0, 0", out(reg) fp) + } + #[cfg(target_arch = "aarch64")] + unsafe { + core::arch::asm!("mov {0}, x29", out(reg) fp) + } + #[cfg(target_arch = "loongarch64")] + unsafe { + core::arch::asm!("move {0}, $fp", out(reg) fp) + } + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "aarch64", + target_arch = "loongarch64" + )))] + { + fp = 0; + } + let mut ips = alloc::vec::Vec::new(); + let mut current_fp = fp; + for _ in 0..MAX_STACK_DEPTH { + if current_fp == 0 { + break; + } + unsafe { + let ip_ptr = current_fp as *const usize; + if ip_ptr.is_null() { + break; + } + let next_fp_ptr = ip_ptr.add(1); + if core::ptr::read(next_fp_ptr) == 0 { + break; + } + ips.push(core::ptr::read(ip_ptr) as u64); + current_fp = core::ptr::read(next_fp_ptr); + } + } + if ips.is_empty() { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + if map.meta().map_type != map_type::STACK_TRACE { + return u64::MAX; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(s) => s, + None => return u64::MAX, + }; + inner.store_trace(&ips) as u64 +} + fn helper_probe_read(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { if dst == 0 || size == 0 { return u64::MAX; @@ -991,27 +1645,115 @@ fn helper_probe_read(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { unsafe { core::ptr::write_bytes(dst as *mut u8, 0, len) }; return 0; } - let src_slice = unsafe { core::slice::from_raw_parts(src as *const u8, len) }; - let dst_slice = unsafe { core::slice::from_raw_parts_mut(dst as *mut u8, len) }; - let copied = { - let buf = unsafe { - core::slice::from_raw_parts_mut( - dst_slice.as_mut_ptr() as *mut core::mem::MaybeUninit, - len, - ) - }; - match starry_vm::vm_read_slice(src as *const u8, buf) { - Ok(()) => len, - Err(_) => { - unsafe { - core::ptr::copy_nonoverlapping(src_slice.as_ptr(), dst_slice.as_mut_ptr(), len) - }; - len + let buf = + unsafe { core::slice::from_raw_parts_mut(dst as *mut core::mem::MaybeUninit, len) }; + match starry_vm::vm_read_slice(src as *const u8, buf) { + Ok(()) => 0, + Err(_) => { + unsafe { core::ptr::write_bytes(dst as *mut u8, 0, len) }; + u64::MAX + } + } +} + +fn helper_probe_read_user(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { + if dst == 0 || size == 0 { + return u64::MAX; + } + let len = size as usize; + if len > 4096 { + return u64::MAX; + } + if src == 0 { + unsafe { core::ptr::write_bytes(dst as *mut u8, 0, len) }; + return 0; + } + let buf = + unsafe { core::slice::from_raw_parts_mut(dst as *mut core::mem::MaybeUninit, len) }; + match starry_vm::vm_read_slice(src as *const u8, buf) { + Ok(()) => 0, + Err(_) => { + unsafe { core::ptr::write_bytes(dst as *mut u8, 0, len) }; + u64::MAX + } + } +} + +unsafe fn probe_read_str_kernel(dst: *mut u8, size: usize, src: *const u8) -> usize { + let mut i = 0; + while i < size { + unsafe { + let byte = *src.add(i); + *dst.add(i) = byte; + if byte == 0 { + return i; } } - }; - let _ = copied; - 0 + i += 1; + } + if size > 0 { + unsafe { *dst.add(size - 1) = 0 }; + } + size +} + +unsafe fn probe_read_str_user(dst: *mut u8, size: usize, src: *const u8) -> usize { + let mut i = 0; + while i < size { + let mut one_buf = core::mem::MaybeUninit::::uninit(); + unsafe { + match starry_vm::vm_read_slice(src.add(i), core::slice::from_mut(&mut one_buf)) { + Ok(()) => { + let byte = one_buf.assume_init(); + *dst.add(i) = byte; + if byte == 0 { + return i; + } + } + Err(_) => { + if i < size { + *dst.add(i) = 0; + } + return i; + } + } + } + i += 1; + } + if size > 0 { + unsafe { *dst.add(size - 1) = 0 }; + } + size +} + +fn helper_probe_read_user_str(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { + if dst == 0 || size == 0 { + return u64::MAX; + } + let len = size as usize; + if len > 4096 { + return u64::MAX; + } + if src == 0 { + unsafe { *(dst as *mut u8) = 0 }; + return u64::MAX; + } + unsafe { probe_read_str_user(dst as *mut u8, len, src as *const u8) as u64 } +} + +fn helper_probe_read_kernel_str(dst: u64, size: u64, src: u64, _a4: u64, _a5: u64) -> u64 { + if dst == 0 || size == 0 { + return u64::MAX; + } + let len = size as usize; + if len > 4096 { + return u64::MAX; + } + if src == 0 { + unsafe { *(dst as *mut u8) = 0 }; + return u64::MAX; + } + unsafe { probe_read_str_kernel(dst as *mut u8, len, src as *const u8) as u64 } } fn helper_ktime_get_ns(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { @@ -1038,8 +1780,14 @@ fn helper_get_current_uid_gid(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) } fn helper_get_prandom_u32(_a1: u64, _a2: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { - use core::sync::atomic::{AtomicU32, Ordering}; - static SEED: AtomicU32 = AtomicU32::new(12345); + use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + static SEED: AtomicU32 = AtomicU32::new(0); + static INITIALIZED: AtomicBool = AtomicBool::new(false); + if !INITIALIZED.load(Ordering::Acquire) { + let ts = ax_runtime::hal::time::monotonic_time_nanos() as u32; + SEED.store(ts.wrapping_add(12345), Ordering::Relaxed); + INITIALIZED.store(true, Ordering::Release); + } let prev = SEED.load(Ordering::Relaxed); let next = prev.wrapping_mul(1103515245).wrapping_add(12345); SEED.store(next, Ordering::Relaxed); @@ -1063,6 +1811,94 @@ fn helper_perf_event_output( } } +fn helper_ringbuf_output(map_fd: u64, data_ptr: u64, data_size: u64, _flags: u64, _a5: u64) -> u64 { + if map_fd == 0 || data_ptr == 0 || data_size == 0 { + return u64::MAX; + } + let data = unsafe { core::slice::from_raw_parts(data_ptr as *const u8, data_size as usize) }; + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return u64::MAX, + }; + if map.meta().map_type != map_type::RINGBUF { + return u64::MAX; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => return u64::MAX, + }; + if inner.output(data) { 0 } else { u64::MAX } +} + +fn helper_ringbuf_reserve(map_fd: u64, size: u64, _flags: u64, _a4: u64, _a5: u64) -> u64 { + if map_fd == 0 || size == 0 || size > 4096 { + return 0; + } + let mut guard = BPF_GLOBAL.lock(); + let map = match guard.get_map(map_fd as u32) { + Ok(m) => m, + Err(_) => return 0, + }; + if map.meta().map_type != map_type::RINGBUF { + return 0; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => return 0, + }; + match inner.reserve(size as usize) { + Some(ptr) => ptr as u64, + None => 0, + } +} + +fn helper_ringbuf_submit(sample_ptr: u64, flags: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if sample_ptr == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + for (_, map) in guard.maps.iter_mut() { + if map.meta().map_type != map_type::RINGBUF { + continue; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => continue, + }; + let buf_start = inner.buf.as_ptr() as u64; + let buf_end = buf_start + inner.buf.len() as u64; + if sample_ptr >= buf_start && sample_ptr < buf_end { + inner.submit(flags); + return 0; + } + } + u64::MAX +} + +fn helper_ringbuf_discard(sample_ptr: u64, flags: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { + if sample_ptr == 0 { + return u64::MAX; + } + let mut guard = BPF_GLOBAL.lock(); + for (_, map) in guard.maps.iter_mut() { + if map.meta().map_type != map_type::RINGBUF { + continue; + } + let inner = match map.inner.as_any_mut().downcast_mut::() { + Some(r) => r, + None => continue, + }; + let buf_start = inner.buf.as_ptr() as u64; + let buf_end = buf_start + inner.buf.len() as u64; + if sample_ptr >= buf_start && sample_ptr < buf_end { + inner.discard(flags); + return 0; + } + } + u64::MAX +} + fn helper_trace_printk(fmt_ptr: u64, fmt_size: u64, _a3: u64, _a4: u64, _a5: u64) -> u64 { if fmt_ptr == 0 || fmt_size == 0 || fmt_size > 128 { return u64::MAX; @@ -1103,7 +1939,6 @@ const BPF_MAX_INSN: usize = 1000000; const BPF_MAX_STACK: usize = 512; struct BpfVm { - #[allow(dead_code)] helpers: alloc::collections::BTreeMap, } @@ -1119,30 +1954,153 @@ impl BpfVm { warn!("bpf verifier: program too large: {} insns", insns.len()); return Err("program too large"); } - let has_exit = insns.iter().any(|insn| { - let class = insn.class(); - class == bpf_insn::BPF_JMP && (insn.code & 0xf0) == bpf_insn::BPF_EXIT - }); - if !has_exit { - warn!("bpf verifier: program has no BPF_EXIT instruction"); - return Err("no BPF_EXIT instruction"); + if insns.is_empty() { + return Err("empty program"); } + + let max_pc = insns.len(); + for (pc, insn) in insns.iter().enumerate() { + let dst = insn.dst_reg() as usize; + let src = insn.src_reg() as usize; + if dst > 10 { + warn!("bpf verifier: invalid dst_reg {dst} at pc={pc}"); + return Err("invalid destination register"); + } + if src > 10 { + warn!("bpf verifier: invalid src_reg {src} at pc={pc}"); + return Err("invalid source register"); + } + } + + let mut visited = alloc::vec![false; max_pc]; + let mut stack = alloc::vec![0usize]; + visited[0] = true; + while let Some(pc) = stack.pop() { + let insn = &insns[pc]; let class = insn.class(); - if class == bpf_insn::BPF_JMP || class == bpf_insn::BPF_JMP32 { + + let successors = Self::insn_successors(insn, pc, max_pc)?; + if successors.is_empty() { let op = insn.code & 0xf0; - if op != bpf_insn::BPF_EXIT && op != 0x80 { - let target = pc as isize + 1 + insn.off as isize; - if target < 0 || target as usize >= insns.len() { - warn!("bpf verifier: jump out of bounds at pc={pc} target={target}"); - return Err("jump out of bounds"); - } + if class == bpf_insn::BPF_JMP && op == bpf_insn::BPF_EXIT { + // terminal + } else { + warn!("bpf verifier: unreachable termination at pc={pc}"); + return Err("instruction has no valid successor"); + } + } + for s in successors { + if !visited[s] { + visited[s] = true; + stack.push(s); } } } + + let mut reachable_with_exit = false; + for (pc, &v) in visited.iter().enumerate() { + if !v { + continue; + } + let insn = &insns[pc]; + let class = insn.class(); + let op = insn.code & 0xf0; + if class == bpf_insn::BPF_JMP && op == bpf_insn::BPF_EXIT { + reachable_with_exit = true; + } + + if class == bpf_insn::BPF_ST || class == bpf_insn::BPF_STX || class == bpf_insn::BPF_LDX + { + let off = insn.off as isize; + let abs_off = if off >= 0 { off } else { -off }; + if abs_off as usize >= BPF_MAX_STACK { + warn!("bpf verifier: stack access out of bounds at pc={pc} off={off}"); + return Err("stack access out of bounds"); + } + } + + if class == bpf_insn::BPF_ALU || class == bpf_insn::BPF_ALU64 { + let alu_op = insn.alu_op(); + // Only reject immediate (BPF_K) division/modulo by zero. + // Register-based (BPF_X) division-by-zero is allowed: the + // interpreter returns 0 at runtime, matching Linux behavior. + if (alu_op == 0x30 || alu_op == 0x90) + && (insn.code & bpf_insn::BPF_X) == 0 + && insn.imm == 0 + { + warn!("bpf verifier: division/modulo by zero immediate at pc={pc}"); + return Err("division/modulo by zero"); + } + } + } + if !reachable_with_exit { + warn!("bpf verifier: no reachable BPF_EXIT instruction"); + return Err("no reachable BPF_EXIT instruction"); + } + Ok(()) } + fn insn_successors( + insn: &bpf_insn::BpfInsn, + pc: usize, + max_pc: usize, + ) -> Result, &'static str> { + let class = insn.class(); + let op = insn.code & 0xf0; + match class { + bpf_insn::BPF_ALU + | bpf_insn::BPF_ALU64 + | bpf_insn::BPF_ST + | bpf_insn::BPF_STX + | bpf_insn::BPF_LDX => { + let next = pc + 1; + if next >= max_pc { + return Ok(alloc::vec![]); + } + Ok(alloc::vec![next]) + } + bpf_insn::BPF_LD => { + if insn.is_ld_dw_imm() { + let next = pc + 2; + if next > max_pc { + return Ok(alloc::vec![]); + } + Ok(alloc::vec![next]) + } else { + Ok(alloc::vec![]) + } + } + bpf_insn::BPF_JMP | bpf_insn::BPF_JMP32 => { + if op == bpf_insn::BPF_EXIT { + return Ok(alloc::vec![]); + } + if op == 0x80 { + let next = pc + 1; + if next >= max_pc { + return Ok(alloc::vec![]); + } + return Ok(alloc::vec![next]); + } + let fallthrough = pc + 1; + let target = (pc as isize + 1 + insn.off as isize) as usize; + let mut succs = alloc::vec![]; + if fallthrough < max_pc { + succs.push(fallthrough); + } + if target < max_pc { + succs.push(target); + } else { + warn!("bpf verifier: jump out of bounds at pc={pc} target={target}"); + return Err("jump out of bounds"); + } + Ok(succs) + } + _ => Ok(alloc::vec![]), + } + } + fn execute(&self, insns: &[bpf_insn::BpfInsn], ctx: u64) -> Result { if insns.is_empty() { return Err("empty program"); @@ -1178,7 +2136,7 @@ impl BpfVm { if op == bpf_insn::BPF_EXIT { return Ok(regs[0]); } - if op == bpf_insn::BPF_CALL_OP { + if op == 0x80 { let helper_id = insn.imm as u32; if let Some(helper_fn) = self.helpers.get(&helper_id) { regs[0] = helper_fn(regs[1], regs[2], regs[3], regs[4], regs[5]); @@ -1186,6 +2144,20 @@ impl BpfVm { warn!("bpf: unknown helper {}", helper_id); regs[0] = u64::MAX; } + if helper_id == helper_id::TAIL_CALL && regs[0] == 0 { + let target_fd = { + let mut tail_target = BPF_TAIL_CALL_TARGET.lock(); + tail_target.take() + }; + if let Some(fd) = target_fd { + let guard = BPF_GLOBAL.lock(); + if let Some(target_prog) = guard.progs.get(&fd) { + let target_insns = target_prog.insns.clone(); + drop(guard); + return self.execute(&target_insns, regs[1]); + } + } + } pc += 1; continue; } @@ -1239,41 +2211,40 @@ impl BpfVm { } fn exec_alu(op: u8, dst: u64, src: u64, is_64: bool) -> u64 { - let mask = if is_64 { !0u64 } else { 0xffffffff }; - let result = match op { - bpf_insn::BPF_ADD => dst.wrapping_add(src), - bpf_insn::BPF_SUB => dst.wrapping_sub(src), - bpf_insn::BPF_MUL => dst.wrapping_mul(src), + let (result, mask) = match op { + bpf_insn::BPF_ADD => (dst.wrapping_add(src), !0), + bpf_insn::BPF_SUB => (dst.wrapping_sub(src), !0), + bpf_insn::BPF_MUL => (dst.wrapping_mul(src), !0), bpf_insn::BPF_DIV => { if src == 0 { return 0; } - dst / src + (dst / src, !0) } - bpf_insn::BPF_OR => dst | src, - bpf_insn::BPF_AND => dst & src, - bpf_insn::BPF_LSH => dst.wrapping_shl(src as u32), + bpf_insn::BPF_OR => (dst | src, !0), + bpf_insn::BPF_AND => (dst & src, !0), + bpf_insn::BPF_LSH => (dst.wrapping_shl(src as u32), !0), bpf_insn::BPF_RSH => { if is_64 { - dst >> src + (dst >> src, !0) } else { - (dst as u32 >> src as u32) as u64 + ((dst as u32 >> src as u32) as u64, 0xffffffff) } } - bpf_insn::BPF_NEG => (-(dst as i64)) as u64, + bpf_insn::BPF_NEG => ((-(dst as i64)) as u64, !0), bpf_insn::BPF_MOD => { if src == 0 { - return dst & mask; + return dst; } - dst % src + (dst % src, !0) } - bpf_insn::BPF_XOR => dst ^ src, - bpf_insn::BPF_MOV => src, + bpf_insn::BPF_XOR => (dst ^ src, !0), + bpf_insn::BPF_MOV => (src, !0), bpf_insn::BPF_ARSH => { if is_64 { - ((dst as i64) >> src) as u64 + (((dst as i64) >> src) as u64, !0) } else { - (((dst as i32) as i64) >> src) as u64 + ((((dst as i32) as i64) >> src) as u64, 0xffffffff) } } _ => return dst, @@ -1376,6 +2347,39 @@ impl BpfVm { } } +#[cfg(any( + target_arch = "x86_64", + target_arch = "riscv64", + target_arch = "aarch64" +))] +pub fn run_bpf_prog(fd: u32, ctx: u64) -> AxResult { + let (insns, prog_type, has_jit) = { + let guard = BPF_GLOBAL.lock(); + let prog = guard.progs.get(&fd).ok_or(AxError::BadFileDescriptor)?; + (prog.insns.clone(), prog.prog_type, prog.jitted.is_some()) + }; + let _ = prog_type; + + if has_jit { + let guard = BPF_GLOBAL.lock(); + let prog = guard.progs.get(&fd).ok_or(AxError::BadFileDescriptor)?; + if let Some(jitted) = &prog.jitted { + return Ok(jitted.execute(ctx)); + } + } + + let vm = BpfVm::new(); + vm.execute(&insns, ctx).map_err(|e| { + warn!("bpf: program execution failed: {e}"); + AxError::Io + }) +} + +#[cfg(not(any( + target_arch = "x86_64", + target_arch = "riscv64", + target_arch = "aarch64" +)))] pub fn run_bpf_prog(fd: u32, ctx: u64) -> AxResult { let (insns, prog_type) = { let guard = BPF_GLOBAL.lock(); @@ -1383,6 +2387,7 @@ pub fn run_bpf_prog(fd: u32, ctx: u64) -> AxResult { (prog.insns.clone(), prog.prog_type) }; let _ = prog_type; + let vm = BpfVm::new(); vm.execute(&insns, ctx).map_err(|e| { warn!("bpf: program execution failed: {e}"); @@ -1483,11 +2488,6 @@ pub fn sys_perf_event_open( crate::perf_event::sys_perf_event_open_impl(attr_uptr, pid, cpu, group_fd, flags) } -pub fn bpf_close_fd(fd: u32) -> AxResult<()> { - let mut guard = BPF_GLOBAL.lock(); - guard.close_fd(fd) -} - pub fn bpf_close_all_fds() { let mut guard = BPF_GLOBAL.lock(); guard.maps.clear(); @@ -1496,6 +2496,12 @@ pub fn bpf_close_all_fds() { guard.free_fds.clear(); } +#[allow(dead_code)] +pub fn bpf_close_fd(fd: u32) -> AxResult<()> { + let mut guard = BPF_GLOBAL.lock(); + guard.close_fd(fd) +} + #[allow(dead_code)] pub fn bpf_fd_exists(fd: u32) -> bool { let guard = BPF_GLOBAL.lock(); diff --git a/os/StarryOS/kernel/src/ebpf/rbpf_jit.rs b/os/StarryOS/kernel/src/ebpf/rbpf_jit.rs new file mode 100644 index 0000000000..de80a9b71d --- /dev/null +++ b/os/StarryOS/kernel/src/ebpf/rbpf_jit.rs @@ -0,0 +1,86 @@ +use alloc::collections::BTreeMap; +use core::alloc::Layout; + +const PAGE_SIZE: usize = 4096; +const NUM_PAGES: usize = 1; + +pub struct RbpfJitBuffer { + memory: *mut u8, + layout: Layout, + pub func: unsafe fn(*mut u8, usize, *mut u8, usize, usize, usize) -> u64, +} + +unsafe impl Send for RbpfJitBuffer {} +unsafe impl Sync for RbpfJitBuffer {} + +impl Drop for RbpfJitBuffer { + fn drop(&mut self) { + unsafe { + alloc::alloc::dealloc(self.memory, self.layout); + } + } +} + +impl core::fmt::Debug for RbpfJitBuffer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "RbpfJitBuffer({:p})", self.memory) + } +} + +pub fn try_jit_compile( + insns: &[crate::ebpf::bpf_insn::BpfInsn], + helpers: &BTreeMap, +) -> Option { + let size = NUM_PAGES * PAGE_SIZE; + let layout = Layout::from_size_align(size, PAGE_SIZE).ok()?; + let memory = unsafe { alloc::alloc::alloc(layout) }; + if memory.is_null() { + return None; + } + + // We need to make the memory executable in tgoskits, but maybe it's already RWX? + // tgoskits memory allocation usually returns RW memory. We might need to map it as RX? + // But since it's an OS, kernel heap might be RWX, or we need to call some mprotect equivalent. + // Let's assume kernel heap allows execution for now (or we can use axalloc). + + let exec_slice = unsafe { core::slice::from_raw_parts_mut(memory, size) }; + + // Convert BpfInsn to u8 slice + let prog_bytes = unsafe { + core::slice::from_raw_parts( + insns.as_ptr() as *const u8, + insns.len() * core::mem::size_of::(), + ) + }; + + // rbpf helpers use hashbrown::HashMap, but wait, BTreeMap is not HashMap! + // rbpf::ebpf::Helper is `fn(u64, u64, u64, u64, u64) -> u64`. + // Wait, the helpers we pass in `try_jit_compile` are `BTreeMap`. + // We need to convert it to hashbrown::HashMap. + let mut rbpf_helpers = hashbrown::HashMap::new(); + for (&k, &v) in helpers.iter() { + rbpf_helpers.insert(k, v); + } + + match rbpf::jit::JitMemory::new(prog_bytes, exec_slice, &rbpf_helpers, false, false) { + Ok(jit) => { + let func = jit.get_prog(); + Some(RbpfJitBuffer { + memory, + layout, + func, + }) + } + Err(e) => { + ax_log::warn!("rbpf JIT compilation failed: {:?}", e); + unsafe { alloc::alloc::dealloc(memory, layout) }; + None + } + } +} + +impl RbpfJitBuffer { + pub fn execute(&self, ctx: u64) -> u64 { + unsafe { (self.func)(ctx as *mut u8, 0, ctx as *mut u8, 0, 0, 0) } + } +}