diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3796fc505..c3d691a6a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -357,6 +357,15 @@ jobs: apk_region: us limit_to_owner: "" main_pr_only: false + - name: Test starry aarch64 buddy-slab qemu + use_container: true + runs_on: '["ubuntu-latest"]' + command: cargo xtask starry test qemu --arch aarch64 -c test-clone-files-race-buddy-slab + cache_key: test-starry-aarch64-buddy-slab + container_image: base + apk_region: us + limit_to_owner: "" + main_pr_only: false - name: Test starry loongarch64 qemu use_container: true runs_on: '["ubuntu-latest"]' diff --git a/Cargo.lock b/Cargo.lock index 94b4877d85..88a87e4e53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -529,6 +529,8 @@ dependencies = [ name = "arceos-wait-queue-remote-wake" version = "0.3.1" dependencies = [ + "ax-driver", + "ax-hal", "ax-std", ] @@ -1813,7 +1815,6 @@ name = "axplat-dyn" version = "0.6.2" dependencies = [ "anyhow", - "ax-alloc", "ax-config-macros", "ax-cpu", "ax-driver", @@ -2288,11 +2289,11 @@ dependencies = [ [[package]] name = "buddy-slab-allocator" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a844607dec426282e05649372acdc1170b08ba57879be9d3216184ed6dbfd3b" +version = "0.4.1" dependencies = [ + "divan", "log", + "rand 0.10.1", "spin", ] @@ -2472,6 +2473,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -2544,6 +2556,7 @@ dependencies = [ "anstyle", "clap_lex", "strsim 0.11.1", + "terminal_size", ] [[package]] @@ -2651,6 +2664,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "console" version = "0.16.3" @@ -3362,6 +3381,31 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "divan" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a405457ec78b8fe08b0e32b4a3570ab5dff6dd16eb9e76a5ee0a9d9cbd898933" +dependencies = [ + "cfg-if", + "clap", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "dma-api" version = "0.7.3" @@ -4097,6 +4141,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -4747,9 +4792,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6835eea34fb6321b9b3aa7b685c2b433948c09447e389dc017fdf687d5d11e65" +checksum = "30457d51cb0e68ee18184b30cd9eb8e1602a20837c321f6ea9706b94f1c681c3" dependencies = [ "jiff-static", "log", @@ -4760,9 +4805,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c22e04db9c58f5136eb1757f3d5c49a7b187f49e52185228cbd2f5acdfcc08c" +checksum = "05f86e4f0326c61ae6c00b04d9009aaeda644d0b5bdfbf6c67247f492f42b3f3" dependencies = [ "proc-macro2", "quote", @@ -6284,6 +6329,8 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ + "chacha20", + "getrandom 0.4.2", "rand_core 0.10.1", ] @@ -6701,6 +6748,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -7873,7 +7926,6 @@ dependencies = [ "ax-errno", "ax-feat", "ax-fs-ng", - "ax-hal", "ax-input", "ax-io", "ax-ipi", @@ -7985,10 +8037,8 @@ dependencies = [ "ax-driver", "ax-feat", "ax-hal", - "ax-plat-riscv64-sg2002", "axbuild", "axplat-dyn", - "axplat-riscv64-visionfive2", "clap", "starry-kernel", "tokio", @@ -8184,6 +8234,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "terminfo" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 6f5a38e290..b1c5ffd97e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "components/axvm", "components/axvmconfig", "components/bitmap-allocator", + "components/buddy-slab-allocator", "components/cap_access", "components/cpumask", "components/crate_interface", @@ -371,7 +372,7 @@ bitflags = "2.11" byte-unit = { version = "5", default-features = false, features = ["byte"] } derive_more = { version = "2.1", default-features = false, features = ["full"] } bindgen = "0.72" -buddy-slab-allocator = "0.4" +buddy-slab-allocator = { version = "0.4", path = "components/buddy-slab-allocator" } cfg-if = "1.0" chrono = { version = "0.4", default-features = false } enum_dispatch = "0.3" diff --git a/components/buddy-slab-allocator/.github/workflows/check.yml b/components/buddy-slab-allocator/.github/workflows/check.yml new file mode 100644 index 0000000000..bc2347e8c5 --- /dev/null +++ b/components/buddy-slab-allocator/.github/workflows/check.yml @@ -0,0 +1,94 @@ +name: Quality Checks + +on: + workflow_call: + +jobs: + fmt: + name: Format Check + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-02-25 + components: rustfmt + + - name: Check code format + run: cargo fmt --all -- --check + + clippy: + name: Clippy Lint + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-02-25 + components: clippy + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Run clippy + run: cargo clippy --all-targets -- -D warnings + + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-02-25 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Build library + run: cargo build + + doc: + name: Documentation + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-02-25 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Build documentation + run: cargo doc --no-deps + + bench-check: + name: Bench Compile Check + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-02-25 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Check benchmarks + run: cargo check --benches diff --git a/components/buddy-slab-allocator/.github/workflows/ci.yml b/components/buddy-slab-allocator/.github/workflows/ci.yml new file mode 100644 index 0000000000..f08e0782bb --- /dev/null +++ b/components/buddy-slab-allocator/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: CI + +on: + pull_request: + push: + branches-ignore: + - main + tags-ignore: + - "*" + +permissions: + contents: read + +jobs: + check: + uses: ./.github/workflows/check.yml + + test: + uses: ./.github/workflows/test.yml + needs: check diff --git a/components/buddy-slab-allocator/.github/workflows/release-plz.yml b/components/buddy-slab-allocator/.github/workflows/release-plz.yml new file mode 100644 index 0000000000..0d36a95690 --- /dev/null +++ b/components/buddy-slab-allocator/.github/workflows/release-plz.yml @@ -0,0 +1,72 @@ +name: Release-plz + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + check: + uses: ./.github/workflows/check.yml + + test: + uses: ./.github/workflows/test.yml + needs: check + + release-plz-release: + name: Release-plz release + runs-on: ubuntu-latest + needs: [check, test] + permissions: + contents: write + pull-requests: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Run release-plz + uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128 + with: + command: release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # If you switch to crates.io trusted publishing, remove this line + # and keep `id-token: write` above. + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + release-plz-pr: + name: Release-plz PR + runs-on: ubuntu-latest + needs: [check, test] + permissions: + contents: write + pull-requests: write + concurrency: + group: release-plz-${{ github.ref }} + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Run release-plz + uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128 + with: + command: release-pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/components/buddy-slab-allocator/.github/workflows/test.yml b/components/buddy-slab-allocator/.github/workflows/test.yml new file mode 100644 index 0000000000..45ecba068f --- /dev/null +++ b/components/buddy-slab-allocator/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: Test + +on: + workflow_call: + +jobs: + host-test: + name: Run Logic Tests (Host) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 # v6.0.2 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Run all tests + run: cargo test + + - name: Run tests serially + run: cargo test -- --test-threads=1 + + - name: Run ignored stress tests + run: cargo test --test stress_test -- --ignored --nocapture diff --git a/components/buddy-slab-allocator/.gitignore b/components/buddy-slab-allocator/.gitignore new file mode 100644 index 0000000000..ed768f33ec --- /dev/null +++ b/components/buddy-slab-allocator/.gitignore @@ -0,0 +1,2 @@ +/target/ +Cargo.lock \ No newline at end of file diff --git a/components/buddy-slab-allocator/CHANGELOG.md b/components/buddy-slab-allocator/CHANGELOG.md new file mode 100644 index 0000000000..1e19def0e6 --- /dev/null +++ b/components/buddy-slab-allocator/CHANGELOG.md @@ -0,0 +1,150 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Changed +- Removed the external allocator-interface dependency and restored crate-local `AllocError` / `AllocResult` +- Stopped exporting the generic allocator traits `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` +- Public allocator APIs now favor concrete allocator methods directly instead of requiring trait imports +- Removed `alloc_pages_at` because the current buddy/slab architecture does not stably support fixed-address allocation + +### Removed +- Removed the external allocator-interface crate from `[dependencies]` + +### Migration Notes +- Replace imports of `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` with direct method calls on `BuddyPageAllocator`, `CompositePageAllocator`, `SlabByteAllocator`, and `GlobalAllocator` +- `PageAllocatorForSlab` remains available for wiring `SlabByteAllocator` to a page allocator + +## [0.2.0] - 2026-03-05 + +### Added +- Added an external allocator-interface dependency + +### Changed +- `AllocError`, `AllocResult`, `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` are now re-exported from the external allocator-interface crate instead of being defined locally +- Updated Rust toolchain to `nightly-2026-02-25` +- Benchmarks no longer require `--features bench`; `criterion` and `rand` moved to `[dev-dependencies]` + +### Removed +- Removed locally defined allocator trait and error type definitions (now provided by the external allocator-interface crate) +- Removed the deprecated `bench` feature flag + +## [0.1.1] - 2026-02-06 + +### Added +- Comprehensive benchmark suite with criterion +- Benchmark documentation in Chinese (`benches/README_CN.md`) +- Benchmark workflow in CI with Rust 1.93.0 toolchain for compatibility + +### Changed +- Increased `MAX_ZONES` from 16 to 32 for more flexible memory region management +- Improved documentation in README with benchmark usage instructions + + +## [0.1.0] - 2025-01-30 + +### Added +- Buddy page allocator implementation for page-level allocation +- Slab byte allocator implementation for small object allocation +- Composite page allocator for unified multi-region page allocation +- Global allocator that coordinates page and byte allocators +- Automatic allocation size selection (≤2048 bytes uses slab, >2048 bytes uses page) +- Zero `std` dependency (`#![no_std]`) for embedded/kernel environments +- Optional `log` feature for logging allocation events +- Optional `tracking` feature for memory usage statistics +- `AddrTranslator` trait for virtual-to-physical address translation +- `BaseAllocator`, `ByteAllocator`, `PageAllocator`, and `IdAllocator` traits +- Comprehensive error handling with `AllocError` enum + +### Features +- O(1) time complexity for small object allocation +- Buddy algorithm for efficient page allocation with automatic merging +- Support for multiple memory regions +- Flexible page size configuration (const generic) +- Memory fragmentation reduction through slab allocation +- Statistics tracking for debugging and profiling + +### Documentation +- Complete API documentation with examples +- README with bilingual (English/Chinese) documentation +- Inline documentation for all public APIs + +### Testing +- Integration tests for page allocator +- Integration tests for slab allocator +- Integration tests for global allocator +- DMA32 pages test cases +- Comprehensive edge case coverage + +## [Unreleased]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...HEAD + +## [0.3.1](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.3.0...v0.3.1) - 2026-04-10 + +### Other + +- Refactor global allocator to support singleton pattern and improve slab management +- Implement per-CPU slab allocator with object-safe interface +- bump version to 0.3.1 and update changelog for enhancements +- enhance region layout handling and alignment in allocator methods + +## [0.3.1](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.3.0...v0.3.1) - 2026-04-09 + +### Other + +- enhance region layout handling and alignment in allocator methods + +## [0.3.0](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...v0.3.0) - 2026-04-09 + +### Added + +- enhance slab allocation by reclaiming full slabs with remote frees and add integration tests for cross-CPU deallocation +- implement Default trait for BuddyAllocator, GlobalAllocator, and SlabAllocator +- remove alloc_pages_at method from buddy allocator and related components; simplify allocation logic +- refactor allocator interfaces and remove deprecated dependencies; update documentation and examples +- enhance logging support by integrating log crate and updating documentation + +### Other + +- add unsafe blocks for improved safety checks in allocator methods +- update changelog for version 0.2.1 and modify Cargo.toml for version bump to 0.3.0 +- format code for better readability in common.rs +- streamline region initialization by introducing SectionInitSpec for better clarity and maintainability +- Refactor GlobalAllocator to support multiple managed sections +- Refactor stress tests for allocator stability +- update allocator initialization to use a mutable slice instead of separate start and size parameters +- Refactor slab allocator benchmarks and improve global allocator initialization +- Refactor integration and stress tests for buddy-slab-allocator +- Refactor integration and stress tests for allocator +- Refactor benchmarks and remove stability tests +- update workflows and dependencies; migrate to actions/checkout@v6 and rand v0.10 + +## [0.2.1](https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.2.0...v0.2.1) - 2026-04-09 + +### Added + +- enhance slab allocation by reclaiming full slabs with remote frees and add integration tests for cross-CPU deallocation +- implement Default trait for BuddyAllocator, GlobalAllocator, and SlabAllocator +- remove alloc_pages_at method from buddy allocator and related components; simplify allocation logic +- refactor allocator interfaces and remove deprecated dependencies; update documentation and examples +- enhance logging support by integrating log crate and updating documentation + +### Other + +- format code for better readability in common.rs +- streamline region initialization by introducing SectionInitSpec for better clarity and maintainability +- Refactor GlobalAllocator to support multiple managed sections +- Refactor stress tests for allocator stability +- update allocator initialization to use a mutable slice instead of separate start and size parameters +- Refactor slab allocator benchmarks and improve global allocator initialization +- Refactor integration and stress tests for buddy-slab-allocator +- Refactor integration and stress tests for allocator +- Refactor benchmarks and remove stability tests +- update workflows and dependencies; migrate to actions/checkout@v6 and rand v0.10 +## [0.2.0]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.1.1...v0.2.0 +## [0.1.1]: https://github.com/arceos-hypervisor/buddy-slab-allocator/compare/v0.1.0...v0.1.1 +## [0.1.0]: https://github.com/arceos-hypervisor/buddy-slab-allocator/releases/tag/v0.1.0 diff --git a/components/buddy-slab-allocator/Cargo.toml b/components/buddy-slab-allocator/Cargo.toml new file mode 100644 index 0000000000..8ab3be9c72 --- /dev/null +++ b/components/buddy-slab-allocator/Cargo.toml @@ -0,0 +1,44 @@ +[package] +authors = [ + "Song Zhiyong ", + "周睿 ", +] +autobenches = false +categories = [ + "memory-management", + "no-std", + "embedded", +] +description = "Memory allocator with Buddy and Slab allocation" +documentation = "https://docs.rs/buddy-slab-allocator" +edition = "2024" +keywords = [ + "buddy", + "slab", + "allocator", +] +license = "Apache-2.0" +name = "buddy-slab-allocator" +readme = "README.md" +repository = "https://github.com/arceos-hypervisor/buddy-slab-allocator" +version = "0.4.1" + +[[bench]] +harness = false +name = "buddy_allocator" + +[[bench]] +harness = false +name = "slab_allocator" + +[[bench]] +harness = false +name = "global_allocator" + +[dependencies] +log = "0.4" +spin = "0.10" + +[dev-dependencies] +divan = "0.1.21" +rand = { version = "0.10", features = ["std_rng"] } diff --git a/components/buddy-slab-allocator/LICENSE.Apache2 b/components/buddy-slab-allocator/LICENSE.Apache2 new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/components/buddy-slab-allocator/LICENSE.Apache2 @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/components/buddy-slab-allocator/README.md b/components/buddy-slab-allocator/README.md new file mode 100644 index 0000000000..e22c1aa971 --- /dev/null +++ b/components/buddy-slab-allocator/README.md @@ -0,0 +1,251 @@ +# buddy-slab-allocator + +A `no_std` two-level allocator for kernel and embedded environments, combining a buddy page allocator with per-CPU slab allocators. + +## Overview + +The current implementation is built from three layers: + +1. `BuddyAllocator` + Manages one or more virtual memory sections in page units, with power-of-two splitting and merging. +2. `SlabAllocator` + Manages small objects up to 2048 bytes with fixed size classes. +3. `GlobalAllocator` + Combines the two, routing small allocations to per-CPU slab caches and large allocations to buddy pages. + +The design details are documented in [docs/design.md](docs/design.md). + +## Architecture + +```mermaid +flowchart TD + GA["GlobalAllocator"] --> B["SpinMutex"] + GA --> EI["eii hooks"] + EI --> SP["slab_pool()"] + + B --> PM["PageMeta[]"] + B --> FL["free_lists by order"] + + SP --> SA["SlabPoolTrait"] + SA --> SC["SlabCache x 9"] + SC --> P["partial"] + SC --> F["full"] + SC --> E["empty"] + SC --> H["SlabPageHeader"] +``` + +### Allocation routing + +```mermaid +flowchart TD + A["GlobalAllocator::alloc(layout)"] --> B{"size <= 2048 and align <= 2048?"} + B -- Yes --> C["Use current CPU slab allocator"] + C --> D{"Allocated from slab?"} + D -- Yes --> E["Return object pointer"] + D -- No --> F["Allocate pages from buddy as a new slab"] + F --> G["add_slab and retry"] + G --> E + B -- No --> H["Allocate pages directly from buddy"] + H --> I["Return page-backed pointer"] +``` + +### Cross-CPU free path + +```mermaid +sequenceDiagram + participant CPU1 as current CPU + participant H as SlabPageHeader + participant CPU0 as owner CPU + participant S as owner SlabAllocator + + CPU1->>H: remote_free(obj_addr) + Note right of H: lock-free CAS push to remote_free_head + CPU1-->>CPU1: return immediately + + CPU0->>S: next local alloc/dealloc under slab lock + S->>H: drain_remote_frees() + H-->>S: slots moved back into local bitmap +``` + +## Features + +- Buddy page allocation with splitting and merging +- Dynamic hot-add of managed regions via `add_region` +- Slab allocation for 9 size classes: `8..=2048` +- Per-CPU slab caches +- Lock-free cross-CPU remote frees +- DMA32 / lowmem page allocation via `alloc_pages_lowmem` +- `no_std` friendly +- Built-in `log` integration +- Standalone `BuddyAllocator` and `SlabAllocator` usage + +## Add dependency + +```toml +[dependencies] +buddy-slab-allocator = "0.2.0" +``` + +## Using `GlobalAllocator` + +```rust +#![feature(extern_item_impls)] + +use buddy_slab_allocator::eii::{slab_pool_impl, virt_to_phys_impl}; +use buddy_slab_allocator::{GlobalAllocator, PerCpuSlab, SlabPoolTrait, StaticSlabPool}; +use core::alloc::Layout; + +const PAGE_SIZE: usize = 0x1000; + +fn current_cpu_id() -> usize { + 0 +} + +static SLAB_POOL: StaticSlabPool = + StaticSlabPool::new([PerCpuSlab::new(0)], current_cpu_id); + +#[virt_to_phys_impl] +fn virt_to_phys(vaddr: usize) -> usize { + vaddr +} + +#[slab_pool_impl] +fn slab_pool() -> &'static dyn SlabPoolTrait { + &SLAB_POOL +} + +let allocator = GlobalAllocator::::new(); +let region_start = 0x8000_0000 as *mut u8; +let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; + +unsafe { + allocator.init(region).unwrap(); +} + +let layout = Layout::from_size_align(64, 8).unwrap(); +let ptr = allocator.alloc(layout).unwrap(); + +unsafe { + allocator.dealloc(ptr, layout); +} + +// More memory can be added later. +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + allocator.add_region(extra_region).unwrap(); +} +``` + +`GlobalAllocator` is designed as a singleton-style system allocator: only one live +instance should be initialized at a time. + +## Using Buddy and Slab separately + +For lower-level control, the two building blocks can be used directly. + +```rust +use buddy_slab_allocator::{ + BuddyAllocator, SlabAllocResult, SlabAllocator, SlabDeallocResult, +}; +use core::alloc::Layout; + +const PAGE_SIZE: usize = 0x1000; +let region_start = 0x8000_0000 as *mut u8; +let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; + +let mut buddy = BuddyAllocator::::new(); +unsafe { + buddy.init(region).unwrap(); +} + +let mut slab = SlabAllocator::::new(); +let layout = Layout::from_size_align(64, 8).unwrap(); + +let ptr = loop { + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => break ptr, + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + } + } +}; + +match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + buddy.dealloc_pages(base, pages); + } +} + +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + buddy.add_region(extra_region).unwrap(); +} +``` + +## Public API summary + +- `GlobalAllocator` + High-level allocator facade that can also implement `GlobalAlloc`, and supports `add_region`, `managed_section_count`, `managed_section`, `managed_bytes`, and `allocated_bytes`. +- `BuddyAllocator` + Standalone multi-section page allocator, supporting `init`, `add_region`, section queries, `managed_bytes`, and `allocated_bytes`. +- `ManagedSection` + Read-only summary for one managed section. +- `SlabAllocator` + Standalone slab allocator. +- `SizeClass` + Fixed object size classes used by slab. +- `SlabAllocResult` + `Allocated(ptr)` or `NeedsSlab { size_class, pages }`. +- `SlabDeallocResult` + `Done` or `FreeSlab { base, pages }`. +- `SlabPoolTrait` + System-global slab pool interface used by `GlobalAllocator`, exposing + object-safe `current_slab()` / `owner_slab()` primitives with default + routing for `alloc` / `add_slab` / `dealloc`. +- `SlabPoolExt` + Callback-style helpers: `with_current_slab()` and `with_owner_slab()`. +- `eii` + Declares `slab_pool()` and `virt_to_phys()` for platform integration. + +`managed_bytes` counts only allocatable heap bytes and excludes region-prefix metadata. +`allocated_bytes` is backend page occupancy, not the exact sum of requested `layout.size()`. + +## Testing + +```bash +# Run normal tests +cargo test + +# Run tests serially +cargo test -- --test-threads=1 + +# Run ignored stress tests +cargo test --test stress_test -- --ignored --nocapture + +# Check benchmarks compile +cargo check --benches + +# Run benchmarks +cargo bench +``` + +More test notes are in [tests/README.md](tests/README.md). + +## License + +Licensed under [Apache-2.0](LICENSE). diff --git a/components/buddy-slab-allocator/README_CN.md b/components/buddy-slab-allocator/README_CN.md new file mode 100644 index 0000000000..7b8d276111 --- /dev/null +++ b/components/buddy-slab-allocator/README_CN.md @@ -0,0 +1,249 @@ +# buddy-slab-allocator 内存分配器 + +这是一个面向内核和嵌入式环境的 `no_std` 两级内存分配器，结合了 buddy 页分配器与 per-CPU slab 小对象分配器。 + +## 总览 + +当前实现由三层组成： + +1. `BuddyAllocator` + 管理一个或多个虚拟内存 section，支持按 2 的幂分裂与合并。 +2. `SlabAllocator` + 管理 `<= 2048` 字节的小对象，采用固定 size class。 +3. `GlobalAllocator` + 将两者组合起来，对小对象走 per-CPU slab，对大对象走 buddy 页分配。 + +更完整的设计细节见 [docs/design.md](docs/design.md)。 + +## 架构图 + +```mermaid +flowchart TD + GA["GlobalAllocator"] --> B["SpinMutex"] + GA --> EI["eii hooks"] + EI --> SP["slab_pool()"] + + B --> PM["PageMeta[]"] + B --> FL["按 order 的 free_lists"] + + SP --> SA["SlabPoolTrait"] + SA --> SC["9 个 SlabCache"] + SC --> P["partial"] + SC --> F["full"] + SC --> E["empty"] + SC --> H["SlabPageHeader"] +``` + +### 分配路由 + +```mermaid +flowchart TD + A["GlobalAllocator::alloc(layout)"] --> B{"size <= 2048 且 align <= 2048?"} + B -- 是 --> C["走当前 CPU 的 slab allocator"] + C --> D{"slab 直接命中?"} + D -- 是 --> E["返回对象指针"] + D -- 否 --> F["从 buddy 申请新 slab 页"] + F --> G["add_slab 后重试"] + G --> E + B -- 否 --> H["直接走 buddy 页分配"] + H --> I["返回页级指针"] +``` + +### 跨 CPU 释放 + +```mermaid +sequenceDiagram + participant CPU1 as 当前 CPU + participant H as SlabPageHeader + participant CPU0 as owner CPU + participant S as owner SlabAllocator + + CPU1->>H: remote_free(obj_addr) + Note right of H: 无锁 CAS 推入 remote_free_head + CPU1-->>CPU1: 立即返回 + + CPU0->>S: 后续本地 alloc/dealloc + S->>H: drain_remote_frees() + H-->>S: 远程释放对象回收到本地 bitmap +``` + +## 特性 + +- Buddy 页分配，支持拆分与合并 +- 支持通过 `add_region` 动态追加可管理 region +- Slab 小对象分配，固定 9 个 size class：`8..=2048` +- per-CPU slab cache +- 跨 CPU 释放走 lock-free remote free +- 支持 `alloc_pages_lowmem` 低地址页分配 +- 适合 `no_std` +- 内置 `log` 日志支持 +- 可分别独立使用 `BuddyAllocator` 与 `SlabAllocator` + +## 添加依赖 + +```toml +[dependencies] +buddy-slab-allocator = "0.2.0" +``` + +## 使用 `GlobalAllocator` + +```rust +#![feature(extern_item_impls)] + +use buddy_slab_allocator::eii::{slab_pool_impl, virt_to_phys_impl}; +use buddy_slab_allocator::{GlobalAllocator, PerCpuSlab, SlabPoolTrait, StaticSlabPool}; +use core::alloc::Layout; + +const PAGE_SIZE: usize = 0x1000; + +fn current_cpu_id() -> usize { + 0 +} + +static SLAB_POOL: StaticSlabPool = + StaticSlabPool::new([PerCpuSlab::new(0)], current_cpu_id); + +#[virt_to_phys_impl] +fn virt_to_phys(vaddr: usize) -> usize { + vaddr +} + +#[slab_pool_impl] +fn slab_pool() -> &'static dyn SlabPoolTrait { + &SLAB_POOL +} + +let allocator = GlobalAllocator::::new(); +let region_start = 0x8000_0000 as *mut u8; +let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; + +unsafe { + allocator.init(region).unwrap(); +} + +let layout = Layout::from_size_align(64, 8).unwrap(); +let ptr = allocator.alloc(layout).unwrap(); + +unsafe { + allocator.dealloc(ptr, layout); +} + +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + allocator.add_region(extra_region).unwrap(); +} +``` + +`GlobalAllocator` 按系统级单例分配器设计：同一时刻只应初始化一套正在使用的实例。 + +## 分别使用 Buddy 与 Slab + +如果需要更底层的控制，也可以分别使用这两个组件。 + +```rust +use buddy_slab_allocator::{ + BuddyAllocator, SlabAllocResult, SlabAllocator, SlabDeallocResult, +}; +use core::alloc::Layout; + +const PAGE_SIZE: usize = 0x1000; +let region_start = 0x8000_0000 as *mut u8; +let region_size = 16 * 1024 * 1024; +let region = unsafe { core::slice::from_raw_parts_mut(region_start, region_size) }; + +let mut buddy = BuddyAllocator::::new(); +unsafe { + buddy.init(region).unwrap(); +} + +let mut slab = SlabAllocator::::new(); +let layout = Layout::from_size_align(64, 8).unwrap(); + +let ptr = loop { + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => break ptr, + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + } + } +}; + +match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + buddy.dealloc_pages(base, pages); + } +} + +let extra_region_start = 0x9000_0000 as *mut u8; +let extra_region_size = 8 * 1024 * 1024; +let extra_region = unsafe { + core::slice::from_raw_parts_mut(extra_region_start, extra_region_size) +}; + +unsafe { + buddy.add_region(extra_region).unwrap(); +} +``` + +## 公开 API 摘要 + +- `GlobalAllocator` + 高层门面，也可以作为 `GlobalAlloc` 使用，并支持 `add_region`、`managed_section_count`、`managed_section`、`managed_bytes` 与 `allocated_bytes`。 +- `BuddyAllocator` + 独立多 section 页分配器，支持 `init`、`add_region`、section 查询、`managed_bytes` 与 `allocated_bytes`。 +- `ManagedSection` + 单个 managed section 的只读摘要。 +- `SlabAllocator` + 独立 slab 分配器。 +- `SizeClass` + slab 使用的固定对象尺寸类。 +- `SlabAllocResult` + `Allocated(ptr)` 或 `NeedsSlab { size_class, pages }`。 +- `SlabDeallocResult` + `Done` 或 `FreeSlab { base, pages }`。 +- `SlabPoolTrait` + `GlobalAllocator` 使用的系统级 slab 池接口，暴露 object-safe 的 + `current_slab()` / `owner_slab()` 原语，并提供 `alloc` / `add_slab` / + `dealloc` 默认路由。 +- `SlabPoolExt` + callback 风格辅助接口：`with_current_slab()` 和 `with_owner_slab()`。 +- `eii` + 声明 `slab_pool()` 与 `virt_to_phys()`，供平台侧实现。 + +`managed_bytes` 只统计可分配 heap，不包含 region 前缀 metadata。 +`allocated_bytes` 表示后端页占用，不是用户请求的 `layout.size()` 精确求和。 + +## 测试 + +```bash +# 常规测试 +cargo test + +# 串行执行，便于排查问题 +cargo test -- --test-threads=1 + +# 运行忽略的压力测试 +cargo test --test stress_test -- --ignored --nocapture + +# 检查 benchmark 是否可编译 +cargo check --benches + +# 运行 benchmark +cargo bench +``` + +更多测试说明见 [tests/README.md](tests/README.md)。 + +## 许可证 + +采用 [Apache-2.0](LICENSE) 许可证。 diff --git a/components/buddy-slab-allocator/benches/README_CN.md b/components/buddy-slab-allocator/benches/README_CN.md new file mode 100644 index 0000000000..586e3b1507 --- /dev/null +++ b/components/buddy-slab-allocator/benches/README_CN.md @@ -0,0 +1,45 @@ +# Benchmark 使用说明 + +本目录包含基于 `divan` 的 benchmark 套件，围绕当前 crate 的三层结构组织： + +- `buddy_allocator.rs` + `BuddyAllocator` 的页分配、对齐分配、碎片恢复、随机页 workload +- `slab_allocator.rs` + `SlabAllocator` 的 size class alloc/free、hot reuse、mixed-size batch、steady-state recycle +- `global_allocator.rs` + `GlobalAllocator` 的小对象、大对象、页接口、混合 workload、cross-CPU free cycle + +共享的 host-side harness 放在 `common.rs`，负责： + +- region / metadata 分配 +- buddy / slab / global 初始化 +- 固定随机种子 +- mock EII 接口 + +## 运行方式 + +```bash +# 仅检查 benchmark 是否可编译 +cargo check --benches + +# 运行全部 benchmark +cargo bench + +# 单独运行某个 suite +cargo bench --bench buddy_allocator +cargo bench --bench slab_allocator +cargo bench --bench global_allocator +``` + +## 设计原则 + +- 使用 `divan::Bencher` 和 `divan::black_box` +- 不再保留旧的 `criterion` 代码路径 +- benchmark 只使用当前公开 API，不依赖历史类型名 +- 尽量让每次迭代在闭环内恢复 allocator 状态，减少跨迭代污染 +- workload 使用固定模式或固定随机种子，便于复现 + +## CI 策略 + +CI 仅执行 `cargo check --benches`，确保 benchmark 工程持续可编译。 +真实性能测量和结果对比保留在本地执行。 diff --git a/components/buddy-slab-allocator/benches/buddy_allocator.rs b/components/buddy-slab-allocator/benches/buddy_allocator.rs new file mode 100644 index 0000000000..82c745b9a9 --- /dev/null +++ b/components/buddy-slab-allocator/benches/buddy_allocator.rs @@ -0,0 +1,105 @@ +//! Benchmarks for the buddy page allocator. + +mod common; + +use common::{ + BuddyHarness, FRAGMENTATION_PAGES, HEAP_SIZE, OPERATIONS_PER_BATCH, PAGE_SIZE, seeded_rng, +}; +use divan::{Bencher, black_box}; +use rand::RngExt; + +fn main() { + divan::main(); +} + +#[divan::bench_group] +mod buddy { + use super::*; + + #[divan::bench(args = [1usize, 2, 4, 16, 64])] + fn page_alloc_free(bencher: Bencher, pages: usize) { + let mut harness = BuddyHarness::new(HEAP_SIZE); + + bencher.bench_local(|| { + let addr = harness + .allocator + .alloc_pages(black_box(pages), PAGE_SIZE) + .unwrap(); + harness.allocator.dealloc_pages(addr, pages); + black_box(addr) + }); + } + + #[divan::bench(args = [PAGE_SIZE, PAGE_SIZE * 2, PAGE_SIZE * 4, PAGE_SIZE * 16])] + fn aligned_page_alloc_free(bencher: Bencher, align: usize) { + let mut harness = BuddyHarness::new(HEAP_SIZE); + + bencher.bench_local(|| { + let addr = harness + .allocator + .alloc_pages(black_box(4), black_box(align)) + .unwrap(); + harness.allocator.dealloc_pages(addr, 4); + black_box(addr) + }); + } + + #[divan::bench] + fn fragmentation_recovery_cycle(bencher: Bencher) { + let mut harness = BuddyHarness::new(HEAP_SIZE); + + bencher.bench_local(|| { + let mut addrs = Vec::with_capacity(FRAGMENTATION_PAGES); + for _ in 0..FRAGMENTATION_PAGES { + addrs.push(harness.allocator.alloc_pages(1, PAGE_SIZE).unwrap()); + } + + for idx in (0..addrs.len()).step_by(2) { + harness.allocator.dealloc_pages(addrs[idx], 1); + } + + let large = harness.allocator.alloc_pages(64, PAGE_SIZE).unwrap(); + harness.allocator.dealloc_pages(large, 64); + + for idx in (1..addrs.len()).step_by(2) { + harness.allocator.dealloc_pages(addrs[idx], 1); + } + + black_box(large) + }); + } + + #[divan::bench] + fn random_page_workload(bencher: Bencher) { + let mut harness = BuddyHarness::new(HEAP_SIZE); + let mut rng = seeded_rng(); + let plan: Vec<(bool, usize, usize)> = (0..OPERATIONS_PER_BATCH) + .map(|_| { + let allocate = rng.random_bool(0.65); + let pages = 1usize << rng.random_range(0..=4); + let free_hint = rng.random_range(0..OPERATIONS_PER_BATCH.max(1)); + (allocate, pages, free_hint) + }) + .collect(); + + bencher.bench_local(|| { + let mut active = Vec::new(); + + for &(allocate, pages, free_hint) in &plan { + if allocate || active.is_empty() { + if let Ok(addr) = harness.allocator.alloc_pages(pages, PAGE_SIZE) { + active.push((addr, pages)); + } + } else { + let idx = free_hint % active.len(); + let (addr, count) = active.swap_remove(idx); + harness.allocator.dealloc_pages(addr, count); + } + } + + for (addr, count) in active { + harness.allocator.dealloc_pages(addr, count); + } + }); + } +} diff --git a/components/buddy-slab-allocator/benches/common.rs b/components/buddy-slab-allocator/benches/common.rs new file mode 100644 index 0000000000..c31137a7b1 --- /dev/null +++ b/components/buddy-slab-allocator/benches/common.rs @@ -0,0 +1,238 @@ +#![allow(dead_code)] + +use core::alloc::Layout; +use std::{ + alloc::{alloc, dealloc}, + sync::{ + Mutex, MutexGuard, OnceLock, + atomic::{AtomicUsize, Ordering}, + }, +}; + +use buddy_slab_allocator::{ + __reset_global_allocator_singleton_for_tests, BuddyAllocator, GlobalAllocator, PerCpuSlab, + SlabAllocResult, SlabAllocator, SlabDeallocResult, SlabPoolTrait, + eii::{slab_pool_impl, virt_to_phys_impl}, +}; +use rand::{SeedableRng, rngs::StdRng}; + +pub const PAGE_SIZE: usize = 0x1000; +pub const HEAP_SIZE: usize = 64 * 1024 * 1024; +pub const OPERATIONS_PER_BATCH: usize = 256; +pub const FRAGMENTATION_PAGES: usize = 512; + +const REGION_ALIGN: usize = 64 * 1024; +const BENCH_PAGE_SIZE: usize = 0x1000; +const MAX_BENCH_CPUS: usize = 64; + +pub struct HostRegion { + ptr: *mut u8, + layout: Layout, +} + +impl HostRegion { + pub fn new(size: usize) -> Self { + let layout = Layout::from_size_align(size, REGION_ALIGN).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "failed to allocate host region"); + Self { ptr, layout } + } + + pub fn addr(&self) -> usize { + self.ptr as usize + } + + pub fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr, self.layout.size()) } + } +} + +impl Drop for HostRegion { + fn drop(&mut self) { + unsafe { dealloc(self.ptr, self.layout) }; + } +} + +pub struct MockOs { + cpu: AtomicUsize, +} + +impl MockOs { + pub const fn new() -> Self { + Self { + cpu: AtomicUsize::new(0), + } + } + + pub fn set_cpu(&self, cpu: usize) { + self.cpu.store(cpu, Ordering::Relaxed); + } +} + +pub struct BenchContext { + _guard: MutexGuard<'static, ()>, +} + +struct BenchSlabPool { + slabs: &'static [PerCpuSlab], +} + +pub static MOCK_OS: MockOs = MockOs::new(); + +fn bench_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) +} + +impl Drop for BenchContext { + fn drop(&mut self) { + __reset_global_allocator_singleton_for_tests(); + } +} + +fn bench_cpu_slabs() -> &'static [PerCpuSlab] { + static SLABS: OnceLock]>> = OnceLock::new(); + SLABS.get_or_init(|| { + (0..MAX_BENCH_CPUS) + .map(|cpu| PerCpuSlab::new(cpu as u16)) + .collect::>() + .into_boxed_slice() + }) +} + +fn reset_cpu_slabs(cpu_count: usize) { + assert!( + cpu_count <= MAX_BENCH_CPUS, + "cpu_count exceeds bench slab pool" + ); + for slab in &bench_cpu_slabs()[..cpu_count] { + slab.reset(); + } +} + +impl SlabPoolTrait for BenchSlabPool { + fn current_slab(&self) -> &dyn buddy_slab_allocator::SlabTrait { + &self.slabs[MOCK_OS.cpu.load(Ordering::Relaxed)] + } + + fn owner_slab(&self, cpu_idx: usize) -> &dyn buddy_slab_allocator::SlabTrait { + &self.slabs[cpu_idx] + } +} + +fn bench_slab_pool_ref() -> &'static BenchSlabPool { + static POOL: OnceLock = OnceLock::new(); + POOL.get_or_init(|| BenchSlabPool { + slabs: bench_cpu_slabs(), + }) +} + +#[virt_to_phys_impl] +fn bench_virt_to_phys(vaddr: usize) -> usize { + vaddr +} + +#[slab_pool_impl] +fn bench_slab_pool() -> &'static dyn SlabPoolTrait { + bench_slab_pool_ref() +} + +pub fn seeded_rng() -> StdRng { + StdRng::from_seed([0; 32]) +} + +pub struct BuddyHarness { + _region: HostRegion, + pub allocator: BuddyAllocator, +} + +impl BuddyHarness { + pub fn new(heap_size: usize) -> Self { + let region_size = + heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4; + let mut region = HostRegion::new(region_size); + let mut allocator = BuddyAllocator::::new(); + unsafe { + allocator.init(region.as_mut_slice()).unwrap(); + } + Self { + _region: region, + allocator, + } + } +} + +pub struct SlabHarness { + _region: HostRegion, + buddy: BuddyAllocator, + slab: SlabAllocator, +} + +impl SlabHarness { + pub fn new(heap_size: usize) -> Self { + let region_size = + heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4; + let mut region = HostRegion::new(region_size); + let mut buddy = BuddyAllocator::::new(); + unsafe { + buddy.init(region.as_mut_slice()).unwrap(); + } + Self { + _region: region, + buddy, + slab: SlabAllocator::new(), + } + } + + pub fn alloc(&mut self, layout: Layout) -> core::ptr::NonNull { + loop { + match self.slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => return ptr, + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let base = self.buddy.alloc_pages(pages, slab_bytes).unwrap(); + self.slab.add_slab(size_class, base, slab_bytes, 0); + } + } + } + } + + pub fn dealloc(&mut self, ptr: core::ptr::NonNull, layout: Layout) { + match self.slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { base, pages } => { + self.buddy.dealloc_pages(base, pages); + } + } + } +} + +pub struct GlobalHarness { + _region: HostRegion, + _ctx: BenchContext, + pub allocator: GlobalAllocator, +} + +impl GlobalHarness { + pub fn new(region_size: usize, cpu_count: usize) -> Self { + assert_eq!( + PAGE_SIZE, BENCH_PAGE_SIZE, + "bench EII slab pool only supports PAGE_SIZE={BENCH_PAGE_SIZE:#x}" + ); + let mut region = HostRegion::new(region_size); + let allocator = GlobalAllocator::::new(); + let guard = bench_lock() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + MOCK_OS.set_cpu(0); + reset_cpu_slabs(cpu_count); + unsafe { + allocator.init(region.as_mut_slice()).unwrap(); + } + Self { + _region: region, + _ctx: BenchContext { _guard: guard }, + allocator, + } + } +} diff --git a/components/buddy-slab-allocator/benches/global_allocator.rs b/components/buddy-slab-allocator/benches/global_allocator.rs new file mode 100644 index 0000000000..f61a3112ec --- /dev/null +++ b/components/buddy-slab-allocator/benches/global_allocator.rs @@ -0,0 +1,112 @@ +//! Benchmarks for the unified global allocator. + +mod common; + +use core::alloc::Layout; + +use common::{GlobalHarness, HEAP_SIZE, MOCK_OS, OPERATIONS_PER_BATCH, PAGE_SIZE}; +use divan::{Bencher, black_box}; + +fn main() { + divan::main(); +} + +#[divan::bench_group] +mod global { + use super::*; + + #[divan::bench(args = [8usize, 64, 512, 2048])] + fn small_alloc_free(bencher: Bencher, size: usize) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + let layout = Layout::from_size_align(size, 8).unwrap(); + + bencher.bench_local(|| { + MOCK_OS.set_cpu(0); + let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; + black_box(ptr) + }); + } + + #[divan::bench(args = [PAGE_SIZE, PAGE_SIZE * 4, PAGE_SIZE * 16])] + fn large_alloc_free(bencher: Bencher, size: usize) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + let layout = Layout::from_size_align(size, PAGE_SIZE).unwrap(); + + bencher.bench_local(|| { + MOCK_OS.set_cpu(0); + let ptr = harness.allocator.alloc(black_box(layout)).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; + black_box(ptr) + }); + } + + #[divan::bench(args = [1usize, 4, 16, 64])] + fn page_interface(bencher: Bencher, pages: usize) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + + bencher.bench_local(|| { + MOCK_OS.set_cpu(0); + let addr = harness + .allocator + .alloc_pages(black_box(pages), PAGE_SIZE) + .unwrap(); + harness.allocator.dealloc_pages(addr, pages); + black_box(addr) + }); + } + + #[divan::bench] + fn mixed_object_page_workload(bencher: Bencher) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + let object_layouts = [ + Layout::from_size_align(64, 8).unwrap(), + Layout::from_size_align(256, 8).unwrap(), + Layout::from_size_align(1024, 8).unwrap(), + Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(), + ]; + let page_counts = [1usize, 2, 4, 8]; + + bencher.bench_local(|| { + for idx in 0..OPERATIONS_PER_BATCH { + MOCK_OS.set_cpu(idx % 2); + let layout = object_layouts[idx % object_layouts.len()]; + let ptr = harness.allocator.alloc(layout).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; + + let pages = page_counts[idx % page_counts.len()]; + let addr = harness.allocator.alloc_pages(pages, PAGE_SIZE).unwrap(); + harness.allocator.dealloc_pages(addr, pages); + + black_box((ptr, addr)); + } + }); + } + + #[divan::bench] + fn remote_free_cycle(bencher: Bencher) { + let harness = GlobalHarness::new(HEAP_SIZE, 2); + let layout = Layout::from_size_align(64, 8).unwrap(); + + bencher.bench_local(|| { + let mut ptrs = Vec::with_capacity(64); + + MOCK_OS.set_cpu(0); + for _ in 0..64 { + ptrs.push(harness.allocator.alloc(layout).unwrap()); + } + + MOCK_OS.set_cpu(1); + for &ptr in &ptrs { + unsafe { harness.allocator.dealloc(ptr, layout) }; + } + + MOCK_OS.set_cpu(0); + for _ in 0..64 { + let ptr = harness.allocator.alloc(layout).unwrap(); + unsafe { harness.allocator.dealloc(ptr, layout) }; + black_box(ptr); + } + }); + } +} diff --git a/components/buddy-slab-allocator/benches/slab_allocator.rs b/components/buddy-slab-allocator/benches/slab_allocator.rs new file mode 100644 index 0000000000..a7b73305b0 --- /dev/null +++ b/components/buddy-slab-allocator/benches/slab_allocator.rs @@ -0,0 +1,102 @@ +//! Benchmarks for the slab allocator with buddy-backed slab refill. + +mod common; + +use core::alloc::Layout; + +use common::{HEAP_SIZE, OPERATIONS_PER_BATCH, SlabHarness}; +use divan::{Bencher, black_box}; + +fn main() { + divan::main(); +} + +#[divan::bench_group] +mod slab { + use super::*; + + #[divan::bench(args = [8usize, 64, 256, 512, 1024, 2048])] + fn size_class_alloc_free(bencher: Bencher, size: usize) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layout = Layout::from_size_align(size, size).unwrap(); + + bencher.bench_local(|| { + let ptr = harness.alloc(black_box(layout)); + harness.dealloc(ptr, layout); + black_box(ptr) + }); + } + + #[divan::bench] + fn hot_reuse(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layout = Layout::from_size_align(128, 128).unwrap(); + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); + + bencher.bench_local(|| { + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); + black_box(ptr) + }); + } + + #[divan::bench] + fn mixed_size_batch(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layouts = [ + Layout::from_size_align(8, 8).unwrap(), + Layout::from_size_align(64, 64).unwrap(), + Layout::from_size_align(256, 256).unwrap(), + Layout::from_size_align(512, 512).unwrap(), + Layout::from_size_align(1024, 1024).unwrap(), + Layout::from_size_align(2048, 2048).unwrap(), + ]; + + bencher.bench_local(|| { + for layout in layouts { + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); + black_box(ptr); + } + }); + } + + #[divan::bench] + fn steady_state_recycle(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layout = Layout::from_size_align(64, 64).unwrap(); + let mut active = Vec::with_capacity(256); + for _ in 0..256 { + active.push(harness.alloc(layout)); + } + + bencher.bench_local(|| { + let ptr = active.pop().unwrap(); + harness.dealloc(ptr, layout); + let new_ptr = harness.alloc(layout); + active.push(new_ptr); + black_box(new_ptr) + }); + } + + #[divan::bench] + fn mixed_size_workload(bencher: Bencher) { + let mut harness = SlabHarness::new(HEAP_SIZE); + let layouts = [ + Layout::from_size_align(8, 8).unwrap(), + Layout::from_size_align(64, 64).unwrap(), + Layout::from_size_align(256, 256).unwrap(), + Layout::from_size_align(1024, 1024).unwrap(), + ]; + + bencher.bench_local(|| { + for idx in 0..OPERATIONS_PER_BATCH { + let layout = layouts[idx % layouts.len()]; + let ptr = harness.alloc(layout); + harness.dealloc(ptr, layout); + black_box(ptr); + } + }); + } +} diff --git a/components/buddy-slab-allocator/docs/design.md b/components/buddy-slab-allocator/docs/design.md new file mode 100644 index 0000000000..6a5fd01de9 --- /dev/null +++ b/components/buddy-slab-allocator/docs/design.md @@ -0,0 +1,611 @@ +# buddy-slab-allocator 设计文档 + +本文档面向阅读和维护当前实现的开发者，描述这个分配器在 `buddy + slab` 两级结构下的核心数据结构、初始化过程、`add_region` 热添加流程、`alloc` / `dealloc` 路径，以及多核并发场景下的协作方式。 + +文中描述以当前仓库实现为准，重点对应以下模块： + +- `src/global.rs` +- `src/buddy/mod.rs` +- `src/slab/mod.rs` +- `src/slab/cache.rs` +- `src/slab/page.rs` + +## 1. 设计目标 + +这个分配器服务于 `no_std`、内核或嵌入式环境，目标是同时满足两类需求： + +- 页级分配：按页返回连续内存，支持 power-of-two 分裂和合并。 +- 小对象分配：对 `<= 2048` 字节的对象提供更低碎片、更低分配开销的快速路径。 + +因此整体采用两级结构： + +- 后端：`BuddyAllocator` + 负责管理一个或多个 section 中的页。 +- 前端：`SlabAllocator` + 负责管理不同 size class 的小对象。 +- 门面：`GlobalAllocator` + 统一对外暴露 `alloc` / `dealloc` / `alloc_pages` / `dealloc_pages` 等接口。 + +## 2. 总体结构 + +```mermaid +flowchart TD + GA["GlobalAllocator"] --> B["BuddyAllocator"] + GA --> SP["SlabPoolTrait"] + GA --> EI["EII hooks"] + EI --> SP + EI --> V2P["virt_to_phys()"] + + SP --> SC8["SlabCache: 8B"] + SP --> SC64["SlabCache: 64B"] + SP --> SC2K["SlabCache: 2048B"] + + SC8 --> SPH["SlabPageHeader"] + SC64 --> SPH + SC2K --> SPH + + B --> SH["BuddySection intrusive list"] + SH --> PM["PageMeta[]"] + SH --> FL["free_lists by order"] +``` + +### 2.1 地址语义 + +当前实现的主语义是“虚拟地址”： + +- `GlobalAllocator::init()` 接收的是一段可写 `&mut [u8]` 区域。 +- `BuddyAllocator` 管理的是一段连续虚拟地址区间。 +- `alloc_pages()`、`alloc()` 返回的也都是虚拟地址。 + +物理地址只在 `alloc_pages_lowmem()` 里参与判断： + +- 候选块的虚拟地址通过 `eii::virt_to_phys()` 翻译成物理地址。 +- 只有物理地址低于 `4 GiB` 的块才会被当作 DMA32 候选。 + +### 2.2 容量统计语义 + +当前公开了两类整体容量统计： + +- `managed_bytes` + 所有 section 中可分配 heap 的总字节数，不包含 region 前缀 metadata。 +- `allocated_bytes` + 后端页占用字节数，按 `managed_bytes - free_pages * PAGE_SIZE` 计算。 + +因此 `allocated_bytes` 的口径是“页后端已经占住多少字节”，不是用户请求的 `layout.size()` 精确累加。 +它会包含： + +- slab 页 +- empty slab cache 保留页 +- 对齐放大 +- buddy / slab 的内部碎片 + +## 3. 初始化、追加 Region 与内存布局 + +当前实现支持两种 region 进入 allocator 的方式： + +- `GlobalAllocator::init(region)` + 注册首个 region；slab 池由外部通过 `eii::slab_pool()` 提供。 +- `GlobalAllocator::add_region(region)` + 在运行时追加新的 region，只扩展 buddy 后端。 + +对 buddy 而言，每个 region 都会被拆成： + +- 前缀：region 自带元数据 +- 后缀：该 section 的 managed heap + +首个 global region 的元数据前缀包含： + +- `BuddySection` +- `PageMeta[]` + +后续追加 region 的元数据前缀只包含： + +- `BuddySection` +- `PageMeta[]` + +```mermaid +flowchart LR + A["region start"] --> B["aligned section_start"] + B --> C["BuddySection"] + C --> D["PageMeta array"] + D --> H["padding to PAGE_SIZE / granule"] + H --> I["section heap start"] + I --> J["managed heap of this section"] +``` + +### 3.1 初始化步骤 + +`GlobalAllocator::init()` 的主要步骤如下： + +1. 根据 `region.len()` 计算首个 section 最多能管理多少页。 +2. 在 region 前缀预留 `BuddySection + PageMeta[]`。 +3. 选择一个页对齐的 section heap 起点。 +4. 将首个 section 注册进 `BuddyAllocator`。 +5. 将 allocator 标记为已初始化。 + +当前实现按“系统里只有唯一一套全局 allocator”建模；slab 池通过 +`eii::slab_pool()` 接入，而不是存放在 `GlobalAllocator` 内部。 + +`GlobalAllocator::add_region()` 的步骤更简单： + +1. 校验 allocator 已初始化。 +2. 在新 region 前缀预留 `BuddySection + PageMeta[]`。 +3. 计算该 section 的可管理 heap。 +4. 将该 section 追加到 buddy 的 intrusive list 尾部。 + +### 3.2 为什么 metadata 放在 region 前缀 + +这样做有几个好处： + +- 不需要额外的启动期堆分配。 +- `BuddySection` 与 `PageMeta[]` 都跟随 region 自身生命周期。 +- 不需要预先声明最大 section 数量。 + +代价是： + +- metadata 会消耗一部分可分配空间。 +- 每个 section 的 heap 起点都可能晚于各自的 region 起点。 +- section 查询与地址归属定位当前采用线性扫描。 + +## 4. BuddyAllocator 设计 + +`BuddyAllocator` 管理页级内存，是整个系统的共享后端。它的基本单位不再是单一 heap，而是一个按注册顺序串起来的 section 链表。 + +### 4.1 核心数据结构 + +```mermaid +classDiagram + class BuddyAllocator { + +sections_head_ptr + +section_count + +os_hook + } + + class BuddySection { + +next_ptr + +meta_ptr + +heap_start + +heap_size + +free_lists_by_order + +free_pages + +total_pages + } + + class PageFlags { + <> + Free + Allocated + Slab + } +``` + +说明： + +- `BuddyAllocator` 自身只维护 section 链表头和 section 数量。 +- 每个 `BuddySection` 都拥有自己的一组 `PageMeta[]` 和 `free_lists`。 +- `PageMeta` 仍然是每页一项的外部元数据。 +- 同一 section 内按 buddy 规则拆分和合并，section 之间不跨界合并。 + +### 4.2 buddy 初始化 + +初始化某个 section 时，buddy 会从低地址到高地址扫描该 section 的整段 heap： + +- 每次尽量切出“当前地址自然对齐、且仍然放得下”的最大 order 块。 +- 将该块挂到对应 order 的 free list。 + +这一步的结果是： + +- 该 section 中的所有页都被覆盖。 +- 该 section 的 free list 从一开始就处于可分裂、可合并的标准 buddy 形态。 + +### 4.3 页分配流程 + +```mermaid +flowchart TD + A["alloc_pages(count, align)"] --> B["计算 order"] + B --> C["计算 align_order"] + C --> D["effective_order = max(order, align_order)"] + D --> E["按注册顺序扫描 section"] + E --> F["在 section 内从 effective_order 向上找非空 free list"] + F --> G{找到块?} + G -- 否 --> H["尝试下一个 section"] + H --> I{还有 section?} + I -- 否 --> J["返回 NoMemory"] + I -- 是 --> F + G -- 是 --> K["弹出大块"] + K --> L["逐级 split 到目标 order"] + L --> M["标记头页为 Allocated"] + M --> N["返回虚拟地址"] +``` + +注意点： + +- `count` 会向上取整到 2 的幂。 +- `align` 也会转成对应的 `align_order`。 +- 真正分配的块大小可能大于用户请求。 + +### 4.4 低地址页分配 + +`alloc_pages_lowmem()` 的差异在于： + +- 不只看 free list 是否有块。 +- 还要检查块的物理地址区间是否完全落在 `DMA32_LIMIT` 下方。 + +流程是： + +1. 逐个扫描候选 free list。 +2. 对每个候选块调用 `os.virt_to_phys(addr)`。 +3. 计算 `phys + block_bytes <= 4GiB` 是否成立。 +4. 找到合格块后执行和普通分配相同的拆分逻辑。 + +### 4.5 页释放流程 + +`dealloc_pages(addr, count)` 的关键点是： + +- `addr` 必须是原始返回地址。 +- 实际释放的 order 以 `PageMeta.order` 为准。 +- `count` 主要用于 debug 断言，确保调用者没有传得比真实块更大。 + +```mermaid +flowchart TD + A["dealloc_pages(addr, count)"] --> B["根据 addr 算出 pfn"] + B --> C["读取 PageMeta.order"] + C --> D["与 buddy pfn 检查是否可合并"] + D --> E{buddy 同 order 且空闲?} + E -- 是 --> F["从 free list 移除 buddy"] + F --> G["合并并提升 order"] + G --> D + E -- 否 --> H["将最终块挂回 free list"] +``` + +## 5. SlabAllocator 设计 + +`SlabAllocator` 不直接申请页，它只管理“已经拿到的 slab page”。 + +### 5.1 size class + +当前固定 9 个 size class： + +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +- 1024 +- 2048 + +布局规则： + +- `layout.size()` 与 `layout.align()` 取较大值。 +- 选择能容纳它的最小 size class。 +- 超过 `2048` 字节则不走 slab。 + +### 5.2 SlabCache 的三条链 + +每个 size class 对应一个 `SlabCache`，内部维护三条 intrusive list： + +- `partial` + 还有空位的 slab，优先分配。 +- `full` + 本地 bitmap 没空位，但可能积累了 remote free。 +- `empty` + 所有对象都空闲的 slab。 + +实现上还有限制： + +- 每个 `SlabCache` 最多缓存 1 个空 slab。 +- 如果又出现新的空 slab，多出来的那个会还给 buddy。 + +这也是压力测试里“页数不一定精确回到初始值”的原因之一：每个 CPU、每个 size class 允许保留一个空 slab 作为缓存。 + +### 5.3 SlabPageHeader + +每个 slab page 起始位置都嵌着一个 `SlabPageHeader`。 + +它包含： + +- `magic` +- `size_class` +- `object_count` +- `local_free_count` +- `owner_cpu` +- `slab_bytes` +- `list_prev/list_next` +- `local_bitmap` +- `remote_free_head` +- `remote_free_count` + +### 5.4 本地分配 + +owner CPU 在持有对应 slab 锁时执行本地分配： + +1. 先看 `partial` 头 slab。 +2. 如果它有 remote free，先 drain 回本地 bitmap。 +3. 从 bitmap 找一个空位。 +4. 如果分完后满了，把 slab 从 `partial` 挪到 `full`。 + +如果 `partial` 不行，则： + +1. 扫 `full`，找是否有 slab 因 remote free 重新出现空位。 +2. 如果有，把它移回 `partial`。 +3. 还不行就尝试复用 `empty` 中缓存的 slab。 +4. 再不行就返回 `NeedsSlab`，要求上层先提供新页。 + +## 6. GlobalAllocator 的分配路径 + +`GlobalAllocator` 是真正对外使用的入口。 + +它把请求分成两类： + +- 小对象：走 slab +- 大对象：走 buddy + +判断条件： + +- `layout.size() <= 2048` +- `layout.align() <= 2048` + +满足才走 slab。 + +### 6.1 alloc 总流程 + +```mermaid +flowchart TD + A["alloc(layout)"] --> B{"已初始化?"} + B -- 否 --> X["NotInitialized"] + B -- 是 --> C{slab eligible?} + C -- 是 --> D["slab_alloc"] + C -- 否 --> E["large_alloc"] +``` + +### 6.2 小对象分配 + +```mermaid +sequenceDiagram + participant Caller + participant GA as GlobalAllocator + participant P as SlabPoolTrait + participant B as BuddyAllocator + + Caller->>GA: alloc(layout) + GA->>P: alloc(layout) + alt Slab 命中 + P-->>GA: Allocated(ptr) + GA-->>Caller: ptr + else 需要新 slab + P-->>GA: NeedsSlab(size_class, pages) + GA->>B: alloc_pages(pages, slab_bytes) + GA->>B: set_page_flags(addr, Slab) + GA->>P: add_slab(...) + GA->>P: alloc(layout) + P-->>GA: Allocated(ptr) + GA-->>Caller: ptr + end +``` + +这里有一个很重要的锁顺序细节： + +- 当 slab 缺页时，`GlobalAllocator` 会先释放 slab 锁，再去拿 buddy 锁。 + +这样可以避免： + +- 持有 slab 锁时阻塞在 buddy 上。 +- 未来演化成更复杂锁图时出现循环等待。 + +### 6.3 大对象分配 + +大对象路径很直接： + +1. 根据 `layout.size()` 计算页数。 +2. 对齐至少是 `PAGE_SIZE`。 +3. 调用 `BuddyAllocator::alloc_pages()`。 + +大对象不会进入 slab，也不会设置 `PageFlags::Slab`。 + +## 7. dealloc 路径 + +### 7.1 大对象释放 + +大对象释放和分配对称： + +1. 根据 `layout.size()` 推导页数。 +2. 直接进入 `BuddyAllocator::dealloc_pages()`。 + +### 7.2 小对象释放：本地路径 + +如果当前 CPU 就是该 slab 的 owner： + +1. 通过对象地址反推出 slab base。 +2. 拿 owner CPU 对应的 slab 锁。 +3. 调用 `slab.dealloc(ptr, layout)`。 +4. 如果结果是 `Done`，结束。 +5. 如果结果是 `FreeSlab { base, pages }`，释放 slab 锁后把整页还给 buddy。 + +为什么空 slab 不总是立刻还给 buddy： + +- `SlabCache` 会保留一个 empty slab 作为热缓存。 +- 只有当已有一个 cached empty slab 时，新的空 slab 才会还给 buddy。 + +### 7.3 小对象释放：远程路径 + +如果当前 CPU 不是 owner CPU，不会直接拿对方的 slab 锁。 + +而是走 lock-free remote free： + +```mermaid +sequenceDiagram + participant CPU1 as current CPU + participant H as SlabPageHeader + participant CPU0 as owner CPU + participant S as owner SlabCache + + CPU1->>H: remote_free(obj_addr) + Note right of H: CAS 推入 remote_free_head\n并增加 remote_free_count + CPU1-->>CPU1: 返回，不加锁 + + CPU0->>S: 下次 alloc/dealloc 时持锁进入 + S->>H: drain_remote_frees() + Note right of H: 将 remote 栈中的对象\n重新记回 local bitmap + S-->>CPU0: 对象重新可分配 +``` + +这条路径的核心特点： + +- 释放方不需要知道 owner CPU 的锁状态。 +- 释放方不抢远端锁。 +- owner CPU 在后续本地操作时，顺手把 remote free 合并回 bitmap。 + +这也是当前实现中“跨 CPU free 无锁，但重新利用对象仍由 owner CPU 控制”的关键设计。 + +## 8. 多线程与并发模型 + +### 8.1 并发边界 + +当前实现中真正共享的状态主要有两类： + +- `BuddyAllocator` + 整体包在一个 `SpinMutex` 里，页级操作串行化。 +- 每 CPU 一个 `SlabAllocator` + 每个都各自包在一个 `SpinMutex` 里。 + +因此并发模型可以概括为： + +- 小对象本地路径：只竞争本 CPU 的 slab 锁。 +- 小对象远程 free：不走锁，直接原子 CAS。 +- 缺页或空 slab 回收：会短暂进入全局 buddy 锁。 + +### 8.2 为什么是 per-CPU slab + +如果所有小对象都走单个全局 slab 锁，会有两个问题： + +- 多核下锁竞争重。 +- 小对象热点路径和页级路径容易互相干扰。 + +per-CPU slab 的好处是： + +- 本地分配/释放命中时只碰本 CPU 锁。 +- 跨 CPU free 通过 remote list 延后处理，不把释放方拖进远端临界区。 + +### 8.3 远程释放的正确性依赖 + +远程释放正确性依赖几个事实： + +- slab 页头里记录了 `owner_cpu`。 +- 远程释放只把对象地址压到无锁链表里。 +- owner CPU 持锁 drain 时，再把对象重新放回本地 bitmap。 +- remote 栈中的 next 指针写在对象自身内存开头，这要求被释放对象已经不再被用户使用。 + +### 8.4 锁顺序 + +当前实现中建议理解为以下顺序： + +- 常规 slab 本地路径：只拿 slab 锁。 +- 常规 buddy 页路径：只拿 buddy 锁。 +- slab 缺页：先释放 slab 锁，再拿 buddy 锁，再重新拿 slab 锁。 +- slab 回收空页：先释放 slab 锁，再拿 buddy 锁。 + +这样可以避免出现“持有 slab 锁后等待 buddy，同时另一路持有 buddy 锁又等待 slab”的循环。 + +## 9. 关键实现细节 + +### 9.1 `PageFlags` + +页状态分三种： + +- `Free` +- `Allocated` +- `Slab` + +含义是： + +- `Free`：在 buddy free list 中。 +- `Allocated`：普通页分配返回的块头页。 +- `Slab`：该块当前作为 slab 页使用。 + +### 9.2 buddy 只在块头记录 order + +`PageMeta.order` 只有块头页有意义。 + +因此： + +- 分配时只标记头页的 `order`。 +- 释放时调用方必须传回原始返回地址。 + +### 9.3 `dealloc_pages(count)` 为什么不完全信任 `count` + +因为分配时请求可能被扩大： + +- `count` 会按 buddy order 向上取整。 +- `align` 也可能把块提升到更高阶。 + +所以释放时真正信任的是 `PageMeta.order`，`count` 只用于做 debug 级别的合理性检查。 + +### 9.4 empty slab 缓存策略 + +每个 `SlabCache` 最多保留一个 empty slab。 + +这是一个很重要的折中： + +- 优点：后续同 size class 再分配时更快，减少 buddy 往返。 +- 代价：空闲页不会总是立刻全部回收到 buddy。 + +所以测试或观测时不能简单假设： + +- “所有对象释放后，buddy free pages 一定精确回到初始值”。 + +更准确的理解应是： + +- 除去每 CPU、每 size class 允许缓存的 empty slab 后，应当没有额外页泄漏。 + +## 10. realloc 行为 + +`GlobalAllocator` 通过 `GlobalAlloc` 实现了 `realloc`： + +1. 构造新的 `Layout` +2. 新分配一块内存 +3. 拷贝 `min(old_size, new_size)` 字节 +4. 释放旧块 + +它不是原地扩容，也没有针对 slab/buddy 做特殊优化。 + +## 11. 当前实现的优点与限制 + +### 11.1 优点 + +- 结构清晰，页级与对象级职责分离。 +- 小对象本地路径开销低。 +- 跨 CPU free 不抢远端锁。 +- 初始化不依赖额外堆分配。 +- 支持低地址页分配。 + +### 11.2 限制 + +- buddy 后端仍然是全局单锁。 +- `remote_free` 的对象复用必须等 owner CPU 后续 drain。 +- empty slab 缓存会牺牲一部分“空闲页立刻回收”的极致性。 +- `realloc` 为通用 copy 语义，不做原地优化。 + +## 12. 阅读源码建议 + +如果想快速建立整体理解，建议按下面顺序读： + +1. `src/lib.rs` + 先看公开模块、EII、`SlabPoolTrait` 与 `SlabPoolExt`。 +2. `src/global.rs` + 先理解门面层如何在 buddy/slab 间路由。 +3. `src/buddy/mod.rs` + 看页级分配、拆分和合并。 +4. `src/slab/mod.rs` + 看 slab 与上层的接口契约。 +5. `src/slab/cache.rs` + 看 `partial/full/empty` 三链切换。 +6. `src/slab/page.rs` + 看 bitmap、remote free 栈与 owner CPU drain。 + +## 13. 一句话总结 + +这个分配器的核心思想可以概括成一句话： + +> 用 buddy 统一管理页，用 per-CPU slab 加速小对象，把跨 CPU free 变成“无锁登记、延迟回收”，从而在实现复杂度、局部性和并发开销之间取得平衡。 diff --git a/components/buddy-slab-allocator/rust-toolchain.toml b/components/buddy-slab-allocator/rust-toolchain.toml new file mode 100644 index 0000000000..81aeeb0cc4 --- /dev/null +++ b/components/buddy-slab-allocator/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-04-01" +components = ["rust-src", "rustfmt", "clippy"] diff --git a/components/buddy-slab-allocator/src/buddy/mod.rs b/components/buddy-slab-allocator/src/buddy/mod.rs new file mode 100644 index 0000000000..8dedf40db6 --- /dev/null +++ b/components/buddy-slab-allocator/src/buddy/mod.rs @@ -0,0 +1,831 @@ +//! Buddy page allocator — page-metadata-based with intrusive free lists. +//! +//! The allocator manages one or more contiguous virtual address ranges ("sections"). +//! Each section stores its own [`BuddySection`] descriptor and [`PageMeta`] array +//! in the caller-provided region prefix, enabling O(1) free-list operations +//! without any dynamic allocation. + +pub mod page_meta; + +use core::ptr; + +pub use page_meta::{PFN_NONE, PageFlags, PageMeta}; +use page_meta::{free_list_push, free_list_remove}; + +use crate::{ + align_up, eii, + error::{AllocError, AllocResult}, + is_aligned, +}; + +/// Maximum buddy order. With 4 KiB pages this gives 2^20 × 4 KiB = 4 GiB blocks. +pub const MAX_ORDER: usize = 20; + +/// DMA32 zone upper bound (4 GiB physical). +const DMA32_LIMIT: usize = 0x1_0000_0000; + +fn normalize_region( + region_start: usize, + region_size: usize, + granule: usize, +) -> Option<(usize, usize)> { + if region_size == 0 || !granule.is_power_of_two() { + return None; + } + let region_end = region_start.checked_add(region_size)?; + let usable_start = align_up(region_start, granule); + let usable_end = region_end & !(granule - 1); + if usable_end <= usable_start { + return None; + } + Some((usable_start, usable_end - usable_start)) +} + +pub(crate) struct RegionLayout { + pub(crate) section_start: usize, + pub(crate) meta_start: usize, + pub(crate) managed_heap_start: usize, + pub(crate) managed_heap_size: usize, +} + +pub(crate) struct SectionInitSpec { + pub(crate) region_start: usize, + pub(crate) region_size: usize, + pub(crate) section_ptr: *mut BuddySection, + pub(crate) meta_ptr: *mut u8, + pub(crate) meta_size: usize, + pub(crate) heap_start: usize, + pub(crate) heap_size: usize, +} + +/// Public read-only summary of a managed section. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ManagedSection { + pub start: usize, + pub size: usize, + pub free_pages: usize, + pub total_pages: usize, +} + +/// Per-region buddy state stored in the region prefix. +#[repr(C)] +pub(crate) struct BuddySection { + pub(crate) next: *mut BuddySection, + pub(crate) region_start: usize, + pub(crate) region_size: usize, + pub(crate) meta: *mut PageMeta, + pub(crate) max_pages: usize, + pub(crate) heap_start: usize, + pub(crate) heap_size: usize, + pub(crate) free_lists: [u32; MAX_ORDER + 1], + pub(crate) free_pages: usize, + pub(crate) total_pages: usize, +} + +impl BuddySection { + const fn metadata_align() -> usize { + let section_align = core::mem::align_of::(); + let meta_align = core::mem::align_of::(); + if section_align > meta_align { + section_align + } else { + meta_align + } + } + + fn metadata_layout_for_pages(pages: usize) -> Option<(usize, usize)> { + let meta_offset = align_up( + core::mem::size_of::(), + core::mem::align_of::(), + ); + let page_meta_size = pages.checked_mul(core::mem::size_of::())?; + let meta_size = meta_offset.checked_add(page_meta_size)?; + Some((meta_offset, meta_size)) + } + + fn available_heap_pages( + region_end: usize, + section_start: usize, + meta_size: usize, + heap_align: usize, + ) -> Option { + let managed_heap_start = align_up(section_start.checked_add(meta_size)?, heap_align); + if managed_heap_start > region_end { + return Some(0); + } + Some((region_end - managed_heap_start) / PAGE_SIZE) + } + + fn can_manage_pages( + region_end: usize, + section_start: usize, + pages: usize, + heap_align: usize, + ) -> bool { + let Some((_, meta_size)) = Self::metadata_layout_for_pages(pages) else { + return false; + }; + let Some(available_pages) = Self::available_heap_pages::( + region_end, + section_start, + meta_size, + heap_align, + ) else { + return false; + }; + available_pages >= pages + } + + pub(crate) fn compute_region_layout_with_heap_align( + region_start: usize, + region_size: usize, + heap_align: usize, + ) -> Option { + if region_size == 0 || !PAGE_SIZE.is_power_of_two() || !heap_align.is_power_of_two() { + return None; + } + + let region_end = region_start.checked_add(region_size)?; + let section_start = align_up(region_start, Self::metadata_align()); + if section_start >= region_end { + return None; + } + + let heap_search_start = align_up( + section_start.checked_add(core::mem::size_of::())?, + PAGE_SIZE, + ); + let max_pages = if heap_search_start >= region_end { + 0 + } else { + (region_end - heap_search_start) / PAGE_SIZE + }; + + let mut low = 0usize; + let mut high = max_pages; + while low < high { + let mid = low + (high - low).div_ceil(2); + if Self::can_manage_pages::(region_end, section_start, mid, heap_align) { + low = mid; + } else { + high = mid - 1; + } + } + + if low == 0 { + return None; + } + + let (meta_offset, meta_size) = Self::metadata_layout_for_pages(low)?; + let meta_start = section_start.checked_add(meta_offset)?; + let managed_heap_start = align_up(section_start.checked_add(meta_size)?, heap_align); + let managed_heap_size = low.checked_mul(PAGE_SIZE)?; + + Some(RegionLayout { + section_start, + meta_start, + managed_heap_start, + managed_heap_size, + }) + } + + fn compute_region_layout( + region_start: usize, + region_size: usize, + ) -> Option { + Self::compute_region_layout_with_heap_align::( + region_start, + region_size, + PAGE_SIZE, + ) + } + + unsafe fn init_at( + section_ptr: *mut BuddySection, + region_start: usize, + region_size: usize, + meta_ptr: *mut u8, + meta_size: usize, + heap_start: usize, + heap_size: usize, + ) -> AllocResult { + unsafe { + if !PAGE_SIZE.is_power_of_two() { + return Err(AllocError::InvalidParam); + } + if !is_aligned(heap_start, PAGE_SIZE) || heap_size == 0 { + return Err(AllocError::InvalidParam); + } + + let total_pages = heap_size / PAGE_SIZE; + let required = BuddyAllocator::::required_meta_size(heap_size); + if meta_size < required { + return Err(AllocError::InvalidParam); + } + + let meta = meta_ptr as *mut PageMeta; + for i in 0..total_pages { + meta.add(i).write(PageMeta::new()); + } + + section_ptr.write(BuddySection { + next: ptr::null_mut(), + region_start, + region_size, + meta, + max_pages: total_pages, + heap_start, + heap_size, + free_lists: [PFN_NONE; MAX_ORDER + 1], + free_pages: 0, + total_pages, + }); + + let section = &mut *section_ptr; + let mut pfn: usize = 0; + while pfn < total_pages { + let mut order = MAX_ORDER; + loop { + let block_pages = 1usize << order; + if block_pages <= total_pages - pfn && (pfn & (block_pages - 1)) == 0 { + break; + } + if order == 0 { + break; + } + order -= 1; + } + let block_pages = 1usize << order; + let m = &mut *section.meta.add(pfn); + m.flags = PageFlags::Free; + m.order = order as u8; + free_list_push(section.meta, &mut section.free_lists, pfn as u32, order); + section.free_pages += block_pages; + pfn += block_pages; + } + + Ok(()) + } + } + + #[inline] + fn contains_heap_addr(&self, addr: usize) -> bool { + addr >= self.heap_start && addr < self.heap_start + self.heap_size + } + + #[inline] + fn summary(&self) -> ManagedSection { + ManagedSection { + start: self.heap_start, + size: self.heap_size, + free_pages: self.free_pages, + total_pages: self.total_pages, + } + } +} + +/// Page-metadata-based buddy allocator. +/// +/// `PAGE_SIZE` must be a power of two (commonly 0x1000 = 4 KiB). +pub struct BuddyAllocator { + sections_head: *mut BuddySection, + sections_tail: *mut BuddySection, + section_count: usize, +} + +// SAFETY: The allocator is designed to be wrapped in a SpinMutex. +// All section pointers point into caller-provided regions whose lifetime is managed externally. +unsafe impl Send for BuddyAllocator {} + +impl BuddyAllocator { + /// Calculate the metadata-region size (in bytes) required for `heap_size` bytes. + pub const fn required_meta_size(heap_size: usize) -> usize { + let pages = heap_size / PAGE_SIZE; + pages * core::mem::size_of::() + } + + /// Create an uninitialised allocator. Call [`init`](Self::init) before use. + pub const fn new() -> Self { + Self { + sections_head: ptr::null_mut(), + sections_tail: ptr::null_mut(), + section_count: 0, + } + } +} + +impl Default for BuddyAllocator { + fn default() -> Self { + Self::new() + } +} + +impl BuddyAllocator { + pub(crate) fn reset(&mut self) { + self.sections_head = ptr::null_mut(); + self.sections_tail = ptr::null_mut(); + self.section_count = 0; + } + + /// Initialise the allocator over the first section. + /// + /// # Safety + /// - `region` must be writable and remain valid for the lifetime of this allocator. + /// - Bytes consumed by metadata become unavailable for allocation. + pub unsafe fn init(&mut self, region: &mut [u8]) -> AllocResult { + unsafe { + self.reset(); + self.add_region(region) + } + } + + /// Add a new managed region after initialisation. + /// + /// # Safety + /// - `region` must be writable and remain valid for the lifetime of this allocator. + /// - The region must not overlap any existing managed region. + pub unsafe fn add_region(&mut self, region: &mut [u8]) -> AllocResult { + unsafe { + let region_start = region.as_mut_ptr() as usize; + let region_size = region.len(); + let (region_start, region_size) = + normalize_region(region_start, region_size, PAGE_SIZE) + .ok_or(AllocError::InvalidParam)?; + let layout = + BuddySection::compute_region_layout::(region_start, region_size) + .ok_or(AllocError::InvalidParam)?; + self.add_region_raw(SectionInitSpec { + region_start, + region_size, + section_ptr: layout.section_start as *mut BuddySection, + meta_ptr: layout.meta_start as *mut u8, + meta_size: Self::required_meta_size(layout.managed_heap_size), + heap_start: layout.managed_heap_start, + heap_size: layout.managed_heap_size, + }) + } + } + + pub(crate) unsafe fn add_region_raw(&mut self, spec: SectionInitSpec) -> AllocResult { + unsafe { + let region_size = spec.region_size; + let region_end = spec + .region_start + .checked_add(region_size) + .ok_or(AllocError::InvalidParam)?; + let heap_end = spec + .heap_start + .checked_add(spec.heap_size) + .ok_or(AllocError::InvalidParam)?; + if heap_end > region_end { + return Err(AllocError::InvalidParam); + } + + let mut section = self.sections_head; + while !section.is_null() { + let existing = &*section; + let existing_end = existing + .region_start + .checked_add(existing.region_size) + .ok_or(AllocError::InvalidParam)?; + if spec.region_start < existing_end && existing.region_start < region_end { + return Err(AllocError::MemoryOverlap); + } + section = existing.next; + } + + BuddySection::init_at::( + spec.section_ptr, + spec.region_start, + spec.region_size, + spec.meta_ptr, + spec.meta_size, + spec.heap_start, + spec.heap_size, + )?; + + if self.sections_head.is_null() { + self.sections_head = spec.section_ptr; + } else { + (*self.sections_tail).next = spec.section_ptr; + } + self.sections_tail = spec.section_ptr; + self.section_count += 1; + + log::debug!( + "BuddyAllocator: add section region {:#x}+{:#x}, heap {:#x}..{:#x}, {} pages", + spec.region_start, + spec.region_size, + spec.heap_start, + heap_end, + spec.heap_size / PAGE_SIZE, + ); + + Ok(()) + } + } + + /// Number of managed sections. + pub fn section_count(&self) -> usize { + self.section_count + } + + /// Read-only summary for a managed section by registration order. + pub fn section(&self, index: usize) -> Option { + let mut current = self.sections_head; + let mut i = 0usize; + while !current.is_null() { + if i == index { + return Some(unsafe { (&*current).summary() }); + } + current = unsafe { (*current).next }; + i += 1; + } + None + } + + /// Total number of pages managed across all sections. + pub fn total_pages(&self) -> usize { + let mut total = 0usize; + let mut current = self.sections_head; + while !current.is_null() { + total += unsafe { (*current).total_pages }; + current = unsafe { (*current).next }; + } + total + } + + /// Total managed heap bytes across all sections. + /// + /// This counts only bytes in allocatable heaps, excluding region-prefix metadata. + pub fn managed_bytes(&self) -> usize { + let mut total = 0usize; + let mut current = self.sections_head; + while !current.is_null() { + total += unsafe { (*current).heap_size }; + current = unsafe { (*current).next }; + } + total + } + + /// Number of currently free pages across all sections. + pub fn free_pages(&self) -> usize { + let mut total = 0usize; + let mut current = self.sections_head; + while !current.is_null() { + total += unsafe { (*current).free_pages }; + current = unsafe { (*current).next }; + } + total + } + + /// Allocated backend bytes across all sections. + /// + /// This is computed as managed heap bytes minus currently free page bytes. + /// It reflects page-level occupancy, so it includes slab pages, alignment + /// amplification, and internal fragmentation. + pub fn allocated_bytes(&self) -> usize { + self.managed_bytes() + .saturating_sub(self.free_pages().saturating_mul(PAGE_SIZE)) + } + + /// Allocate `count` contiguous pages, returning the virtual address. + pub fn alloc_pages(&mut self, count: usize, align: usize) -> AllocResult { + if count == 0 { + return Err(AllocError::InvalidParam); + } + let align = if align == 0 { PAGE_SIZE } else { align }; + if !align.is_power_of_two() || align < PAGE_SIZE { + return Err(AllocError::InvalidParam); + } + + let order = count.next_power_of_two().trailing_zeros() as usize; + if order > MAX_ORDER { + return Err(AllocError::InvalidParam); + } + + let mut section = self.sections_head; + while !section.is_null() { + if let Ok(addr) = + unsafe { Self::alloc_from_section_aligned(&mut *section, order, align) } + { + return Ok(addr); + } + section = unsafe { (*section).next }; + } + + Err(AllocError::NoMemory) + } + + fn alloc_from_section_aligned( + section: &mut BuddySection, + order: usize, + align: usize, + ) -> AllocResult { + for search_order in order..=MAX_ORDER { + let mut pfn_u32 = section.free_lists[search_order]; + while pfn_u32 != PFN_NONE { + let block_pfn = pfn_u32 as usize; + if let Some(target_pfn) = Self::find_aligned_pfn_in_block( + section.heap_start, + block_pfn, + search_order, + order, + align, + ) { + unsafe { + free_list_remove( + section.meta, + &mut section.free_lists, + pfn_u32, + search_order, + ); + } + + let mut current_order = search_order; + let mut current_pfn = block_pfn; + while current_order > order { + current_order -= 1; + let left_pfn = current_pfn; + let right_pfn = current_pfn + (1 << current_order); + let (next_pfn, free_pfn) = if target_pfn >= right_pfn { + (right_pfn, left_pfn) + } else { + (left_pfn, right_pfn) + }; + unsafe { + let bm = &mut *section.meta.add(free_pfn); + bm.flags = PageFlags::Free; + bm.order = current_order as u8; + free_list_push( + section.meta, + &mut section.free_lists, + free_pfn as u32, + current_order, + ); + } + current_pfn = next_pfn; + } + + unsafe { + let m = &mut *section.meta.add(current_pfn); + m.flags = PageFlags::Allocated; + m.order = order as u8; + } + + section.free_pages -= 1 << order; + return Ok(section.heap_start + current_pfn * PAGE_SIZE); + } + pfn_u32 = unsafe { (*section.meta.add(pfn_u32 as usize)).next }; + } + } + + Err(AllocError::NoMemory) + } + + fn find_aligned_pfn_in_block( + heap_start: usize, + block_pfn: usize, + block_order: usize, + alloc_order: usize, + align: usize, + ) -> Option { + let subblock_pages = 1usize << alloc_order; + let align_pages = align / PAGE_SIZE; + let heap_page_offset = (heap_start / PAGE_SIZE) & (align_pages - 1); + let offset = (align_pages - heap_page_offset) & (align_pages - 1); + + let candidate = if align_pages <= subblock_pages { + if !heap_start.is_multiple_of(align) { + return None; + } + block_pfn + } else { + if !offset.is_multiple_of(subblock_pages) { + return None; + } + let rem = block_pfn & (align_pages - 1); + let delta = (offset + align_pages - rem) & (align_pages - 1); + block_pfn + delta + }; + + let block_pages = 1usize << block_order; + let last_start = block_pfn + block_pages - subblock_pages; + (candidate <= last_start).then_some(candidate) + } + + /// Allocate pages whose *physical* address is below 4 GiB (DMA32 zone). + pub fn alloc_pages_lowmem(&mut self, count: usize, align: usize) -> AllocResult { + if count == 0 { + return Err(AllocError::InvalidParam); + } + let align = if align == 0 { PAGE_SIZE } else { align }; + if !align.is_power_of_two() || align < PAGE_SIZE { + return Err(AllocError::InvalidParam); + } + + let order = count.next_power_of_two().trailing_zeros() as usize; + if order > MAX_ORDER { + return Err(AllocError::InvalidParam); + } + + let mut section = self.sections_head; + while !section.is_null() { + if let Ok(addr) = + unsafe { Self::alloc_lowmem_from_section(&mut *section, order, align) } + { + return Ok(addr); + } + section = unsafe { (*section).next }; + } + + Err(AllocError::NoMemory) + } + + fn alloc_lowmem_from_section( + section: &mut BuddySection, + alloc_order: usize, + align: usize, + ) -> AllocResult { + for search_order in alloc_order..=MAX_ORDER { + let mut pfn_u32 = section.free_lists[search_order]; + while pfn_u32 != PFN_NONE { + let block_pfn = pfn_u32 as usize; + let Some(target_pfn) = Self::find_aligned_pfn_in_block( + section.heap_start, + block_pfn, + search_order, + alloc_order, + align, + ) else { + pfn_u32 = unsafe { (*section.meta.add(pfn_u32 as usize)).next }; + continue; + }; + let addr = section.heap_start + target_pfn * PAGE_SIZE; + let phys = eii::virt_to_phys(addr); + let block_bytes = (1usize << alloc_order) * PAGE_SIZE; + if phys + block_bytes <= DMA32_LIMIT && addr.is_multiple_of(align) { + unsafe { + free_list_remove( + section.meta, + &mut section.free_lists, + pfn_u32, + search_order, + ); + } + + let mut current_order = search_order; + let mut current_pfn = block_pfn; + while current_order > alloc_order { + current_order -= 1; + let left_pfn = current_pfn; + let right_pfn = current_pfn + (1 << current_order); + let (next_pfn, free_pfn) = if target_pfn >= right_pfn { + (right_pfn, left_pfn) + } else { + (left_pfn, right_pfn) + }; + unsafe { + let bm = &mut *section.meta.add(free_pfn); + bm.flags = PageFlags::Free; + bm.order = current_order as u8; + free_list_push( + section.meta, + &mut section.free_lists, + free_pfn as u32, + current_order, + ); + } + current_pfn = next_pfn; + } + + unsafe { + let m = &mut *section.meta.add(current_pfn); + m.flags = PageFlags::Allocated; + m.order = alloc_order as u8; + } + section.free_pages -= 1 << alloc_order; + return Ok(addr); + } + pfn_u32 = unsafe { (*section.meta.add(pfn_u32 as usize)).next }; + } + } + + Err(AllocError::NoMemory) + } + + /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages). + /// + /// `addr` must be the exact address returned by alloc. The allocator frees + /// the full block size recorded in page metadata, which may be larger than + /// `count` if the original allocation was rounded up for buddy order or alignment. + pub fn dealloc_pages(&mut self, addr: usize, count: usize) { + let Some(section) = self.find_section_by_addr_mut(addr) else { + debug_assert!( + false, + "dealloc_pages called with address outside all sections" + ); + return; + }; + + debug_assert!(is_aligned(addr, PAGE_SIZE)); + debug_assert!(count > 0); + + let pfn = (addr - section.heap_start) / PAGE_SIZE; + debug_assert!(pfn < section.max_pages); + let stored = unsafe { &*section.meta.add(pfn) }; + debug_assert!( + stored.flags == PageFlags::Allocated || stored.flags == PageFlags::Slab, + "dealloc_pages called on non-allocated block" + ); + + let expected_order = count.next_power_of_two().trailing_zeros() as usize; + let order = stored.order as usize; + debug_assert!( + expected_order <= order, + "dealloc_pages count implies larger order than the allocated block" + ); + Self::dealloc_in_section(section, pfn, order); + } + + /// Mark the page at `addr` with the given flags (used by slab to tag pages). + /// + /// # Safety + /// The caller must ensure `addr` is valid and properly allocated. + pub unsafe fn set_page_flags(&mut self, addr: usize, flags: PageFlags) -> AllocResult { + unsafe { + let section = self + .find_section_by_addr_mut(addr) + .ok_or(AllocError::NotFound)?; + let pfn = (addr - section.heap_start) / PAGE_SIZE; + (*section.meta.add(pfn)).flags = flags; + Ok(()) + } + } + + /// Read the flags of the page containing `addr`. + pub fn page_flags(&self, addr: usize) -> AllocResult { + let section = self + .find_section_by_addr(addr) + .ok_or(AllocError::NotFound)?; + let pfn = (addr - section.heap_start) / PAGE_SIZE; + Ok(unsafe { (*section.meta.add(pfn)).flags }) + } + + fn dealloc_in_section(section: &mut BuddySection, mut pfn: usize, mut order: usize) { + let freed_pages = 1usize << order; + + while order < MAX_ORDER { + let buddy_pfn = pfn ^ (1 << order); + if buddy_pfn >= section.max_pages { + break; + } + let buddy = unsafe { &*section.meta.add(buddy_pfn) }; + if buddy.flags != PageFlags::Free || buddy.order as usize != order { + break; + } + unsafe { + free_list_remove( + section.meta, + &mut section.free_lists, + buddy_pfn as u32, + order, + ); + } + pfn = pfn.min(buddy_pfn); + order += 1; + } + + unsafe { + let m = &mut *section.meta.add(pfn); + m.flags = PageFlags::Free; + m.order = order as u8; + free_list_push(section.meta, &mut section.free_lists, pfn as u32, order); + } + section.free_pages += freed_pages; + } + + fn find_section_by_addr(&self, addr: usize) -> Option<&BuddySection> { + let mut section = self.sections_head; + while !section.is_null() { + let current = unsafe { &*section }; + if current.contains_heap_addr(addr) { + return Some(current); + } + section = current.next; + } + None + } + + fn find_section_by_addr_mut(&mut self, addr: usize) -> Option<&mut BuddySection> { + let mut section = self.sections_head; + while !section.is_null() { + let current = unsafe { &mut *section }; + if current.contains_heap_addr(addr) { + return Some(current); + } + section = current.next; + } + None + } +} diff --git a/components/buddy-slab-allocator/src/buddy/page_meta.rs b/components/buddy-slab-allocator/src/buddy/page_meta.rs new file mode 100644 index 0000000000..07e1657aec --- /dev/null +++ b/components/buddy-slab-allocator/src/buddy/page_meta.rs @@ -0,0 +1,137 @@ +//! Per-page metadata stored in the external metadata region. +//! +//! Each page frame in the heap has a corresponding [`PageMeta`] entry. +//! Free pages are linked together via intrusive doubly-linked lists using PFN indices. + +/// Sentinel value indicating "no page" in free-list links. +pub const PFN_NONE: u32 = u32::MAX; + +/// Page state flags. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum PageFlags { + /// Page is free and sits in a buddy free list. + Free = 0, + /// Page is allocated (head page of a buddy block). + Allocated = 1, + /// Page is used as a slab page. + Slab = 2, +} + +/// Metadata for a single page frame (12 bytes). +/// +/// Head pages carry the `order` of the entire block. +/// Tail pages within a buddy block are marked `Allocated` (or `Slab`) with order 0. +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct PageMeta { + /// Current state of this page. + pub flags: PageFlags, + /// Order of the block (only meaningful on head pages). + pub order: u8, + /// Reserved padding. + pub _pad: u16, + /// Previous PFN in the same-order free list (`PFN_NONE` if head or not free). + pub prev: u32, + /// Next PFN in the same-order free list (`PFN_NONE` if tail or not free). + pub next: u32, +} + +const _: () = assert!(core::mem::size_of::() == 12); + +impl Default for PageMeta { + fn default() -> Self { + Self::new() + } +} + +impl PageMeta { + /// Create a zeroed (free, order-0) page meta. + pub const fn new() -> Self { + Self { + flags: PageFlags::Free, + order: 0, + _pad: 0, + prev: PFN_NONE, + next: PFN_NONE, + } + } +} + +// --------------------------------------------------------------------------- +// Free-list helpers operating on a `*mut PageMeta` array + head array +// --------------------------------------------------------------------------- + +/// Push `pfn` onto the front of `free_lists[order]`. +/// +/// # Safety +/// `meta` must point to an array with at least `pfn + 1` entries. +/// `pfn` must not already be in any free list. +#[inline] +pub unsafe fn free_list_push(meta: *mut PageMeta, free_lists: &mut [u32], pfn: u32, order: usize) { + unsafe { + let old_head = free_lists[order]; + let m = &mut *meta.add(pfn as usize); + m.prev = PFN_NONE; + m.next = old_head; + if old_head != PFN_NONE { + (*meta.add(old_head as usize)).prev = pfn; + } + free_lists[order] = pfn; + } +} + +/// Pop the first PFN from `free_lists[order]`, returning `PFN_NONE` if empty. +/// +/// # Safety +/// `meta` must be a valid metadata array. +#[inline] +pub unsafe fn free_list_pop(meta: *mut PageMeta, free_lists: &mut [u32], order: usize) -> u32 { + unsafe { + let head = free_lists[order]; + if head == PFN_NONE { + return PFN_NONE; + } + let m = &mut *meta.add(head as usize); + let next = m.next; + m.prev = PFN_NONE; + m.next = PFN_NONE; + if next != PFN_NONE { + (*meta.add(next as usize)).prev = PFN_NONE; + } + free_lists[order] = next; + head + } +} + +/// Remove `pfn` from the free list at `order`. +/// +/// # Safety +/// `pfn` must currently be in `free_lists[order]`. +#[inline] +pub unsafe fn free_list_remove( + meta: *mut PageMeta, + free_lists: &mut [u32], + pfn: u32, + order: usize, +) { + unsafe { + let m = &*meta.add(pfn as usize); + let prev = m.prev; + let next = m.next; + + if prev != PFN_NONE { + (*meta.add(prev as usize)).next = next; + } else { + // pfn was the head + free_lists[order] = next; + } + if next != PFN_NONE { + (*meta.add(next as usize)).prev = prev; + } + + let m = &mut *meta.add(pfn as usize); + m.prev = PFN_NONE; + m.next = PFN_NONE; + } +} diff --git a/components/buddy-slab-allocator/src/error.rs b/components/buddy-slab-allocator/src/error.rs new file mode 100644 index 0000000000..561ccf5877 --- /dev/null +++ b/components/buddy-slab-allocator/src/error.rs @@ -0,0 +1,37 @@ +use core::fmt; + +/// The error type used for allocation operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AllocError { + /// Invalid size, alignment, or other input parameter. + InvalidParam, + /// A global allocator instance has already been initialized. + AlreadyInitialized, + /// A region overlaps with an existing managed region. + MemoryOverlap, + /// Not enough memory is available to satisfy the request. + NoMemory, + /// Attempted to deallocate memory that was not allocated. + NotAllocated, + /// The allocator has not been initialized. + NotInitialized, + /// The requested address or entity was not found in any managed region. + NotFound, +} + +impl fmt::Display for AllocError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidParam => write!(f, "invalid parameter"), + Self::AlreadyInitialized => write!(f, "allocator already initialized"), + Self::MemoryOverlap => write!(f, "memory regions overlap"), + Self::NoMemory => write!(f, "out of memory"), + Self::NotAllocated => write!(f, "memory not allocated"), + Self::NotInitialized => write!(f, "allocator not initialized"), + Self::NotFound => write!(f, "not found"), + } + } +} + +/// A [`Result`] alias with [`AllocError`] as the error type. +pub type AllocResult = Result; diff --git a/components/buddy-slab-allocator/src/global.rs b/components/buddy-slab-allocator/src/global.rs new file mode 100644 index 0000000000..63ae9dce9c --- /dev/null +++ b/components/buddy-slab-allocator/src/global.rs @@ -0,0 +1,331 @@ +/// Global allocator composing buddy (pages) + per-CPU slab (objects). +/// +/// Implements [`core::alloc::GlobalAlloc`] so it can serve as `#[global_allocator]`. +/// Cross-CPU frees are lock-free via [`SlabPageHeader::remote_free`]. +use core::alloc::{GlobalAlloc, Layout}; +use core::{ + ptr::{self, NonNull}, + sync::atomic::{AtomicBool, Ordering}, +}; + +use spin::Mutex as SpinMutex; + +use crate::{ + align_up, + buddy::{BuddyAllocator, BuddySection, ManagedSection, PageFlags, SectionInitSpec}, + eii, + error::{AllocError, AllocResult}, + slab::{ + SlabAllocResult, + page::{SLAB_MAGIC, SlabPageHeader}, + size_class::{SLAB_MAX_SIZE, SizeClass}, + }, +}; + +const REGION_GRANULE: usize = 2 * 1024 * 1024; +static GLOBAL_ALLOCATOR_LIVE: AtomicBool = AtomicBool::new(false); + +#[doc(hidden)] +pub fn __reset_global_allocator_singleton_for_tests() { + GLOBAL_ALLOCATOR_LIVE.store(false, Ordering::Release); +} + +/// Unified allocator: buddy page allocator + per-CPU slab caches. +pub struct GlobalAllocator { + buddy: SpinMutex>, + initialized: AtomicBool, +} + +// SAFETY: All mutable state is behind SpinMutex or AtomicBool. +unsafe impl Sync for GlobalAllocator {} +unsafe impl Send for GlobalAllocator {} + +impl GlobalAllocator { + /// Create an uninitialised global allocator. + pub const fn new() -> Self { + Self { + buddy: SpinMutex::new(BuddyAllocator::new()), + initialized: AtomicBool::new(false), + } + } +} + +impl Default for GlobalAllocator { + fn default() -> Self { + Self::new() + } +} + +impl GlobalAllocator { + /// Initialise the allocator over the first region. + /// + /// # Safety + /// - `region` must be writable and remain valid for the lifetime of this allocator. + /// - Any bytes consumed by metadata or alignment padding become unavailable for allocation. + pub unsafe fn init(&self, region: &mut [u8]) -> AllocResult { + unsafe { + if self.initialized.load(Ordering::Acquire) { + return Err(AllocError::AlreadyInitialized); + } + if GLOBAL_ALLOCATOR_LIVE + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(AllocError::AlreadyInitialized); + } + + let raw_region_start = region.as_mut_ptr() as usize; + let raw_region_size = region.len(); + let layout = match BuddySection::compute_region_layout_with_heap_align::( + raw_region_start, + raw_region_size, + REGION_GRANULE, + ) + .ok_or(AllocError::InvalidParam) + { + Ok(layout) => layout, + Err(err) => { + GLOBAL_ALLOCATOR_LIVE.store(false, Ordering::Release); + return Err(err); + } + }; + + let mut buddy = self.buddy.lock(); + buddy.reset(); + if let Err(err) = buddy.add_region_raw(SectionInitSpec { + region_start: raw_region_start, + region_size: raw_region_size, + section_ptr: layout.section_start as *mut BuddySection, + meta_ptr: layout.meta_start as *mut u8, + meta_size: BuddyAllocator::::required_meta_size( + layout.managed_heap_size, + ), + heap_start: layout.managed_heap_start, + heap_size: layout.managed_heap_size, + }) { + GLOBAL_ALLOCATOR_LIVE.store(false, Ordering::Release); + return Err(err); + } + drop(buddy); + + self.initialized.store(true, Ordering::Release); + + log::debug!( + "GlobalAllocator: region {:#x}+{:#x}, section {:#x}, first heap {:#x}+{:#x}", + raw_region_start, + raw_region_size, + layout.section_start, + layout.managed_heap_start, + layout.managed_heap_size, + ); + + Ok(()) + } + } + + /// Add a new managed region after [`init`](Self::init). + /// + /// # Safety + /// - `region` must be writable and remain valid for the lifetime of this allocator. + /// - The region must not overlap any already managed region. + pub unsafe fn add_region(&self, region: &mut [u8]) -> AllocResult { + unsafe { + if !self.initialized.load(Ordering::Acquire) { + return Err(AllocError::NotInitialized); + } + let region_start = region.as_mut_ptr() as usize; + let region_size = region.len(); + let Some(layout) = BuddySection::compute_region_layout_with_heap_align::( + region_start, + region_size, + REGION_GRANULE, + ) else { + log::info!( + "GlobalAllocator: skip region {:#x}+{:#x}, no allocator-visible memory after \ + {} alignment", + region_start, + region_size, + REGION_GRANULE, + ); + return Ok(()); + }; + self.buddy.lock().add_region_raw(SectionInitSpec { + region_start, + region_size, + section_ptr: layout.section_start as *mut BuddySection, + meta_ptr: layout.meta_start as *mut u8, + meta_size: BuddyAllocator::::required_meta_size( + layout.managed_heap_size, + ), + heap_start: layout.managed_heap_start, + heap_size: layout.managed_heap_size, + }) + } + } + + /// Number of managed sections. + pub fn managed_section_count(&self) -> usize { + self.buddy.lock().section_count() + } + + /// Read-only summary for a managed section. + pub fn managed_section(&self, index: usize) -> Option { + self.buddy.lock().section(index) + } + + /// Total managed heap bytes across all sections. + /// + /// This excludes region-prefix metadata such as `BuddySection` and `PageMeta[]`. + pub fn managed_bytes(&self) -> usize { + self.buddy.lock().managed_bytes() + } + + /// Allocated backend bytes across all sections. + /// + /// This is page-level occupancy, not the exact sum of requested layout sizes. + pub fn allocated_bytes(&self) -> usize { + self.buddy.lock().allocated_bytes() + } + + /// Allocate contiguous pages. Returns the virtual start address. + pub fn alloc_pages(&self, count: usize, align: usize) -> AllocResult { + self.buddy.lock().alloc_pages(count, align) + } + + /// Free pages previously obtained via [`alloc_pages`](Self::alloc_pages). + pub fn dealloc_pages(&self, addr: usize, count: usize) { + self.buddy.lock().dealloc_pages(addr, count); + } + + /// Allocate pages with physical address below 4 GiB. + pub fn alloc_pages_lowmem(&self, count: usize, align: usize) -> AllocResult { + self.buddy.lock().alloc_pages_lowmem(count, align) + } + + /// Allocate memory for `layout`. Returns a pointer on success. + pub fn alloc(&self, layout: Layout) -> AllocResult> { + if !self.initialized.load(Ordering::Acquire) { + return Err(AllocError::NotInitialized); + } + + if self.is_slab_eligible(&layout) { + self.slab_alloc(layout) + } else { + self.large_alloc(layout) + } + } + + /// Deallocate memory previously returned by [`alloc`](Self::alloc). + /// + /// # Safety + /// `ptr` must have been returned by a prior `alloc` with the same `layout`. + pub unsafe fn dealloc(&self, ptr: NonNull, layout: Layout) { + unsafe { + if self.is_slab_eligible(&layout) { + self.slab_dealloc(ptr, layout); + } else { + self.large_dealloc(ptr, layout); + } + } + } + + #[inline] + fn is_slab_eligible(&self, layout: &Layout) -> bool { + layout.size() <= SLAB_MAX_SIZE && layout.align() <= SLAB_MAX_SIZE + } + + fn slab_alloc(&self, layout: Layout) -> AllocResult> { + let pool = eii::slab_pool(); + + match pool.alloc(layout)? { + SlabAllocResult::Allocated(ptr) => Ok(ptr), + SlabAllocResult::NeedsSlab { size_class, pages } => { + let bytes = pages * PAGE_SIZE; + let addr = self.buddy.lock().alloc_pages(pages, bytes)?; + unsafe { + self.buddy.lock().set_page_flags(addr, PageFlags::Slab)?; + } + pool.add_slab(size_class, addr, bytes); + match pool.alloc(layout)? { + SlabAllocResult::Allocated(ptr) => Ok(ptr), + SlabAllocResult::NeedsSlab { .. } => Err(AllocError::NoMemory), + } + } + } + } + + unsafe fn slab_dealloc(&self, ptr: NonNull, layout: Layout) { + unsafe { + let sc = SizeClass::from_layout(layout).expect("layout exceeds slab"); + let slab_bytes = sc.slab_pages(PAGE_SIZE) * PAGE_SIZE; + let base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + let hdr = &*(base as *const SlabPageHeader); + debug_assert_eq!(hdr.magic, SLAB_MAGIC); + + match eii::slab_pool().dealloc(ptr, layout, hdr.owner_cpu as usize) { + crate::SlabPoolDeallocResult::Done => {} + crate::SlabPoolDeallocResult::RemoteQueued => {} + crate::SlabPoolDeallocResult::FreeSlab { base, pages } => { + self.buddy.lock().dealloc_pages(base, pages); + } + } + } + } + + fn large_alloc(&self, layout: Layout) -> AllocResult> { + let pages = align_up(layout.size(), PAGE_SIZE) / PAGE_SIZE; + let align = layout.align().max(PAGE_SIZE); + let addr = self.buddy.lock().alloc_pages(pages, align)?; + Ok(unsafe { NonNull::new_unchecked(addr as *mut u8) }) + } + + unsafe fn large_dealloc(&self, ptr: NonNull, layout: Layout) { + let pages = align_up(layout.size(), PAGE_SIZE) / PAGE_SIZE; + self.buddy + .lock() + .dealloc_pages(ptr.as_ptr() as usize, pages); + } +} + +impl Drop for GlobalAllocator { + fn drop(&mut self) { + if self.initialized.swap(false, Ordering::AcqRel) { + GLOBAL_ALLOCATOR_LIVE.store(false, Ordering::Release); + } + } +} + +unsafe impl GlobalAlloc for GlobalAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + match self.alloc(layout) { + Ok(ptr) => ptr.as_ptr(), + Err(_) => ptr::null_mut(), + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { + if let Some(nn) = NonNull::new(ptr) { + self.dealloc(nn, layout); + } + } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + unsafe { + let new_layout = match Layout::from_size_align(new_size, layout.align()) { + Ok(l) => l, + Err(_) => return ptr::null_mut(), + }; + + let new_ptr = ::alloc(self, new_layout); + if !new_ptr.is_null() { + let copy_size = layout.size().min(new_size); + ptr::copy_nonoverlapping(ptr, new_ptr, copy_size); + ::dealloc(self, ptr, layout); + } + new_ptr + } + } +} diff --git a/components/buddy-slab-allocator/src/lib.rs b/components/buddy-slab-allocator/src/lib.rs new file mode 100644 index 0000000000..e811d2a26f --- /dev/null +++ b/components/buddy-slab-allocator/src/lib.rs @@ -0,0 +1,111 @@ +//! # buddy-slab-allocator +//! +//! A `#![no_std]` memory allocator featuring: +//! +//! - **Buddy page allocator** — page-metadata-based with intrusive free lists +//! - **Slab allocator** — bitmap-based with lock-free cross-CPU freeing (Linux SLUB inspired) +//! - **Global allocator** — composes buddy + per-CPU slab, implements [`core::alloc::GlobalAlloc`] +//! +//! Both buddy and slab allocators can be used standalone. + +#![no_std] +#![feature(extern_item_impls)] + +mod error; +pub use error::{AllocError, AllocResult}; + +pub mod buddy; +pub use buddy::{BuddyAllocator, ManagedSection}; + +pub mod slab; +pub use slab::{ + PerCpuSlab, SizeClass, SlabAllocResult, SlabAllocator, SlabDeallocResult, + SlabPoolDeallocResult, SlabPoolTrait, SlabTrait, StaticSlabPool, +}; + +pub mod global; +#[doc(hidden)] +pub use global::__reset_global_allocator_singleton_for_tests; +pub use global::GlobalAllocator; + +/// External interface items supplied by the platform / allocator integrator. +pub mod eii { + /// Translate a virtual address to a physical address. + #[eii(virt_to_phys_impl)] + pub fn virt_to_phys(vaddr: usize) -> usize; + + /// Return the system-global slab pool. + #[eii(slab_pool_impl)] + pub fn slab_pool() -> &'static dyn crate::SlabPoolTrait; +} + +// --------------------------------------------------------------------------- +// Utility helpers (crate-internal) +// --------------------------------------------------------------------------- + +#[inline] +pub(crate) const fn align_up(pos: usize, align: usize) -> usize { + (pos + align - 1) & !(align - 1) +} + +#[inline] +pub(crate) const fn is_aligned(addr: usize, align: usize) -> bool { + addr & (align - 1) == 0 +} + +#[cfg(test)] +mod test_eii_impls { + use core::{alloc::Layout, ptr::NonNull}; + + use super::{ + AllocError, AllocResult, SizeClass, SlabAllocResult, SlabPoolTrait, SlabTrait, + eii::{slab_pool_impl, virt_to_phys_impl}, + }; + + struct NullSlabPool; + struct NullSlab; + + impl SlabTrait for NullSlab { + fn cpu_id(&self) -> usize { + 0 + } + + fn page_size(&self) -> usize { + 0x1000 + } + + fn alloc(&self, _layout: Layout) -> AllocResult { + Err(AllocError::NotInitialized) + } + + fn add_slab(&self, _size_class: SizeClass, _base: usize, _bytes: usize) {} + + fn dealloc_local(&self, _ptr: NonNull, _layout: Layout) -> super::SlabDeallocResult { + super::SlabDeallocResult::Done + } + } + + static NULL_SLAB: NullSlab = NullSlab; + + impl SlabPoolTrait for NullSlabPool { + fn current_slab(&self) -> &dyn SlabTrait { + &NULL_SLAB + } + + fn owner_slab(&self, _cpu_idx: usize) -> &dyn SlabTrait { + &NULL_SLAB + } + } + + static NULL_SLAB_POOL: NullSlabPool = NullSlabPool; + + #[virt_to_phys_impl] + fn test_virt_to_phys(vaddr: usize) -> usize { + vaddr + } + + #[slab_pool_impl] + fn test_slab_pool() -> &'static dyn SlabPoolTrait { + &NULL_SLAB_POOL + } +} diff --git a/components/buddy-slab-allocator/src/slab/cache.rs b/components/buddy-slab-allocator/src/slab/cache.rs new file mode 100644 index 0000000000..22de3cc174 --- /dev/null +++ b/components/buddy-slab-allocator/src/slab/cache.rs @@ -0,0 +1,275 @@ +/// Per-size-class slab cache. +/// +/// Maintains three intrusive doubly-linked lists of slab pages: +/// - **partial**: some objects free (preferred for allocation) +/// - **full**: no objects free +/// - **empty**: all objects free (at most one cached; rest returned to buddy) +use super::page::{SlabListState, SlabPageHeader}; +use super::size_class::SizeClass; + +/// Intrusive list head (address of the first `SlabPageHeader`, 0 = empty). +#[derive(Debug, Clone, Copy)] +struct ListHead { + first: usize, + kind: SlabListState, +} + +impl ListHead { + const fn empty(kind: SlabListState) -> Self { + Self { first: 0, kind } + } + + fn is_empty(&self) -> bool { + self.first == 0 + } + + /// Push a slab page onto the front of the list. + /// + /// # Safety + /// `base` must point to a valid `SlabPageHeader`. + unsafe fn push_front(&mut self, base: usize) { + unsafe { + let hdr = &mut *(base as *mut SlabPageHeader); + debug_assert_eq!( + hdr.list_state, + SlabListState::None, + "pushing slab onto {:?} while it is on {:?}", + self.kind, + hdr.list_state + ); + debug_assert_eq!(hdr.list_prev, 0, "pushing slab with stale list_prev"); + debug_assert_eq!(hdr.list_next, 0, "pushing slab with stale list_next"); + hdr.list_state = self.kind; + hdr.list_prev = 0; + hdr.list_next = self.first; + if self.first != 0 { + let old = &mut *(self.first as *mut SlabPageHeader); + debug_assert_eq!( + old.list_state, self.kind, + "list head points to slab with wrong membership" + ); + old.list_prev = base; + } + self.first = base; + } + } + + /// Remove a slab page from this list. + /// + /// # Safety + /// `base` must be in this list. + unsafe fn remove(&mut self, base: usize) { + unsafe { + let hdr = &*(base as *const SlabPageHeader); + debug_assert_eq!( + hdr.list_state, self.kind, + "removing slab from {:?} while it is on {:?}", + self.kind, hdr.list_state + ); + if hdr.list_prev == 0 { + debug_assert_eq!(self.first, base, "front slab is not this list's head"); + } else { + debug_assert_eq!( + (*(hdr.list_prev as *const SlabPageHeader)).list_next, + base, + "previous slab does not link back to removed slab" + ); + } + if hdr.list_next != 0 { + debug_assert_eq!( + (*(hdr.list_next as *const SlabPageHeader)).list_prev, + base, + "next slab does not link back to removed slab" + ); + } + let prev = hdr.list_prev; + let next = hdr.list_next; + + if prev != 0 { + (*(prev as *mut SlabPageHeader)).list_next = next; + } else { + self.first = next; + } + if next != 0 { + (*(next as *mut SlabPageHeader)).list_prev = prev; + } + // Clear links + let hdr = &mut *(base as *mut SlabPageHeader); + hdr.list_prev = 0; + hdr.list_next = 0; + hdr.list_state = SlabListState::None; + } + } + + /// Pop the first page from the list. Returns 0 if empty. + unsafe fn pop_front(&mut self) -> usize { + unsafe { + if self.first == 0 { + return 0; + } + let base = self.first; + self.remove(base); + base + } + } +} + +/// Cache for a single [`SizeClass`]. +pub struct SlabCache { + pub size_class: SizeClass, + partial: ListHead, + full: ListHead, + empty: ListHead, + /// Number of empty slabs cached (we keep at most 1). + empty_count: usize, +} + +/// Result of a per-cache deallocation. +pub enum CacheDeallocResult { + /// Object freed, slab stays. + Done, + /// Slab became empty and should be returned to the page allocator. + FreeSlab { base: usize, pages: usize }, +} + +impl SlabCache { + pub const fn new(size_class: SizeClass) -> Self { + Self { + size_class, + partial: ListHead::empty(SlabListState::Partial), + full: ListHead::empty(SlabListState::Full), + empty: ListHead::empty(SlabListState::Empty), + empty_count: 0, + } + } + + /// Try to allocate one object. Returns `Some(obj_addr)` or `None` if no slabs available. + pub fn alloc_object(&mut self) -> Option { + // 1. Try the first partial slab (drain remote frees first). + if let Some(addr) = self.try_alloc_from_partial::() { + return Some(addr); + } + + // 2. A full slab may have gained free objects via lock-free remote frees. + if let Some(base) = self.reclaim_full_with_remote_frees() { + unsafe { self.partial.push_front(base) }; + return self.try_alloc_from_partial::(); + } + + // 3. Try recycling an empty slab. + if !self.empty.is_empty() { + let base = unsafe { self.empty.pop_front() }; + self.empty_count -= 1; + // Move to partial and alloc from it. + unsafe { self.partial.push_front(base) }; + return self.try_alloc_from_partial::(); + } + + None + } + + /// Drain remote frees from the first full slab that has them and move it + /// back to the partial list. + fn reclaim_full_with_remote_frees(&mut self) -> Option { + let mut base = self.full.first; + while base != 0 { + let next = unsafe { (*(base as *const SlabPageHeader)).list_next }; + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + if hdr.has_remote_frees() { + hdr.drain_remote_frees(base); + unsafe { self.full.remove(base) }; + return Some(base); + } + base = next; + } + None + } + + /// Attempt allocation from the first partial slab. + fn try_alloc_from_partial(&mut self) -> Option { + let base = self.partial.first; + if base == 0 { + return None; + } + + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + + // Drain any remote frees first. + if hdr.has_remote_frees() { + hdr.drain_remote_frees(base); + } + + if let Some(idx) = hdr.local_alloc() { + let obj_addr = hdr.object_addr(base, idx); + // If slab is now full, move to full list. + if hdr.is_local_full() && !hdr.has_remote_frees() { + unsafe { + self.partial.remove(base); + self.full.push_front(base); + } + } + return Some(obj_addr); + } + None + } + + /// Free an object back to this cache (local CPU path — under lock). + /// + /// Returns whether the slab should be returned to the page allocator. + pub fn dealloc_object( + &mut self, + obj_addr: usize, + ) -> CacheDeallocResult { + let slab_bytes = self.size_class.slab_pages(PAGE_SIZE) * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(obj_addr, slab_bytes); + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + let was_full = hdr.list_state == SlabListState::Full; + + let idx = hdr.object_index(base, obj_addr); + hdr.local_free(idx); + + if was_full { + // Move from full to partial. + unsafe { + self.full.remove(base); + self.partial.push_front(base); + } + } + + // Check if slab is now completely empty. + // First drain remote frees so we have an accurate count. + if hdr.has_remote_frees() { + hdr.drain_remote_frees(base); + } + + if hdr.is_all_free() { + if self.empty_count == 0 { + // Cache one empty slab for reuse. + unsafe { + self.partial.remove(base); + self.empty.push_front(base); + } + self.empty_count += 1; + CacheDeallocResult::Done + } else { + // Already have a cached empty slab — return this one. + unsafe { self.partial.remove(base) }; + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + hdr.prepare_for_buddy_free(); + CacheDeallocResult::FreeSlab { + base, + pages: self.size_class.slab_pages(PAGE_SIZE), + } + } + } else { + CacheDeallocResult::Done + } + } + + /// Register a newly allocated slab page (from the buddy allocator). + pub fn add_slab(&mut self, base: usize, bytes: usize, owner_cpu: u16) { + let hdr = unsafe { &mut *(base as *mut SlabPageHeader) }; + hdr.init(self.size_class, bytes, owner_cpu); + unsafe { self.partial.push_front(base) }; + } +} diff --git a/components/buddy-slab-allocator/src/slab/mod.rs b/components/buddy-slab-allocator/src/slab/mod.rs new file mode 100644 index 0000000000..e4a81142d6 --- /dev/null +++ b/components/buddy-slab-allocator/src/slab/mod.rs @@ -0,0 +1,298 @@ +//! Slab allocator — bitmap-based with lock-free cross-CPU freeing. +//! +//! The [`SlabAllocator`] is a standalone component that manages object allocation +//! within pre-supplied slab pages. It does **not** allocate pages itself; instead +//! it returns [`SlabAllocResult::NeedsSlab`] to request pages from the caller. +//! +//! Cross-CPU frees go through the lock-free [`SlabPageHeader::remote_free`] path. + +pub mod cache; +pub mod page; +pub mod size_class; + +use core::{alloc::Layout, ptr::NonNull}; + +use cache::{CacheDeallocResult, SlabCache}; +pub use page::SlabPageHeader; +pub use size_class::SizeClass; +use spin::Mutex as SpinMutex; + +use crate::error::{AllocError, AllocResult}; + +/// Result of a slab allocation attempt. +pub enum SlabAllocResult { + /// Object successfully allocated. + Allocated(NonNull), + /// The slab cache for this size class has no free objects. + /// The caller should allocate `pages` pages from the buddy allocator, + /// call [`SlabAllocator::add_slab`], and retry. + NeedsSlab { size_class: SizeClass, pages: usize }, +} + +/// Result of a slab deallocation. +pub enum SlabDeallocResult { + /// Object freed, nothing else to do. + Done, + /// The slab page at `base` became empty and should be returned to the buddy. + FreeSlab { base: usize, pages: usize }, +} + +/// Result of a pool-mediated slab deallocation. +pub enum SlabPoolDeallocResult { + /// Object freed on the local CPU path. + Done, + /// Object was queued onto the owner's remote-free list. + RemoteQueued, + /// The slab page at `base` became empty and should be returned to the buddy. + FreeSlab { base: usize, pages: usize }, +} + +/// Object-safe slab interface used by [`crate::GlobalAllocator`] EII hooks. +pub trait SlabTrait: Sync { + /// Logical CPU id this slab belongs to. + fn cpu_id(&self) -> usize; + + /// Page size used by this slab. + fn page_size(&self) -> usize; + + /// Allocate one object. + fn alloc(&self, layout: Layout) -> AllocResult; + + /// Register a freshly allocated slab page. + fn add_slab(&self, size_class: SizeClass, base: usize, bytes: usize); + + /// Free an object on the owner CPU path. + fn dealloc_local(&self, ptr: NonNull, layout: Layout) -> SlabDeallocResult; + + /// Free an object on the remote CPU path. + fn dealloc_remote(&self, ptr: NonNull) { + let owner_cpu = u16::try_from(self.cpu_id()).expect("CPU id exceeds slab owner range"); + unsafe { SlabPageHeader::remote_free_object(ptr, owner_cpu, self.page_size()) }; + } +} + +/// Object-safe slab-pool interface used by [`crate::GlobalAllocator`] EII hooks. +pub trait SlabPoolTrait: Sync { + /// Return the slab belonging to the current CPU. + fn current_slab(&self) -> &dyn SlabTrait; + + /// Return the owner slab for the given CPU. + fn owner_slab(&self, cpu_idx: usize) -> &dyn SlabTrait; + + /// Logical CPU id of the current CPU. + fn current_cpu_id(&self) -> usize { + self.current_slab().cpu_id() + } + + /// Allocate one object from the current CPU's slab. + fn alloc(&self, layout: Layout) -> AllocResult { + self.current_slab().alloc(layout) + } + + /// Register a freshly allocated slab page in the current CPU's slab. + fn add_slab(&self, size_class: SizeClass, base: usize, bytes: usize) { + self.current_slab().add_slab(size_class, base, bytes) + } + + /// Free an object, routing to local or remote slab ownership as needed. + fn dealloc(&self, ptr: NonNull, layout: Layout, owner_cpu: usize) -> SlabPoolDeallocResult { + if owner_cpu == self.current_cpu_id() { + match self.current_slab().dealloc_local(ptr, layout) { + SlabDeallocResult::Done => SlabPoolDeallocResult::Done, + SlabDeallocResult::FreeSlab { base, pages } => { + SlabPoolDeallocResult::FreeSlab { base, pages } + } + } + } else { + self.owner_slab(owner_cpu).dealloc_remote(ptr); + SlabPoolDeallocResult::RemoteQueued + } + } +} + +/// Convenience helpers for callback-style slab access. +pub trait SlabPoolExt: SlabPoolTrait { + /// Access the current CPU's slab via a callback. + fn with_current_slab(&self, f: impl FnOnce(&dyn SlabTrait) -> R) -> R { + f(self.current_slab()) + } + + /// Access the given owner's slab via a callback. + fn with_owner_slab(&self, cpu_idx: usize, f: impl FnOnce(&dyn SlabTrait) -> R) -> R { + f(self.owner_slab(cpu_idx)) + } +} + +impl SlabPoolExt for T {} + +/// Standalone slab allocator (one per CPU or standalone use). +pub struct SlabAllocator { + caches: [SlabCache; SizeClass::COUNT], +} + +/// Default per-CPU slab wrapper used by EII integrators. +pub struct PerCpuSlab { + cpu_id: u16, + inner: SpinMutex>, +} + +/// Default static slab-pool wrapper used by EII integrators. +pub struct StaticSlabPool { + slabs: [PerCpuSlab; N], + current_cpu_id: fn() -> usize, +} + +impl PerCpuSlab { + /// Create an empty per-CPU slab wrapper for `cpu_id`. + pub const fn new(cpu_id: u16) -> Self { + Self { + cpu_id, + inner: SpinMutex::new(SlabAllocator::new()), + } + } + + /// Reset the inner slab allocator to an empty state. + pub fn reset(&self) { + *self.inner.lock() = SlabAllocator::new(); + } + + /// Return this slab's logical CPU id. + pub const fn cpu_id(&self) -> usize { + self.cpu_id as usize + } + + /// Allocate one object. + pub fn alloc(&self, layout: Layout) -> AllocResult { + self.inner.lock().alloc(layout) + } + + /// Register a freshly allocated slab page. + pub fn add_slab(&self, size_class: SizeClass, base: usize, bytes: usize) { + self.inner + .lock() + .add_slab(size_class, base, bytes, self.cpu_id); + } + + /// Free an object on the owner CPU path. + pub fn dealloc_local(&self, ptr: NonNull, layout: Layout) -> SlabDeallocResult { + self.inner.lock().dealloc(ptr, layout) + } + + /// Queue an object onto this slab's remote-free list. + pub fn dealloc_remote(&self, ptr: NonNull) { + unsafe { SlabPageHeader::remote_free_object(ptr, self.cpu_id, PAGE_SIZE) }; + } +} + +impl StaticSlabPool { + /// Create a static slab pool from pre-built per-CPU slabs and a CPU-id hook. + pub const fn new(slabs: [PerCpuSlab; N], current_cpu_id: fn() -> usize) -> Self { + Self { + slabs, + current_cpu_id, + } + } +} + +impl SlabAllocator { + /// Create a new (empty) slab allocator. No pages are owned yet. + pub const fn new() -> Self { + Self { + caches: [ + SlabCache::new(SizeClass::Bytes8), + SlabCache::new(SizeClass::Bytes16), + SlabCache::new(SizeClass::Bytes32), + SlabCache::new(SizeClass::Bytes64), + SlabCache::new(SizeClass::Bytes128), + SlabCache::new(SizeClass::Bytes256), + SlabCache::new(SizeClass::Bytes512), + SlabCache::new(SizeClass::Bytes1024), + SlabCache::new(SizeClass::Bytes2048), + ], + } + } +} + +impl Default for SlabAllocator { + fn default() -> Self { + Self::new() + } +} + +impl SlabAllocator { + /// Try to allocate an object matching `layout`. + /// + /// If the matching cache is exhausted, [`SlabAllocResult::NeedsSlab`] is returned + /// so the caller can supply pages and retry. + pub fn alloc(&mut self, layout: Layout) -> AllocResult { + let sc = SizeClass::from_layout(layout).ok_or(AllocError::InvalidParam)?; + let cache = &mut self.caches[sc.index()]; + + match cache.alloc_object::() { + Some(addr) => { + // SAFETY: `addr` is non-null, aligned, and within a live slab page. + let ptr = unsafe { NonNull::new_unchecked(addr as *mut u8) }; + Ok(SlabAllocResult::Allocated(ptr)) + } + None => Ok(SlabAllocResult::NeedsSlab { + size_class: sc, + pages: sc.slab_pages(PAGE_SIZE), + }), + } + } + + /// Free an object previously allocated with [`alloc`](Self::alloc). + /// + /// This is the **local** (owner-CPU) path. Cross-CPU frees should go through + /// [`SlabPageHeader::remote_free`] directly (see [`GlobalAllocator`]). + pub fn dealloc(&mut self, ptr: NonNull, layout: Layout) -> SlabDeallocResult { + let sc = SizeClass::from_layout(layout).expect("layout exceeds slab size"); + let cache = &mut self.caches[sc.index()]; + + match cache.dealloc_object::(ptr.as_ptr() as usize) { + CacheDeallocResult::Done => SlabDeallocResult::Done, + CacheDeallocResult::FreeSlab { base, pages } => { + SlabDeallocResult::FreeSlab { base, pages } + } + } + } + + /// Supply a freshly allocated slab page to the given size class. + /// + /// `base` is the virtual address of the page(s), `bytes` = pages × PAGE_SIZE. + pub fn add_slab(&mut self, size_class: SizeClass, base: usize, bytes: usize, owner_cpu: u16) { + self.caches[size_class.index()].add_slab(base, bytes, owner_cpu); + } +} + +impl SlabTrait for PerCpuSlab { + fn cpu_id(&self) -> usize { + PerCpuSlab::cpu_id(self) + } + + fn page_size(&self) -> usize { + PAGE_SIZE + } + + fn alloc(&self, layout: Layout) -> AllocResult { + PerCpuSlab::alloc(self, layout) + } + + fn add_slab(&self, size_class: SizeClass, base: usize, bytes: usize) { + PerCpuSlab::add_slab(self, size_class, base, bytes) + } + + fn dealloc_local(&self, ptr: NonNull, layout: Layout) -> SlabDeallocResult { + PerCpuSlab::dealloc_local(self, ptr, layout) + } +} + +impl SlabPoolTrait for StaticSlabPool { + fn current_slab(&self) -> &dyn SlabTrait { + &self.slabs[(self.current_cpu_id)()] + } + + fn owner_slab(&self, cpu_idx: usize) -> &dyn SlabTrait { + &self.slabs[cpu_idx] + } +} diff --git a/components/buddy-slab-allocator/src/slab/page.rs b/components/buddy-slab-allocator/src/slab/page.rs new file mode 100644 index 0000000000..d8ef5c4453 --- /dev/null +++ b/components/buddy-slab-allocator/src/slab/page.rs @@ -0,0 +1,329 @@ +/// Slab page header, bitmap-based object tracking, and lock-free remote free. +/// +/// Each slab page starts with a [`SlabPageHeader`] followed by the object array. +/// Local (owner-CPU) operations use a bitmap under the slab lock. +/// Remote (cross-CPU) frees use an atomic CAS stack — no lock required. +use core::ptr::NonNull; +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use super::size_class::SizeClass; + +/// Magic number written at the start of every slab page header. +pub const SLAB_MAGIC: u32 = 0x534C_4142; // "SLAB" + +/// Maximum objects per slab page (512 = 8 × u64 bitmap words). +pub const MAX_OBJECTS_PER_SLAB: usize = 512; + +/// Number of u64 words in the local bitmap. +pub const BITMAP_WORDS: usize = MAX_OBJECTS_PER_SLAB / 64; // 8 + +/// Maximum number of pages a slab may span. +pub const MAX_SLAB_PAGES: usize = 4; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SlabListState { + None, + Partial, + Full, + Empty, +} + +/// Header placed at the very start of each slab page. +/// +/// Object data starts at `header_end`, aligned to `size_class.size()`. +#[repr(C)] +pub struct SlabPageHeader { + /// Magic number for integrity checks. + pub magic: u32, + /// Which size class this slab serves. + pub size_class: SizeClass, + /// Total number of objects that fit. + pub object_count: u16, + /// Number of objects free in `local_bitmap`. + pub local_free_count: u16, + /// The CPU that owns this slab page (local alloc/dealloc go through this CPU's lock). + pub owner_cpu: u16, + _pad: u16, + /// Total usable bytes (slab pages × PAGE_SIZE). + pub slab_bytes: u32, + + // --- Intrusive doubly-linked list pointers (used by SlabCache) --- + pub list_prev: usize, + pub list_next: usize, + pub(crate) list_state: SlabListState, + + // --- Local bitmap (under slab lock, owner CPU only) --- + // A set bit means the slot is FREE. + pub local_bitmap: [u64; BITMAP_WORDS], + + // --- Lock-free remote free stack (any CPU) --- + /// Head of the remote-free linked list (object virtual address, 0 = empty). + /// Each freed object stores `next` at its start (reuses object memory). + pub remote_free_head: AtomicUsize, + /// Number of objects in the remote-free stack. + pub remote_free_count: AtomicU32, +} + +impl SlabPageHeader { + /// Size of the header in bytes. + pub const HEADER_SIZE: usize = core::mem::size_of::(); + + /// Initialise a slab page header. All objects are marked free in the bitmap. + /// + /// `base` is the virtual address of the page, `bytes` is the total slab size. + pub fn init(&mut self, size_class: SizeClass, bytes: usize, owner_cpu: u16) { + let obj_size = size_class.size(); + let data_start = Self::data_offset(obj_size); + let usable = bytes.saturating_sub(data_start); + let count = (usable / obj_size).min(MAX_OBJECTS_PER_SLAB); + + self.magic = SLAB_MAGIC; + self.size_class = size_class; + self.object_count = count as u16; + self.local_free_count = count as u16; + self.owner_cpu = owner_cpu; + self._pad = 0; + self.slab_bytes = bytes as u32; + self.list_prev = 0; + self.list_next = 0; + self.list_state = SlabListState::None; + self.local_bitmap = [0u64; BITMAP_WORDS]; + self.remote_free_head = AtomicUsize::new(0); + self.remote_free_count = AtomicU32::new(0); + + // Mark first `count` bits as 1 (free). + let full_words = count / 64; + let remaining_bits = count % 64; + for w in self.local_bitmap.iter_mut().take(full_words) { + *w = u64::MAX; + } + if remaining_bits > 0 { + self.local_bitmap[full_words] = (1u64 << remaining_bits) - 1; + } + } + + /// Byte offset from the start of the page to the first object. + /// Aligned up to `obj_size` for natural alignment. + pub fn data_offset(obj_size: usize) -> usize { + let raw = Self::HEADER_SIZE; + // Align to obj_size (which is always a power of two). + (raw + obj_size - 1) & !(obj_size - 1) + } + + /// Virtual address of the object region start (given `base` = page start). + #[inline] + pub fn data_start(&self, base: usize) -> usize { + base + Self::data_offset(self.size_class.size()) + } + + /// Virtual address of object at `index`. + #[inline] + pub fn object_addr(&self, base: usize, index: usize) -> usize { + self.data_start(base) + index * self.size_class.size() + } + + /// Index of the object whose address is `addr`. + #[inline] + pub fn object_index(&self, base: usize, addr: usize) -> usize { + (addr - self.data_start(base)) / self.size_class.size() + } + + /// Base address (slab start) from an object address. + /// + /// Searches backward by page because the slab base may no longer have + /// absolute `slab_bytes` alignment after metadata is carved from a region + /// prefix by [`GlobalAllocator`](crate::GlobalAllocator). + #[inline] + pub fn base_from_obj_addr(addr: usize, slab_bytes: usize) -> usize { + let slab_pages = slab_bytes / PAGE_SIZE; + debug_assert!(slab_pages > 0); + + let page_base = addr & !(PAGE_SIZE - 1); + for page_idx in 0..slab_pages { + let Some(candidate) = page_base.checked_sub(page_idx * PAGE_SIZE) else { + break; + }; + let hdr = unsafe { &*(candidate as *const SlabPageHeader) }; + if hdr.magic == SLAB_MAGIC + && hdr.slab_bytes as usize == slab_bytes + && addr >= candidate + && addr < candidate + slab_bytes + { + return candidate; + } + } + + debug_assert!(false, "object address does not belong to a live slab"); + page_base + } + + fn base_from_obj_addr_unknown_with_page_size(addr: usize, page_size: usize) -> Option { + if page_size == 0 || !page_size.is_power_of_two() { + return None; + } + let page_base = addr & !(page_size - 1); + for page_idx in 0..MAX_SLAB_PAGES { + let Some(candidate) = page_base.checked_sub(page_idx * page_size) else { + break; + }; + let hdr = unsafe { &*(candidate as *const SlabPageHeader) }; + let slab_bytes = hdr.slab_bytes as usize; + if hdr.magic != SLAB_MAGIC + || slab_bytes == 0 + || !slab_bytes.is_multiple_of(page_size) + || slab_bytes / page_size > MAX_SLAB_PAGES + { + continue; + } + if addr >= candidate && addr < candidate + slab_bytes { + return Some(candidate); + } + } + None + } + + /// Base address (slab start) from an object address without knowing the slab size. + /// + /// Searches backward up to [`MAX_SLAB_PAGES`] pages and validates each candidate + /// against the header's `slab_bytes`. + #[inline] + pub fn base_from_obj_addr_unknown(addr: usize) -> Option { + Self::base_from_obj_addr_unknown_with_page_size(addr, PAGE_SIZE) + } + + /// Queue the object on its owner's remote-free list. + /// + /// # Safety + /// - `ptr` must point to a valid live slab object. + /// - `owner_cpu` must match the slab header's owner CPU. + /// - `page_size` must be the slab allocator's page size. + pub unsafe fn remote_free_object(ptr: NonNull, owner_cpu: u16, page_size: usize) { + let obj_addr = ptr.as_ptr() as usize; + let Some(base) = Self::base_from_obj_addr_unknown_with_page_size(obj_addr, page_size) + else { + debug_assert!(false, "object address does not belong to a live slab"); + return; + }; + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + debug_assert_eq!(hdr.magic, SLAB_MAGIC); + debug_assert_eq!(hdr.owner_cpu, owner_cpu); + unsafe { hdr.remote_free(obj_addr) }; + } + + // ------------------------------------------------------------------ + // Local allocation (under slab lock) + // ------------------------------------------------------------------ + + /// Allocate one object from the local bitmap. Returns the slot index or `None`. + pub fn local_alloc(&mut self) -> Option { + for (wi, word) in self.local_bitmap.iter_mut().enumerate() { + if *word != 0 { + let bit = word.trailing_zeros() as usize; + *word &= !(1u64 << bit); + self.local_free_count -= 1; + return Some(wi * 64 + bit); + } + } + None + } + + /// Free an object back to the local bitmap. + pub fn local_free(&mut self, index: usize) { + let wi = index / 64; + let bit = index % 64; + debug_assert!(self.local_bitmap[wi] & (1u64 << bit) == 0, "double free"); + self.local_bitmap[wi] |= 1u64 << bit; + self.local_free_count += 1; + } + + /// Whether this slab has any free objects (local bitmap only). + #[inline] + pub fn has_local_free(&self) -> bool { + self.local_free_count > 0 + } + + /// Whether every object in this slab is free (local bitmap only). + #[inline] + pub fn is_all_free(&self) -> bool { + self.local_free_count == self.object_count + } + + /// Whether the local bitmap is completely full (zero free objects locally). + #[inline] + pub fn is_local_full(&self) -> bool { + self.local_free_count == 0 + } + + pub(crate) fn prepare_for_buddy_free(&mut self) { + assert_eq!( + self.remote_free_head.load(Ordering::Acquire), + 0, + "returning slab with pending remote frees" + ); + self.list_prev = 0; + self.list_next = 0; + self.list_state = SlabListState::None; + self.magic = 0; + } + + // ------------------------------------------------------------------ + // Remote free (lock-free, any CPU) + // ------------------------------------------------------------------ + + /// Push `obj_addr` onto the remote-free stack (lock-free CAS). + /// + /// # Safety + /// - `obj_addr` must point to a previously allocated object within this slab. + /// - The object's first `size_of::()` bytes will be overwritten with + /// the next-pointer. + pub unsafe fn remote_free(&self, obj_addr: usize) { + unsafe { + loop { + let old_head = self.remote_free_head.load(Ordering::Acquire); + // Store "next" pointer inside the freed object. + (obj_addr as *mut usize).write(old_head); + if self + .remote_free_head + .compare_exchange_weak(old_head, obj_addr, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + self.remote_free_count.fetch_add(1, Ordering::Relaxed); + return; + } + } + } + } + + /// Drain all remote frees back into the local bitmap. + /// + /// Must be called under the owner-CPU slab lock. + pub fn drain_remote_frees(&mut self, base: usize) { + let head = self.remote_free_head.swap(0, Ordering::AcqRel); + if head == 0 { + return; + } + // Also zero the count (we'll re-add to local). + self.remote_free_count.store(0, Ordering::Relaxed); + + let mut ptr = head; + while ptr != 0 { + let next = unsafe { *(ptr as *const usize) }; + let idx = self.object_index(base, ptr); + let wi = idx / 64; + let bit = idx % 64; + debug_assert!( + self.local_bitmap[wi] & (1u64 << bit) == 0, + "remote double free" + ); + self.local_bitmap[wi] |= 1u64 << bit; + self.local_free_count += 1; + ptr = next; + } + } + + /// Whether any remote frees are pending. + #[inline] + pub fn has_remote_frees(&self) -> bool { + self.remote_free_head.load(Ordering::Acquire) != 0 + } +} diff --git a/components/buddy-slab-allocator/src/slab/size_class.rs b/components/buddy-slab-allocator/src/slab/size_class.rs new file mode 100644 index 0000000000..8f16fb8069 --- /dev/null +++ b/components/buddy-slab-allocator/src/slab/size_class.rs @@ -0,0 +1,91 @@ +/// Object size classes for the slab allocator. +/// +/// Each size class corresponds to a fixed object size. +/// Allocations are rounded up to the nearest size class. +use core::alloc::Layout; + +/// Number of distinct size classes. +pub const SIZE_CLASS_COUNT: usize = 9; + +/// Fixed set of object sizes served by the slab allocator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum SizeClass { + Bytes8 = 0, + Bytes16 = 1, + Bytes32 = 2, + Bytes64 = 3, + Bytes128 = 4, + Bytes256 = 5, + Bytes512 = 6, + Bytes1024 = 7, + Bytes2048 = 8, +} + +/// Maximum object size handled by the slab. +pub const SLAB_MAX_SIZE: usize = 2048; + +/// Ordered table of (object_size, index) for all classes. +const CLASS_SIZES: [usize; SIZE_CLASS_COUNT] = [8, 16, 32, 64, 128, 256, 512, 1024, 2048]; + +impl SizeClass { + /// All size classes in ascending order. + pub const ALL: [SizeClass; SIZE_CLASS_COUNT] = [ + SizeClass::Bytes8, + SizeClass::Bytes16, + SizeClass::Bytes32, + SizeClass::Bytes64, + SizeClass::Bytes128, + SizeClass::Bytes256, + SizeClass::Bytes512, + SizeClass::Bytes1024, + SizeClass::Bytes2048, + ]; + + /// Number of distinct size classes. + pub const COUNT: usize = SIZE_CLASS_COUNT; + + /// Select the smallest size class that can satisfy `layout`. + /// + /// Returns `None` if the requested size or alignment exceeds the slab's capability. + pub fn from_layout(layout: Layout) -> Option { + let size = layout.size().max(layout.align()); + if size > SLAB_MAX_SIZE { + return None; + } + for (i, &class_size) in CLASS_SIZES.iter().enumerate() { + if size <= class_size { + return Some(SizeClass::ALL[i]); + } + } + None + } + + /// Object size in bytes. + pub const fn size(self) -> usize { + CLASS_SIZES[self as usize] + } + + /// Array index (0-based). + pub const fn index(self) -> usize { + self as usize + } + + /// How many pages are needed for a single slab of this class. + /// + /// Smaller classes use 1 page, larger classes may use more to amortise the + /// per-page header overhead. + pub const fn slab_pages(self, page_size: usize) -> usize { + let obj_size = self.size(); + if obj_size <= 256 { + 1 + } else if obj_size <= 1024 { + 2 + } else { + // 2048-byte objects: 4 pages → header + room for objects + let v = 16 * page_size / (obj_size * 8); + let v = if v < 4 { v } else { 4 }; + if v < 1 { 1 } else { v } + } + } +} diff --git a/components/buddy-slab-allocator/tests/README.md b/components/buddy-slab-allocator/tests/README.md new file mode 100644 index 0000000000..227f624869 --- /dev/null +++ b/components/buddy-slab-allocator/tests/README.md @@ -0,0 +1,48 @@ +# Allocator Test Suite + +本目录包含 allocator 的集成测试与压力测试。 + +## 结构 + +- `integration_test.rs` + 常规集成测试，覆盖全局分配器、Buddy、Slab 以及多模块协同行为。 +- `dma32_pages_test.rs` + 低地址页分配相关测试。 +- `stress_test.rs` + 长时间随机、耗尽恢复、碎片化恢复，以及真实多线程跨 CPU 压力测试。 + 这些测试默认使用 `#[ignore]`，不会进入常规 `cargo test` 路径。 +- `common/` + 共享测试辅助模块，提供宿主堆管理、线程本地 CPU mock、固定种子 RNG 和通用初始化逻辑。 + +单元测试仍位于 `src/**/*.rs` 的 `#[cfg(test)]` 模块中，文档测试位于公共 API 注释中。 + +## 常用命令 + +```bash +# 常规测试 +cargo test + +# 串行执行，便于排查测试交互 +cargo test -- --test-threads=1 + +# 仅运行常规集成测试 +cargo test --test integration_test + +# 仅运行压力测试 +cargo test --test stress_test -- --ignored --nocapture + +# 串行运行压力测试，便于定位多线程问题 +cargo test --test stress_test -- --ignored --nocapture --test-threads=1 +``` + +## 设计原则 + +- 常规测试应保持快速、稳定，可直接进入 CI。 +- 压力测试用于长时间 workload、真实多线程 cross-CPU 交互、耗尽恢复、碎片化恢复与统计不变量检查。 +- 性能测量移至 `benches/`，不在测试中混入 benchmark 逻辑。 + +## 注意事项 + +1. 所有测试都使用宿主分配器申请一块测试堆。 +2. 压力测试默认被忽略，需要显式运行。 +3. 多线程压力测试默认被 `#[ignore]` 标记，需要显式运行。 diff --git a/components/buddy-slab-allocator/tests/common/mod.rs b/components/buddy-slab-allocator/tests/common/mod.rs new file mode 100644 index 0000000000..2939707523 --- /dev/null +++ b/components/buddy-slab-allocator/tests/common/mod.rs @@ -0,0 +1,190 @@ +#![allow(dead_code)] + +use core::ptr::NonNull; +use std::{ + alloc::{Layout, alloc, dealloc}, + cell::Cell, + sync::{Mutex, MutexGuard, OnceLock}, +}; + +use buddy_slab_allocator::{ + __reset_global_allocator_singleton_for_tests, GlobalAllocator, PerCpuSlab, SlabPoolTrait, + eii::{slab_pool_impl, virt_to_phys_impl}, +}; +use rand::{SeedableRng, rngs::StdRng}; + +thread_local! { + static CURRENT_CPU: Cell = const { Cell::new(0) }; +} + +const TEST_PAGE_SIZE: usize = 0x1000; +const MAX_TEST_CPUS: usize = 64; + +fn lowmem_map(vaddr: usize) -> usize { + vaddr & 0x0FFF_FFFF +} + +pub struct GlobalTestContext { + _guard: MutexGuard<'static, ()>, +} + +struct TestSlabPool { + slabs: &'static [PerCpuSlab], +} + +fn global_test_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) +} + +impl Drop for GlobalTestContext { + fn drop(&mut self) { + __reset_global_allocator_singleton_for_tests(); + } +} + +fn test_cpu_slabs() -> &'static [PerCpuSlab] { + static SLABS: OnceLock]>> = OnceLock::new(); + SLABS.get_or_init(|| { + (0..MAX_TEST_CPUS) + .map(|cpu| PerCpuSlab::new(cpu as u16)) + .collect::>() + .into_boxed_slice() + }) +} + +fn reset_cpu_slabs(cpu_count: usize) { + assert!( + cpu_count <= MAX_TEST_CPUS, + "cpu_count exceeds test slab pool" + ); + for slab in &test_cpu_slabs()[..cpu_count] { + slab.reset(); + } +} + +impl SlabPoolTrait for TestSlabPool { + fn current_slab(&self) -> &dyn buddy_slab_allocator::SlabTrait { + &self.slabs[CURRENT_CPU.with(|slot| slot.get())] + } + + fn owner_slab(&self, cpu_idx: usize) -> &dyn buddy_slab_allocator::SlabTrait { + &self.slabs[cpu_idx] + } +} + +fn test_slab_pool_ref() -> &'static TestSlabPool { + static POOL: OnceLock = OnceLock::new(); + POOL.get_or_init(|| TestSlabPool { + slabs: test_cpu_slabs(), + }) +} + +#[virt_to_phys_impl] +fn test_virt_to_phys(vaddr: usize) -> usize { + lowmem_map(vaddr) +} + +#[slab_pool_impl] +fn test_slab_pool() -> &'static dyn SlabPoolTrait { + test_slab_pool_ref() +} + +pub fn set_current_cpu(cpu: usize) { + CURRENT_CPU.with(|slot| slot.set(cpu)); +} + +pub fn seeded_rng(seed: u64) -> StdRng { + StdRng::seed_from_u64(seed) +} + +pub struct HostRegion { + ptr: *mut u8, + layout: Layout, +} + +impl HostRegion { + pub fn new(size: usize, align: usize) -> Self { + let layout = Layout::from_size_align(size, align).unwrap(); + let ptr = unsafe { alloc(layout) }; + assert!(!ptr.is_null(), "host alloc failed"); + Self { ptr, layout } + } + + pub fn addr(&self) -> usize { + self.ptr as usize + } + + pub fn len(&self) -> usize { + self.layout.size() + } + + pub fn as_mut_ptr(&mut self) -> *mut u8 { + self.ptr + } + + pub fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.layout.size()) } + } + + pub unsafe fn subslice(&mut self, offset: usize, len: usize) -> &mut [u8] { + unsafe { std::slice::from_raw_parts_mut(self.ptr.add(offset), len) } + } +} + +impl Drop for HostRegion { + fn drop(&mut self) { + unsafe { dealloc(self.ptr, self.layout) }; + } +} + +pub fn init_global_slice( + allocator: &GlobalAllocator, + region: &mut [u8], + cpu_count: usize, +) -> GlobalTestContext { + let ctx = global_test_context::(cpu_count); + unsafe { allocator.init(region).unwrap() }; + ctx +} + +pub fn init_global( + allocator: &GlobalAllocator, + region: &mut HostRegion, + cpu_count: usize, +) -> GlobalTestContext { + init_global_slice(allocator, region.as_mut_slice(), cpu_count) +} + +pub fn global_test_context(cpu_count: usize) -> GlobalTestContext { + assert_eq!( + PAGE_SIZE, TEST_PAGE_SIZE, + "test EII slab pool only supports PAGE_SIZE={TEST_PAGE_SIZE:#x}" + ); + let guard = global_test_lock() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + set_current_cpu(0); + reset_cpu_slabs(cpu_count); + GlobalTestContext { _guard: guard } +} + +pub fn count_free_pages(allocator: &GlobalAllocator) -> usize { + let mut addrs = Vec::new(); + while let Ok(addr) = allocator.alloc_pages(1, PAGE_SIZE) { + addrs.push(addr); + } + let count = addrs.len(); + for addr in addrs { + allocator.dealloc_pages(addr, 1); + } + count +} + +pub fn nonnull_from_addr(addr: usize) -> NonNull { + unsafe { NonNull::new_unchecked(addr as *mut u8) } +} + +pub fn virt_to_phys(vaddr: usize) -> usize { + lowmem_map(vaddr) +} diff --git a/components/buddy-slab-allocator/tests/dma32_pages_test.rs b/components/buddy-slab-allocator/tests/dma32_pages_test.rs new file mode 100644 index 0000000000..c4420aa045 --- /dev/null +++ b/components/buddy-slab-allocator/tests/dma32_pages_test.rs @@ -0,0 +1,101 @@ +//! Tests for lowmem (DMA32) page allocation via GlobalAllocator. + +extern crate buddy_slab_allocator; + +mod common; + +use buddy_slab_allocator::GlobalAllocator; +use common::{GlobalTestContext, HostRegion, init_global, virt_to_phys}; + +const PAGE_SIZE: usize = 0x1000; +const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; + +fn init_allocator( + allocator: &GlobalAllocator, + region: &mut HostRegion, +) -> GlobalTestContext { + init_global(allocator, region, 1) +} + +#[test] +fn test_lowmem_basic() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_allocator(&allocator, &mut region); + let section = allocator.managed_section(0).unwrap(); + let managed_start = section.start; + let managed_end = managed_start + section.size; + + let addr1 = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + let addr2 = allocator.alloc_pages_lowmem(4, PAGE_SIZE).unwrap(); + + assert!(addr1 >= managed_start && addr1 < managed_end); + assert!(addr2 >= managed_start && addr2 < managed_end); + assert_eq!(addr1 % PAGE_SIZE, 0); + assert_eq!(addr2 % PAGE_SIZE, 0); + + allocator.dealloc_pages(addr1, 1); + allocator.dealloc_pages(addr2, 4); +} + +#[test] +fn test_lowmem_aligned() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 2); + let allocator = GlobalAllocator::::new(); + let _ctx = init_allocator(&allocator, &mut region); + + let addr = allocator.alloc_pages_lowmem(1, 2 * PAGE_SIZE).unwrap(); + assert_eq!(addr % (2 * PAGE_SIZE), 0); + allocator.dealloc_pages(addr, 1); +} + +#[test] +fn test_lowmem_vs_normal() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_allocator(&allocator, &mut region); + + let addr_low = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + let addr_normal = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); + + assert!(addr_low >= allocator.managed_section(0).unwrap().start); + assert!(addr_normal >= allocator.managed_section(0).unwrap().start); + + allocator.dealloc_pages(addr_low, 1); + allocator.dealloc_pages(addr_normal, 1); +} + +#[test] +fn test_lowmem_stress() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_allocator(&allocator, &mut region); + + let mut addrs = Vec::new(); + for _ in 0..32 { + let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + addrs.push(addr); + } + for addr in addrs { + allocator.dealloc_pages(addr, 1); + } +} + +#[test] +fn global_add_region_unaligned_lowmem_alignment() { + const ALIGN_2M: usize = 2 * 1024 * 1024; + + let mut first = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); + let mut second = HostRegion::new(8 * ALIGN_2M + 0x1234 + PAGE_SIZE, ALIGN_2M); + let allocator = GlobalAllocator::::new(); + let _ctx = init_allocator(&allocator, &mut first); + + let second_len = second.len(); + let second_slice = unsafe { second.subslice(0x1234, second_len - 0x1234 - PAGE_SIZE / 3) }; + unsafe { allocator.add_region(second_slice).unwrap() }; + + let addr = allocator.alloc_pages_lowmem(1, ALIGN_2M).unwrap(); + assert_eq!(addr % ALIGN_2M, 0); + assert!(virt_to_phys(addr) + PAGE_SIZE <= 0x1000_0000); + allocator.dealloc_pages(addr, 1); +} diff --git a/components/buddy-slab-allocator/tests/integration_test.rs b/components/buddy-slab-allocator/tests/integration_test.rs new file mode 100644 index 0000000000..0c82e098d2 --- /dev/null +++ b/components/buddy-slab-allocator/tests/integration_test.rs @@ -0,0 +1,1266 @@ +//! Integration tests for the buddy-slab-allocator crate. + +extern crate buddy_slab_allocator; +mod common; + +use core::{alloc::Layout, ptr::NonNull}; +use std::collections::BTreeSet; + +use buddy_slab_allocator::{ + AllocError, BuddyAllocator, GlobalAllocator, ManagedSection, SizeClass, SlabAllocResult, + SlabAllocator, SlabDeallocResult, slab::SlabPageHeader, +}; +use common::{ + GlobalTestContext, HostRegion, count_free_pages, global_test_context, + init_global as init_global_allocator, init_global_slice, set_current_cpu, +}; + +const PAGE_SIZE: usize = 0x1000; +const TEST_HEAP_SIZE: usize = 16 * 1024 * 1024; // 16 MiB + +fn buddy_region_size(heap_size: usize) -> usize { + heap_size + BuddyAllocator::::required_meta_size(heap_size) + PAGE_SIZE * 4 +} + +fn init_buddy(buddy: &mut BuddyAllocator, region: &mut HostRegion) -> ManagedSection { + unsafe { buddy.init(region.as_mut_slice()).unwrap() }; + buddy.section(0).unwrap() +} + +fn init_buddy_with_heap_alignment( + buddy: &mut BuddyAllocator, + region: &mut HostRegion, + heap_align: usize, +) -> ManagedSection { + for offset in (0..heap_align).step_by(PAGE_SIZE) { + if region.len() <= offset { + break; + } + let slice = unsafe { region.subslice(offset, region.len() - offset) }; + if unsafe { buddy.init(slice) }.is_ok() { + let section = buddy.section(0).unwrap(); + if section.start.is_multiple_of(heap_align) { + return section; + } + } + } + panic!("failed to find test region with heap alignment {heap_align:#x}"); +} + +fn primary_section(allocator: &GlobalAllocator) -> ManagedSection { + allocator.managed_section(0).unwrap() +} + +fn irregular_region(size: usize, offset: usize, trim: usize, host_align: usize) -> HostRegion { + HostRegion::new(size + offset + trim + PAGE_SIZE, host_align) +} + +// ====================================================================== +// Buddy allocator (standalone) tests +// ====================================================================== + +#[test] +fn buddy_basic_alloc_dealloc() { + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let section = init_buddy(&mut buddy, &mut region); + + let addr1 = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr1 >= section.start && addr1 < section.start + section.size); + assert_eq!(addr1 % PAGE_SIZE, 0); + + let addr4 = buddy.alloc_pages(4, PAGE_SIZE).unwrap(); + assert_eq!(addr4 % PAGE_SIZE, 0); + + let free_before = buddy.free_pages(); + buddy.dealloc_pages(addr1, 1); + buddy.dealloc_pages(addr4, 4); + assert!(buddy.free_pages() > free_before); +} + +#[test] +fn buddy_alignment() { + // Heap must be aligned to the highest alignment we test (PAGE_SIZE * 4) + let mut region = HostRegion::new( + buddy_region_size(TEST_HEAP_SIZE) + PAGE_SIZE * 4, + PAGE_SIZE * 4, + ); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy_with_heap_alignment(&mut buddy, &mut region, PAGE_SIZE * 4); + + let addr2 = buddy.alloc_pages(1, PAGE_SIZE * 2).unwrap(); + assert_eq!(addr2 % (PAGE_SIZE * 2), 0); + + let addr4 = buddy.alloc_pages(1, PAGE_SIZE * 4).unwrap(); + assert_eq!(addr4 % (PAGE_SIZE * 4), 0); + + buddy.dealloc_pages(addr2, 1); + buddy.dealloc_pages(addr4, 1); +} + +#[test] +fn buddy_add_region_unaligned_start_preserves_4k_alignment() { + const ALIGN_2M: usize = 2 * 1024 * 1024; + let mut first = HostRegion::new(buddy_region_size(32 * PAGE_SIZE) + ALIGN_2M, ALIGN_2M); + let mut second = irregular_region( + buddy_region_size(64 * PAGE_SIZE), + 0x1234, + PAGE_SIZE / 3, + 2 * 1024 * 1024, + ); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy_with_heap_alignment(&mut buddy, &mut first, ALIGN_2M); + + let second_len = second.len(); + let second_slice = unsafe { second.subslice(0x1234, second_len - 0x1234 - PAGE_SIZE / 3) }; + + unsafe { buddy.add_region(second_slice).unwrap() }; + + let addr = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert_eq!(addr % PAGE_SIZE, 0); + buddy.dealloc_pages(addr, 1); +} + +#[test] +fn buddy_add_region_unaligned_start_preserves_2m_alignment() { + const ALIGN_2M: usize = 2 * 1024 * 1024; + + let mut first = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut second = irregular_region( + buddy_region_size(4 * ALIGN_2M), + PAGE_SIZE / 2, + PAGE_SIZE / 3, + ALIGN_2M, + ); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy(&mut buddy, &mut first); + + let second_len = second.len(); + let second_slice = + unsafe { second.subslice(PAGE_SIZE / 2, second_len - PAGE_SIZE / 2 - PAGE_SIZE / 3) }; + + unsafe { buddy.add_region(second_slice).unwrap() }; + + let addr = buddy.alloc_pages(1, ALIGN_2M).unwrap(); + assert_eq!(addr % ALIGN_2M, 0); + buddy.dealloc_pages(addr, 1); +} + +#[test] +fn buddy_aligned_alloc_dealloc_uses_recorded_order() { + let heap_size = 64 * PAGE_SIZE; + let mut region = HostRegion::new( + buddy_region_size(heap_size) + PAGE_SIZE * 16, + PAGE_SIZE * 16, + ); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy_with_heap_alignment(&mut buddy, &mut region, PAGE_SIZE * 16); + + let free_before = buddy.free_pages(); + let addr = buddy.alloc_pages(4, PAGE_SIZE * 16).unwrap(); + buddy.dealloc_pages(addr, 4); + assert_eq!(buddy.free_pages(), free_before); +} + +#[test] +fn buddy_exhaust_and_recover() { + let heap_size = 64 * PAGE_SIZE; // Small heap + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region); + + let mut addrs = Vec::new(); + while let Ok(addr) = buddy.alloc_pages(1, PAGE_SIZE) { + addrs.push(addr); + } + assert_eq!(buddy.free_pages(), 0); + + // Free half + for addr in addrs.drain(..addrs.len() / 2) { + buddy.dealloc_pages(addr, 1); + } + assert!(buddy.free_pages() > 0); + + // Allocate again + let addr = buddy.alloc_pages(1, PAGE_SIZE); + assert!(addr.is_ok()); + + // Cleanup + if let Ok(a) = addr { + buddy.dealloc_pages(a, 1); + } + for a in addrs { + buddy.dealloc_pages(a, 1); + } +} + +#[test] +fn buddy_merge_coalescing() { + let heap_size = 16 * PAGE_SIZE; + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region); + + let initial_free = buddy.free_pages(); + + // Allocate two single pages + let a = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + let b = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + buddy.dealloc_pages(a, 1); + buddy.dealloc_pages(b, 1); + + // After freeing both, free_pages should return to initial + assert_eq!(buddy.free_pages(), initial_free); +} + +#[test] +fn buddy_fragmentation_blocks_high_order_then_recovers() { + let heap_size = 32 * PAGE_SIZE; + let mut region = HostRegion::new(buddy_region_size(heap_size), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let section = init_buddy(&mut buddy, &mut region); + + let mut addrs = Vec::new(); + while let Ok(addr) = buddy.alloc_pages(1, PAGE_SIZE) { + addrs.push(addr); + } + assert_eq!(addrs.len(), section.total_pages); + + for &addr in addrs.iter().step_by(2) { + buddy.dealloc_pages(addr, 1); + } + assert!(buddy.alloc_pages(2, PAGE_SIZE).is_err()); + + for &addr in addrs.iter().skip(1).step_by(2) { + buddy.dealloc_pages(addr, 1); + } + + let addr = buddy.alloc_pages(8, PAGE_SIZE).unwrap(); + buddy.dealloc_pages(addr, 8); + assert_eq!(buddy.free_pages(), section.total_pages); +} + +#[test] +fn buddy_high_order_full_cycle_restores_free_pages() { + let heap_size = 256 * PAGE_SIZE; + let mut region = HostRegion::new( + buddy_region_size(heap_size) + PAGE_SIZE * 16, + PAGE_SIZE * 16, + ); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy_with_heap_alignment(&mut buddy, &mut region, PAGE_SIZE * 16); + + let initial_free = buddy.free_pages(); + let requests = [ + (1usize, PAGE_SIZE), + (2, 2 * PAGE_SIZE), + (3, 4 * PAGE_SIZE), + (8, 8 * PAGE_SIZE), + (5, PAGE_SIZE), + (16, 16 * PAGE_SIZE), + ]; + let mut allocations = Vec::new(); + + for (count, align) in requests { + let addr = buddy.alloc_pages(count, align).unwrap(); + allocations.push((addr, count)); + } + assert!(buddy.free_pages() < initial_free); + + for (addr, count) in allocations.into_iter().rev() { + buddy.dealloc_pages(addr, count); + } + assert_eq!(buddy.free_pages(), initial_free); +} + +#[test] +fn buddy_add_region_enables_second_section_allocation() { + let mut first = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(64 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first); + + while buddy.alloc_pages(1, PAGE_SIZE).is_ok() {} + assert_eq!(buddy.free_pages(), 0); + + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + assert_eq!(buddy.section_count(), 2); + let second_section = buddy.section(1).unwrap(); + + let addr = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); + assert!(addr < first_section.start || addr >= first_section.start + first_section.size); +} + +#[test] +fn buddy_add_region_overlap_rejected() { + let mut region = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region); + + let overlap = unsafe { region.subslice(1, region.len() - 1) }; + let err = unsafe { buddy.add_region(overlap) }.unwrap_err(); + assert_eq!(err, AllocError::MemoryOverlap); +} + +#[test] +fn buddy_alloc_pages_first_fit_by_registration_order() { + let mut first = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(64 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first); + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + + let addr = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= first_section.start && addr < first_section.start + first_section.size); +} + +#[test] +fn buddy_lowmem_scans_across_sections() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy(&mut buddy, &mut first); + + while buddy.alloc_pages_lowmem(1, PAGE_SIZE).is_ok() {} + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let second_section = buddy.section(1).unwrap(); + + let addr = buddy.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); +} + +#[test] +fn buddy_dealloc_pages_finds_correct_section() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy(&mut buddy, &mut first); + while buddy.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let baseline = buddy.free_pages(); + let second_section = buddy.section(1).unwrap(); + + let addr = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); + buddy.dealloc_pages(addr, 1); + + assert_eq!(buddy.free_pages(), baseline); +} + +#[test] +fn buddy_total_and_free_pages_are_aggregated() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first); + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let second_section = buddy.section(1).unwrap(); + + assert_eq!( + buddy.total_pages(), + first_section.total_pages + second_section.total_pages + ); + assert_eq!( + buddy.free_pages(), + first_section.free_pages + second_section.free_pages + ); +} + +#[test] +fn buddy_managed_bytes_matches_all_sections() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let first_section = init_buddy(&mut buddy, &mut first); + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + let second_section = buddy.section(1).unwrap(); + + assert_eq!( + buddy.managed_bytes(), + first_section.size + second_section.size + ); +} + +#[test] +fn buddy_allocated_bytes_changes_with_page_alloc_free() { + let mut region = HostRegion::new(buddy_region_size(64 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region); + + assert_eq!(buddy.allocated_bytes(), 0); + + let a = buddy.alloc_pages(1, PAGE_SIZE).unwrap(); + assert_eq!(buddy.allocated_bytes(), PAGE_SIZE); + + let b = buddy.alloc_pages(4, PAGE_SIZE).unwrap(); + assert_eq!(buddy.allocated_bytes(), 5 * PAGE_SIZE); + + buddy.dealloc_pages(a, 1); + assert_eq!(buddy.allocated_bytes(), 4 * PAGE_SIZE); + + buddy.dealloc_pages(b, 4); + assert_eq!(buddy.allocated_bytes(), 0); +} + +#[test] +fn buddy_allocated_bytes_zero_when_all_free() { + let mut region = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region); + + assert_eq!(buddy.allocated_bytes(), 0); +} + +#[test] +fn buddy_allocated_bytes_aggregates_across_sections() { + let mut first = HostRegion::new(buddy_region_size(16 * PAGE_SIZE), PAGE_SIZE); + let mut second = HostRegion::new(buddy_region_size(32 * PAGE_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _first_section = init_buddy(&mut buddy, &mut first); + while buddy.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { buddy.add_region(second.as_mut_slice()).unwrap() }; + + let addr = buddy.alloc_pages(8, PAGE_SIZE).unwrap(); + assert_eq!( + buddy.allocated_bytes(), + buddy.managed_bytes() - buddy.free_pages() * PAGE_SIZE + ); + buddy.dealloc_pages(addr, 8); + assert_eq!( + buddy.allocated_bytes(), + buddy.managed_bytes() - buddy.free_pages() * PAGE_SIZE + ); +} + +// ====================================================================== +// Slab allocator (standalone) tests +// ====================================================================== + +#[test] +fn slab_basic() { + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region); + + let mut slab = SlabAllocator::::new(); + + let layout = Layout::from_size_align(64, 8).unwrap(); + // First alloc should request pages + match slab.alloc(layout).unwrap() { + SlabAllocResult::NeedsSlab { size_class, pages } => { + let addr = buddy.alloc_pages(pages, PAGE_SIZE).unwrap(); + slab.add_slab(size_class, addr, pages * PAGE_SIZE, 0); + } + SlabAllocResult::Allocated(_) => panic!("should need slab first"), + } + + // Now allocation should succeed + let ptr = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(p) => p, + _ => panic!("expected allocated"), + }; + + // Dealloc + match slab.dealloc(ptr, layout) { + SlabDeallocResult::Done => {} + SlabDeallocResult::FreeSlab { .. } => {} // also valid + } +} + +#[test] +fn slab_many_objects() { + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE) + 0x10000, 0x10000); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy_with_heap_alignment(&mut buddy, &mut region, 0x10000); + + let mut slab = SlabAllocator::::new(); + let layout = Layout::from_size_align(32, 8).unwrap(); + + let mut ptrs = Vec::new(); + for _ in 0..200 { + loop { + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(p) => { + ptrs.push(p); + break; + } + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + } + } + } + } + + assert_eq!(ptrs.len(), 200); + for ptr in ptrs { + let _ = slab.dealloc(ptr, layout); + } +} + +#[test] +fn slab_all_size_classes() { + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE) + 0x10000, 0x10000); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy_with_heap_alignment(&mut buddy, &mut region, 0x10000); + + let mut slab = SlabAllocator::::new(); + let mut allocations = Vec::new(); + + for sc in SizeClass::ALL { + let layout = Layout::from_size_align(sc.size(), sc.size()).unwrap(); + loop { + match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(p) => { + allocations.push((p, layout)); + break; + } + SlabAllocResult::NeedsSlab { size_class, pages } => { + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + } + } + } + } + + assert_eq!(allocations.len(), SizeClass::COUNT); + for (ptr, layout) in allocations { + let _ = slab.dealloc(ptr, layout); + } +} + +#[test] +fn slab_reuses_freed_objects_same_size_class() { + let mut region = HostRegion::new(buddy_region_size(TEST_HEAP_SIZE), PAGE_SIZE * 4); + let mut buddy = BuddyAllocator::::new(); + let _section = init_buddy(&mut buddy, &mut region); + + let mut slab = SlabAllocator::::new(); + let layout = Layout::from_size_align(64, 8).unwrap(); + let (size_class, pages) = match slab.alloc(layout).unwrap() { + SlabAllocResult::NeedsSlab { size_class, pages } => (size_class, pages), + SlabAllocResult::Allocated(_) => panic!("should need slab first"), + }; + let slab_bytes = pages * PAGE_SIZE; + let addr = buddy.alloc_pages(pages, slab_bytes).unwrap(); + slab.add_slab(size_class, addr, slab_bytes, 0); + + let first = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => ptr, + SlabAllocResult::NeedsSlab { .. } => panic!("expected allocation from fresh slab"), + }; + let base = SlabPageHeader::base_from_obj_addr::(first.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + let object_count = hdr.object_count as usize; + + let mut ptrs = Vec::with_capacity(object_count); + ptrs.push(first); + for _ in 1..object_count { + let ptr = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => ptr, + SlabAllocResult::NeedsSlab { .. } => panic!("expected same slab to satisfy alloc"), + }; + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + ptrs.push(ptr); + } + assert!(matches!( + slab.alloc(layout).unwrap(), + SlabAllocResult::NeedsSlab { .. } + )); + + let freed_ptrs: Vec<_> = ptrs.iter().copied().step_by(2).collect(); + let freed_addrs: BTreeSet<_> = freed_ptrs.iter().map(|ptr| ptr.as_ptr() as usize).collect(); + for &ptr in &freed_ptrs { + assert!(matches!(slab.dealloc(ptr, layout), SlabDeallocResult::Done)); + } + + let mut reused_addrs = BTreeSet::new(); + for _ in 0..freed_addrs.len() { + let ptr = match slab.alloc(layout).unwrap() { + SlabAllocResult::Allocated(ptr) => ptr, + SlabAllocResult::NeedsSlab { .. } => panic!("expected reuse from freed slots"), + }; + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + reused_addrs.insert(ptr.as_ptr() as usize); + } + assert_eq!(reused_addrs, freed_addrs); + + for ptr in ptrs { + let addr = ptr.as_ptr() as usize; + if !freed_addrs.contains(&addr) { + let _ = slab.dealloc(ptr, layout); + } + } + for addr in reused_addrs { + let ptr = unsafe { NonNull::new_unchecked(addr as *mut u8) }; + let _ = slab.dealloc(ptr, layout); + } +} + +#[test] +fn global_init_with_unaligned_region_preserves_large_alloc_alignment() { + const ALIGN_2M: usize = 2 * 1024 * 1024; + + let mut region = irregular_region(12 * ALIGN_2M, 0x1234, PAGE_SIZE / 3, ALIGN_2M); + let allocator = GlobalAllocator::::new(); + let region_len = region.len(); + let region_slice = unsafe { region.subslice(0x1234, region_len - 0x1234 - PAGE_SIZE / 3) }; + + let _ctx = init_global_slice(&allocator, region_slice, 1); + + let layout = Layout::from_size_align(ALIGN_2M, ALIGN_2M).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + assert_eq!((ptr.as_ptr() as usize) % ALIGN_2M, 0); + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_add_region_with_unaligned_slice_preserves_large_alloc_alignment() { + const ALIGN_2M: usize = 2 * 1024 * 1024; + + let allocator = GlobalAllocator::::new(); + let mut first = HostRegion::new(12 * ALIGN_2M, ALIGN_2M); + let _ctx = init_global_allocator(&allocator, &mut first, 1); + + let mut second = irregular_region(12 * ALIGN_2M, 0x1234, PAGE_SIZE / 3, ALIGN_2M); + let second_len = second.len(); + let second_slice = unsafe { second.subslice(0x1234, second_len - 0x1234 - PAGE_SIZE / 3) }; + unsafe { allocator.add_region(second_slice).unwrap() }; + + let layout = Layout::from_size_align(ALIGN_2M, ALIGN_2M).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + assert_eq!((ptr.as_ptr() as usize) % ALIGN_2M, 0); + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_add_region_unaligned_does_not_break_small_alloc() { + let allocator = GlobalAllocator::::new(); + let mut first = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE); + let _ctx = init_global_allocator(&allocator, &mut first, 1); + + let mut second = irregular_region( + buddy_region_size(TEST_HEAP_SIZE), + 0x1234, + PAGE_SIZE / 3, + PAGE_SIZE, + ); + let second_len = second.len(); + let second_slice = unsafe { second.subslice(0x1234, second_len - 0x1234 - PAGE_SIZE / 3) }; + unsafe { allocator.add_region(second_slice).unwrap() }; + + for layout in [ + Layout::from_size_align(64, 8).unwrap(), + Layout::from_size_align(128, 16).unwrap(), + Layout::from_size_align(512, 64).unwrap(), + ] { + let ptr = allocator.alloc(layout).unwrap(); + unsafe { + ptr.as_ptr().write_bytes(0x5a, layout.size()); + allocator.dealloc(ptr, layout); + } + } +} + +// ====================================================================== +// Global allocator tests +// ====================================================================== + +fn init_global( + allocator: &GlobalAllocator, + region: &mut HostRegion, + cpu_count: usize, +) -> GlobalTestContext { + init_global_allocator(allocator, region, cpu_count) +} + +#[test] +fn global_reinit_same_instance_rejected() { + let allocator = GlobalAllocator::::new(); + let _ctx = global_test_context::(1); + let mut first = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut second = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + + unsafe { allocator.init(first.as_mut_slice()).unwrap() }; + let err = unsafe { allocator.init(second.as_mut_slice()) }.unwrap_err(); + assert_eq!(err, AllocError::AlreadyInitialized); +} + +#[test] +fn global_second_live_instance_rejected() { + let first_allocator = GlobalAllocator::::new(); + let second_allocator = GlobalAllocator::::new(); + let _ctx = global_test_context::(1); + let mut first = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let mut second = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + + unsafe { first_allocator.init(first.as_mut_slice()).unwrap() }; + let err = unsafe { second_allocator.init(second.as_mut_slice()) }.unwrap_err(); + assert_eq!(err, AllocError::AlreadyInitialized); +} + +#[test] +fn global_failed_init_rolls_back_singleton() { + let bad_allocator = GlobalAllocator::::new(); + let good_allocator = GlobalAllocator::::new(); + let _ctx = global_test_context::(1); + let mut bad = HostRegion::new(PAGE_SIZE - 1, PAGE_SIZE); + let mut good = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + + let err = unsafe { bad_allocator.init(bad.as_mut_slice()) }.unwrap_err(); + assert_eq!(err, AllocError::InvalidParam); + unsafe { good_allocator.init(good.as_mut_slice()).unwrap() }; +} + +#[test] +fn global_page_alloc() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let region_addr = region.addr(); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + let section = primary_section(&allocator); + let managed_start = section.start; + let managed_end = managed_start + section.size; + + let addr = allocator.alloc_pages(4, PAGE_SIZE).unwrap(); + assert!(managed_start > region_addr); + assert!(addr >= managed_start && addr < managed_end); + assert_eq!(addr % PAGE_SIZE, 0); + allocator.dealloc_pages(addr, 4); +} + +#[test] +fn global_small_alloc() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + let layout = Layout::from_size_align(64, 8).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_large_alloc() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + let layout = Layout::from_size_align(8192, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_mixed_alloc() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + let sizes: &[(usize, usize)] = &[ + (8, 8), + (64, 8), + (1024, 8), + (4096, PAGE_SIZE), + (8192, PAGE_SIZE), + ]; + let mut allocations = Vec::new(); + for &(size, align) in sizes { + let layout = Layout::from_size_align(size, align).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + allocations.push((ptr, layout)); + } + for (ptr, layout) in allocations { + unsafe { allocator.dealloc(ptr, layout) }; + } +} + +#[test] +fn global_cross_cpu_free() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 2); + + // Allocate on CPU 0 + set_current_cpu(0); + let layout = Layout::from_size_align(64, 8).unwrap(); + let mut ptrs = Vec::new(); + for _ in 0..10 { + ptrs.push(allocator.alloc(layout).unwrap()); + } + + // Free from CPU 1 (triggers remote free path) + set_current_cpu(1); + for ptr in ptrs { + unsafe { allocator.dealloc(ptr, layout) }; + } + + // Allocate on CPU 0 again — should drain remote frees and succeed + set_current_cpu(0); + let ptr = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_cross_cpu_free_drains_remote_queue() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 2); + + set_current_cpu(0); + let layout = Layout::from_size_align(64, 8).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + + let slab_bytes = SizeClass::from_layout(layout) + .unwrap() + .slab_pages(PAGE_SIZE) + * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + assert_eq!(hdr.owner_cpu, 0); + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + + set_current_cpu(1); + unsafe { allocator.dealloc(ptr, layout) }; + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 1 + ); + assert_ne!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + + set_current_cpu(0); + let ptr2 = allocator.alloc(layout).unwrap(); + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + assert_eq!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + unsafe { allocator.dealloc(ptr2, layout) }; +} + +#[test] +fn global_cross_cpu_free_multiple_rounds_same_slab() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 2); + + let layout = Layout::from_size_align(64, 8).unwrap(); + + set_current_cpu(0); + let first = allocator.alloc(layout).unwrap(); + let slab_bytes = SizeClass::from_layout(layout) + .unwrap() + .slab_pages(PAGE_SIZE) + * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(first.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + let object_count = hdr.object_count as usize; + let mut ptrs = Vec::with_capacity(object_count); + ptrs.push(first); + for _ in 1..object_count { + let ptr = allocator.alloc(layout).unwrap(); + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + ptrs.push(ptr); + } + + set_current_cpu(1); + for &ptr in &ptrs { + unsafe { allocator.dealloc(ptr, layout) }; + } + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed) as usize, + object_count + ); + + set_current_cpu(0); + let mut drained = Vec::with_capacity(object_count); + for _ in 0..object_count { + drained.push(allocator.alloc(layout).unwrap()); + } + assert_eq!( + hdr.remote_free_count + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + assert_eq!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Relaxed), + 0 + ); + + for ptr in drained { + unsafe { allocator.dealloc(ptr, layout) }; + } +} + +#[test] +fn global_full_slab_remote_then_local_free_reuses_without_list_cycle() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 2); + + let layout = Layout::from_size_align(64, 8).unwrap(); + + set_current_cpu(0); + let first = allocator.alloc(layout).unwrap(); + let slab_bytes = SizeClass::from_layout(layout) + .unwrap() + .slab_pages(PAGE_SIZE) + * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(first.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + let object_count = hdr.object_count as usize; + + let mut ptrs = Vec::with_capacity(object_count); + ptrs.push(first); + for _ in 1..object_count { + let ptr = allocator.alloc(layout).unwrap(); + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + ptrs.push(ptr); + } + assert_eq!(hdr.local_free_count, 0); + + let remote_count = 4; + set_current_cpu(1); + for &ptr in &ptrs[..remote_count] { + unsafe { allocator.dealloc(ptr, layout) }; + } + assert_ne!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Acquire), + 0 + ); + + set_current_cpu(0); + for &ptr in &ptrs[remote_count..] { + unsafe { allocator.dealloc(ptr, layout) }; + } + assert_eq!( + hdr.remote_free_head + .load(core::sync::atomic::Ordering::Acquire), + 0 + ); + + let mut reused = Vec::with_capacity(object_count); + for _ in 0..object_count { + let ptr = allocator.alloc(layout).unwrap(); + let ptr_base = + SlabPageHeader::base_from_obj_addr::(ptr.as_ptr() as usize, slab_bytes); + assert_eq!(ptr_base, base); + reused.push(ptr); + } + + assert_ne!(hdr.list_prev, base, "slab list_prev points to itself"); + assert_ne!(hdr.list_next, base, "slab list_next points to itself"); + + for ptr in reused { + unsafe { allocator.dealloc(ptr, layout) }; + } +} + +#[test] +fn global_small_object_churn_then_large_alloc() { + const REGION_SIZE: usize = 8 * 1024 * 1024; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + let small_layout = Layout::from_size_align(2048, 8).unwrap(); + let warmup = allocator.alloc(small_layout).unwrap(); + unsafe { allocator.dealloc(warmup, small_layout) }; + let baseline = count_free_pages(&allocator); + let mut ptrs = Vec::new(); + while let Ok(ptr) = allocator.alloc(small_layout) { + ptrs.push(ptr); + } + assert!(!ptrs.is_empty()); + + for ptr in ptrs { + unsafe { allocator.dealloc(ptr, small_layout) }; + } + + let large_layout = Layout::from_size_align(16 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(large_layout).unwrap(); + unsafe { allocator.dealloc(ptr, large_layout) }; + assert_eq!(count_free_pages(&allocator), baseline); +} + +#[test] +fn global_cross_cpu_free_all_objects_recovers_backend_pages() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 2); + + let layout = Layout::from_size_align(64, 8).unwrap(); + set_current_cpu(0); + let warmup = allocator.alloc(layout).unwrap(); + unsafe { allocator.dealloc(warmup, layout) }; + let baseline = count_free_pages(&allocator); + + let first = allocator.alloc(layout).unwrap(); + let slab_bytes = SizeClass::from_layout(layout) + .unwrap() + .slab_pages(PAGE_SIZE) + * PAGE_SIZE; + let base = SlabPageHeader::base_from_obj_addr::(first.as_ptr() as usize, slab_bytes); + let hdr = unsafe { &*(base as *const SlabPageHeader) }; + let object_count = hdr.object_count as usize; + + let mut ptrs = Vec::with_capacity(object_count); + ptrs.push(first); + for _ in 1..object_count { + ptrs.push(allocator.alloc(layout).unwrap()); + } + + set_current_cpu(1); + for &ptr in &ptrs { + unsafe { allocator.dealloc(ptr, layout) }; + } + + set_current_cpu(0); + let mut drained = Vec::with_capacity(object_count); + for _ in 0..object_count { + drained.push(allocator.alloc(layout).unwrap()); + } + for ptr in drained { + unsafe { allocator.dealloc(ptr, layout) }; + } + + assert_eq!(count_free_pages(&allocator), baseline); +} + +#[test] +fn global_lowmem_fragmentation_recovery() { + const REGION_SIZE: usize = 8 * 1024 * 1024; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global_allocator(&allocator, &mut region, 1); + + let mut addrs = Vec::new(); + while let Ok(addr) = allocator.alloc_pages_lowmem(1, PAGE_SIZE) { + addrs.push(addr); + } + assert!(addrs.len() > 8); + + for &addr in addrs.iter().step_by(2) { + allocator.dealloc_pages(addr, 1); + } + assert!(allocator.alloc_pages_lowmem(2, 2 * PAGE_SIZE).is_err()); + + for &addr in addrs.iter().skip(1).step_by(2) { + allocator.dealloc_pages(addr, 1); + } + + let addr = allocator.alloc_pages_lowmem(2, 2 * PAGE_SIZE).unwrap(); + allocator.dealloc_pages(addr, 2); +} + +#[test] +fn global_lowmem_pages() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global_allocator(&allocator, &mut region, 1); + + let addr = allocator.alloc_pages_lowmem(1, PAGE_SIZE).unwrap(); + assert!(addr >= primary_section(&allocator).start); + allocator.dealloc_pages(addr, 1); +} + +#[test] +fn global_unaligned_region_start() { + let mut region = HostRegion::new(TEST_HEAP_SIZE + PAGE_SIZE, PAGE_SIZE * 4); + let region_start = region.addr() + 1; + let region_size = TEST_HEAP_SIZE; + let allocator = GlobalAllocator::::new(); + let unaligned_region = unsafe { region.subslice(1, region_size) }; + let _ctx = init_global_slice(&allocator, unaligned_region, 1); + + let section = primary_section(&allocator); + let managed_start = section.start; + let managed_end = managed_start + section.size; + + assert_eq!(managed_start % PAGE_SIZE, 0); + assert!(managed_start >= region_start); + assert!(managed_end <= region_start + region_size); + + let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= managed_start && addr < managed_end); + allocator.dealloc_pages(addr, 1); +} + +#[test] +fn global_rejects_region_without_one_managed_page() { + let region_size = PAGE_SIZE - 1; + let mut region = HostRegion::new(region_size, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = global_test_context::(1); + + let err = unsafe { allocator.init(region.as_mut_slice()) }.unwrap_err(); + assert_eq!(err, AllocError::InvalidParam); +} + +#[test] +fn global_add_region_after_init_expands_capacity() { + let mut first = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(8 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut first, 1); + + let before = count_free_pages(&allocator); + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + let after = count_free_pages(&allocator); + + assert!(after > before); + assert_eq!(allocator.managed_section_count(), 2); +} + +#[test] +fn global_add_region_supports_discontiguous_regions() { + let mut first = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(8 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut first, 1); + + while allocator.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + let second_section = allocator.managed_section(1).unwrap(); + + let addr = allocator.alloc_pages(1, PAGE_SIZE).unwrap(); + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); +} + +#[test] +fn global_large_alloc_can_come_from_added_region() { + let mut first = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(8 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut first, 1); + + while allocator.alloc_pages(1, PAGE_SIZE).is_ok() {} + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + let second_section = allocator.managed_section(1).unwrap(); + + let layout = Layout::from_size_align(8 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + let addr = ptr.as_ptr() as usize; + assert!(addr >= second_section.start && addr < second_section.start + second_section.size); + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_managed_section_queries_report_all_sections() { + let mut first = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(8 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut first, 1); + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + + assert_eq!(allocator.managed_section_count(), 2); + let first_section = allocator.managed_section(0).unwrap(); + let second_section = allocator.managed_section(1).unwrap(); + assert!(first_section.size > 0); + assert!(second_section.size > 0); +} + +#[test] +fn global_add_region_overlap_rejected() { + let mut first = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut first, 1); + + let overlap = unsafe { first.subslice(1, first.len() - 1) }; + let err = unsafe { allocator.add_region(overlap) }.unwrap_err(); + assert_eq!(err, AllocError::MemoryOverlap); +} + +#[test] +fn global_managed_bytes_matches_all_sections() { + let mut first = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(8 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut first, 1); + unsafe { allocator.add_region(second.as_mut_slice()).unwrap() }; + + let expected = (0..allocator.managed_section_count()) + .map(|i| allocator.managed_section(i).unwrap().size) + .sum::(); + assert_eq!(allocator.managed_bytes(), expected); +} + +#[test] +fn global_allocated_bytes_changes_with_large_alloc() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + assert_eq!(allocator.allocated_bytes(), 0); + + let layout = Layout::from_size_align(3 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + assert_eq!(allocator.allocated_bytes(), 4 * PAGE_SIZE); + + unsafe { allocator.dealloc(ptr, layout) }; + assert_eq!(allocator.allocated_bytes(), 0); +} + +#[test] +fn global_allocated_bytes_reflects_slab_pages() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + assert_eq!(allocator.allocated_bytes(), 0); + + let layout = Layout::from_size_align(64, 8).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + assert!(allocator.allocated_bytes() >= PAGE_SIZE); + + unsafe { allocator.dealloc(ptr, layout) }; +} + +#[test] +fn global_allocated_bytes_not_zero_until_cached_empty_slab_released() { + let mut region = HostRegion::new(TEST_HEAP_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + + let layout = Layout::from_size_align(64, 8).unwrap(); + let ptr = allocator.alloc(layout).unwrap(); + let allocated_after_refill = allocator.allocated_bytes(); + assert!(allocated_after_refill >= PAGE_SIZE); + + unsafe { allocator.dealloc(ptr, layout) }; + + // One empty slab may remain cached, so backend occupancy need not drop to zero. + assert!(allocator.allocated_bytes() <= allocated_after_refill); + assert!(allocator.allocated_bytes() >= PAGE_SIZE); +} diff --git a/components/buddy-slab-allocator/tests/stress_test.rs b/components/buddy-slab-allocator/tests/stress_test.rs new file mode 100644 index 0000000000..20c0ceb663 --- /dev/null +++ b/components/buddy-slab-allocator/tests/stress_test.rs @@ -0,0 +1,485 @@ +//! Stress tests for allocator stability. + +mod common; + +use std::{ + alloc::Layout, + sync::{ + Barrier, Mutex, + atomic::{AtomicBool, Ordering}, + }, + thread, +}; + +use buddy_slab_allocator::{GlobalAllocator, SizeClass}; +use common::{ + HostRegion, count_free_pages, init_global, nonnull_from_addr, seeded_rng, set_current_cpu, +}; +use rand::RngExt; + +const PAGE_SIZE: usize = 0x1000; +const HEAP_SIZE: usize = 64 * 1024 * 1024; +const WORKERS: usize = 4; + +fn assert_recovered_with_cached_slabs( + allocator: &GlobalAllocator, + baseline: usize, + cpu_count: usize, + cached_classes: &[SizeClass], +) { + let recovered = count_free_pages(allocator); + let retained_pages = cached_classes + .iter() + .map(|sc| sc.slab_pages(PAGE_SIZE)) + .sum::() + * cpu_count; + assert!( + recovered + retained_pages >= baseline, + "recovered {recovered} pages, baseline {baseline}, retained allowance {retained_pages}", + ); +} + +#[test] +#[ignore = "stress test"] +fn stress_random_mixed_alloc_free() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 2); + let mut rng = seeded_rng(0); + let mut allocated: Vec<(usize, Layout)> = Vec::new(); + + for i in 0..10_000 { + set_current_cpu(i % 2); + if allocated.is_empty() || rng.random_bool(0.65) { + let size: usize = rng.random_range(8..8193); + let layout = if size <= 2048 { + Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() + } else { + let aligned = size.div_ceil(PAGE_SIZE) * PAGE_SIZE; + Layout::from_size_align(aligned, PAGE_SIZE).unwrap() + }; + + if let Ok(ptr) = allocator.alloc(layout) { + allocated.push((ptr.as_ptr() as usize, layout)); + } + } else { + let idx = rng.random_range(0..allocated.len()); + let (addr, layout) = allocated.swap_remove(idx); + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + } + + for (addr, layout) in allocated { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } +} + +#[test] +#[ignore = "stress test"] +fn stress_exhaustion_recovery() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 1); + let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); + let mut allocated = Vec::new(); + + while let Ok(ptr) = allocator.alloc(layout) { + allocated.push(ptr.as_ptr() as usize); + } + + for addr in allocated.drain(..allocated.len() / 4) { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + + let recovered = allocator.alloc(layout); + assert!(recovered.is_ok()); + + if let Ok(ptr) = recovered { + unsafe { allocator.dealloc(ptr, layout) }; + } + + for addr in allocated { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } +} + +#[test] +#[ignore = "stress test"] +fn stress_fragmentation_recovery() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, 2); + let small_layout = Layout::from_size_align(64, 8).unwrap(); + let mut small_ptrs = Vec::new(); + + for i in 0..4000 { + set_current_cpu(i % 2); + if let Ok(ptr) = allocator.alloc(small_layout) { + small_ptrs.push(ptr.as_ptr() as usize); + } + } + + for i in (0..small_ptrs.len()).step_by(2) { + unsafe { allocator.dealloc(nonnull_from_addr(small_ptrs[i]), small_layout) }; + } + + let large_layout = Layout::from_size_align(PAGE_SIZE * 16, PAGE_SIZE).unwrap(); + let large = allocator.alloc(large_layout); + + for addr in small_ptrs.into_iter().skip(1).step_by(2) { + unsafe { allocator.dealloc(nonnull_from_addr(addr), small_layout) }; + } + + if let Ok(ptr) = large { + unsafe { allocator.dealloc(ptr, large_layout) }; + } +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_mixed_alloc_free() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, WORKERS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + scope.spawn(move || { + set_current_cpu(cpu); + barrier.wait(); + + let mut rng = seeded_rng(0x1000 + cpu as u64); + let mut live: Vec<(usize, Layout)> = Vec::new(); + for _ in 0..4_000 { + if live.is_empty() || rng.random_bool(0.65) { + let layout = if rng.random_bool(0.7) { + let size: usize = rng.random_range(8..=2048); + Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() + } else { + let page_counts = [1usize, 2, 4, 8]; + let pages = page_counts[rng.random_range(0..page_counts.len())]; + Layout::from_size_align(pages * PAGE_SIZE, PAGE_SIZE).unwrap() + }; + + if let Ok(ptr) = allocator.alloc(layout) { + live.push((ptr.as_ptr() as usize, layout)); + } + } else { + let idx = rng.random_range(0..live.len()); + let (addr, layout) = live.swap_remove(idx); + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + } + + for (addr, layout) in live { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + }); + } + }); + + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &SizeClass::ALL); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_remote_free() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, WORKERS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + let layout = Layout::from_size_align(64, 8).unwrap(); + let queues: Vec<_> = (0..WORKERS) + .map(|_| Mutex::new(Vec::::new())) + .collect(); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + let queues = &queues; + scope.spawn(move || { + set_current_cpu(cpu); + let mut local = Vec::new(); + for _ in 0..256 { + local.push(allocator.alloc(layout).unwrap().as_ptr() as usize); + } + + let target = (cpu + 1) % WORKERS; + queues[target].lock().unwrap().extend(local); + barrier.wait(); + + let remote = { + let mut queue = queues[cpu].lock().unwrap(); + queue.drain(..).collect::>() + }; + for addr in remote { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + barrier.wait(); + + let mut drained = Vec::new(); + for _ in 0..256 { + drained.push(allocator.alloc(layout).unwrap()); + } + for ptr in drained { + unsafe { allocator.dealloc(ptr, layout) }; + } + barrier.wait(); + }); + } + }); + + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &[SizeClass::Bytes64]); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_page_alloc_free() { + let mut region = HostRegion::new(HEAP_SIZE, PAGE_SIZE); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, WORKERS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + scope.spawn(move || { + set_current_cpu(cpu); + barrier.wait(); + + let mut rng = seeded_rng(0x2000 + cpu as u64); + let page_counts = [1usize, 2, 4, 8]; + let alignments = [PAGE_SIZE, 2 * PAGE_SIZE, 4 * PAGE_SIZE, 8 * PAGE_SIZE]; + let mut live = Vec::new(); + + for _ in 0..2_000 { + if live.is_empty() || rng.random_bool(0.6) { + let count = page_counts[rng.random_range(0..page_counts.len())]; + let align = alignments[rng.random_range(0..alignments.len())]; + if let Ok(addr) = allocator.alloc_pages(count, align.max(PAGE_SIZE)) { + live.push((addr, count)); + } + } else { + let idx = rng.random_range(0..live.len()); + let (addr, count) = live.swap_remove(idx); + allocator.dealloc_pages(addr, count); + } + } + + for (addr, count) in live { + allocator.dealloc_pages(addr, count); + } + }); + } + }); + + assert_eq!(count_free_pages(allocator), baseline); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_fragmentation_recovery() { + const REGION_SIZE: usize = 4 * 1024 * 1024; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, WORKERS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + let before_cleanup_failed = AtomicBool::new(false); + let partial_cleanup_failed = AtomicBool::new(false); + let large_layout = Layout::from_size_align(32 * PAGE_SIZE, PAGE_SIZE).unwrap(); + let small_layout = Layout::from_size_align(64, 8).unwrap(); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + let before_cleanup_failed = &before_cleanup_failed; + let partial_cleanup_failed = &partial_cleanup_failed; + scope.spawn(move || { + set_current_cpu(cpu); + let mut live = Vec::new(); + while let Ok(ptr) = allocator.alloc(small_layout) { + live.push(ptr.as_ptr() as usize); + } + + barrier.wait(); + if cpu == 0 { + match allocator.alloc(large_layout) { + Ok(ptr) => unsafe { + allocator.dealloc(ptr, large_layout); + }, + Err(_) => before_cleanup_failed.store(true, Ordering::Relaxed), + } + } + + barrier.wait(); + let mut retained = Vec::new(); + for (idx, addr) in live.into_iter().enumerate() { + if idx % 2 == 0 { + unsafe { allocator.dealloc(nonnull_from_addr(addr), small_layout) }; + } else { + retained.push(addr); + } + } + + barrier.wait(); + if cpu == 0 { + match allocator.alloc(large_layout) { + Ok(ptr) => unsafe { + allocator.dealloc(ptr, large_layout); + }, + Err(_) => partial_cleanup_failed.store(true, Ordering::Relaxed), + } + } + + barrier.wait(); + for addr in retained { + unsafe { allocator.dealloc(nonnull_from_addr(addr), small_layout) }; + } + + barrier.wait(); + if cpu == 0 { + let ptr = allocator.alloc(large_layout).unwrap(); + unsafe { allocator.dealloc(ptr, large_layout) }; + } + barrier.wait(); + }); + } + }); + + assert!(before_cleanup_failed.load(Ordering::Relaxed)); + assert!(partial_cleanup_failed.load(Ordering::Relaxed)); + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &[SizeClass::Bytes64]); +} + +#[test] +#[ignore = "stress test"] +fn stress_multithread_exhaustion_recovery() { + const REGION_SIZE: usize = 8 * 1024 * 1024; + + let mut region = HostRegion::new(REGION_SIZE, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut region, WORKERS); + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + let exhausted = AtomicBool::new(false); + let recovered = AtomicBool::new(false); + let layout = Layout::from_size_align(PAGE_SIZE, PAGE_SIZE).unwrap(); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + let exhausted = &exhausted; + let recovered = &recovered; + scope.spawn(move || { + set_current_cpu(cpu); + let mut live = Vec::new(); + while let Ok(ptr) = allocator.alloc(layout) { + live.push(ptr.as_ptr() as usize); + } + + barrier.wait(); + if cpu == 0 { + exhausted.store(allocator.alloc(layout).is_err(), Ordering::Relaxed); + } + + barrier.wait(); + let mut retained = Vec::new(); + for (idx, addr) in live.into_iter().enumerate() { + if idx % 4 == 0 { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } else { + retained.push(addr); + } + } + + barrier.wait(); + if cpu == 0 + && let Ok(ptr) = allocator.alloc(layout) + { + recovered.store(true, Ordering::Relaxed); + unsafe { allocator.dealloc(ptr, layout) }; + } + + barrier.wait(); + for addr in retained { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + barrier.wait(); + }); + } + }); + + assert!(exhausted.load(Ordering::Relaxed)); + assert!(recovered.load(Ordering::Relaxed)); + assert_eq!(count_free_pages(allocator), baseline); +} + +#[test] +#[ignore = "stress test"] +fn stress_add_region_then_multithread_alloc_free() { + let mut first = HostRegion::new(2 * 1024 * 1024, PAGE_SIZE * 4); + let mut second = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let mut third = HostRegion::new(4 * 1024 * 1024, PAGE_SIZE * 4); + let allocator = GlobalAllocator::::new(); + let _ctx = init_global(&allocator, &mut first, WORKERS); + unsafe { + allocator.add_region(second.as_mut_slice()).unwrap(); + allocator.add_region(third.as_mut_slice()).unwrap(); + } + assert_eq!(allocator.managed_section_count(), 3); + + let baseline = count_free_pages(&allocator); + let allocator = &allocator; + let barrier = Barrier::new(WORKERS); + + thread::scope(|scope| { + for cpu in 0..WORKERS { + let barrier = &barrier; + scope.spawn(move || { + set_current_cpu(cpu); + barrier.wait(); + + let mut rng = seeded_rng(0x3000 + cpu as u64); + let mut live: Vec<(usize, Layout)> = Vec::new(); + for _ in 0..5_000 { + if live.is_empty() || rng.random_bool(0.65) { + let layout = if rng.random_bool(0.75) { + let size: usize = rng.random_range(8..=2048); + Layout::from_size_align(size.next_power_of_two().min(2048), 8).unwrap() + } else { + let pages = [1usize, 2, 4, 8][rng.random_range(0..4)]; + Layout::from_size_align(pages * PAGE_SIZE, PAGE_SIZE).unwrap() + }; + + if let Ok(ptr) = allocator.alloc(layout) { + live.push((ptr.as_ptr() as usize, layout)); + } + } else { + let idx = rng.random_range(0..live.len()); + let (addr, layout) = live.swap_remove(idx); + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + } + + for (addr, layout) in live { + unsafe { allocator.dealloc(nonnull_from_addr(addr), layout) }; + } + }); + } + }); + + assert_eq!(allocator.managed_section_count(), 3); + assert_recovered_with_cached_slabs(allocator, baseline, WORKERS, &SizeClass::ALL); +} diff --git a/drivers/intc/arm-gic-driver/src/version/v2/mod.rs b/drivers/intc/arm-gic-driver/src/version/v2/mod.rs index 16fe5adfb6..1d1b1d44ce 100644 --- a/drivers/intc/arm-gic-driver/src/version/v2/mod.rs +++ b/drivers/intc/arm-gic-driver/src/version/v2/mod.rs @@ -526,6 +526,24 @@ impl CpuInterface { self.gicd().ISPENDR.get_irq_bit(id.into()) } + /// Send a Software Generated Interrupt (SGI) to target CPUs. + pub fn send_sgi(&self, sgi_id: IntId, target: SGITarget) { + let sgi_id = sgi_id.to_u32(); + assert!(sgi_id < 16, "Invalid SGI ID: {sgi_id}"); + let (filter, target_list) = match target { + SGITarget::TargetList(list) => ( + gicd::SGIR::TargetListFilter::TargetList, + list.as_u8() as u32, + ), + SGITarget::AllOther => (gicd::SGIR::TargetListFilter::AllOther, 0), + SGITarget::Current => (gicd::SGIR::TargetListFilter::Current, 0), + }; + + self.gicd().SGIR.write( + gicd::SGIR::SGIINTID.val(sgi_id) + gicd::SGIR::CPUTargetList.val(target_list) + filter, + ); + } + pub fn set_cfg(&self, id: IntId, trigger: Trigger) { self.gicd().set_cfg(id, trigger); } diff --git a/os/StarryOS/configs/board/licheerv-nano-sg2002.toml b/os/StarryOS/configs/board/licheerv-nano-sg2002.toml index 899576e482..273b184467 100644 --- a/os/StarryOS/configs/board/licheerv-nano-sg2002.toml +++ b/os/StarryOS/configs/board/licheerv-nano-sg2002.toml @@ -1,7 +1,6 @@ env = { UIMAGE = "y" } features = [ "sg2002", - "myplat", ] log = "Info" plat_dyn = false diff --git a/os/StarryOS/kernel/Cargo.toml b/os/StarryOS/kernel/Cargo.toml index fbcf3b9aa4..1531ea32af 100644 --- a/os/StarryOS/kernel/Cargo.toml +++ b/os/StarryOS/kernel/Cargo.toml @@ -36,6 +36,7 @@ sg2002 = [ "dep:tock-registers", ] stack-guard-page = ["ax-feat/stack-guard-page"] +buddy-slab = ["ax-feat/buddy-slab"] [dependencies] ax-feat = { workspace = true, features = [ @@ -60,7 +61,6 @@ ax-alloc = { workspace = true, features = ["default"] } ax-config.workspace = true ax-display.workspace = true ax-fs = { version = "0.5.15", path = "../../arceos/modules/axfs-ng", package = "ax-fs-ng" } -ax-hal.workspace = true ax-input = { workspace = true, optional = true } ax-lazyinit.workspace = true ax-log.workspace = true diff --git a/os/StarryOS/kernel/src/entry.rs b/os/StarryOS/kernel/src/entry.rs index c0c11460a8..0e00b1728b 100644 --- a/os/StarryOS/kernel/src/entry.rs +++ b/os/StarryOS/kernel/src/entry.rs @@ -4,7 +4,7 @@ use alloc::{ }; use ax_fs::FS_CONTEXT; -use ax_hal::uspace::UserContext; +use ax_runtime::hal::cpu::uspace::UserContext; use ax_sync::Mutex; use ax_task::{AxTaskExt, spawn_task}; use starry_process::{Pid, Process}; diff --git a/os/StarryOS/kernel/src/file/memfd.rs b/os/StarryOS/kernel/src/file/memfd.rs index c3b2a3607f..a3f45191e9 100644 --- a/os/StarryOS/kernel/src/file/memfd.rs +++ b/os/StarryOS/kernel/src/file/memfd.rs @@ -30,10 +30,10 @@ use core::{ use ax_errno::{AxError, AxResult}; use ax_fs::FileFlags; -use ax_hal::paging::MappingFlags; use ax_io::{IoBuf, SeekFrom, prelude::*}; use ax_memory_addr::VirtAddr; use ax_memory_set::MemoryArea; +use ax_runtime::hal::paging::MappingFlags; use ax_sync::Mutex; use axpoll::{IoEvents, Pollable}; diff --git a/os/StarryOS/kernel/src/file/timerfd.rs b/os/StarryOS/kernel/src/file/timerfd.rs index ee519307d4..c19fed63b5 100644 --- a/os/StarryOS/kernel/src/file/timerfd.rs +++ b/os/StarryOS/kernel/src/file/timerfd.rs @@ -33,7 +33,7 @@ use core::{ }; use ax_errno::{AxError, AxResult}; -use ax_hal::time::{TimeValue, monotonic_time, wall_time}; +use ax_runtime::hal::time::{TimeValue, monotonic_time, wall_time}; use ax_sync::Mutex; use ax_task::future::{block_on, poll_io, timeout_at}; use axpoll::{IoEvents, PollSet, Pollable}; diff --git a/os/StarryOS/kernel/src/mm/access.rs b/os/StarryOS/kernel/src/mm/access.rs index 220af50103..1eacc6a51e 100644 --- a/os/StarryOS/kernel/src/mm/access.rs +++ b/os/StarryOS/kernel/src/mm/access.rs @@ -8,9 +8,12 @@ use core::{ }; use ax_errno::{AxError, AxResult}; -use ax_hal::{asm::user_copy, paging::MappingFlags, trap::page_fault_handler}; use ax_io::prelude::*; use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr}; +use ax_runtime::hal::{ + cpu::{asm::user_copy, trap::page_fault_handler}, + paging::MappingFlags, +}; use ax_task::{current, might_sleep}; use extern_trait::extern_trait; use starry_vm::{VmError, VmIo, VmResult, vm_load_until_nul, vm_read_slice, vm_write_slice}; @@ -25,7 +28,7 @@ use crate::{ #[track_caller] pub fn access_user_memory(f: impl FnOnce() -> R) -> R { assert!( - ax_hal::asm::irqs_enabled(), + ax_runtime::hal::cpu::asm::irqs_enabled(), "faultable user memory access requires IRQs enabled" ); @@ -508,7 +511,7 @@ pub fn write_kernel_text(addr: VirtAddr, data: &[u8]) -> AxResult<()> { } #[cfg(target_arch = "aarch64")] - ax_hal::asm::clean_dcache_range_to_pou(addr, data.len()); + ax_runtime::hal::cpu::asm::clean_dcache_range_to_pou(addr, data.len()); guard.protect(aligned_addr, aligned_length, original_flags)?; Ok(()) @@ -519,12 +522,12 @@ pub fn write_kernel_text(addr: VirtAddr, data: &[u8]) -> AxResult<()> { fn flush_tlb_range(start: VirtAddr, size: usize) { for offset in (0..size).step_by(PAGE_SIZE_4K) { - ax_hal::asm::flush_tlb(Some(start + offset)); + ax_runtime::hal::cpu::asm::flush_tlb(Some(start + offset)); } } fn sync_modified_kernel_text(start: VirtAddr, size: usize) { flush_tlb_range(start, size); - ax_hal::asm::flush_icache_all(); + ax_runtime::hal::cpu::asm::flush_icache_all(); } diff --git a/os/StarryOS/kernel/src/mm/aspace/backend/cow.rs b/os/StarryOS/kernel/src/mm/aspace/backend/cow.rs index abfbea8fd0..012a2b4a93 100644 --- a/os/StarryOS/kernel/src/mm/aspace/backend/cow.rs +++ b/os/StarryOS/kernel/src/mm/aspace/backend/cow.rs @@ -7,12 +7,12 @@ use core::slice; use ax_errno::{AxError, AxResult}; use ax_fs::FileBackend; -use ax_hal::{ +use ax_kspin::SpinNoIrq; +use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, PhysAddr, VirtAddr, VirtAddrRange, align_down_4k}; +use ax_runtime::hal::{ mem::phys_to_virt, paging::{MappingFlags, PageSize, PageTableCursor, PagingError}, }; -use ax_kspin::SpinNoIrq; -use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, PhysAddr, VirtAddr, VirtAddrRange, align_down_4k}; use ax_sync::Mutex; use super::{ diff --git a/os/StarryOS/kernel/src/mm/aspace/backend/file.rs b/os/StarryOS/kernel/src/mm/aspace/backend/file.rs index 4221f25f3f..36df4b6274 100644 --- a/os/StarryOS/kernel/src/mm/aspace/backend/file.rs +++ b/os/StarryOS/kernel/src/mm/aspace/backend/file.rs @@ -8,8 +8,8 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use ax_errno::{AxError, AxResult}; use ax_fs::{CachedFile, FileFlags}; -use ax_hal::paging::{MappingFlags, PageSize, PageTableCursor, PagingError}; use ax_memory_addr::{PAGE_SIZE_4K, VirtAddr, VirtAddrRange}; +use ax_runtime::hal::paging::{MappingFlags, PageSize, PageTableCursor, PagingError}; use ax_sync::Mutex; use axfs_ng_vfs::Location; use weak_map::StrongRef; diff --git a/os/StarryOS/kernel/src/mm/aspace/backend/linear.rs b/os/StarryOS/kernel/src/mm/aspace/backend/linear.rs index f9954f359c..d2ee7bc33d 100644 --- a/os/StarryOS/kernel/src/mm/aspace/backend/linear.rs +++ b/os/StarryOS/kernel/src/mm/aspace/backend/linear.rs @@ -1,8 +1,8 @@ use alloc::sync::Arc; use ax_errno::AxResult; -use ax_hal::paging::{MappingFlags, PageSize, PageTableCursor, PagingError}; use ax_memory_addr::{PhysAddr, PhysAddrRange, VirtAddr, VirtAddrRange}; +use ax_runtime::hal::paging::{MappingFlags, PageSize, PageTableCursor, PagingError}; use ax_sync::Mutex; use super::{AddrSpace, Backend, BackendOps, pages_in}; diff --git a/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs b/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs index a70caf4715..d0c13f5c20 100644 --- a/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs +++ b/os/StarryOS/kernel/src/mm/aspace/backend/mod.rs @@ -7,12 +7,12 @@ use alloc::{ use ax_alloc::{UsageKind, global_allocator}; use ax_errno::{AxError, AxResult}; -use ax_hal::{ +use ax_memory_addr::{DynPageIter, PAGE_SIZE_4K, PhysAddr, VirtAddr, VirtAddrRange}; +use ax_memory_set::MappingBackend; +use ax_runtime::hal::{ mem::{phys_to_virt, virt_to_phys}, paging::{MappingFlags, PageSize, PageTable, PageTableCursor}, }; -use ax_memory_addr::{DynPageIter, PAGE_SIZE_4K, PhysAddr, VirtAddr, VirtAddrRange}; -use ax_memory_set::MappingBackend; use ax_sync::Mutex; use enum_dispatch::enum_dispatch; diff --git a/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs b/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs index 5ab9d670af..564ba4e5d5 100644 --- a/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs +++ b/os/StarryOS/kernel/src/mm/aspace/backend/shared.rs @@ -2,8 +2,8 @@ use alloc::{sync::Arc, vec::Vec}; use core::ops::Deref; use ax_errno::AxResult; -use ax_hal::paging::{MappingFlags, PageSize, PageTableCursor, PagingError}; use ax_memory_addr::{MemoryAddr, PhysAddr, VirtAddr, VirtAddrRange}; +use ax_runtime::hal::paging::{MappingFlags, PageSize, PageTableCursor, PagingError}; use ax_sync::Mutex; use super::{AddrSpace, Backend, BackendOps, alloc_frame, dealloc_frame, divide_page, pages_in}; diff --git a/os/StarryOS/kernel/src/mm/aspace/mod.rs b/os/StarryOS/kernel/src/mm/aspace/mod.rs index 5bf5887c4d..791b0426c4 100644 --- a/os/StarryOS/kernel/src/mm/aspace/mod.rs +++ b/os/StarryOS/kernel/src/mm/aspace/mod.rs @@ -6,15 +6,15 @@ use core::{ }; use ax_errno::{AxError, AxResult, ax_bail}; -use ax_hal::{ - mem::phys_to_virt, - paging::{MappingFlags, PageSize, PageTable, PageTableCursor}, - trap::PageFaultFlags, -}; use ax_memory_addr::{ MemoryAddr, PAGE_SIZE_4K, PageIter4K, PhysAddr, VirtAddr, VirtAddrRange, is_aligned_4k, }; use ax_memory_set::{MemoryArea, MemorySet}; +use ax_runtime::hal::{ + mem::phys_to_virt, + paging::{MappingFlags, PageSize, PageTable, PageTableCursor}, + trap::PageFaultFlags, +}; use ax_sync::Mutex; mod backend; diff --git a/os/StarryOS/kernel/src/mm/loader.rs b/os/StarryOS/kernel/src/mm/loader.rs index 515469d9b6..31b0c08427 100644 --- a/os/StarryOS/kernel/src/mm/loader.rs +++ b/os/StarryOS/kernel/src/mm/loader.rs @@ -5,11 +5,11 @@ use core::{ffi::CStr, iter}; use ax_errno::{AxError, AxResult}; use ax_fs::{CachedFile, FS_CONTEXT, FileBackend}; -use ax_hal::{ +use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr}; +use ax_runtime::hal::{ mem::virt_to_phys, paging::{MappingFlags, PageSize}, }; -use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr}; use ax_sync::Mutex; use axfs_ng_vfs::Location; use kernel_elf_parser::{ diff --git a/os/StarryOS/kernel/src/pseudofs/dev/card0.rs b/os/StarryOS/kernel/src/pseudofs/dev/card0.rs index ce7c4f8fd4..24e02ab314 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/card0.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/card0.rs @@ -37,8 +37,8 @@ use core::{ task::Context, }; -use ax_hal::{mem::virt_to_phys, paging::PageSize, time::monotonic_time}; use ax_memory_addr::{PhysAddr, PhysAddrRange, VirtAddr}; +use ax_runtime::hal::{mem::virt_to_phys, paging::PageSize, time::monotonic_time}; use ax_sync::Mutex; use axfs_ng_vfs::{NodeFlags, VfsError, VfsResult}; use axpoll::{IoEvents, PollSet, Pollable}; @@ -490,7 +490,7 @@ impl Card0 { fn dealloc_dumb_pages(&self, paddr: PhysAddr, size: usize) { use ax_alloc::{UsageKind, global_allocator}; - use ax_hal::mem::phys_to_virt; + use ax_runtime::hal::mem::phys_to_virt; let vaddr = phys_to_virt(paddr); let num_pages = size / (PageSize::Size4K as usize); global_allocator().dealloc_pages(vaddr.as_usize(), num_pages, UsageKind::VirtMem); @@ -511,7 +511,7 @@ impl Card0 { let info = ax_display::framebuffer_info(); let src_size = buf.size.min(info.fb_size as u64) as usize; if let Some(src_paddr) = buf.paddr { - use ax_hal::mem::phys_to_virt; + use ax_runtime::hal::mem::phys_to_virt; let src = phys_to_virt(src_paddr); let dst = VirtAddr::from(info.fb_base_vaddr); let copy_bytes = src_size.min(info.fb_size); diff --git a/os/StarryOS/kernel/src/pseudofs/dev/card1.rs b/os/StarryOS/kernel/src/pseudofs/dev/card1.rs index 66ff22ffd8..30ca807ec5 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/card1.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/card1.rs @@ -9,8 +9,8 @@ use core::{ use ax_driver::rknpu::{self, RknpuAction, RknpuMemCreate, RknpuMemMap, RknpuMemSync, RknpuSubmit}; use ax_errno::{AxError, AxResult}; -use ax_hal::mem::virt_to_phys; use ax_memory_addr::PhysAddrRange; +use ax_runtime::hal::mem::virt_to_phys; use axfs_ng_vfs::{DeviceId, NodeFlags, VfsError, VfsResult}; use axpoll::{IoEvents, Pollable}; use linux_raw_sys::general::O_CLOEXEC; diff --git a/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs b/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs index dd2698a881..0da31bbb27 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/cvi_camera.rs @@ -3,9 +3,9 @@ use alloc::{collections::vec_deque::VecDeque, vec::Vec}; use core::{any::Any, time::Duration}; use ax_errno::{AxError, LinuxError}; -use ax_hal::mem::phys_to_virt; use ax_kspin::SpinNoIrq as Mutex; use ax_memory_addr::PhysAddr; +use ax_runtime::hal::mem::phys_to_virt; use ax_task::sleep; use axfs_ng_vfs::{NodeFlags, VfsResult}; use sg200x_bsp::{ @@ -219,7 +219,7 @@ impl CameraProtocol { fn read_slip_frame(&mut self, timeout_ms: u64) -> Result, CameraError> { use core::time::Duration; - use ax_hal::time::wall_time; + use ax_runtime::hal::time::wall_time; let deadline = wall_time() + Duration::from_millis(timeout_ms); let mut tmp = [0u8; 0x1200]; loop { @@ -321,7 +321,7 @@ impl UartTransport for Uart3 { fn read_bytes(&mut self, buf: &mut [u8], _timeout_ms: u64) -> Result { sleep(Duration::from_millis(3)); - ax_hal::irq::set_enable(47, false); + ax_runtime::hal::irq::set_enable(47, false); let n = { let mut cache_buf = CAMERA_UART_BUF.lock(); let n = cache_buf.len().min(buf.len()); @@ -336,7 +336,7 @@ impl UartTransport for Uart3 { // Always re-enable the IRQ before returning, otherwise the ESP32's // reply traffic stops landing in CAMERA_UART_BUF and every subsequent // poll sees an empty queue forever. - ax_hal::irq::set_enable(47, true); + ax_runtime::hal::irq::set_enable(47, true); if n == 0 { sleep(Duration::from_millis(1)); } @@ -373,7 +373,7 @@ impl CviCamera { dw_apb_uart::DW8250::new(phys_to_virt(PhysAddr::from(UART3_ADDR)).as_usize()); uart3.init_with_baud(1500000); uart3.set_ier(true); - ax_hal::irq::register(47, |_irq| { + ax_runtime::hal::irq::register(47, |_irq| { let mut uart3 = dw_apb_uart::DW8250::new(phys_to_virt(PhysAddr::from(UART3_ADDR)).as_usize()); let mut buf = CAMERA_UART_BUF.lock(); @@ -386,7 +386,7 @@ impl CviCamera { } uart3.set_ier(true); }); - ax_hal::irq::set_enable(47, true); + ax_runtime::hal::irq::set_enable(47, true); Self { inner: Mutex::new(CameraProtocol::new_default(Uart3)), } diff --git a/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs b/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs index 09a0ba5b61..df48a44dd6 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/cvi_usb_camera.rs @@ -2,12 +2,12 @@ use core::{any::Any, time::Duration}; use ax_config::plat::PHYS_VIRT_OFFSET; use ax_errno::AxError; -use ax_hal::{ +use ax_kspin::SpinNoIrq as Mutex; +use ax_memory_addr::{PhysAddr, VirtAddr}; +use ax_runtime::hal::{ mem::{phys_to_virt, virt_to_phys}, time::busy_wait, }; -use ax_kspin::SpinNoIrq as Mutex; -use ax_memory_addr::{PhysAddr, VirtAddr}; use axfs_ng_vfs::{NodeFlags, VfsResult}; use sg200x_bsp::{ gpio::{Direction, GPIO, GPIO1_BASE}, diff --git a/os/StarryOS/kernel/src/pseudofs/dev/event.rs b/os/StarryOS/kernel/src/pseudofs/dev/event.rs index 1d3bc2fc86..853fb8dffb 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/event.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/event.rs @@ -17,8 +17,8 @@ pub fn input_device_count() -> u32 { } use ax_errno::{AxError, AxResult}; -use ax_hal::time::wall_time; use ax_input::{ErasedInputDevice, Event, EventType, InputDevice, InputDeviceId, InputError}; +use ax_runtime::hal::time::wall_time; use ax_sync::Mutex; use axfs_ng_vfs::{DeviceId, NodeFlags, NodeType, VfsResult}; use axpoll::{IoEvents, Pollable}; diff --git a/os/StarryOS/kernel/src/pseudofs/dev/fb.rs b/os/StarryOS/kernel/src/pseudofs/dev/fb.rs index 0bf8df5502..492ea4d277 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/fb.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/fb.rs @@ -1,8 +1,8 @@ use core::{any::Any, slice}; use ax_errno::AxError; -use ax_hal::mem::virt_to_phys; use ax_memory_addr::{PhysAddrRange, VirtAddr}; +use ax_runtime::hal::mem::virt_to_phys; use axfs_ng_vfs::{NodeFlags, VfsError, VfsResult}; use starry_vm::VmMutPtr; diff --git a/os/StarryOS/kernel/src/pseudofs/dev/rknpu_card.rs b/os/StarryOS/kernel/src/pseudofs/dev/rknpu_card.rs index 02b64942b4..c0f5295590 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/rknpu_card.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/rknpu_card.rs @@ -4,7 +4,7 @@ use core::{ ffi::{c_char, c_ulong}, }; -use ax_hal::asm::user_copy; +use ax_runtime::hal::cpu::asm::user_copy; use axfs_ng_vfs::{DeviceId, NodeFlags, VfsError, VfsResult}; use super::rknpu_drm::DrmVersion; diff --git a/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs b/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs index c1082a7454..dfc41d8c1e 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/tty/ntty.rs @@ -17,12 +17,12 @@ pub type NTtyDriver = Tty; pub struct Console; impl TtyRead for Console { fn read(&mut self, buf: &mut [u8]) -> usize { - ax_hal::console::read_bytes(buf) + ax_runtime::hal::console::read_bytes(buf) } } impl TtyWrite for Console { fn write(&self, buf: &[u8]) { - ax_hal::console::write_bytes(buf); + ax_runtime::hal::console::write_bytes(buf); } } @@ -31,11 +31,11 @@ pub static N_TTY: Lazy> = Lazy::new(new_n_tty); static CONSOLE_INPUT_SOURCE: Lazy> = Lazy::new(|| Arc::new(PollSet::new())); fn handle_console_input_irq(_irq_num: usize) { - let events = ax_hal::console::handle_irq(); + let events = ax_runtime::hal::console::handle_irq(); if events.intersects( - ax_hal::console::ConsoleIrqEvent::RX_READY - | ax_hal::console::ConsoleIrqEvent::RX_ERROR - | ax_hal::console::ConsoleIrqEvent::OVERRUN, + ax_runtime::hal::console::ConsoleIrqEvent::RX_READY + | ax_runtime::hal::console::ConsoleIrqEvent::RX_ERROR + | ax_runtime::hal::console::ConsoleIrqEvent::OVERRUN, ) { CONSOLE_INPUT_SOURCE.wake(); } @@ -86,19 +86,19 @@ fn new_n_tty() -> Arc { /// task is spawned, so there is no concurrent consumer racing on the /// UART receive FIFO. fn query_console_size() -> Option<(u16, u16)> { - ax_hal::console::write_bytes(b"\x1b7\x1b[9999;9999H\x1b[6n\x1b8"); + ax_runtime::hal::console::write_bytes(b"\x1b7\x1b[9999;9999H\x1b[6n\x1b8"); let mut buf = [0u8; 32]; let mut len = 0usize; - // Spin up to ~100 ms (in wall time, polled via ax_hal::time::wall_time) + // Spin up to ~100 ms (in wall time, polled via ax_runtime::hal::time::wall_time) // for the `R` terminator. Hosts that ignore CPR (jcode running under // a non-interactive serial, automated CI runners) will time out and // we fall back to the 24x80 default without blocking boot further. - let deadline = ax_hal::time::wall_time() + core::time::Duration::from_millis(100); - 'collect: while ax_hal::time::wall_time() < deadline { + let deadline = ax_runtime::hal::time::wall_time() + core::time::Duration::from_millis(100); + 'collect: while ax_runtime::hal::time::wall_time() < deadline { let mut tmp = [0u8; 1]; - if ax_hal::console::read_bytes(&mut tmp) > 0 { + if ax_runtime::hal::console::read_bytes(&mut tmp) > 0 { if len < buf.len() { buf[len] = tmp[0]; len += 1; @@ -131,12 +131,12 @@ fn query_console_size() -> Option<(u16, u16)> { } fn console_irq_mode() -> Option { - let irq = ax_hal::console::irq_num()?; - if !ax_hal::irq::register(irq, handle_console_input_irq) { + let irq = ax_runtime::hal::console::irq_num()?; + if !ax_runtime::hal::irq::register(irq, handle_console_input_irq) { warn!("Failed to register console IRQ handler for irq {irq}, falling back to polling mode"); return None; } - ax_hal::console::set_input_irq_enabled(true); + ax_runtime::hal::console::set_input_irq_enabled(true); Some(ProcessMode::InterruptDriven(CONSOLE_INPUT_SOURCE.clone())) } diff --git a/os/StarryOS/kernel/src/pseudofs/dev/tty_serial.rs b/os/StarryOS/kernel/src/pseudofs/dev/tty_serial.rs index 8c8bb21d0d..4225a7fc00 100644 --- a/os/StarryOS/kernel/src/pseudofs/dev/tty_serial.rs +++ b/os/StarryOS/kernel/src/pseudofs/dev/tty_serial.rs @@ -2,9 +2,9 @@ use alloc::collections::vec_deque::VecDeque; use core::{any::Any, task::Context}; use ax_errno::{AxError, LinuxError}; -use ax_hal::mem::phys_to_virt; use ax_kspin::SpinNoIrq; use ax_memory_addr::{PhysAddr, pa}; +use ax_runtime::hal::mem::phys_to_virt; use ax_sync::Mutex; use ax_task::future::{block_on, poll_io}; use axfs_ng_vfs::{NodeFlags, VfsResult}; @@ -134,8 +134,8 @@ impl TtySerial { let mut uart = DW8250::new(vaddr); uart.init_with_baud(baud); uart.set_ier(true); - ax_hal::irq::register(irq, irq_handler); - ax_hal::irq::set_enable(irq, true); + ax_runtime::hal::irq::register(irq, irq_handler); + ax_runtime::hal::irq::set_enable(irq, true); Self { paddr, irq, @@ -153,7 +153,7 @@ impl TtySerial { let mut uart = DW8250::new(vaddr); uart.init_with_baud(baud); uart.set_ier(true); - ax_hal::irq::set_enable(self.irq, true); + ax_runtime::hal::irq::set_enable(self.irq, true); } } diff --git a/os/StarryOS/kernel/src/pseudofs/proc.rs b/os/StarryOS/kernel/src/pseudofs/proc.rs index c639257a97..4dd7d4cc8e 100644 --- a/os/StarryOS/kernel/src/pseudofs/proc.rs +++ b/os/StarryOS/kernel/src/pseudofs/proc.rs @@ -14,11 +14,11 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; -use ax_hal::{ +use ax_memory_addr::PAGE_SIZE_4K; +use ax_runtime::hal::{ paging::MappingFlags, time::{monotonic_time, wall_time}, }; -use ax_memory_addr::PAGE_SIZE_4K; use ax_task::{AxCpuMask, AxTaskRef, TaskState, WeakAxTaskRef, current}; use axfs_ng_vfs::{DeviceId, Filesystem, NodePermission, NodeType, VfsError, VfsResult}; use starry_process::{Pid, Process}; @@ -61,7 +61,7 @@ fn procfs_lookup_process(pid: Pid) -> VfsResult> { } fn render_meminfo() -> String { - let total = ax_hal::mem::total_ram_size(); + let total = ax_runtime::hal::mem::total_ram_size(); let usages = ax_alloc::global_allocator().usages(); // Sum all allocator categories to estimate kernel-consumed memory. let used = usages.get(ax_alloc::UsageKind::RustHeap) @@ -113,7 +113,7 @@ fn render_meminfo() -> String { } fn render_cpuinfo() -> String { - let cpu_count = ax_hal::cpu_num(); + let cpu_count = ax_runtime::hal::cpu_num(); let mut buf = String::new(); for i in 0..cpu_count { render_cpu_entry(&mut buf, i); @@ -185,7 +185,7 @@ fn render_cpu_entry(buf: &mut String, idx: usize) { fn render_stat() -> String { let up = monotonic_time(); - let cpu_count = ax_hal::cpu_num() as u64; + let cpu_count = ax_runtime::hal::cpu_num() as u64; // Total CPU-time budget in jiffies across all CPUs (USER_HZ = 100). let up_jiffies = up.as_secs() * 100 + (up.subsec_millis() / 10) as u64; let total_budget = up_jiffies.saturating_mul(cpu_count); @@ -339,7 +339,7 @@ fn task_status(task: &AxTaskRef) -> String { &cred, num_threads, task.cpumask(), - ax_hal::cpu_num(), + ax_runtime::hal::cpu_num(), ) } @@ -678,7 +678,7 @@ impl SimpleDirOps for ThreadDir { &cred, num_threads, task.cpumask(), - ax_hal::cpu_num(), + ax_runtime::hal::cpu_num(), )) } else { Ok(task_status(&task)) @@ -859,7 +859,7 @@ fn builder(fs: Arc) -> DirMaker { let secs = up.as_secs(); let cs = up.subsec_millis() / 10; // Approximate total idle as uptime × cpu_count (no per-CPU idle accounting yet). - let idle_secs = secs.saturating_mul(ax_hal::cpu_num() as u64); + let idle_secs = secs.saturating_mul(ax_runtime::hal::cpu_num() as u64); Ok(format!("{secs}.{cs:02} {idle_secs}.00\n")) }), ); diff --git a/os/StarryOS/kernel/src/pseudofs/sysfs.rs b/os/StarryOS/kernel/src/pseudofs/sysfs.rs index dd41bacff0..44fa030137 100644 --- a/os/StarryOS/kernel/src/pseudofs/sysfs.rs +++ b/os/StarryOS/kernel/src/pseudofs/sysfs.rs @@ -396,7 +396,7 @@ impl SimpleDirOps for SystemCpuDir { Cow::Borrowed("possible"), Cow::Borrowed("present"), ]; - names.extend((0..ax_hal::cpu_num()).map(|cpu| Cow::Owned(format!("cpu{cpu}")))); + names.extend((0..ax_runtime::hal::cpu_num()).map(|cpu| Cow::Owned(format!("cpu{cpu}")))); Box::new(names.into_iter()) } @@ -411,7 +411,7 @@ impl SimpleDirOps for SystemCpuDir { .strip_prefix("cpu") .and_then(|s| s.parse::().ok()) .ok_or(VfsError::NotFound)?; - if cpu >= ax_hal::cpu_num() { + if cpu >= ax_runtime::hal::cpu_num() { return Err(VfsError::NotFound); } NodeOpsMux::Dir(SimpleDir::new_maker( @@ -436,7 +436,7 @@ impl SimpleDirOps for SystemCpuEntryDir { fn lookup_child(&self, name: &str) -> VfsResult { match name { "online" => { - let online = if self.cpu < ax_hal::cpu_num() { + let online = if self.cpu < ax_runtime::hal::cpu_num() { "1\n" } else { "0\n" @@ -449,7 +449,7 @@ impl SimpleDirOps for SystemCpuEntryDir { } fn cpu_range_string() -> String { - let cpu_num = ax_hal::cpu_num(); + let cpu_num = ax_runtime::hal::cpu_num(); if cpu_num <= 1 { "0".to_owned() } else { diff --git a/os/StarryOS/kernel/src/pseudofs/usbfs/irq.rs b/os/StarryOS/kernel/src/pseudofs/usbfs/irq.rs index b1d72b0a72..96e3673434 100644 --- a/os/StarryOS/kernel/src/pseudofs/usbfs/irq.rs +++ b/os/StarryOS/kernel/src/pseudofs/usbfs/irq.rs @@ -92,7 +92,7 @@ pub(super) fn init_globals(manager: Arc, pending_slots: Vec usize { for (_, irq_num) in failed_device_ids { if let Some(irq_num) = irq_num { - let _ = ax_hal::irq::unregister(irq_num); + let _ = ax_runtime::hal::irq::unregister(irq_num); } } diff --git a/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs b/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs index 66d5ae56fb..78f4a0d8b6 100644 --- a/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs +++ b/os/StarryOS/kernel/src/pseudofs/usbfs/mod.rs @@ -1348,7 +1348,7 @@ fn cleanup_submitted_urbs( mut submitted_urbs: Vec, timeout: Option, ) -> Vec { - let deadline = timeout.map(|timeout| ax_hal::time::wall_time() + timeout); + let deadline = timeout.map(|timeout| ax_runtime::hal::time::wall_time() + timeout); for submitted in &submitted_urbs { if let Err(err) = submitted.cancel() { debug!( @@ -1376,7 +1376,7 @@ fn cleanup_submitted_urbs( } if !submitted_urbs.is_empty() { - if deadline.is_some_and(|deadline| ax_hal::time::wall_time() >= deadline) { + if deadline.is_some_and(|deadline| ax_runtime::hal::time::wall_time() >= deadline) { break; } ax_task::sleep(Duration::from_millis(1)); diff --git a/os/StarryOS/kernel/src/stop_machine.rs b/os/StarryOS/kernel/src/stop_machine.rs index b90b7bd6be..d36c279f87 100644 --- a/os/StarryOS/kernel/src/stop_machine.rs +++ b/os/StarryOS/kernel/src/stop_machine.rs @@ -4,10 +4,10 @@ use core::{ sync::atomic::{AtomicU8, AtomicUsize, Ordering}, }; -use ax_hal::{cpu_num, percpu::this_cpu_id, time::monotonic_time_nanos}; use ax_ipi::run_on_cpu; use ax_kernel_guard::NoPreemptIrqSave; use ax_kspin::SpinNoIrq; +use ax_runtime::hal::{cpu_num, percpu::this_cpu_id, time::monotonic_time_nanos}; static STOP_MACHINE_LOCK: SpinNoIrq<()> = SpinNoIrq::new(()); diff --git a/os/StarryOS/kernel/src/syscall/fs/ctl.rs b/os/StarryOS/kernel/src/syscall/fs/ctl.rs index b120ed7da4..81014f6c2e 100644 --- a/os/StarryOS/kernel/src/syscall/fs/ctl.rs +++ b/os/StarryOS/kernel/src/syscall/fs/ctl.rs @@ -12,7 +12,7 @@ use core::{ use ax_errno::{AxError, AxResult, LinuxError}; use ax_fs::{FS_CONTEXT, FsContext}; -use ax_hal::time::wall_time; +use ax_runtime::hal::time::wall_time; use ax_task::current; use axfs_ng_vfs::{DeviceId, MetadataUpdate, NodePermission, NodeType, path::Path}; use linux_raw_sys::{ diff --git a/os/StarryOS/kernel/src/syscall/io_mpx/poll.rs b/os/StarryOS/kernel/src/syscall/io_mpx/poll.rs index 57b0725556..1ad8434caa 100644 --- a/os/StarryOS/kernel/src/syscall/io_mpx/poll.rs +++ b/os/StarryOS/kernel/src/syscall/io_mpx/poll.rs @@ -2,7 +2,7 @@ use alloc::vec::Vec; use core::mem::{MaybeUninit, offset_of}; use ax_errno::{AxError, AxResult}; -use ax_hal::time::TimeValue; +use ax_runtime::hal::time::TimeValue; use ax_task::{ current, future::{self, block_on, poll_io}, diff --git a/os/StarryOS/kernel/src/syscall/ipc/msg.rs b/os/StarryOS/kernel/src/syscall/ipc/msg.rs index 295fde88cb..87c0720f89 100644 --- a/os/StarryOS/kernel/src/syscall/ipc/msg.rs +++ b/os/StarryOS/kernel/src/syscall/ipc/msg.rs @@ -1,7 +1,7 @@ use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use ax_errno::{AxError, AxResult, LinuxError}; -use ax_hal::time::monotonic_time_nanos; +use ax_runtime::hal::time::monotonic_time_nanos; use ax_sync::Mutex; use ax_task::current; use bytemuck::AnyBitPattern; diff --git a/os/StarryOS/kernel/src/syscall/ipc/shm.rs b/os/StarryOS/kernel/src/syscall/ipc/shm.rs index 78756b8448..2e746e1793 100644 --- a/os/StarryOS/kernel/src/syscall/ipc/shm.rs +++ b/os/StarryOS/kernel/src/syscall/ipc/shm.rs @@ -1,11 +1,11 @@ use alloc::{collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; use ax_errno::{AxError, AxResult}; -use ax_hal::{ +use ax_memory_addr::{PAGE_SIZE_4K, VirtAddr, VirtAddrRange}; +use ax_runtime::hal::{ paging::{MappingFlags, PageSize}, time::monotonic_time_nanos, }; -use ax_memory_addr::{PAGE_SIZE_4K, VirtAddr, VirtAddrRange}; use ax_sync::Mutex; use ax_task::current; use linux_raw_sys::{ctypes::c_ushort, general::*}; diff --git a/os/StarryOS/kernel/src/syscall/mm/brk.rs b/os/StarryOS/kernel/src/syscall/mm/brk.rs index 6f7e2855da..55cbf9a3b2 100644 --- a/os/StarryOS/kernel/src/syscall/mm/brk.rs +++ b/os/StarryOS/kernel/src/syscall/mm/brk.rs @@ -1,6 +1,6 @@ use ax_errno::AxResult; -use ax_hal::paging::{MappingFlags, PageSize}; use ax_memory_addr::{VirtAddr, align_up_4k}; +use ax_runtime::hal::paging::{MappingFlags, PageSize}; use ax_task::current; use linux_raw_sys::general::RLIMIT_DATA; diff --git a/os/StarryOS/kernel/src/syscall/mm/mincore.rs b/os/StarryOS/kernel/src/syscall/mm/mincore.rs index 8ab77bac2b..08441b2e65 100644 --- a/os/StarryOS/kernel/src/syscall/mm/mincore.rs +++ b/os/StarryOS/kernel/src/syscall/mm/mincore.rs @@ -9,8 +9,8 @@ use alloc::vec; use ax_errno::{AxError, AxResult}; -use ax_hal::paging::MappingFlags; use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr}; +use ax_runtime::hal::paging::MappingFlags; use ax_task::current; use starry_vm::vm_write_slice; diff --git a/os/StarryOS/kernel/src/syscall/mm/mmap.rs b/os/StarryOS/kernel/src/syscall/mm/mmap.rs index 0be64134f7..9f1d03e712 100644 --- a/os/StarryOS/kernel/src/syscall/mm/mmap.rs +++ b/os/StarryOS/kernel/src/syscall/mm/mmap.rs @@ -2,8 +2,8 @@ use alloc::sync::Arc; use ax_errno::{AxError, AxResult}; use ax_fs::{FileBackend, FileFlags}; -use ax_hal::paging::{MappingFlags, PageSize}; use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, VirtAddr, VirtAddrRange, align_up_4k}; +use ax_runtime::hal::paging::{MappingFlags, PageSize}; use ax_task::current; use linux_raw_sys::general::*; diff --git a/os/StarryOS/kernel/src/syscall/mod.rs b/os/StarryOS/kernel/src/syscall/mod.rs index 9321b6a562..5d173fa906 100644 --- a/os/StarryOS/kernel/src/syscall/mod.rs +++ b/os/StarryOS/kernel/src/syscall/mod.rs @@ -11,7 +11,7 @@ mod task; mod time; use ax_errno::{AxError, LinuxError}; -use ax_hal::uspace::UserContext; +use ax_runtime::hal::cpu::uspace::UserContext; use syscalls::Sysno; pub use self::{ diff --git a/os/StarryOS/kernel/src/syscall/net/io.rs b/os/StarryOS/kernel/src/syscall/net/io.rs index ad9b1436fa..53fa23fc42 100644 --- a/os/StarryOS/kernel/src/syscall/net/io.rs +++ b/os/StarryOS/kernel/src/syscall/net/io.rs @@ -2,8 +2,8 @@ use alloc::{boxed::Box, vec::Vec}; use core::{net::Ipv4Addr, time::Duration}; use ax_errno::{AxError, AxResult}; -use ax_hal::time::wall_time; use ax_io::prelude::*; +use ax_runtime::hal::time::wall_time; use axnet::{CMsgData, RecvFlags, RecvOptions, SendFlags, SendOptions, SocketAddrEx, SocketOps}; use linux_raw_sys::{ general::timespec, diff --git a/os/StarryOS/kernel/src/syscall/resources.rs b/os/StarryOS/kernel/src/syscall/resources.rs index 95740cd78d..719146a88e 100644 --- a/os/StarryOS/kernel/src/syscall/resources.rs +++ b/os/StarryOS/kernel/src/syscall/resources.rs @@ -1,5 +1,5 @@ use ax_errno::{AxError, AxResult}; -use ax_hal::time::TimeValue; +use ax_runtime::hal::time::TimeValue; use ax_task::current; use linux_raw_sys::general::{__kernel_old_timeval, RLIM_NLIMITS, rlimit64, rusage}; use starry_process::Pid; diff --git a/os/StarryOS/kernel/src/syscall/signal.rs b/os/StarryOS/kernel/src/syscall/signal.rs index 32c3d0cf84..546c97d5b1 100644 --- a/os/StarryOS/kernel/src/syscall/signal.rs +++ b/os/StarryOS/kernel/src/syscall/signal.rs @@ -1,7 +1,7 @@ use core::{future::poll_fn, task::Poll}; use ax_errno::{AxError, AxResult, LinuxError}; -use ax_hal::uspace::UserContext; +use ax_runtime::hal::cpu::uspace::UserContext; use ax_task::{ current, future::{self, block_on}, diff --git a/os/StarryOS/kernel/src/syscall/sync/futex.rs b/os/StarryOS/kernel/src/syscall/sync/futex.rs index d2a0d0f6a0..19955ea27d 100644 --- a/os/StarryOS/kernel/src/syscall/sync/futex.rs +++ b/os/StarryOS/kernel/src/syscall/sync/futex.rs @@ -1,7 +1,7 @@ use core::mem::align_of; use ax_errno::{AxError, AxResult}; -use ax_hal::time::{TimeValue, monotonic_time, wall_time}; +use ax_runtime::hal::time::{TimeValue, monotonic_time, wall_time}; use ax_task::current; use linux_raw_sys::general::{ FUTEX_CLOCK_REALTIME, FUTEX_CMP_REQUEUE, FUTEX_REQUEUE, FUTEX_WAIT, FUTEX_WAIT_BITSET, diff --git a/os/StarryOS/kernel/src/syscall/sys.rs b/os/StarryOS/kernel/src/syscall/sys.rs index 6e4183f7a3..5c50ca931b 100644 --- a/os/StarryOS/kernel/src/syscall/sys.rs +++ b/os/StarryOS/kernel/src/syscall/sys.rs @@ -566,7 +566,7 @@ pub fn sys_uname(name: *mut new_utsname) -> AxResult { pub fn sys_sysinfo(info: *mut sysinfo) -> AxResult { let mut kinfo: sysinfo = unsafe { core::mem::zeroed() }; - let total = ax_hal::mem::total_ram_size(); + let total = ax_runtime::hal::mem::total_ram_size(); let usages = ax_alloc::global_allocator().usages(); let used = usages.get(ax_alloc::UsageKind::RustHeap) + usages.get(ax_alloc::UsageKind::VirtMem) @@ -575,7 +575,7 @@ pub fn sys_sysinfo(info: *mut sysinfo) -> AxResult { + usages.get(ax_alloc::UsageKind::Dma) + usages.get(ax_alloc::UsageKind::Global); let free = total.saturating_sub(used); - let uptime = ax_hal::time::monotonic_time(); + let uptime = ax_runtime::hal::time::monotonic_time(); kinfo.uptime = uptime.as_secs() as _; kinfo.totalram = total as _; diff --git a/os/StarryOS/kernel/src/syscall/task/clone.rs b/os/StarryOS/kernel/src/syscall/task/clone.rs index e17de3264e..214519f46c 100644 --- a/os/StarryOS/kernel/src/syscall/task/clone.rs +++ b/os/StarryOS/kernel/src/syscall/task/clone.rs @@ -2,8 +2,8 @@ use alloc::sync::Arc; use ax_errno::{AxError, AxResult}; use ax_fs::FS_CONTEXT; -use ax_hal::uspace::UserContext; use ax_kspin::SpinNoIrq; +use ax_runtime::hal::cpu::uspace::UserContext; use ax_task::{AxTaskExt, current, spawn_task}; use bitflags::bitflags; use linux_raw_sys::general::*; diff --git a/os/StarryOS/kernel/src/syscall/task/clone3.rs b/os/StarryOS/kernel/src/syscall/task/clone3.rs index 41d8224208..453e425b6b 100644 --- a/os/StarryOS/kernel/src/syscall/task/clone3.rs +++ b/os/StarryOS/kernel/src/syscall/task/clone3.rs @@ -1,7 +1,7 @@ use core::mem::{self, MaybeUninit}; use ax_errno::{AxError, AxResult}; -use ax_hal::uspace::UserContext; +use ax_runtime::hal::cpu::uspace::UserContext; use bytemuck::AnyBitPattern; use starry_vm::vm_read_slice; diff --git a/os/StarryOS/kernel/src/syscall/task/execve.rs b/os/StarryOS/kernel/src/syscall/task/execve.rs index 02e0a8b066..0a57d45581 100644 --- a/os/StarryOS/kernel/src/syscall/task/execve.rs +++ b/os/StarryOS/kernel/src/syscall/task/execve.rs @@ -7,7 +7,7 @@ use core::{ffi::c_char, future::poll_fn, iter, task::Poll}; use ax_errno::{AxError, AxResult}; use ax_fs::FS_CONTEXT; -use ax_hal::uspace::UserContext; +use ax_runtime::hal::cpu::uspace::UserContext; use ax_sync::Mutex; use ax_task::{current, future::block_on, yield_now}; use starry_process::Pid; diff --git a/os/StarryOS/kernel/src/syscall/task/schedule.rs b/os/StarryOS/kernel/src/syscall/task/schedule.rs index 535b19786a..7e87f814b0 100644 --- a/os/StarryOS/kernel/src/syscall/task/schedule.rs +++ b/os/StarryOS/kernel/src/syscall/task/schedule.rs @@ -1,7 +1,7 @@ use alloc::{sync::Arc, vec::Vec}; use ax_errno::{AxError, AxResult}; -use ax_hal::time::TimeValue; +use ax_runtime::hal::time::TimeValue; use ax_task::{ AxCpuMask, current, future::{block_on, interruptible, sleep}, @@ -43,7 +43,7 @@ pub fn sys_nanosleep(req: *const timespec, rem: *mut timespec) -> AxResult rem: {diff:?}"); @@ -63,8 +63,8 @@ pub fn sys_clock_nanosleep( rem: *mut timespec, ) -> AxResult { let clock = match clock_id as u32 { - CLOCK_REALTIME => ax_hal::time::wall_time, - CLOCK_MONOTONIC => ax_hal::time::monotonic_time, + CLOCK_REALTIME => ax_runtime::hal::time::wall_time, + CLOCK_MONOTONIC => ax_runtime::hal::time::monotonic_time, _ => { warn!("Unsupported clock_id: {clock_id}"); return Err(AxError::InvalidInput); @@ -94,7 +94,7 @@ pub fn sys_clock_nanosleep( } pub fn sys_sched_getaffinity(pid: i32, cpusetsize: usize, user_mask: *mut u8) -> AxResult { - if cpusetsize * 8 < ax_hal::cpu_num() { + if cpusetsize * 8 < ax_runtime::hal::cpu_num() { return Err(AxError::InvalidInput); } @@ -108,11 +108,11 @@ pub fn sys_sched_getaffinity(pid: i32, cpusetsize: usize, user_mask: *mut u8) -> } pub fn sys_sched_setaffinity(pid: i32, cpusetsize: usize, user_mask: *const u8) -> AxResult { - let size = cpusetsize.min(ax_hal::cpu_num().div_ceil(8)); + let size = cpusetsize.min(ax_runtime::hal::cpu_num().div_ceil(8)); let user_mask = vm_load(user_mask, size)?; let mut cpu_mask = AxCpuMask::new(); - for i in 0..(size * 8).min(ax_hal::cpu_num()) { + for i in 0..(size * 8).min(ax_runtime::hal::cpu_num()) { if user_mask[i / 8] & (1 << (i % 8)) != 0 { cpu_mask.set(i, true); } diff --git a/os/StarryOS/kernel/src/syscall/task/thread.rs b/os/StarryOS/kernel/src/syscall/task/thread.rs index e1ec225983..47be16d532 100644 --- a/os/StarryOS/kernel/src/syscall/task/thread.rs +++ b/os/StarryOS/kernel/src/syscall/task/thread.rs @@ -30,7 +30,7 @@ pub fn sys_gettid() -> AxResult { /// current CPU id and node 0 (single NUMA node); the obsolete `tcache` arg is /// ignored. Either pointer may be NULL. pub fn sys_getcpu(cpu: *mut u32, node: *mut u32, _tcache: usize) -> AxResult { - use ax_hal::percpu::this_cpu_id; + use ax_runtime::hal::percpu::this_cpu_id; use starry_vm::VmMutPtr; if !cpu.is_null() { @@ -77,7 +77,7 @@ pub fn sys_set_tid_address(clear_child_tid: usize) -> AxResult { #[cfg(target_arch = "x86_64")] pub fn sys_arch_prctl( - uctx: &mut ax_hal::uspace::UserContext, + uctx: &mut ax_runtime::hal::cpu::uspace::UserContext, code: i32, addr: usize, ) -> AxResult { diff --git a/os/StarryOS/kernel/src/syscall/time.rs b/os/StarryOS/kernel/src/syscall/time.rs index 4c815556ce..3c59a8cb17 100644 --- a/os/StarryOS/kernel/src/syscall/time.rs +++ b/os/StarryOS/kernel/src/syscall/time.rs @@ -1,5 +1,5 @@ use ax_errno::{AxError, AxResult}; -use ax_hal::time::{ +use ax_runtime::hal::time::{ NANOS_PER_SEC, TimeValue, monotonic_time, monotonic_time_nanos, nanos_to_ticks, wall_time, }; use ax_task::current; diff --git a/os/StarryOS/kernel/src/task/mod.rs b/os/StarryOS/kernel/src/task/mod.rs index 6d61abe40b..63620a38e4 100644 --- a/os/StarryOS/kernel/src/task/mod.rs +++ b/os/StarryOS/kernel/src/task/mod.rs @@ -17,7 +17,7 @@ use core::{ sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU32, AtomicUsize, Ordering}, }; -use ax_hal::time::TimeValue; +use ax_runtime::hal::time::TimeValue; use ax_sync::{Mutex, spin::SpinNoIrq}; use ax_task::{TaskExt, TaskInner}; use axpoll::PollSet; diff --git a/os/StarryOS/kernel/src/task/ops.rs b/os/StarryOS/kernel/src/task/ops.rs index 34d4d974fd..f8ef2ac024 100644 --- a/os/StarryOS/kernel/src/task/ops.rs +++ b/os/StarryOS/kernel/src/task/ops.rs @@ -6,7 +6,7 @@ use alloc::{ use core::ffi::c_long; use ax_errno::{AxError, AxResult}; -use ax_hal::time::TimeValue; +use ax_runtime::hal::time::TimeValue; use ax_task::{AxTaskRef, TaskInner, WeakAxTaskRef, current}; use bytemuck::AnyBitPattern; use linux_raw_sys::general::ROBUST_LIST_LIMIT; diff --git a/os/StarryOS/kernel/src/task/posix_timer.rs b/os/StarryOS/kernel/src/task/posix_timer.rs index 6fd711eb1e..a8c9f7b55d 100644 --- a/os/StarryOS/kernel/src/task/posix_timer.rs +++ b/os/StarryOS/kernel/src/task/posix_timer.rs @@ -7,8 +7,8 @@ use core::{ }; use ax_errno::{AxError, AxResult}; -use ax_hal::time::{NANOS_PER_SEC, monotonic_time_nanos, wall_time}; use ax_kspin::SpinNoIrq as Mutex; +use ax_runtime::hal::time::{NANOS_PER_SEC, monotonic_time_nanos, wall_time}; use linux_raw_sys::general::{ CLOCK_BOOTTIME, CLOCK_MONOTONIC, CLOCK_MONOTONIC_COARSE, CLOCK_MONOTONIC_RAW, CLOCK_PROCESS_CPUTIME_ID, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, CLOCK_THREAD_CPUTIME_ID, diff --git a/os/StarryOS/kernel/src/task/signal.rs b/os/StarryOS/kernel/src/task/signal.rs index 588fbcc818..1360c1f4ff 100644 --- a/os/StarryOS/kernel/src/task/signal.rs +++ b/os/StarryOS/kernel/src/task/signal.rs @@ -1,7 +1,7 @@ use core::{future::poll_fn, task::Poll}; use ax_errno::{AxError, AxResult}; -use ax_hal::uspace::UserContext; +use ax_runtime::hal::cpu::uspace::UserContext; use ax_task::{TaskInner, current, future::block_on}; use linux_raw_sys::general::{CLD_CONTINUED, CLD_STOPPED}; use starry_process::Pid; diff --git a/os/StarryOS/kernel/src/task/timer.rs b/os/StarryOS/kernel/src/task/timer.rs index 2f5dfc8b68..9f6d511e4c 100644 --- a/os/StarryOS/kernel/src/task/timer.rs +++ b/os/StarryOS/kernel/src/task/timer.rs @@ -3,8 +3,8 @@ use alloc::{borrow::ToOwned, collections::binary_heap::BinaryHeap, sync::Arc}; use core::{mem, time::Duration}; -use ax_hal::time::{NANOS_PER_SEC, TimeValue, monotonic_time_nanos, wall_time}; use ax_kspin::SpinNoIrq as Mutex; +use ax_runtime::hal::time::{NANOS_PER_SEC, TimeValue, monotonic_time_nanos, wall_time}; use ax_task::{ WeakAxTaskRef, current, future::{block_on, timeout_at}, diff --git a/os/StarryOS/kernel/src/task/user.rs b/os/StarryOS/kernel/src/task/user.rs index 0d74819766..e6d5bc324b 100644 --- a/os/StarryOS/kernel/src/task/user.rs +++ b/os/StarryOS/kernel/src/task/user.rs @@ -1,4 +1,4 @@ -use ax_hal::uspace::{ExceptionInfo, ExceptionKind, ReturnReason, UserContext}; +use ax_runtime::hal::cpu::uspace::{ExceptionInfo, ExceptionKind, ReturnReason, UserContext}; use ax_task::TaskInner; use starry_process::Pid; use starry_signal::{SignalInfo, Signo}; diff --git a/os/StarryOS/kernel/src/time.rs b/os/StarryOS/kernel/src/time.rs index 4743cb6f4f..4e0d4d190c 100644 --- a/os/StarryOS/kernel/src/time.rs +++ b/os/StarryOS/kernel/src/time.rs @@ -1,5 +1,5 @@ use ax_errno::{AxError, AxResult}; -use ax_hal::time::TimeValue; +use ax_runtime::hal::time::TimeValue; use linux_raw_sys::general::{ __kernel_old_timespec, __kernel_old_timeval, __kernel_sock_timeval, __kernel_timespec, timespec, timeval, diff --git a/os/StarryOS/kernel/src/tracepoint/mod.rs b/os/StarryOS/kernel/src/tracepoint/mod.rs index 5ccea5bc6a..875b1c9714 100644 --- a/os/StarryOS/kernel/src/tracepoint/mod.rs +++ b/os/StarryOS/kernel/src/tracepoint/mod.rs @@ -7,9 +7,9 @@ use alloc::{collections::BTreeMap, string::ToString, sync::Arc, vec::Vec}; use core::{num::NonZero, ops::Deref}; use ax_errno::{AxError, AxResult}; -use ax_hal::{percpu::this_cpu_id, time::monotonic_time_nanos}; use ax_lazyinit::LazyInit; use ax_memory_addr::VirtAddr; +use ax_runtime::hal::{percpu::this_cpu_id, time::monotonic_time_nanos}; use ax_sync::Mutex; use ax_task::current; use axfs_ng_vfs::NodePermission; diff --git a/os/StarryOS/kernel/src/trap.rs b/os/StarryOS/kernel/src/trap.rs index ea3fbdb290..694b766660 100644 --- a/os/StarryOS/kernel/src/trap.rs +++ b/os/StarryOS/kernel/src/trap.rs @@ -1,10 +1,10 @@ -#[ax_hal::trap::breakpoint_handler] -fn default_breakpoint_handler(_tf: &mut ax_hal::context::TrapFrame) -> bool { +#[ax_runtime::hal::cpu::trap::breakpoint_handler] +fn default_breakpoint_handler(_tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool { false } #[cfg(target_arch = "x86_64")] -#[ax_hal::trap::debug_handler] -fn default_debug_handler(_tf: &mut ax_hal::context::TrapFrame) -> bool { +#[ax_runtime::hal::cpu::trap::debug_handler] +fn default_debug_handler(_tf: &mut ax_runtime::hal::cpu::TrapFrame) -> bool { false } diff --git a/os/StarryOS/starryos/Cargo.toml b/os/StarryOS/starryos/Cargo.toml index 500da6e231..c9fb4538aa 100644 --- a/os/StarryOS/starryos/Cargo.toml +++ b/os/StarryOS/starryos/Cargo.toml @@ -24,10 +24,10 @@ qemu = [ "starry-kernel/input", "starry-kernel/vsock", # auxilary features ] -smp = ["ax-feat/smp", "axplat-dyn?/smp", "axplat-riscv64-visionfive2?/smp"] +smp = ["ax-feat/smp", "axplat-dyn?/smp", "ax-hal/smp"] sg2002 = [ - "dep:ax-plat-riscv64-sg2002", + "ax-hal/riscv64-sg2002", "starry-kernel/sg2002", ] @@ -49,7 +49,8 @@ cntv-timer = ["ax-feat/aarch64-cntv-timer"] aarch64-hvf = ["gic-v3", "cntv-timer"] rknpu = ["ax-driver/rknpu", "starry-kernel/rknpu"] -vf2 = ["dep:axplat-riscv64-visionfive2"] +vf2 = ["ax-hal/riscv64-visionfive2"] +buddy-slab = ["starry-kernel/buddy-slab"] [[bin]] name = "starryos" @@ -66,10 +67,6 @@ ax-hal.workspace = true axplat-dyn = { workspace = true, optional = true } starry-kernel = { workspace = true, features = ["dev-log", "ext4"] } -[target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies] -ax-plat-riscv64-sg2002 = { workspace = true, features = ["fp-simd", "irq", "rtc"], optional = true } -axplat-riscv64-visionfive2 = { workspace = true, features = ["fp-simd", "irq", "rtc"], optional = true } - [target.'cfg(any(windows,unix))'.dependencies] anyhow = "1.0" axbuild = { workspace = true } diff --git a/os/StarryOS/starryos/src/main.rs b/os/StarryOS/starryos/src/main.rs index 531d66c44c..14a4a4723c 100644 --- a/os/StarryOS/starryos/src/main.rs +++ b/os/StarryOS/starryos/src/main.rs @@ -19,12 +19,3 @@ fn main() { starry_kernel::entry::init(&args, &envs); } - -#[cfg(all( - feature = "sg2002", - any(target_arch = "riscv32", target_arch = "riscv64") -))] -extern crate ax_plat_riscv64_sg2002; - -#[cfg(all(feature = "vf2", target_arch = "riscv64"))] -extern crate axplat_riscv64_visionfive2; diff --git a/os/arceos/api/axfeat/Cargo.toml b/os/arceos/api/axfeat/Cargo.toml index 835eea21f7..d8582ceb4d 100644 --- a/os/arceos/api/axfeat/Cargo.toml +++ b/os/arceos/api/axfeat/Cargo.toml @@ -48,6 +48,7 @@ alloc = ["ax-alloc", "ax-runtime/alloc"] alloc-tlsf = ["ax-alloc/tlsf"] alloc-slab = ["ax-alloc/slab"] alloc-buddy = ["ax-alloc/buddy"] +buddy-slab = ["alloc", "ax-runtime/buddy-slab"] page-alloc-64g = ["ax-alloc/page-alloc-64g"] # up to 64G memory capacity page-alloc-4g = ["ax-alloc/page-alloc-4g"] # up to 4G memory capacity paging = ["alloc", "ax-hal/paging", "ax-runtime/paging"] diff --git a/os/arceos/modules/axalloc/src/buddy_slab.rs b/os/arceos/modules/axalloc/src/buddy_slab.rs index e2f46d7e0a..6af3242956 100644 --- a/os/arceos/modules/axalloc/src/buddy_slab.rs +++ b/os/arceos/modules/axalloc/src/buddy_slab.rs @@ -165,11 +165,10 @@ impl GlobalAllocator { /// Allocate arbitrary number of bytes. Returns the left bound of the /// allocated region. pub fn alloc(&self, layout: Layout) -> AllocResult> { - let result = self - .inner - .lock() - .alloc(layout) - .map_err(crate::AllocError::from); + let result = { + let inner = self.inner.lock(); + inner.alloc(layout).map_err(crate::AllocError::from) + }; if result.is_ok() { self.usages.lock().alloc(UsageKind::RustHeap, layout.size()); } @@ -178,10 +177,13 @@ impl GlobalAllocator { /// Gives back the allocated region to the byte allocator. pub fn dealloc(&self, pos: NonNull, layout: Layout) { + { + let inner = self.inner.lock(); + unsafe { inner.dealloc(pos, layout) }; + } self.usages .lock() .dealloc(UsageKind::RustHeap, layout.size()); - unsafe { self.inner.lock().dealloc(pos, layout) }; } /// Allocates contiguous pages. @@ -191,11 +193,12 @@ impl GlobalAllocator { alignment: usize, kind: UsageKind, ) -> AllocResult { - let result = self - .inner - .lock() - .alloc_pages(num_pages, alignment) - .map_err(crate::AllocError::from); + let result = { + let inner = self.inner.lock(); + inner + .alloc_pages(num_pages, alignment) + .map_err(crate::AllocError::from) + }; if result.is_ok() { self.usages.lock().alloc(kind, num_pages * PAGE_SIZE); } @@ -209,11 +212,12 @@ impl GlobalAllocator { alignment: usize, kind: UsageKind, ) -> AllocResult { - let result = self - .inner - .lock() - .alloc_pages_lowmem(num_pages, alignment) - .map_err(crate::AllocError::from); + let result = { + let inner = self.inner.lock(); + inner + .alloc_pages_lowmem(num_pages, alignment) + .map_err(crate::AllocError::from) + }; if result.is_ok() { self.usages.lock().alloc(kind, num_pages * PAGE_SIZE); } @@ -233,8 +237,11 @@ impl GlobalAllocator { /// Gives back the allocated pages starts from `pos` to the page allocator. pub fn dealloc_pages(&self, pos: usize, num_pages: usize, kind: UsageKind) { + { + let inner = self.inner.lock(); + inner.dealloc_pages(pos, num_pages); + } self.usages.lock().dealloc(kind, num_pages * PAGE_SIZE); - self.inner.lock().dealloc_pages(pos, num_pages); } /// Returns the number of allocated bytes in the allocator backend. @@ -346,6 +353,9 @@ pub fn global_allocator() -> &'static GlobalAllocator { } /// Initializes the per-CPU slab for the current CPU. +/// +/// Must run after per-CPU storage is initialized and before scheduler, IPI, or +/// IRQ paths can allocate on this CPU. pub fn init_percpu_slab(cpu_id: usize) { PERCPU_SLAB.with_current(|slab| slab.init(cpu_id)); } diff --git a/os/arceos/modules/axhal/src/lib.rs b/os/arceos/modules/axhal/src/lib.rs index 2ee0cc0aa0..c531538599 100644 --- a/os/arceos/modules/axhal/src/lib.rs +++ b/os/arceos/modules/axhal/src/lib.rs @@ -89,6 +89,7 @@ pub mod context { pub use ax_cpu::{TaskContext, TrapFrame}; } +pub use ax_cpu as cpu; pub use ax_cpu::asm; #[cfg(feature = "uspace")] pub use ax_cpu::uspace; diff --git a/os/arceos/modules/axruntime/Cargo.toml b/os/arceos/modules/axruntime/Cargo.toml index 88da7cad43..fed7af39b8 100644 --- a/os/arceos/modules/axruntime/Cargo.toml +++ b/os/arceos/modules/axruntime/Cargo.toml @@ -22,7 +22,7 @@ smp = ["alloc", "ax-hal/smp", "ax-task?/smp"] stack-guard-page = ["multitask", "paging", "ax-task/stack-guard-page"] tls = ["ax-hal/tls", "ax-task?/tls"] -plat-dyn = ["ax-hal/plat-dyn", "paging", "buddy-slab"] +plat-dyn = ["ax-hal/plat-dyn", "paging"] display = ["dep:ax-display", "ax-driver/display"] fs = [ diff --git a/os/arceos/modules/axruntime/src/lib.rs b/os/arceos/modules/axruntime/src/lib.rs index 31f957e60e..cfe1f462c9 100644 --- a/os/arceos/modules/axruntime/src/lib.rs +++ b/os/arceos/modules/axruntime/src/lib.rs @@ -51,6 +51,8 @@ mod klib; mod devices; mod registers; +pub use ax_hal as hal; + #[cfg(feature = "smp")] pub use self::mp::rust_main_secondary; @@ -165,6 +167,7 @@ pub fn rust_main(cpu_id: usize, arg: usize) -> ! { }; ax_hal::percpu::init_primary(cpu_id); #[cfg(all(feature = "alloc", feature = "buddy-slab"))] + // After per-CPU init, before scheduler/IPI/IRQ paths can allocate. ax_alloc::init_percpu_slab(cpu_id); ax_hal::init_early(cpu_id, arg); let log_level = option_env!("AX_LOG").unwrap_or("info"); diff --git a/os/arceos/modules/axruntime/src/mp.rs b/os/arceos/modules/axruntime/src/mp.rs index 2d3239fde7..7194510827 100644 --- a/os/arceos/modules/axruntime/src/mp.rs +++ b/os/arceos/modules/axruntime/src/mp.rs @@ -138,6 +138,7 @@ pub fn rust_main_secondary(cpu_id: usize) -> ! { } ax_hal::percpu::init_secondary(cpu_id); #[cfg(feature = "buddy-slab")] + // After per-CPU init, before scheduler/IPI/IRQ paths can allocate. ax_alloc::init_percpu_slab(cpu_id); ax_hal::init_early_secondary(cpu_id); diff --git a/platform/axplat-dyn/Cargo.toml b/platform/axplat-dyn/Cargo.toml index 78f20be7e8..df69d8cfe2 100644 --- a/platform/axplat-dyn/Cargo.toml +++ b/platform/axplat-dyn/Cargo.toml @@ -20,7 +20,6 @@ hv = ["somehal/hv", "ax-cpu/arm-el2"] [dependencies] anyhow = { version = "1", default-features = false } -ax-alloc = { workspace = true, features = ["buddy-slab"] } ax-config-macros = { workspace = true } ax-cpu.workspace = true ax-driver = { workspace = true, features = ["fdt", "pci-fdt"] } diff --git a/platform/axplat-dyn/src/irq.rs b/platform/axplat-dyn/src/irq.rs index c3371cf9c8..254eaa3253 100644 --- a/platform/axplat-dyn/src/irq.rs +++ b/platform/axplat-dyn/src/irq.rs @@ -51,8 +51,17 @@ impl IrqIf for IrqIfImpl { Some(irq.raw()) } - fn send_ipi(_id: usize, _target: ax_plat::irq::IpiTarget) { - todo!() + fn send_ipi(id: usize, target: ax_plat::irq::IpiTarget) { + let target = match target { + ax_plat::irq::IpiTarget::Current { cpu_id } => { + somehal::irq::IpiTarget::Current { cpu_id } + } + ax_plat::irq::IpiTarget::Other { cpu_id } => somehal::irq::IpiTarget::Other { cpu_id }, + ax_plat::irq::IpiTarget::AllExceptCurrent { cpu_id, cpu_num } => { + somehal::irq::IpiTarget::AllExceptCurrent { cpu_id, cpu_num } + } + }; + somehal::irq::send_ipi(id.into(), target); } } diff --git a/platform/axplat-dyn/src/lib.rs b/platform/axplat-dyn/src/lib.rs index 8d8614d897..ffb0a89ee8 100644 --- a/platform/axplat-dyn/src/lib.rs +++ b/platform/axplat-dyn/src/lib.rs @@ -1,6 +1,5 @@ #![no_std] #![cfg(not(any(windows, unix)))] -#![feature(used_with_arg)] extern crate alloc; extern crate ax_driver as _; diff --git a/platform/somehal/src/arch/aarch64/gic/mod.rs b/platform/somehal/src/arch/aarch64/gic/mod.rs index 2d2e781cdf..221bdc28ff 100644 --- a/platform/somehal/src/arch/aarch64/gic/mod.rs +++ b/platform/somehal/src/arch/aarch64/gic/mod.rs @@ -5,11 +5,40 @@ use someboot::irq::IrqId; mod v2; mod v3; +use core::sync::atomic::{AtomicU8, Ordering}; + +#[derive(Clone, Copy, Eq, PartialEq)] +enum GicBackend { + None = 0, + V2 = 2, + V3 = 3, +} + +static GIC_BACKEND: AtomicU8 = AtomicU8::new(GicBackend::None as u8); + +fn set_backend(backend: GicBackend) { + GIC_BACKEND.store(backend as u8, Ordering::Release); +} + +fn backend() -> GicBackend { + match GIC_BACKEND.load(Ordering::Acquire) { + 2 => GicBackend::V2, + 3 => GicBackend::V3, + _ => GicBackend::None, + } +} + pub fn init_current_cpu() { - if v3::is_support_icc() { - v3::init_cpu(); - } else { - v2::init_cpu(); + match backend() { + GicBackend::V2 => v2::init_cpu(), + GicBackend::V3 => v3::init_cpu(), + GicBackend::None => { + if v3::is_support_icc() { + v3::init_cpu(); + } else { + v2::init_cpu(); + } + } } } @@ -19,8 +48,33 @@ fn get_gicd() -> Device { pub fn irq_set_enable(irq: rdrive::IrqId, enable: bool) { let raw = irq.into(); - v2::irq_set_enable(raw, enable); - v3::irq_set_enable(raw, enable); + match backend() { + GicBackend::V2 => v2::irq_set_enable(raw, enable), + GicBackend::V3 => v3::irq_set_enable(raw, enable), + GicBackend::None => { + v2::irq_set_enable(raw, enable); + v3::irq_set_enable(raw, enable); + } + } +} + +pub fn send_ipi(irq: rdrive::IrqId, target: crate::irq::IpiTarget) { + let raw = irq.into(); + match backend() { + GicBackend::V2 => v2::send_ipi(raw, target), + GicBackend::V3 => v3::send_ipi(raw, target), + GicBackend::None => { + if v3::is_support_icc() { + v3::send_ipi(raw, target); + } else { + v2::send_ipi(raw, target); + } + } + } +} + +fn hardware_cpu_id(cpu_id: usize) -> usize { + someboot::smp::cpu_idx_to_id(cpu_id).unwrap_or(cpu_id) } #[unsafe(no_mangle)] @@ -29,10 +83,16 @@ fn __aarch64_irq_handler() { } pub(crate) fn irq_handler() -> someboot::irq::IrqId { - if v3::is_support_icc() { - v3::handle_irq() - } else { - v2::handle_irq() + match backend() { + GicBackend::V2 => v2::handle_irq(), + GicBackend::V3 => v3::handle_irq(), + GicBackend::None => { + if v3::is_support_icc() { + v3::handle_irq() + } else { + v2::handle_irq() + } + } } } diff --git a/platform/somehal/src/arch/aarch64/gic/v2.rs b/platform/somehal/src/arch/aarch64/gic/v2.rs index 83d97a6ba1..ef47b00962 100644 --- a/platform/somehal/src/arch/aarch64/gic/v2.rs +++ b/platform/somehal/src/arch/aarch64/gic/v2.rs @@ -75,6 +75,7 @@ fn probe_gic(info: FdtInfo<'_>, dev: PlatformDevice) -> Result<(), OnProbeError> let trap = cpu.trap_operations(); CPU_IF.init(cpu); TRAP.init(trap); + super::set_backend(super::GicBackend::V2); init_cpu(); @@ -120,3 +121,16 @@ pub fn irq_set_enable(raw: usize, enable: bool) { gic.set_irq_enable(unsafe { IntId::raw(raw as _) }, enable); }); } + +pub fn send_ipi(raw: usize, target: crate::irq::IpiTarget) { + let sgi = IntId::sgi(raw as u32); + let target = match target { + crate::irq::IpiTarget::Current { cpu_id: _ } => SGITarget::Current, + crate::irq::IpiTarget::Other { cpu_id } => { + let target_cpu = super::hardware_cpu_id(cpu_id); + SGITarget::TargetList(TargetList::new(&mut core::iter::once(target_cpu))) + } + crate::irq::IpiTarget::AllExceptCurrent { .. } => SGITarget::AllOther, + }; + CPU_IF.send_sgi(sgi, target); +} diff --git a/platform/somehal/src/arch/aarch64/gic/v3.rs b/platform/somehal/src/arch/aarch64/gic/v3.rs index 7054df4775..2b6d7e39b1 100644 --- a/platform/somehal/src/arch/aarch64/gic/v3.rs +++ b/platform/somehal/src/arch/aarch64/gic/v3.rs @@ -51,6 +51,7 @@ fn probe_gic(info: FdtInfo<'_>, dev: PlatformDevice) -> Result<(), OnProbeError> gic.init(); let cpu = gic.cpu_interface(); CPU_IF.init(cpu); + super::set_backend(super::GicBackend::V3); init_cpu(); @@ -87,6 +88,27 @@ pub fn irq_set_enable(raw: usize, enable: bool) { }); } +pub fn send_ipi(raw: usize, target: crate::irq::IpiTarget) { + let sgi = IntId::sgi(raw as u32); + let target = match target { + crate::irq::IpiTarget::Current { cpu_id: _ } => SGITarget::current(), + crate::irq::IpiTarget::Other { cpu_id } => { + SGITarget::list([affinity_from_mpidr(super::hardware_cpu_id(cpu_id))]) + } + crate::irq::IpiTarget::AllExceptCurrent { .. } => SGITarget::All, + }; + CPU_IF.send_sgi(sgi, target); +} + +fn affinity_from_mpidr(mpidr: usize) -> Affinity { + Affinity { + aff0: (mpidr & 0xff) as u8, + aff1: ((mpidr >> 8) & 0xff) as u8, + aff2: ((mpidr >> 16) & 0xff) as u8, + aff3: ((mpidr >> 32) & 0xff) as u8, + } +} + pub fn init_cpu() { unsafe { CPU_IF.update(|cpu| { diff --git a/platform/somehal/src/arch/aarch64/mod.rs b/platform/somehal/src/arch/aarch64/mod.rs index b3b9e5e98d..063fb5ae98 100644 --- a/platform/somehal/src/arch/aarch64/mod.rs +++ b/platform/somehal/src/arch/aarch64/mod.rs @@ -10,6 +10,10 @@ impl PlatOp for Plat { gic::irq_set_enable(irq, enable); } + fn send_ipi(irq: rdrive::IrqId, target: crate::irq::IpiTarget) { + gic::send_ipi(irq, target); + } + fn systick_irq() -> rdrive::IrqId { systick::systick_irq() } diff --git a/platform/somehal/src/common.rs b/platform/somehal/src/common.rs index 6fe98b084a..f58cc8e5a8 100644 --- a/platform/somehal/src/common.rs +++ b/platform/somehal/src/common.rs @@ -6,6 +6,10 @@ use crate::setup::MmioRaw; pub trait PlatOp { fn irq_set_enable(irq: IrqId, enable: bool); + fn send_ipi(_irq: IrqId, _target: crate::irq::IpiTarget) { + panic!("IPI is not implemented for this dynamic platform"); + } + fn irq_handler() -> someboot::irq::IrqId; fn systick_irq() -> IrqId; diff --git a/platform/somehal/src/irq.rs b/platform/somehal/src/irq.rs index f5760c638d..8d976368e6 100644 --- a/platform/somehal/src/irq.rs +++ b/platform/somehal/src/irq.rs @@ -5,6 +5,28 @@ pub use someboot::irq::*; use crate::{arch::Plat, common::PlatOp}; +/// Target specification for inter-processor interrupts. +#[derive(Clone, Copy, Debug)] +pub enum IpiTarget { + /// Send to the current CPU. + Current { + /// The logical CPU ID of the current CPU. + cpu_id: usize, + }, + /// Send to a specific CPU. + Other { + /// The logical CPU ID of the target CPU. + cpu_id: usize, + }, + /// Send to all other CPUs. + AllExceptCurrent { + /// The logical CPU ID of the current CPU. + cpu_id: usize, + /// The total number of CPUs. + cpu_num: usize, + }, +} + pub fn irq_setup_by_fdt(irq_parent: DeviceId, irq_cell: &[u32]) -> IrqId { let mut intc = rdrive::get::(irq_parent).unwrap().lock().unwrap(); debug!("Setting up IRQ {:?}", irq_cell); @@ -17,6 +39,10 @@ pub fn irq_set_enable(irq: IrqId, enable: bool) { Plat::irq_set_enable(irq, enable); } +pub fn send_ipi(irq: IrqId, target: IpiTarget) { + Plat::send_ipi(irq, target); +} + pub fn systick_irq() -> IrqId { Plat::systick_irq() } diff --git a/scripts/axbuild/src/starry/build.rs b/scripts/axbuild/src/starry/build.rs index f92bb3007b..7664f5bc3b 100644 --- a/scripts/axbuild/src/starry/build.rs +++ b/scripts/axbuild/src/starry/build.rs @@ -86,6 +86,7 @@ pub(crate) fn load_cargo_config(request: &ResolvedStarryRequest) -> anyhow::Resu &makefile_features, metadata, ); + normalize_starry_platform_features(&mut build_info.features); if let Some(smp) = request.smp { build_info.max_cpu_num = Some(smp); } @@ -99,18 +100,33 @@ pub(crate) fn load_cargo_config(request: &ResolvedStarryRequest) -> anyhow::Resu Ok(cargo) } +fn normalize_starry_platform_features(features: &mut Vec) { + let has_sg2002 = features.iter().any(|feature| feature == "sg2002"); + let has_vf2 = features.iter().any(|feature| feature == "vf2"); + + if has_sg2002 { + features.push("ax-hal/riscv64-sg2002".to_string()); + } + if has_vf2 { + features.push("ax-hal/riscv64-visionfive2".to_string()); + } + + features.sort(); + features.dedup(); +} + fn patch_starry_cargo_config( cargo: &mut Cargo, request: &ResolvedStarryRequest, metadata: &Metadata, ) -> anyhow::Result<()> { let platform = crate::context::starry_default_platform_for_arch_checked(&request.arch)?; - let static_defplat = uses_static_default_platform(&cargo.features); + let uses_default_qemu_platform = uses_default_qemu_platform(&cargo.features); cargo.package = request.package.clone(); ensure_starry_bin_arg(&mut cargo.args, &request.package, metadata)?; remove_qemu_feature_for_dynamic_platform(cargo); - if static_defplat { + if uses_default_qemu_platform { cargo.features.push("qemu".to_string()); cargo.features.sort(); cargo.features.dedup(); @@ -122,7 +138,7 @@ fn patch_starry_cargo_config( cargo .env .insert("AX_TARGET".to_string(), request.target.clone()); - if static_defplat { + if uses_default_qemu_platform { cargo .env .entry("AX_PLATFORM".to_string()) @@ -173,14 +189,12 @@ fn remove_qemu_feature_for_dynamic_platform(cargo: &mut Cargo) { } } -fn uses_static_default_platform(features: &[String]) -> bool { +fn uses_default_qemu_platform(features: &[String]) -> bool { let has_static_platform = features.iter().any(|feature| { matches!( feature.as_str(), "defplat" | "ax-feat/defplat" | "ax-std/defplat" - ) - }) || features.iter().any(|feature| { - feature.starts_with("ax-hal/") && feature != "ax-hal/plat-dyn" && feature != "ax-hal/myplat" + ) || default_starry_qemu_platform_feature(feature).is_some() }); let has_dynamic = features.iter().any(|feature| { matches!( @@ -198,6 +212,15 @@ fn uses_static_default_platform(features: &[String]) -> bool { has_static_platform && !has_dynamic && !has_custom } +fn default_starry_qemu_platform_feature(feature: &str) -> Option<&str> { + match feature.strip_prefix("ax-hal/")? { + "x86-pc" | "aarch64-qemu-virt" | "riscv64-qemu-virt" | "loongarch64-qemu-virt" => { + Some(feature) + } + _ => None, + } +} + fn ensure_starry_bin_arg( args: &mut Vec, package: &str, @@ -558,6 +581,39 @@ HELLO = "world" assert!(!cargo.env.contains_key("AX_PLATFORM")); } + #[test] + fn load_cargo_config_treats_sg2002_as_explicit_platform_feature() { + let mut request = request( + PathBuf::from("/tmp/.build.toml"), + "riscv64", + "riscv64gc-unknown-none-elf", + ); + request.build_info_override = Some(StarryBuildInfo { + features: vec!["sg2002".to_string()], + plat_dyn: false, + ..default_starry_build_info_for_target("riscv64gc-unknown-none-elf") + }); + + let cargo = load_cargo_config(&request).unwrap(); + + assert!(cargo.features.contains(&"sg2002".to_string())); + assert!( + cargo + .features + .contains(&"ax-hal/riscv64-sg2002".to_string()) + ); + assert!( + !cargo + .features + .contains(&"ax-hal/riscv64-qemu-virt".to_string()) + ); + assert!(!cargo.features.contains(&"qemu".to_string())); + assert_eq!( + cargo.env.get("AX_PLATFORM").map(String::as_str), + Some("riscv64-sg2002") + ); + } + #[test] fn resolve_build_info_path_supports_starry_subworkspace_root() { let root = tempdir().unwrap(); diff --git a/test-suit/starryos/normal/board-licheerv-nano-sg2002/build-riscv64gc-unknown-none-elf.toml b/test-suit/starryos/normal/board-licheerv-nano-sg2002/build-riscv64gc-unknown-none-elf.toml index 6b3ab7b2b6..f4d0d68008 100644 --- a/test-suit/starryos/normal/board-licheerv-nano-sg2002/build-riscv64gc-unknown-none-elf.toml +++ b/test-suit/starryos/normal/board-licheerv-nano-sg2002/build-riscv64gc-unknown-none-elf.toml @@ -1,7 +1,7 @@ # Build-time config template for StarryOS on SG2002. target = "riscv64gc-unknown-none-elf" env = {} -features = ["sg2002", "myplat"] +features = ["sg2002"] log = "Info" max_cpu_num = 1 plat_dyn = false diff --git a/test-suit/starryos/normal/qemu-smp4-buddy-slab/build-aarch64-unknown-none-softfloat.toml b/test-suit/starryos/normal/qemu-smp4-buddy-slab/build-aarch64-unknown-none-softfloat.toml new file mode 100644 index 0000000000..f24592654f --- /dev/null +++ b/test-suit/starryos/normal/qemu-smp4-buddy-slab/build-aarch64-unknown-none-softfloat.toml @@ -0,0 +1,21 @@ +env = {AX_IP = "10.0.2.15", AX_GW = "10.0.2.2"} +features = [ + "ax-hal/plat-dyn", + "ax-feat/display", + "ax-feat/rtc", + "ax-driver/pci", + "ax-driver/rtc", + "ax-driver/virtio-blk", + "ax-driver/virtio-net", + "ax-driver/virtio-gpu", + "ax-driver/virtio-input", + "ax-driver/virtio-socket", + "starry-kernel/input", + "starry-kernel/plat-dyn", + "starry-kernel/vsock", + "starryos/buddy-slab", +] +max_cpu_num = 4 +log = "Warn" +plat_dyn = true +target = "aarch64-unknown-none-softfloat" diff --git a/test-suit/starryos/normal/qemu-smp4-buddy-slab/test-clone-files-race-buddy-slab/c/CMakeLists.txt b/test-suit/starryos/normal/qemu-smp4-buddy-slab/test-clone-files-race-buddy-slab/c/CMakeLists.txt new file mode 100644 index 0000000000..0a23e19e50 --- /dev/null +++ b/test-suit/starryos/normal/qemu-smp4-buddy-slab/test-clone-files-race-buddy-slab/c/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.20) +project(test-clone-files-race-buddy-slab C) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) + +set(TEST_CLONE_FILES_RACE_SRC + "${CMAKE_CURRENT_LIST_DIR}/../../../qemu-smp4/test-clone-files-race/c/src/main.c") + +add_executable(test-clone-files-race ${TEST_CLONE_FILES_RACE_SRC}) +target_compile_options(test-clone-files-race PRIVATE -Wall -Wextra -Werror) +install(TARGETS test-clone-files-race RUNTIME DESTINATION usr/bin) diff --git a/test-suit/starryos/normal/qemu-smp4-buddy-slab/test-clone-files-race-buddy-slab/qemu-aarch64.toml b/test-suit/starryos/normal/qemu-smp4-buddy-slab/test-clone-files-race-buddy-slab/qemu-aarch64.toml new file mode 100644 index 0000000000..3411897f61 --- /dev/null +++ b/test-suit/starryos/normal/qemu-smp4-buddy-slab/test-clone-files-race-buddy-slab/qemu-aarch64.toml @@ -0,0 +1,16 @@ +args = [ + "-nographic", "-cpu", "cortex-a53", + "-m", "512M", + "-smp", "4", + "-device", "virtio-blk-pci,drive=disk0", + "-drive", "id=disk0,if=none,format=raw,file=${workspace}/tmp/axbuild/rootfs/rootfs-aarch64-alpine.img", + "-device", "virtio-net-pci,netdev=net0", + "-netdev", "user,id=net0", +] +uefi = false +to_bin = true +shell_prefix = "root@starry:" +shell_init_cmd = "/usr/bin/test-clone-files-race" +success_regex = ['(?m)^ DONE: \d+ pass, 0 fail'] +fail_regex = ['(?i)\bpanic(?:ked)?\b', '(?m)FAIL'] +timeout = 360 diff --git a/test-suit/starryos/normal/qemu-smp4/test-clone-files-race/c/src/main.c b/test-suit/starryos/normal/qemu-smp4/test-clone-files-race/c/src/main.c index a29db4b116..f7bbbcb76d 100644 --- a/test-suit/starryos/normal/qemu-smp4/test-clone-files-race/c/src/main.c +++ b/test-suit/starryos/normal/qemu-smp4/test-clone-files-race/c/src/main.c @@ -29,7 +29,8 @@ #include #include -#define N_ITERATIONS 1000 +/* Keep the normal qemu case bounded; larger stress counts belong in stress. */ +#define N_ITERATIONS 64 #define STACK_SIZE (64 * 1024) static int g_pipefd[2];