diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fcc18f06..0e4cc561e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Allow input, interned, and tracked structs to select 128-, 256-, 512-, or 1024-slot table pages + with the `page_size` macro option. The default remains 1024 slots. + ## [0.27.2](https://github.com/salsa-rs/salsa/compare/salsa-v0.27.1...salsa-v0.27.2) - 2026-06-25 ### Fixed diff --git a/book/src/tuning.md b/book/src/tuning.md index 0353e2c33..2f4d8da4f 100644 --- a/book/src/tuning.md +++ b/book/src/tuning.md @@ -1,5 +1,23 @@ # Tuning Salsa +## Table page sizes + +Input, interned, and tracked structs can select their table page capacity in slots: + +```rust +#[salsa::input(page_size = 128)] +struct Project { + name: String, +} +``` + +The supported capacities are `128`, `256`, `512`, and `1024`. Omitting the option is equivalent +to `page_size = 1024`. The option applies to Salsa structs, not tracked functions. + +Smaller pages waste less capacity for sparse ingredients; larger pages use fewer page objects for +dense ingredients. Page size is part of the persisted ID encoding, so serialized databases cannot +be reused after changing it. + ## Cache Eviction (LRU) Salsa supports Least Recently Used (LRU) cache eviction for tracked functions. diff --git a/components/salsa-macro-rules/src/setup_input_struct.rs b/components/salsa-macro-rules/src/setup_input_struct.rs index e8fb1924b..2cdfbe187 100644 --- a/components/salsa-macro-rules/src/setup_input_struct.rs +++ b/components/salsa-macro-rules/src/setup_input_struct.rs @@ -53,6 +53,8 @@ macro_rules! setup_input_struct { // The function used to implement `C::heap_size`. heap_size_fn: $($heap_size_fn:path)?, + page_size: $page_size:ident, + // If `true`, `serialize_fn` and `deserialize_fn` have been provided. persist: $persist:tt, @@ -105,6 +107,7 @@ macro_rules! setup_input_struct { const PERSIST: bool = $persist; + type PageSize = $zalsa::$page_size; type Singleton = $zalsa::macro_if! {if $is_singleton {$zalsa::input::Singleton} else {$zalsa::input::NotSingleton}}; type Struct = $Struct; diff --git a/components/salsa-macro-rules/src/setup_interned_struct.rs b/components/salsa-macro-rules/src/setup_interned_struct.rs index ae3dc77d8..844eb1e62 100644 --- a/components/salsa-macro-rules/src/setup_interned_struct.rs +++ b/components/salsa-macro-rules/src/setup_interned_struct.rs @@ -69,6 +69,8 @@ macro_rules! setup_interned_struct { // The function used to implement `C::heap_size`. heap_size_fn: $($heap_size_fn:path)?, + page_size: $page_size:ident, + // If `true`, `serialize_fn` and `deserialize_fn` have been provided. persist: $persist:tt, @@ -157,6 +159,7 @@ macro_rules! setup_interned_struct { const DEBUG_NAME: &'static str = stringify!($Struct); const PERSIST: bool = $persist; + type PageSize = $zalsa::$page_size; $( const REVISIONS: ::core::num::NonZeroUsize = ::core::num::NonZeroUsize::new($revisions).unwrap(); )? diff --git a/components/salsa-macro-rules/src/setup_tracked_fn.rs b/components/salsa-macro-rules/src/setup_tracked_fn.rs index 5fb3142bb..6d978ba28 100644 --- a/components/salsa-macro-rules/src/setup_tracked_fn.rs +++ b/components/salsa-macro-rules/src/setup_tracked_fn.rs @@ -208,6 +208,8 @@ macro_rules! setup_tracked_fn { const DEBUG_NAME: &'static str = concat!($(stringify!($self_ty), "::",)? stringify!($fn_name), "::interned_arguments"); const PERSIST: bool = $persist; + type PageSize = $zalsa::PageSize1024; + type Fields<$db_lt> = ($($interned_input_ty),*); type Struct<$db_lt> = $InternedData<$db_lt>; diff --git a/components/salsa-macro-rules/src/setup_tracked_struct.rs b/components/salsa-macro-rules/src/setup_tracked_struct.rs index fa8ae20fb..4254300d6 100644 --- a/components/salsa-macro-rules/src/setup_tracked_struct.rs +++ b/components/salsa-macro-rules/src/setup_tracked_struct.rs @@ -84,6 +84,8 @@ macro_rules! setup_tracked_struct { // The function used to implement `C::heap_size`. heap_size_fn: $($heap_size_fn:path)?, + page_size: $page_size:ident, + // If `true`, `serialize_fn` and `deserialize_fn` have been provided. persist: $persist:tt, @@ -147,6 +149,7 @@ macro_rules! setup_tracked_struct { const PERSIST: bool = $persist; + type PageSize = $zalsa::$page_size; type Fields<$db_lt> = ($($field_ty,)*); type Revisions = [$zalsa::AtomicRevision; $N]; diff --git a/components/salsa-macros/CHANGELOG.md b/components/salsa-macros/CHANGELOG.md index 9f12e0421..c2e154d24 100644 --- a/components/salsa-macros/CHANGELOG.md +++ b/components/salsa-macros/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add `page_size = 128 | 256 | 512 | 1024` to input, interned, and tracked struct macros. + ## [0.27.1](https://github.com/salsa-rs/salsa/compare/salsa-macros-v0.27.0...salsa-macros-v0.27.1) - 2026-06-24 ### Fixed diff --git a/components/salsa-macros/src/input.rs b/components/salsa-macros/src/input.rs index 6449f0316..6bf55a61c 100644 --- a/components/salsa-macros/src/input.rs +++ b/components/salsa-macros/src/input.rs @@ -67,6 +67,8 @@ impl AllowedOptions for InputStruct { const HEAP_SIZE: bool = true; + const PAGE_SIZE: bool = true; + const SELF_TY: bool = false; const PERSIST: AllowedPersistOptions = AllowedPersistOptions::AllowedValue; @@ -115,6 +117,7 @@ impl Macro { let is_singleton = self.args.singleton.is_some(); let generate_debug_impl = salsa_struct.generate_debug_impl(); let heap_size_fn = self.args.heap_size_fn.iter(); + let page_size = format_ident!("PageSize{}", self.args.page_size.unwrap_or(1024)); let persist = self.args.persist(); let serialize_fn = salsa_struct.serialize_fn(); @@ -148,6 +151,7 @@ impl Macro { is_singleton: #is_singleton, generate_debug_impl: #generate_debug_impl, heap_size_fn: #(#heap_size_fn)*, + page_size: #page_size, persist: #persist, serialize_fn: #(#serialize_fn)*, deserialize_fn: #(#deserialize_fn)*, diff --git a/components/salsa-macros/src/interned.rs b/components/salsa-macros/src/interned.rs index 30c5f2d73..e8aa8ad71 100644 --- a/components/salsa-macros/src/interned.rs +++ b/components/salsa-macros/src/interned.rs @@ -67,6 +67,8 @@ impl AllowedOptions for InternedStruct { const HEAP_SIZE: bool = true; + const PAGE_SIZE: bool = true; + const SELF_TY: bool = false; const PERSIST: AllowedPersistOptions = AllowedPersistOptions::AllowedValue; @@ -138,6 +140,7 @@ impl Macro { let deserialize_fn = salsa_struct.deserialize_fn(); let heap_size_fn = self.args.heap_size_fn.iter(); + let page_size = format_ident!("PageSize{}", self.args.page_size.unwrap_or(1024)); let zalsa = self.hygiene.ident("zalsa"); let zalsa_struct = self.hygiene.ident("zalsa_struct"); @@ -176,6 +179,7 @@ impl Macro { num_fields: #num_fields, generate_debug_impl: #generate_debug_impl, heap_size_fn: #(#heap_size_fn)*, + page_size: #page_size, persist: #persist, serialize_fn: #(#serialize_fn)*, deserialize_fn: #(#deserialize_fn)*, diff --git a/components/salsa-macros/src/options.rs b/components/salsa-macros/src/options.rs index 9585f45e5..cf2c7e340 100644 --- a/components/salsa-macros/src/options.rs +++ b/components/salsa-macros/src/options.rs @@ -113,6 +113,9 @@ pub(crate) struct Options { /// If this is `Some`, the value is the provided `heap_size` function. pub heap_size_fn: Option, + /// The physical table page capacity in slots. + pub page_size: Option, + /// The `self_ty = ` option is used to set the the self type of the tracked impl for tracked /// functions. This is merely used to refine the query name. pub self_ty: Option, @@ -157,6 +160,7 @@ impl Default for Options { id: Default::default(), revisions: Default::default(), heap_size_fn: Default::default(), + page_size: Default::default(), self_ty: Default::default(), persist: Default::default(), } @@ -182,6 +186,7 @@ pub(crate) trait AllowedOptions { const ID: bool; const REVISIONS: bool; const HEAP_SIZE: bool; + const PAGE_SIZE: bool = false; const SELF_TY: bool; const PERSIST: AllowedPersistOptions; } @@ -519,6 +524,38 @@ impl syn::parse::Parse for Options { "`heap_size` option not allowed here", )); } + } else if ident == "page_size" { + if !A::PAGE_SIZE { + return Err(syn::Error::new( + ident.span(), + "`page_size` option not allowed here", + )); + } + let _eq = Equals::parse(input)?; + let expr = syn::Expr::parse(input)?; + let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Int(lit), + .. + }) = expr + else { + return Err(syn::Error::new( + expr.span(), + "`page_size` must be an integer literal", + )); + }; + let value = lit.base10_parse::()?; + if !matches!(value, 128 | 256 | 512 | 1024) { + return Err(syn::Error::new( + lit.span(), + "expected 128, 256, 512, or 1024 slots", + )); + } + if options.page_size.replace(value).is_some() { + return Err(syn::Error::new( + ident.span(), + "option `page_size` provided twice", + )); + } } else if ident == "self_ty" { if A::SELF_TY { let _eq = Equals::parse(input)?; @@ -572,6 +609,7 @@ impl quote::ToTokens for Options { id, revisions, heap_size_fn, + page_size, self_ty, persist, phantom: _, @@ -627,6 +665,9 @@ impl quote::ToTokens for Options { if let Some(heap_size_fn) = heap_size_fn { tokens.extend(quote::quote! { heap_size = #heap_size_fn, }); } + if let Some(page_size) = page_size { + tokens.extend(quote::quote! { page_size = #page_size, }); + } if let Some(self_ty) = self_ty { tokens.extend(quote::quote! { self_ty = #self_ty, }); } diff --git a/components/salsa-macros/src/tracked_struct.rs b/components/salsa-macros/src/tracked_struct.rs index fb645cb5e..2c8e92c54 100644 --- a/components/salsa-macros/src/tracked_struct.rs +++ b/components/salsa-macros/src/tracked_struct.rs @@ -63,6 +63,8 @@ impl AllowedOptions for TrackedStruct { const HEAP_SIZE: bool = true; + const PAGE_SIZE: bool = true; + const SELF_TY: bool = false; const PERSIST: AllowedPersistOptions = AllowedPersistOptions::AllowedValue; @@ -149,6 +151,7 @@ impl Macro { let deserialize_fn = salsa_struct.deserialize_fn(); let heap_size_fn = self.args.heap_size_fn.iter(); + let page_size = format_ident!("PageSize{}", self.args.page_size.unwrap_or(1024)); let num_tracked_fields = salsa_struct.num_tracked_fields(); let generate_debug_impl = salsa_struct.generate_debug_impl(); @@ -199,6 +202,7 @@ impl Macro { generate_debug_impl: #generate_debug_impl, heap_size_fn: #(#heap_size_fn)*, + page_size: #page_size, persist: #persist, serialize_fn: #(#serialize_fn)*, diff --git a/src/database.rs b/src/database.rs index 100665104..3e879be9d 100644 --- a/src/database.rs +++ b/src/database.rs @@ -415,12 +415,14 @@ mod memory_usage { let mut queries = HashMap::new(); let mut structs = Vec::new(); let mut page_infos = self.zalsa().table().page_infos(); - let page_capacity = self.zalsa().table().page_capacity(); for input_ingredient in self.zalsa().ingredients() { let Some(input_info) = input_ingredient.memory_usage(self) else { continue; }; + let page_capacity = input_ingredient + .page_capacity() + .expect("struct ingredients must provide a table page capacity"); let mut size_of_fields = 0; let mut size_of_metadata = 0; diff --git a/src/id.rs b/src/id.rs index bc2565410..5ccd60f6d 100644 --- a/src/id.rs +++ b/src/id.rs @@ -7,8 +7,9 @@ use crate::zalsa::Zalsa; /// The `Id` of a salsa struct in the database [`Table`](`crate::table::Table`). /// /// The high-order bits of an `Id` store a 32-bit generation counter, while -/// the low-order bits pack a [`PageIndex`](`crate::table::PageIndex`) and -/// [`SlotIndex`](`crate::table::SlotIndex`) within the page. +/// the low-order bits pack a page-size class, a [`PageIndex`](`crate::table::PageIndex`), and +/// [`SlotIndex`](`crate::table::SlotIndex`) within the page. The page-size class is selected by +/// the Salsa struct's `page_size` option. /// /// The low-order bits of `Id` are a `u32` ranging from `0..Id::MAX_U32`. /// The maximum range is smaller than a standard `u32` to leave diff --git a/src/ingredient.rs b/src/ingredient.rs index 23cb11478..242a59b9a 100644 --- a/src/ingredient.rs +++ b/src/ingredient.rs @@ -239,6 +239,12 @@ pub trait Ingredient: Any + fmt::Debug + Send + Sync { None } + /// Returns the table page capacity for struct ingredients. + #[cfg(feature = "salsa_unstable")] + fn page_capacity(&self) -> Option { + None + } + /// Whether this ingredient will be persisted with the database. fn is_persistable(&self) -> bool { false diff --git a/src/input.rs b/src/input.rs index ee02fece5..b5f17f4ce 100644 --- a/src/input.rs +++ b/src/input.rs @@ -30,6 +30,10 @@ pub trait Configuration: Any { /// Whether this struct should be persisted with the database. const PERSIST: bool; + /// Physical table page-size policy. + #[doc(hidden)] + type PageSize: crate::table::PageSize; + /// The singleton state for this input if any. type Singleton: SingletonChoice + Send + Sync; @@ -366,6 +370,11 @@ impl Ingredient for IngredientImpl { Some(memory_usage) } + #[cfg(feature = "salsa_unstable")] + fn page_capacity(&self) -> Option { + Some(crate::table::page_capacity::()) + } + fn is_persistable(&self) -> bool { C::PERSIST } @@ -473,6 +482,8 @@ unsafe impl Slot for Value where C: Configuration, { + type PageSize = C::PageSize; + #[inline(always)] unsafe fn memos( this: *const Self, @@ -610,7 +621,8 @@ mod persistence { while let Some((id, value)) = access.next_entry::>()? { let id = Id::from_bits(id); - let (page_idx, _) = crate::table::split_id(id); + let (page_idx, _) = crate::table::try_split_typed_id::(id) + .ok_or_else(|| de::Error::custom("serialized ID has the wrong page size"))?; let value = Value:: { fields: value.fields.0, diff --git a/src/interned.rs b/src/interned.rs index 1e750d0ef..ff12db594 100644 --- a/src/interned.rs +++ b/src/interned.rs @@ -44,6 +44,10 @@ pub trait Configuration: Sized + 'static { /// Whether this struct should be persisted with the database. const PERSIST: bool; + /// Physical table page-size policy. + #[doc(hidden)] + type PageSize: crate::table::PageSize; + // The minimum number of revisions that must pass before a stale value is garbage collected. const REVISIONS: NonZeroUsize = NonZeroUsize::new(DEFAULT_REVISIONS).unwrap(); @@ -1045,6 +1049,11 @@ where Some(memory_usage) } + #[cfg(feature = "salsa_unstable")] + fn page_capacity(&self) -> Option { + Some(crate::table::page_capacity::()) + } + fn is_persistable(&self) -> bool { C::PERSIST } @@ -1096,6 +1105,8 @@ unsafe impl Slot for Value where C: Configuration, { + type PageSize = C::PageSize; + #[inline(always)] unsafe fn memos( this: *const Self, @@ -1690,7 +1701,8 @@ mod persistence { while let Some((id, value)) = access.next_entry::>()? { let id = Id::from_bits(id); - let (page_idx, _) = crate::table::split_id(id); + let (page_idx, _) = crate::table::try_split_typed_id::(id) + .ok_or_else(|| de::Error::custom("serialized ID has the wrong page size"))?; // Determine the value shard. let hash = ingredient.hasher.hash_one(&value.fields.0); @@ -1788,6 +1800,7 @@ mod tests { const PERSIST: bool = false; const REVISIONS: NonZeroUsize = NonZeroUsize::new(2).unwrap(); + type PageSize = crate::table::PageSize1024; type Fields<'db> = (); type Struct<'db> = Id; diff --git a/src/lib.rs b/src/lib.rs index a3994a48c..1e4e3eb5d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -352,6 +352,7 @@ pub mod plumbing { pub use crate::salsa_struct::{SalsaStructInDb, assert_supertype_no_overlap}; pub use crate::storage::{HasStorage, Storage}; pub use crate::table::memo::MemoTableWithTypes; + pub use crate::table::{PageSize, PageSize128, PageSize256, PageSize512, PageSize1024}; pub use crate::tracked_struct::TrackedStructInDb; pub use crate::update::helper::{Dispatch as UpdateDispatch, Fallback as UpdateFallback}; pub use crate::update::{Update, always_update}; diff --git a/src/table.rs b/src/table.rs index 09cea09f4..f2e055db1 100644 --- a/src/table.rs +++ b/src/table.rs @@ -16,16 +16,61 @@ use crate::{Id, IngredientIndex, Revision}; pub(crate) mod memo; -const PAGE_LEN_BITS: usize = 10; -const PAGE_LEN_MASK: usize = PAGE_LEN - 1; -const PAGE_LEN: usize = 1 << PAGE_LEN_BITS; -const MAX_PAGES: usize = 1 << (u32::BITS as usize - PAGE_LEN_BITS); +const PAGE_CLASS_SHIFT: usize = 30; +const PAGE_CLASS_MASK: usize = 3 << PAGE_CLASS_SHIFT; +const PAGE_CLASS_LEN: usize = 1 << PAGE_CLASS_SHIFT; +const PAGE_CLASS_COUNT: usize = 4; + +mod sealed { + pub trait Sealed { + const CAPACITY: usize; + } +} + +/// Type-level table page-size policy used by Salsa's generated code. +#[doc(hidden)] +pub trait PageSize: sealed::Sealed + Send + Sync + 'static {} + +#[doc(hidden)] +pub struct PageSizeConst; + +impl sealed::Sealed for PageSizeConst { + const CAPACITY: usize = N; +} + +impl PageSize for PageSizeConst<128> {} +impl PageSize for PageSizeConst<256> {} +impl PageSize for PageSizeConst<512> {} +impl PageSize for PageSizeConst<1024> {} + +#[doc(hidden)] +pub type PageSize128 = PageSizeConst<128>; +#[doc(hidden)] +pub type PageSize256 = PageSizeConst<256>; +#[doc(hidden)] +pub type PageSize512 = PageSizeConst<512>; +#[doc(hidden)] +pub type PageSize1024 = PageSizeConst<1024>; + +#[inline] +pub(crate) const fn page_capacity() -> usize { +

::CAPACITY +} + +#[inline] +const fn page_class() -> usize { + 10 - page_capacity::

().trailing_zeros() as usize +} + +const fn class_capacity(class: usize) -> usize { + 1024 >> class +} /// A typed [`Page`] view. pub(crate) struct PageView<'p, T: Slot>(&'p Page, PhantomData<&'p T>); pub struct Table { - pages: boxcar::Vec, + pages: [boxcar::Vec; PAGE_CLASS_COUNT], /// Map from ingredient to non-full pages that are up for grabs non_full_pages: Mutex>>, } @@ -35,6 +80,8 @@ pub struct Table { /// Implementors of this trait need to make sure that their type is unique with respect to /// their owning ingredient as the allocation strategy relies on this. pub unsafe trait Slot: Any + Send + Sync { + type PageSize: PageSize; + /// Access the [`MemoTable`][] for this slot. /// /// # Safety condition @@ -62,9 +109,8 @@ struct SlotVTable { memos_mut: SlotMemosMutFnErased, /// The type name of what is stored as entries in data. type_name: fn() -> &'static str, - /// A drop impl to call when the own page drops - /// SAFETY: The caller is required to supply a valid pointer to a `Box>`, and - /// the correct initialized length and memo types. + /// Drops the page allocation and its initialized prefix. + /// SAFETY: The pointer, slot type, initialized length, and memo types must match. drop_impl: unsafe fn(data: *mut (), initialized: usize, memo_types: &MemoTableTypes), } @@ -73,16 +119,8 @@ impl SlotVTable { const { &Self { drop_impl: |data, initialized, memo_types| { - // SAFETY: The caller is required to provide a valid data pointer. - let data = unsafe { Box::from_raw(data.cast::>()) }; - for i in 0..initialized { - let item = data[i].get().cast::(); - // SAFETY: The caller is required to provide a valid initialized length. - unsafe { - memo_types.attach_memos_mut((*item).memos_mut()).drop(); - ptr::drop_in_place(item); - } - } + // SAFETY: `Page` supplies the allocation and initialized length belonging to `T`. + unsafe { drop_page::(NonNull::new_unchecked(data), initialized, memo_types) } }, layout: Layout::new::(), type_name: std::any::type_name::, @@ -98,7 +136,32 @@ impl SlotVTable { } type PageDataEntry = UnsafeCell>; -type PageData = [PageDataEntry; PAGE_LEN]; + +fn allocate_page() -> NonNull<()> { + // A boxed slice avoids materializing a large page on the stack. + let data = (0..page_capacity::()) + .map(|_| UnsafeCell::new(MaybeUninit::uninit())) + .collect::]>>(); + let data = Box::into_raw(data) as *mut PageDataEntry; + NonNull::new(data).unwrap().cast() +} + +unsafe fn drop_page(data: NonNull<()>, initialized: usize, memo_types: &MemoTableTypes) { + let data = ptr::slice_from_raw_parts_mut( + data.cast::>().as_ptr(), + page_capacity::(), + ); + // SAFETY: `Page::new` allocated this slice for `T` and its page policy. + let data = unsafe { Box::from_raw(data) }; + for entry in &data[..initialized] { + let item = entry.get().cast::(); + // SAFETY: `allocated` tracks the initialized prefix. + unsafe { + memo_types.attach_memos_mut((*item).memos_mut()).drop(); + ptr::drop_in_place(item); + } + } +} struct Page { /// The ingredient for elements on this page. @@ -128,18 +191,24 @@ unsafe impl Send for Page /* where for M: Send */ {} unsafe impl Sync for Page /* where for M: Sync */ {} #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct PageIndex(usize); +pub struct PageIndex { + class: usize, + index: usize, +} impl PageIndex { #[inline] - fn new(idx: usize) -> Self { - debug_assert!(idx < MAX_PAGES); - Self(idx) + fn new(index: usize) -> Self { + let class = page_class::

(); + let tag = class << PAGE_CLASS_SHIFT; + let limit = (tag + PAGE_CLASS_LEN).min(Id::MAX_USIZE); + assert!(index < (limit - tag) / page_capacity::

()); + Self { class, index } } #[allow(dead_code)] pub fn as_usize(&self) -> usize { - self.0 + self.index } } @@ -148,8 +217,8 @@ pub struct SlotIndex(usize); impl SlotIndex { #[inline] - fn new(idx: usize) -> Self { - debug_assert!(idx < PAGE_LEN); + fn new(idx: usize) -> Self { + debug_assert!(idx < page_capacity::

()); Self(idx) } } @@ -157,7 +226,7 @@ impl SlotIndex { impl Default for Table { fn default() -> Self { Self { - pages: boxcar::Vec::new(), + pages: std::array::from_fn(|_| boxcar::Vec::new()), non_full_pages: Default::default(), } } @@ -167,8 +236,8 @@ impl Table { /// Returns the [`IngredientIndex`] for an [`Id`]. #[inline] pub fn ingredient_index(&self, id: Id) -> IngredientIndex { - let (page_idx, _) = split_id(id); - self.pages[page_idx.0].ingredient + let (page, _) = split_erased_id(id); + self.pages[page.class][page.index].ingredient } /// Get a reference to the data for `id`, which must have been allocated from this table with type `T`. @@ -177,7 +246,7 @@ impl Table { /// /// If `id` is out of bounds or the does not have the type `T`. pub(crate) fn get(&self, id: Id) -> &T { - let (page, slot) = split_id(id); + let (page, slot) = split_typed_id::(id); let page_ref = self.page::(page); &page_ref.data()[slot.0] } @@ -192,36 +261,38 @@ impl Table { /// /// See [`Page::get_raw`][]. pub(crate) fn get_raw(&self, id: Id) -> *mut T { - let (page, slot) = split_id(id); + let (page, slot) = split_typed_id::(id); let page_ref = self.page::(page); page_ref.page_data()[slot.0].get().cast::() } /// Returns the number of pages that have been allocated. pub fn page_count(&self) -> usize { - self.pages.count() - } - - #[cfg(feature = "salsa_unstable")] - pub(crate) fn page_capacity(&self) -> usize { - PAGE_LEN + self.pages.iter().map(boxcar::Vec::count).sum() } #[cfg(feature = "salsa_unstable")] pub(crate) fn page_infos(&self) -> FxHashMap { - let mut page_fills: FxHashMap> = FxHashMap::default(); - - for (_, page) in self.pages.iter() { - let fill = page.allocated.load(Ordering::Acquire); - if fill > 0 { - page_fills.entry(page.ingredient).or_default().push(fill); + let mut page_fills: FxHashMap)> = FxHashMap::default(); + + for (class, pages) in self.pages.iter().enumerate() { + let capacity = class_capacity(class); + for (_, page) in pages.iter() { + let fill = page.allocated.load(Ordering::Acquire); + if fill > 0 { + let (ingredient_capacity, fills) = page_fills + .entry(page.ingredient) + .or_insert_with(|| (capacity, Vec::new())); + debug_assert_eq!(*ingredient_capacity, capacity); + fills.push(fill); + } } } page_fills .into_iter() - .filter_map(|(ingredient, fills)| { - crate::database::PageInfo::from_page_fills(PAGE_LEN, fills) + .filter_map(|(ingredient, (capacity, fills))| { + crate::database::PageInfo::from_page_fills(capacity, fills) .map(|page_info| (ingredient, page_info)) }) .collect() @@ -234,7 +305,8 @@ impl Table { /// If `page` is out of bounds or the type `T` is incorrect. #[inline] pub(crate) fn page(&self, page: PageIndex) -> PageView<'_, T> { - self.pages[page.0].assert_type::() + debug_assert_eq!(page.class, page_class::()); + self.pages[page.class][page.index].assert_type::() } /// Force initialize the page at the given index. @@ -255,12 +327,14 @@ impl Table { ingredient: IngredientIndex, memo_types: &Arc, ) { - let page = self.pages.get_mut(page_idx.0); + let class = page_class::(); + assert_eq!(page_idx.class, class); + let page = self.pages[class].get_mut(page_idx.index); match page { Some(page) => { // Initialize the page if was created using `push_uninit_page`. - if page.slot_type_id == TypeId::of::() { + if page.slot_type_id == TypeId::of::>() { *page = Page::new::(ingredient, memo_types.clone()); } @@ -270,10 +344,10 @@ impl Table { None => { // Create dummy pages until we reach the page we want. - while self.page_count() < page_idx.as_usize() { + while self.pages[class].count() < page_idx.as_usize() { // We make sure not to claim any intermediary pages for ourselves, as they may // be required by a different ingredient when it is deserialized. - self.push_uninit_page(); + self.push_uninit_page::(); } let allocated_idx = self.push_page::(ingredient, memo_types.clone()); @@ -292,16 +366,20 @@ impl Table { ingredient: IngredientIndex, memo_types: Arc, ) -> PageIndex { - PageIndex::new(self.pages.push(Page::new::(ingredient, memo_types))) + let class = page_class::(); + PageIndex::new::( + self.pages[class].push(Page::new::(ingredient, memo_types)), + ) } /// Allocate an uninitialized page. #[inline] #[allow(dead_code)] - pub(crate) fn push_uninit_page(&self) -> PageIndex { + pub(crate) fn push_uninit_page(&self) -> PageIndex { // Note that `DummySlot` is a ZST, so the memory wasted by any pages of ingredients // that were not serialized should be negligible. - PageIndex::new(self.pages.push(Page::new::( + let class = page_class::

(); + PageIndex::new::

(self.pages[class].push(Page::new::>( IngredientIndex::new(0), Arc::new(MemoTableTypes::default()), ))) @@ -322,8 +400,8 @@ impl Table { id: Id, current_revision: Revision, ) -> MemoTableWithTypes<'_> { - let (page, slot) = split_id(id); - let page = self.pages[page.0].assert_type::(); + let (page, slot) = split_typed_id::(id); + let page = self.pages[page.class][page.index].assert_type::(); let slot = &page.data()[slot.0]; // SAFETY: The caller is required to pass the `current_revision`. @@ -343,8 +421,8 @@ impl Table { /// The parameter `current_revision` must be the current revision of the owner of database /// owning this table. pub unsafe fn dyn_memos(&self, id: Id, current_revision: Revision) -> MemoTableWithTypes<'_> { - let (page, slot) = split_id(id); - let page = &self.pages[page.0]; + let (page, slot) = split_erased_id(id); + let page = &self.pages[page.class][page.index]; // SAFETY: We supply a proper slot pointer and the caller is required to pass the `current_revision`. let memos = unsafe { &*(page.slot_vtable.memos)(page.get(slot), current_revision) }; // SAFETY: The `Page` keeps the correct memo types. @@ -353,10 +431,9 @@ impl Table { /// Get the memo table associated with `id` pub(crate) fn memos_mut(&mut self, id: Id) -> MemoTableWithTypesMut<'_> { - let (page, slot) = split_id(id); - let page_index = page.0; - let page = self - .pages + let (page, slot) = split_erased_id(id); + let page_index = page.index; + let page = self.pages[page.class] .get_mut(page_index) .unwrap_or_else(|| panic!("index `{page_index}` is uninitialized")); // SAFETY: We supply a proper slot pointer and the caller is required to pass the `current_revision`. @@ -366,8 +443,10 @@ impl Table { } pub(crate) fn slots_of(&self) -> impl Iterator + '_ { + let class = page_class::(); ErasedSlots { - pages: self.pages.iter(), + class, + pages: self.pages[class].iter(), current_page: None, slot_type_id: TypeId::of::(), } @@ -386,6 +465,7 @@ impl Table { memo_types: impl FnOnce() -> Arc, ) -> PageIndex { if let Some(page) = self.take_non_full_page(ingredient) { + debug_assert_eq!(page.class, page_class::()); return page; } @@ -409,6 +489,7 @@ impl Table { } struct ErasedSlots<'db> { + class: usize, pages: boxcar::vec::Iter<'db, Page>, current_page: Option<(PageIndex, &'db Page, std::ops::Range)>, slot_type_id: TypeId, @@ -421,8 +502,8 @@ impl Iterator for ErasedSlots<'_> { loop { if let Some((page_index, page, slots)) = &mut self.current_page { if let Some(slot_index) = slots.next() { - let slot_index = SlotIndex::new(slot_index); - let id = make_id(*page_index, slot_index); + let slot_index = SlotIndex(slot_index); + let id = make_erased_id(*page_index, slot_index); // SAFETY: `slot_index` is below the initialized length captured when the page // became current, so the resulting pointer is within the page allocation. @@ -439,7 +520,10 @@ impl Iterator for ErasedSlots<'_> { self.current_page = self.pages.find_map(|(page_index, page)| { (page.slot_type_id == self.slot_type_id).then(|| { ( - PageIndex::new(page_index), + PageIndex { + class: self.class, + index: page_index, + }, page, 0..page.allocated.load(Ordering::Acquire), ) @@ -477,12 +561,12 @@ impl<'db, T: Slot> PageView<'db, T> { V: FnOnce(Id) -> T, { let index = self.0.allocated.load(Ordering::Acquire); - if index >= PAGE_LEN { + if index >= page_capacity::() { return Err(value); } // Initialize entry `index` - let id = make_id(page, SlotIndex::new(index)); + let id = make_id::(page, SlotIndex::new::(index)); let data = self.0.data.cast::>(); // SAFETY: `index` is also guaranteed to be in bounds as per the check above. @@ -505,30 +589,13 @@ impl<'db, T: Slot> PageView<'db, T> { impl Page { #[inline] fn new(ingredient: IngredientIndex, memo_types: Arc) -> Self { - #[cfg(not(feature = "shuttle"))] - let data: Box> = - Box::new([const { UnsafeCell::new(MaybeUninit::uninit()) }; PAGE_LEN]); - - #[cfg(feature = "shuttle")] - let data = { - // Avoid stack overflows when using larger shuttle types. - let data = (0..PAGE_LEN) - .map(|_| UnsafeCell::new(MaybeUninit::uninit())) - .collect::]>>(); - - let data: *mut [PageDataEntry] = Box::into_raw(data); - - // SAFETY: `*mut PageDataEntry` and `*mut [PageDataEntry; N]` have the same layout. - unsafe { Box::from_raw(data.cast::>().cast::>()) } - }; - Self { ingredient, memo_types, slot_vtable: SlotVTable::of::(), slot_type_id: TypeId::of::(), allocated: AtomicUsize::new(0), - data: NonNull::from(Box::leak(data)).cast::<()>(), + data: allocate_page::(), } } @@ -582,10 +649,12 @@ impl Drop for Page { } /// A placeholder type representing the slots of an uninitialized `Page`. -struct DummySlot; +struct DummySlot

(PhantomData

); // SAFETY: The `DummySlot type is private. -unsafe impl Slot for DummySlot { +unsafe impl Slot for DummySlot

{ + type PageSize = P; + unsafe fn memos(_: *const Self, _: Revision) -> *const MemoTable { unreachable!() } @@ -595,17 +664,47 @@ unsafe impl Slot for DummySlot { } } -fn make_id(page: PageIndex, slot: SlotIndex) -> Id { - let page = page.0 as u32; - let slot = slot.0 as u32; - // SAFETY: `slot` is guaranteed to be small enough that the resulting Id won't be bigger than `Id::MAX_U32` - unsafe { Id::from_index((page << PAGE_LEN_BITS) | slot) } +fn make_id(page: PageIndex, slot: SlotIndex) -> Id { + debug_assert_eq!(page.class, page_class::

()); + debug_assert!(slot.0 < page_capacity::

()); + make_erased_id(page, slot) +} + +fn make_erased_id(page: PageIndex, slot: SlotIndex) -> Id { + let capacity = class_capacity(page.class); + debug_assert!(slot.0 < capacity); + let index = (page.class << PAGE_CLASS_SHIFT) + page.index * capacity + slot.0; + // SAFETY: `PageIndex::new` keeps the complete page below `Id::MAX_U32`. + unsafe { Id::from_index(index as u32) } +} + +#[inline] +fn split_erased_id(id: Id) -> (PageIndex, SlotIndex) { + let class = id.index() as usize >> PAGE_CLASS_SHIFT; + split_id(id, class, class_capacity(class)) +} + +#[inline] +fn split_typed_id(id: Id) -> (PageIndex, SlotIndex) { + let class = page_class::

(); + debug_assert_eq!(id.index() as usize >> PAGE_CLASS_SHIFT, class); + split_id(id, class, page_capacity::

()) } #[inline] -pub fn split_id(id: Id) -> (PageIndex, SlotIndex) { - let index = id.index() as usize; - let slot = index & PAGE_LEN_MASK; - let page = index >> PAGE_LEN_BITS; - (PageIndex::new(page), SlotIndex::new(slot)) +fn split_id(id: Id, class: usize, capacity: usize) -> (PageIndex, SlotIndex) { + let offset = id.index() as usize & !PAGE_CLASS_MASK; + ( + PageIndex { + class, + index: offset / capacity, + }, + SlotIndex(offset % capacity), + ) +} + +#[cfg(feature = "persistence")] +pub(crate) fn try_split_typed_id(id: Id) -> Option<(PageIndex, SlotIndex)> { + (id.index() < Id::MAX_U32 && id.index() as usize >> PAGE_CLASS_SHIFT == page_class::

()) + .then(|| split_typed_id::

(id)) } diff --git a/src/tracked_struct.rs b/src/tracked_struct.rs index dbe3b0d6c..dc30c3ce9 100644 --- a/src/tracked_struct.rs +++ b/src/tracked_struct.rs @@ -49,6 +49,10 @@ pub trait Configuration: Sized + 'static { /// Whether this struct should be persisted with the database. const PERSIST: bool; + /// Physical table page-size policy. + #[doc(hidden)] + type PageSize: crate::table::PageSize; + /// A (possibly empty) tuple of the fields for this struct. type Fields<'db>: Send + Sync; @@ -1059,6 +1063,11 @@ where Some(memory_usage) } + #[cfg(feature = "salsa_unstable")] + fn page_capacity(&self) -> Option { + Some(crate::table::page_capacity::()) + } + fn is_persistable(&self) -> bool { C::PERSIST } @@ -1176,6 +1185,8 @@ unsafe impl Slot for Value where C: Configuration, { + type PageSize = C::PageSize; + #[inline(always)] unsafe fn memos( this: *const Self, @@ -1430,7 +1441,8 @@ mod persistence { while let Some((id, value)) = access.next_entry::>()? { let id = Id::from_bits(id); - let (page_idx, _) = crate::table::split_id(id); + let (page_idx, _) = crate::table::try_split_typed_id::(id) + .ok_or_else(|| de::Error::custom("serialized ID has the wrong page size"))?; let value = Value:: { updated_at: value.updated_at, diff --git a/tests/compile-fail/invalid_page_sizes.rs b/tests/compile-fail/invalid_page_sizes.rs new file mode 100644 index 000000000..2a110db2d --- /dev/null +++ b/tests/compile-fail/invalid_page_sizes.rs @@ -0,0 +1,11 @@ +#[salsa::interned(page_size = 127)] +struct NonPowerOfTwoPageSize<'db> { + value: usize, +} + +#[salsa::input(page_size = 64 * 2)] +struct ExpressionPageSize { + value: usize, +} + +fn main() {} diff --git a/tests/compile-fail/invalid_page_sizes.stderr b/tests/compile-fail/invalid_page_sizes.stderr new file mode 100644 index 000000000..3d905ae9a --- /dev/null +++ b/tests/compile-fail/invalid_page_sizes.stderr @@ -0,0 +1,11 @@ +error: expected 128, 256, 512, or 1024 slots + --> tests/compile-fail/invalid_page_sizes.rs:1:31 + | +1 | #[salsa::interned(page_size = 127)] + | ^^^ + +error: `page_size` must be an integer literal + --> tests/compile-fail/invalid_page_sizes.rs:6:28 + | +6 | #[salsa::input(page_size = 64 * 2)] + | ^^ diff --git a/tests/interned-structs_self_ref.rs b/tests/interned-structs_self_ref.rs index ec8880201..5c7954095 100644 --- a/tests/interned-structs_self_ref.rs +++ b/tests/interned-structs_self_ref.rs @@ -87,6 +87,7 @@ const _: () = { line: line!(), }; const DEBUG_NAME: &'static str = "InternedString"; + type PageSize = zalsa_::PageSize1024; type Fields<'a> = StructData<'a>; type Struct<'a> = InternedString<'a>; diff --git a/tests/page-sizes.rs b/tests/page-sizes.rs new file mode 100644 index 000000000..2a3263aa4 --- /dev/null +++ b/tests/page-sizes.rs @@ -0,0 +1,136 @@ +#![cfg(feature = "inventory")] + +use salsa::plumbing::{AsId, ZalsaDatabase}; + +#[salsa::input(page_size = 128)] +struct Input128 { + value: usize, +} + +#[salsa::input(page_size = 256)] +struct Input256 { + value: usize, +} + +#[salsa::input(page_size = 512)] +struct Input512 { + value: usize, +} + +#[salsa::input(page_size = 256)] +struct EmptyInput256 { + value: usize, +} + +#[cfg(feature = "persistence")] +#[salsa::input(page_size = 128, persist)] +struct PersistedInput128 { + value: usize, +} + +#[cfg(all(feature = "persistence", not(feature = "shuttle")))] +#[test] +fn persistence_preserves_page_class() { + let mut db = salsa::DatabaseImpl::new(); + let expected = PersistedInput128::new(&db, 128); + let json = serde_json::to_string(&::as_serialize(&mut db)).unwrap(); + + let mut restored = salsa::DatabaseImpl::new(); + ::deserialize( + &mut restored, + &mut serde_json::Deserializer::from_str(&json), + ) + .unwrap(); + let actual = PersistedInput128::ingredient(&restored) + .entries(restored.zalsa()) + .last() + .unwrap() + .as_struct(); + assert_eq!(actual.as_id(), expected.as_id()); + assert_eq!(*actual.value(&restored), 128); +} + +#[salsa::input] +struct Input1024 { + value: usize, +} + +#[salsa::interned(page_size = 128)] +struct Interned128<'db> { + value: usize, +} + +#[salsa::tracked(page_size = 512)] +struct Tracked512<'db> { + value: usize, +} + +#[salsa::tracked] +fn create_tracked(db: &dyn salsa::Database, input: Input128) -> Tracked512<'_> { + let base = input.value(db); + (0..513) + .map(|value| Tracked512::new(db, base + value)) + .last() + .unwrap() +} + +#[test] +fn configurable_page_sizes() { + #[cfg(feature = "shuttle")] + shuttle::check_random(configurable_page_sizes_impl, 1); + #[cfg(not(feature = "shuttle"))] + configurable_page_sizes_impl(); +} + +fn configurable_page_sizes_impl() { + let db = salsa::DatabaseImpl::new(); + let _ = EmptyInput256::ingredient(&db); + let input128 = (0..129).map(|v| Input128::new(&db, v)).last().unwrap(); + let input256 = (0..257).map(|v| Input256::new(&db, v)).last().unwrap(); + let input512 = (0..513).map(|v| Input512::new(&db, v)).last().unwrap(); + let input1024 = (0..1025).map(|v| Input1024::new(&db, v)).last().unwrap(); + let interned = (0..129).map(|v| Interned128::new(&db, v)).last().unwrap(); + let tracked = create_tracked(&db, input128); + + for (id, class, name) in [ + (input1024.as_id(), 0, "Input1024"), + (input512.as_id(), 1, "Input512"), + (tracked.as_id(), 1, "Tracked512"), + (input256.as_id(), 2, "Input256"), + (input128.as_id(), 3, "Input128"), + (interned.as_id(), 3, "Interned128"), + ] { + assert_eq!(id.index() >> 30, class); + assert_eq!( + db.zalsa() + .lookup_ingredient(db.zalsa().ingredient_index(id)) + .debug_name(), + name + ); + } + + assert_eq!(*input1024.value(&db), 1024); + assert_eq!(*input512.value(&db), 512); + assert_eq!(*tracked.value(&db), 640); + assert_eq!(*input256.value(&db), 256); + assert_eq!(*input128.value(&db), 128); + assert_eq!(*interned.value(&db), 128); + + let memory_usage = ::memory_usage(&db); + for (name, capacity) in [ + ("Input1024", 1024), + ("Input512", 512), + ("Tracked512", 512), + ("Input256", 256), + ("EmptyInput256", 256), + ("Input128", 128), + ("Interned128", 128), + ] { + let ingredient = memory_usage + .structs + .iter() + .find(|ingredient| ingredient.debug_name() == name) + .unwrap(); + assert_eq!(ingredient.page_info().unwrap().page_capacity(), capacity); + } +}