From f68bac50bc35c42e727742acd2a4c265b78011c2 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Tue, 30 Jun 2026 17:05:05 +0200 Subject: [PATCH 01/10] refactor: generalize eviction policy interface --- .../salsa-macro-rules/src/setup_tracked_fn.rs | 4 +- src/function.rs | 61 ++++++++++++++----- src/function/eviction.rs | 46 ++++++++++---- src/function/eviction/lru.rs | 8 +-- src/function/eviction/noop.rs | 9 +-- src/function/memo.rs | 20 +++++- src/lib.rs | 4 +- 7 files changed, 109 insertions(+), 43 deletions(-) diff --git a/components/salsa-macro-rules/src/setup_tracked_fn.rs b/components/salsa-macro-rules/src/setup_tracked_fn.rs index 5fb3142bb..6cdcb8870 100644 --- a/components/salsa-macro-rules/src/setup_tracked_fn.rs +++ b/components/salsa-macro-rules/src/setup_tracked_fn.rs @@ -484,14 +484,14 @@ macro_rules! setup_tracked_fn { } } - /// Sets the lru capacity + /// Sets the eviction policy's tuning value. /// /// **WARNING:** Just like an ordinary write, this method triggers /// cancellation. If you invoke it while a snapshot exists, it /// will block until that snapshot is dropped -- if that snapshot /// is owned by the current thread, this could trigger deadlock. fn set_lru_capacity(db: &mut dyn $Db, value: usize) where for<'trivial_bounds> $Eviction: $zalsa::function::HasCapacity { - $Configuration::fn_ingredient_mut(db).set_capacity(value); + $Configuration::fn_ingredient_mut(db).set_tuning(value); } $zalsa::macro_if! { $needs_interner => diff --git a/src/function.rs b/src/function.rs index d66cf2f96..c477a1f4c 100644 --- a/src/function.rs +++ b/src/function.rs @@ -37,7 +37,7 @@ mod memo; mod specify; mod sync; -pub use eviction::{EvictionPolicy, HasCapacity, Lru, NoopEviction}; +pub use eviction::{EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction}; pub type Memo = memo::Memo<'static, C>; @@ -239,12 +239,14 @@ where DatabaseKeyIndex::new(self.index, key) } - /// Set eviction capacity. Only available when eviction policy supports it. - pub fn set_capacity(&mut self, capacity: usize) + /// Sets the eviction policy's tuning value. + /// + /// Only available when the eviction policy supports runtime tuning. + pub fn set_tuning(&mut self, tuning: usize) where C::Eviction: HasCapacity, { - self.eviction.set_capacity(capacity); + self.eviction.set_tuning(tuning); } /// Returns a reference to the memo value that lives as long as self. @@ -276,11 +278,18 @@ where // We convert to a `NonNull` here as soon as possible because we are going to alias // into the `Box`, which is a `noalias` type. // FIXME: Use `Box::into_non_null` once stable + let has_value = memo.value.is_some(); let memo = NonNull::from(Box::leak(Box::new(memo))); - if let Some(old_value) = - self.insert_memo_into_table_for(zalsa, id, memo, memo_ingredient_index) - { + let old_value = self.insert_memo_into_table_for(zalsa, id, memo, memo_ingredient_index); + let became_resident = has_value + && old_value.is_none_or(|old_value| { + // SAFETY: The old memo remains allocated in `deleted_entries` until the next + // revision, so it is valid to inspect here. + unsafe { old_value.as_ref().value.is_none() } + }); + + if let Some(old_value) = old_value { // In case there is a reference to the old memo out there, we have to store it // in the deleted entries. This will get cleared when a new revision starts. // @@ -288,6 +297,9 @@ where // memo contents, and so it will be safe to free. unsafe { self.deleted_entries.push(old_value) }; } + if became_resident { + self.eviction.record_insert(id); + } // SAFETY: memo has been inserted into the table unsafe { self.extend_memo_lifetime(memo.as_ref()) } } @@ -465,13 +477,11 @@ where } fn reset_for_new_revision(&mut self, table: &mut Table) { - self.eviction.for_each_evicted(|evict| { - let ingredient_index = table.ingredient_index(evict); - Self::evict_value_from_memo_for( - table.memos_mut(evict), - self.memo_ingredient_indices.get(ingredient_index), - ) - }); + let mut context = FunctionEvictionContext:: { + table, + memo_ingredient_indices: &self.memo_ingredient_indices, + }; + self.eviction.start_new_revision(&mut context); self.deleted_entries.clear(); } @@ -557,6 +567,29 @@ where } } +struct FunctionEvictionContext<'a, C: Configuration> { + table: &'a mut Table, + memo_ingredient_indices: &'a as SalsaStructInDb>::MemoIngredientMap, +} + +impl EvictionContext for FunctionEvictionContext<'_, C> { + fn last_verified_at(&mut self, id: Id) -> Option { + let ingredient_index = self.table.ingredient_index(id); + IngredientImpl::::last_verified_at_for( + self.table.memos_mut(id), + self.memo_ingredient_indices.get(ingredient_index), + ) + } + + fn evict_value(&mut self, id: Id) { + let ingredient_index = self.table.ingredient_index(id); + IngredientImpl::::evict_value_from_memo_for( + self.table.memos_mut(id), + self.memo_ingredient_indices.get(ingredient_index), + ) + } +} + impl memo::MemoHeader { fn collect_minimum_serialized_edges( &self, diff --git a/src/function/eviction.rs b/src/function/eviction.rs index 33be09ba3..bef1c97e3 100644 --- a/src/function/eviction.rs +++ b/src/function/eviction.rs @@ -9,32 +9,52 @@ mod noop; pub use lru::Lru; pub use noop::NoopEviction; -use crate::Id; +use crate::{Id, Revision}; /// Trait for cache eviction strategies. /// /// Implementations control when memoized values are evicted from the cache. /// The eviction policy is selected at compile time via the `Configuration` trait. pub trait EvictionPolicy: Send + Sync { - /// Create a new eviction policy with the given capacity. - fn new(capacity: usize) -> Self; + /// Creates a policy from its configured tuning value. + /// + /// A value of zero disables eviction. + fn new(tuning: usize) -> Self; + + /// Records that `id` transitioned from having no cached value to having one. + fn record_insert(&self, _id: Id) {} + + /// Records that a resident value was used. + /// + /// Implementations may treat this as a best-effort hint. + fn record_use(&self, _id: Id) {} - /// Record that an item was accessed. - fn record_use(&self, id: Id); + /// Changes the policy's tuning value. + /// + /// A value of zero disables eviction. + fn set_tuning(&mut self, tuning: usize); - /// Set the maximum capacity. - fn set_capacity(&mut self, capacity: usize); + /// Processes the start of a new revision. + /// + /// The policy may update its state, inspect memo metadata, and evict + /// resident values through `context`. + fn start_new_revision(&mut self, context: &mut impl EvictionContext); +} - /// Iterate over items that should be evicted. +/// Memo operations available when starting a new revision. +pub trait EvictionContext { + /// Returns the revision in which `id`'s resident value was last verified. /// - /// Called once per revision during `reset_for_new_revision`. - /// The callback `cb` should be invoked for each item to evict. - fn for_each_evicted(&mut self, cb: impl FnMut(Id)); + /// Returns `None` if the memo no longer contains a resident value. + fn last_verified_at(&mut self, id: Id) -> Option; + + /// Evicts `id`'s cached value while retaining its memo metadata. + fn evict_value(&mut self, id: Id); } -/// Marker trait for eviction policies that have a configurable capacity. +/// Marker trait for eviction policies whose tuning can be changed at runtime. /// /// This trait is used to conditionally generate the `set_lru_capacity` method /// on tracked functions. Only policies that implement this trait will expose -/// runtime capacity configuration. +/// runtime tuning. pub trait HasCapacity: EvictionPolicy {} diff --git a/src/function/eviction/lru.rs b/src/function/eviction/lru.rs index 66e589848..7225e5806 100644 --- a/src/function/eviction/lru.rs +++ b/src/function/eviction/lru.rs @@ -9,7 +9,7 @@ use crate::Id; use crate::hash::FxLinkedHashSet; use crate::sync::Mutex; -use super::{EvictionPolicy, HasCapacity}; +use super::{EvictionContext, EvictionPolicy, HasCapacity}; /// Least Recently Used eviction policy. /// @@ -43,21 +43,21 @@ impl EvictionPolicy for Lru { } } - fn set_capacity(&mut self, capacity: usize) { + fn set_tuning(&mut self, capacity: usize) { self.capacity = NonZeroUsize::new(capacity); if self.capacity.is_none() { self.set.get_mut().clear(); } } - fn for_each_evicted(&mut self, mut cb: impl FnMut(Id)) { + fn start_new_revision(&mut self, context: &mut impl EvictionContext) { let Some(cap) = self.capacity else { return; }; let set = self.set.get_mut(); while set.len() > cap.get() { if let Some(id) = set.pop_front() { - cb(id); + context.evict_value(id); } } } diff --git a/src/function/eviction/noop.rs b/src/function/eviction/noop.rs index ea9046882..7dc6d248d 100644 --- a/src/function/eviction/noop.rs +++ b/src/function/eviction/noop.rs @@ -2,7 +2,7 @@ //! //! This is the default eviction policy when no LRU capacity is specified. -use crate::{Id, function::EvictionPolicy}; +use crate::function::{EvictionContext, EvictionPolicy}; /// No eviction - cache grows unbounded. pub struct NoopEviction; @@ -13,11 +13,8 @@ impl EvictionPolicy for NoopEviction { } #[inline(always)] - fn record_use(&self, _id: Id) {} + fn set_tuning(&mut self, _capacity: usize) {} #[inline(always)] - fn set_capacity(&mut self, _capacity: usize) {} - - #[inline(always)] - fn for_each_evicted(&mut self, _cb: impl FnMut(Id)) {} + fn start_new_revision(&mut self, _context: &mut impl EvictionContext) {} } diff --git a/src/function/memo.rs b/src/function/memo.rs index 1d7aac538..40c643500 100644 --- a/src/function/memo.rs +++ b/src/function/memo.rs @@ -56,9 +56,9 @@ impl IngredientImpl { Some(unsafe { transmute::<&Memo<'static, C>, &'db Memo<'db, C>>(static_memo.as_ref()) }) } - /// Evicts the existing memo for the given key, replacing it - /// with an equivalent memo that has no value. If the memo is untracked - /// or has values assigned as output of another query, this has no effect. + /// Evicts the value from the existing memo for the given key. + /// If the memo is untracked or has values assigned as output of another query, + /// this has no effect. pub(super) fn evict_value_from_memo_for( table: MemoTableWithTypesMut<'_>, memo_ingredient_index: MemoIngredientIndex, @@ -72,6 +72,20 @@ impl IngredientImpl { table.map_memo(memo_ingredient_index, map) } + + /// Returns when the resident value was last verified. + pub(super) fn last_verified_at_for( + table: MemoTableWithTypesMut<'_>, + memo_ingredient_index: MemoIngredientIndex, + ) -> Option { + let mut last_verified_at = None; + table.map_memo(memo_ingredient_index, |memo: &mut Memo<'static, C>| { + if memo.value.is_some() { + last_verified_at = Some(memo.header.verified_at.load()); + } + }); + last_verified_at + } } #[derive(Debug)] diff --git a/src/lib.rs b/src/lib.rs index a3994a48c..9ba870aac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -401,7 +401,9 @@ pub mod plumbing { pub use crate::function::Configuration; pub use crate::function::IngredientImpl; pub use crate::function::Memo; - pub use crate::function::{EvictionPolicy, HasCapacity, Lru, NoopEviction}; + pub use crate::function::{ + EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction, + }; pub use crate::table::memo::MemoEntryType; } From 7c6cafbbd10e8145cfd22d7306973f7669aa049e Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Thu, 2 Jul 2026 20:13:43 +0200 Subject: [PATCH 02/10] Add SIEVE eviction policy --- benches/eviction.rs | 8 +- benches/eviction/support.rs | 9 + benches/eviction_walltime.rs | 14 +- components/salsa-macros/src/accumulator.rs | 1 + components/salsa-macros/src/input.rs | 2 + components/salsa-macros/src/interned.rs | 2 + components/salsa-macros/src/options.rs | 25 + components/salsa-macros/src/tracked_fn.rs | 20 +- components/salsa-macros/src/tracked_struct.rs | 2 + src/function.rs | 13 +- src/function/accumulated.rs | 3 +- src/function/eviction.rs | 7 +- src/function/eviction/sieve.rs | 515 ++++++++++++++++++ src/lib.rs | 2 +- src/table.rs | 8 +- tests/sieve.rs | 102 ++++ 16 files changed, 702 insertions(+), 31 deletions(-) create mode 100644 src/function/eviction/sieve.rs create mode 100644 tests/sieve.rs diff --git a/benches/eviction.rs b/benches/eviction.rs index 0e6ac1441..a8d1abdda 100644 --- a/benches/eviction.rs +++ b/benches/eviction.rs @@ -35,7 +35,7 @@ fn main() { /// small per-hit costs can add up. Every result remains resident, which isolates /// hit-path bookkeeping. `NoEviction` shows the tracked-query cost without an /// eviction policy. -#[divan::bench(args = [Policy::NoEviction, Policy::Lru])] +#[divan::bench(args = [Policy::NoEviction, Policy::Lru, Policy::Sieve])] fn fast_path(bencher: divan::Bencher, policy: Policy) { const ITEMS: usize = 1_024; @@ -59,7 +59,7 @@ fn fast_path(bencher: divan::Bencher, policy: Policy) { /// tracked function. This measures the remaining disabled-policy branch on the /// hit path. `NoEviction` is the policy-free reference for how cheap this path /// could be. -#[divan::bench(args = [Policy::NoEviction, Policy::Lru])] +#[divan::bench(args = [Policy::NoEviction, Policy::Lru, Policy::Sieve])] fn disabled_eviction(bencher: divan::Bencher, policy: Policy) { const ITEMS: usize = 1_024; @@ -82,7 +82,7 @@ fn disabled_eviction(bencher: divan::Bencher, policy: Policy) { /// reads a fully resident working set and then triggers a sweep, separating /// steady-state sweep bookkeeping from victim selection and destruction. #[divan::bench( - args = [Policy::NoEviction, Policy::Lru], + args = [Policy::NoEviction, Policy::Lru, Policy::Sieve], sample_count = 20, sample_size = 1 )] @@ -116,7 +116,7 @@ fn fast_path_and_sweep(bencher: divan::Bencher, policy: Policy) { /// selection, and value destruction. `NoEviction` provides the computation /// baseline; setup and final database destruction remain outside timing. #[divan::bench( - args = [Policy::NoEviction, Policy::Lru], + args = [Policy::NoEviction, Policy::Lru, Policy::Sieve], sample_count = 20, sample_size = 1 )] diff --git a/benches/eviction/support.rs b/benches/eviction/support.rs index 979697632..2a0526eae 100644 --- a/benches/eviction/support.rs +++ b/benches/eviction/support.rs @@ -19,10 +19,16 @@ pub(crate) fn lru_value(db: &dyn salsa::Database, item: Item) -> Value { compute_value(item.value(db)) } +#[salsa::tracked(returns(copy), sieve = 4096)] +pub(crate) fn sieve_value(db: &dyn salsa::Database, item: Item) -> Value { + compute_value(item.value(db)) +} + #[derive(Clone, Copy)] pub(crate) enum Policy { NoEviction, Lru, + Sieve, } impl Policy { @@ -30,6 +36,7 @@ impl Policy { match self { Self::NoEviction => {} Self::Lru => lru_value::set_lru_capacity(db, capacity), + Self::Sieve => sieve_value::set_lru_capacity(db, capacity), } } } @@ -39,6 +46,7 @@ impl std::fmt::Display for Policy { formatter.write_str(match self { Self::NoEviction => "NoEviction", Self::Lru => "Lru", + Self::Sieve => "Sieve", }) } } @@ -52,6 +60,7 @@ pub(crate) fn access_all( match policy { Policy::NoEviction => access_all_with(db, items, no_eviction_value), Policy::Lru => access_all_with(db, items, lru_value), + Policy::Sieve => access_all_with(db, items, sieve_value), } } diff --git a/benches/eviction_walltime.rs b/benches/eviction_walltime.rs index cdf74c692..75f943f58 100644 --- a/benches/eviction_walltime.rs +++ b/benches/eviction_walltime.rs @@ -22,7 +22,7 @@ mod support; use support::{ Item, Policy, Value, access_all, access_all_with, lru_value, new_items, no_eviction_value, - prewarm, + prewarm, sieve_value, }; fn main() { @@ -64,7 +64,7 @@ fn main() { /// active entries that did not survive and how quickly the policy stabilizes /// around the smaller working set. #[divan::bench( - args = [Policy::NoEviction, Policy::Lru], + args = [Policy::NoEviction, Policy::Lru, Policy::Sieve], sample_count = 20, sample_size = 1 )] @@ -125,7 +125,7 @@ fn project_check_then_incremental(bencher: divan::Bencher, policy: Policy) { /// recurring-only round charges for entries displaced by the last scan. /// `NoEviction` shows the cost when recurring values are never displaced. #[divan::bench( - args = [Policy::NoEviction, Policy::Lru], + args = [Policy::NoEviction, Policy::Lru, Policy::Sieve], sample_count = 20, sample_size = 1 )] @@ -201,7 +201,7 @@ fn scan_resistance(bencher: divan::Bencher, policy: Policy) { /// provides the computation baseline; retaining cold entries cannot create hits /// because they are never accessed again. #[divan::bench( - args = [Policy::NoEviction, Policy::Lru], + args = [Policy::NoEviction, Policy::Lru, Policy::Sieve], sample_count = 20, sample_size = 1 )] @@ -256,7 +256,7 @@ fn one_hit_wonders(bencher: divan::Bencher, policy: Policy) { /// the policy-free baseline: it retains both phases, but only the second phase /// is accessed after the switch. #[divan::bench( - args = [Policy::NoEviction, Policy::Lru], + args = [Policy::NoEviction, Policy::Lru, Policy::Sieve], sample_count = 20, sample_size = 1 )] @@ -295,7 +295,7 @@ fn phase_change(bencher: divan::Bencher, policy: Policy) { /// misses and contention on the query itself. This isolates contention in the /// eviction policy's hit bookkeeping; `NoEviction` provides the baseline. #[divan::bench( - args = [Policy::NoEviction, Policy::Lru], + args = [Policy::NoEviction, Policy::Lru, Policy::Sieve], sample_count = 20, sample_size = 1 )] @@ -349,6 +349,7 @@ impl Workload { match policy { Policy::NoEviction => self.run_with(no_eviction_value), Policy::Lru => self.run_with(lru_value), + Policy::Sieve => self.run_with(sieve_value), } } @@ -373,6 +374,7 @@ fn parallel_access_repeated( parallel_access_repeated_with(jobs, accesses_per_item, no_eviction_value) } Policy::Lru => parallel_access_repeated_with(jobs, accesses_per_item, lru_value), + Policy::Sieve => parallel_access_repeated_with(jobs, accesses_per_item, sieve_value), } } diff --git a/components/salsa-macros/src/accumulator.rs b/components/salsa-macros/src/accumulator.rs index abc694809..58a722988 100644 --- a/components/salsa-macros/src/accumulator.rs +++ b/components/salsa-macros/src/accumulator.rs @@ -42,6 +42,7 @@ impl AllowedOptions for Accumulator { const CYCLE_INITIAL: bool = false; const CYCLE_RESULT: bool = false; const LRU: bool = false; + const SIEVE: bool = false; const CONSTRUCTOR_NAME: bool = false; const ID: bool = false; const REVISIONS: bool = false; diff --git a/components/salsa-macros/src/input.rs b/components/salsa-macros/src/input.rs index 6449f0316..545352eea 100644 --- a/components/salsa-macros/src/input.rs +++ b/components/salsa-macros/src/input.rs @@ -59,6 +59,8 @@ impl AllowedOptions for InputStruct { const LRU: bool = false; + const SIEVE: bool = false; + const CONSTRUCTOR_NAME: bool = true; const ID: bool = false; diff --git a/components/salsa-macros/src/interned.rs b/components/salsa-macros/src/interned.rs index 30c5f2d73..24e742ad6 100644 --- a/components/salsa-macros/src/interned.rs +++ b/components/salsa-macros/src/interned.rs @@ -59,6 +59,8 @@ impl AllowedOptions for InternedStruct { const LRU: bool = false; + const SIEVE: bool = false; + const CONSTRUCTOR_NAME: bool = true; const ID: bool = true; diff --git a/components/salsa-macros/src/options.rs b/components/salsa-macros/src/options.rs index 9585f45e5..5804504a1 100644 --- a/components/salsa-macros/src/options.rs +++ b/components/salsa-macros/src/options.rs @@ -89,6 +89,11 @@ pub(crate) struct Options { /// If this is `Some`, the value is the ``. pub lru: Option, + /// The `sieve = ` option is used to set the SIEVE capacity for a tracked function. + /// + /// If this is `Some`, the value is the ``. + pub sieve: Option, + /// The `constructor = ` option lets the user specify the name of /// the constructor of a salsa struct. /// @@ -153,6 +158,7 @@ impl Default for Options { constructor_name: Default::default(), phantom: Default::default(), lru: Default::default(), + sieve: Default::default(), singleton: Default::default(), id: Default::default(), revisions: Default::default(), @@ -178,6 +184,7 @@ pub(crate) trait AllowedOptions { const CYCLE_INITIAL: bool; const CYCLE_RESULT: bool; const LRU: bool; + const SIEVE: bool; const CONSTRUCTOR_NAME: bool; const ID: bool; const REVISIONS: bool; @@ -460,6 +467,20 @@ impl syn::parse::Parse for Options { "`lru` option not allowed here", )); } + } else if ident == "sieve" { + if A::SIEVE { + let _eq = Equals::parse(input)?; + let lit = syn::LitInt::parse(input)?; + let value = lit.base10_parse::()?; + if let Some(old) = options.sieve.replace(value) { + return Err(syn::Error::new(old.span(), "option `sieve` provided twice")); + } + } else { + return Err(syn::Error::new( + ident.span(), + "`sieve` option not allowed here", + )); + } } else if ident == "constructor" { if A::CONSTRUCTOR_NAME { let _eq = Equals::parse(input)?; @@ -568,6 +589,7 @@ impl quote::ToTokens for Options { cycle_result, data, lru, + sieve, constructor_name, id, revisions, @@ -615,6 +637,9 @@ impl quote::ToTokens for Options { if let Some(lru) = lru { tokens.extend(quote::quote! { lru = #lru, }); } + if let Some(sieve) = sieve { + tokens.extend(quote::quote! { sieve = #sieve, }); + } if let Some(constructor_name) = constructor_name { tokens.extend(quote::quote! { constructor = #constructor_name, }); } diff --git a/components/salsa-macros/src/tracked_fn.rs b/components/salsa-macros/src/tracked_fn.rs index 28f4ef3bc..49c87a365 100644 --- a/components/salsa-macros/src/tracked_fn.rs +++ b/components/salsa-macros/src/tracked_fn.rs @@ -52,6 +52,8 @@ impl AllowedOptions for TrackedFn { const LRU: bool = true; + const SIEVE: bool = true; + const CONSTRUCTOR_NAME: bool = false; const ID: bool = false; @@ -161,16 +163,32 @@ impl Macro { )); } + if let (Some(_), Some(token)) = (&self.args.sieve, &self.args.specify) { + return Err(syn::Error::new_spanned( + token, + "the `specify` and `sieve` options cannot be used together", + )); + } + + if let (Some(_), Some(token)) = (&self.args.lru, &self.args.sieve) { + return Err(syn::Error::new_spanned( + token, + "the `lru` and `sieve` options cannot be used together", + )); + } + let needs_interner = match function_type { FunctionType::Constant | FunctionType::RequiresInterning => true, FunctionType::SalsaStruct => false, }; - let lru = Literal::usize_unsuffixed(self.args.lru.unwrap_or(0)); + let lru = Literal::usize_unsuffixed(self.args.lru.or(self.args.sieve).unwrap_or(0)); // Determine the eviction policy type based on whether LRU capacity is specified let eviction_type = if self.args.lru.is_some() { quote!(::salsa::plumbing::function::Lru) + } else if self.args.sieve.is_some() { + quote!(::salsa::plumbing::function::Sieve) } else { quote!(::salsa::plumbing::function::NoopEviction) }; diff --git a/components/salsa-macros/src/tracked_struct.rs b/components/salsa-macros/src/tracked_struct.rs index fb645cb5e..79d6c97c4 100644 --- a/components/salsa-macros/src/tracked_struct.rs +++ b/components/salsa-macros/src/tracked_struct.rs @@ -55,6 +55,8 @@ impl AllowedOptions for TrackedStruct { const LRU: bool = false; + const SIEVE: bool = false; + const CONSTRUCTOR_NAME: bool = true; const ID: bool = false; diff --git a/src/function.rs b/src/function.rs index c477a1f4c..64022195f 100644 --- a/src/function.rs +++ b/src/function.rs @@ -37,7 +37,7 @@ mod memo; mod specify; mod sync; -pub use eviction::{EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction}; +pub use eviction::{EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction, Sieve}; pub type Memo = memo::Memo<'static, C>; @@ -278,17 +278,9 @@ where // We convert to a `NonNull` here as soon as possible because we are going to alias // into the `Box`, which is a `noalias` type. // FIXME: Use `Box::into_non_null` once stable - let has_value = memo.value.is_some(); let memo = NonNull::from(Box::leak(Box::new(memo))); let old_value = self.insert_memo_into_table_for(zalsa, id, memo, memo_ingredient_index); - let became_resident = has_value - && old_value.is_none_or(|old_value| { - // SAFETY: The old memo remains allocated in `deleted_entries` until the next - // revision, so it is valid to inspect here. - unsafe { old_value.as_ref().value.is_none() } - }); - if let Some(old_value) = old_value { // In case there is a reference to the old memo out there, we have to store it // in the deleted entries. This will get cleared when a new revision starts. @@ -297,9 +289,6 @@ where // memo contents, and so it will be safe to free. unsafe { self.deleted_entries.push(old_value) }; } - if became_resident { - self.eviction.record_insert(id); - } // SAFETY: memo has been inserted into the table unsafe { self.extend_memo_lifetime(memo.as_ref()) } } diff --git a/src/function/accumulated.rs b/src/function/accumulated.rs index 72f4ccae5..dd4cf482d 100644 --- a/src/function/accumulated.rs +++ b/src/function/accumulated.rs @@ -1,5 +1,6 @@ use crate::accumulator::accumulated_map::{AccumulatedMap, InputAccumulatedValues}; use crate::accumulator::{self}; +use crate::function::eviction::EvictionPolicy; use crate::function::{Configuration, IngredientImpl}; use crate::hash::FxHashSet; use crate::zalsa::ZalsaDatabase; @@ -93,8 +94,8 @@ where key: Id, ) -> (Option<&'db AccumulatedMap>, InputAccumulatedValues) { let (zalsa, zalsa_local) = db.zalsas(); - // NEXT STEP: stash and refactor `fetch` to return an `&Memo` so we can make this work let memo = self.refresh_memo(db, zalsa, zalsa_local, key); + self.eviction.record_use(key); ( memo.header.revisions.accumulated(), memo.header.revisions.accumulated_inputs.load(), diff --git a/src/function/eviction.rs b/src/function/eviction.rs index bef1c97e3..0726e2f32 100644 --- a/src/function/eviction.rs +++ b/src/function/eviction.rs @@ -5,9 +5,11 @@ mod lru; mod noop; +mod sieve; pub use lru::Lru; pub use noop::NoopEviction; +pub use sieve::Sieve; use crate::{Id, Revision}; @@ -21,10 +23,7 @@ pub trait EvictionPolicy: Send + Sync { /// A value of zero disables eviction. fn new(tuning: usize) -> Self; - /// Records that `id` transitioned from having no cached value to having one. - fn record_insert(&self, _id: Id) {} - - /// Records that a resident value was used. + /// Records that a value was used. /// /// Implementations may treat this as a best-effort hint. fn record_use(&self, _id: Id) {} diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs new file mode 100644 index 000000000..a259495a0 --- /dev/null +++ b/src/function/eviction/sieve.rs @@ -0,0 +1,515 @@ +//! SIEVE eviction policy. +//! +//! This policy keeps resident values in FIFO order and records a single visited +//! bit for values used after admission. At eviction time, a moving hand gives +//! visited values one second chance by clearing their bit and continuing toward +//! newer entries. + +use std::num::NonZeroUsize; +use std::ptr; + +use crate::Id; +use crate::sync::Mutex; +use crate::sync::atomic::{AtomicPtr, AtomicU64, Ordering}; +use crate::table::{PAGE_LEN, PAGE_LEN_BITS, split_id}; + +use super::{EvictionContext, EvictionPolicy, HasCapacity}; +use boxcar::buckets::{Buckets, Index, MaybeZeroable, buckets_for_index_bits}; + +const SLOT_STATE_BITS: usize = 2; +const SLOTS_PER_STATE_WORD: usize = u64::BITS as usize / SLOT_STATE_BITS; +const STATE_WORDS: usize = PAGE_LEN.div_ceil(SLOTS_PER_STATE_WORD); + +type PageStates = Buckets; +type PageStateIndex = Index<{ buckets_for_index_bits(u32::BITS - PAGE_LEN_BITS as u32) }>; + +/// SIEVE eviction policy. +/// +/// Values enter at the front of a FIFO queue. Uses set a page-indexed atomic +/// visited bit; they do not move the value in the queue. +pub struct Sieve { + capacity: Option, + state: Mutex, + page_states: PageStates, +} + +impl EvictionPolicy for Sieve { + fn new(capacity: usize) -> Self { + Self { + capacity: NonZeroUsize::new(capacity), + state: Mutex::default(), + page_states: PageStates::new(), + } + } + + fn record_use(&self, id: Id) { + if let Some(capacity) = self.capacity { + if let Some(page) = self.page_if_allocated(id) { + if page.record_use(slot_offset(id)) { + return; + } + } + + self.record_admission(id, capacity.get()); + } + } + + fn set_tuning(&mut self, capacity: usize) { + self.capacity = NonZeroUsize::new(capacity); + let page_states = &self.page_states; + let state = self.state.get_mut(); + if let Some(capacity) = self.capacity { + state.schedule_evictions(capacity.get(), page_states); + } else { + for id in state + .residents + .resident_ids() + .chain(state.pending_evictions.iter().copied()) + { + if let Some(page) = page_if_allocated(page_states, id) { + page.clear(slot_offset(id)); + } + } + *state = State::default(); + } + } + + fn start_new_revision(&mut self, context: &mut impl EvictionContext) { + if self.capacity.is_none() { + return; + } + + let state = self.state.get_mut(); + for id in state.pending_evictions.drain(..) { + if !page_if_allocated(&self.page_states, id) + .is_some_and(|page| page.is_resident(slot_offset(id))) + { + context.evict_value(id); + } + } + } +} + +impl HasCapacity for Sieve {} + +impl Sieve { + fn page_if_allocated(&self, id: Id) -> Option<&PageState> { + page_if_allocated(&self.page_states, id) + } + + fn record_admission(&self, id: Id, capacity: usize) { + let page = page(&self.page_states, id); + let slot = slot_offset(id); + let mut state = self.state.lock(); + + if page.is_resident(slot) { + page.record_use(slot); + return; + } + + state.insert(id, capacity, &self.page_states); + } +} + +#[derive(Default)] +struct State { + residents: Residents, + /// Index of the next resident candidate to inspect. `0` is the sentinel. + hand: ResidentIndex, + /// Victims selected by SIEVE admission, waiting for an eviction context. + pending_evictions: Vec, +} + +impl State { + fn insert(&mut self, id: Id, capacity: usize, page_states: &PageStates) { + let node = self.residents.push_front(id); + page(page_states, id).admit(slot_offset(id)); + + if self.hand == 0 { + self.hand = node; + } + + if self.residents.len() > capacity { + assert!( + self.schedule_eviction(page_states), + "resident list should have an eviction candidate after insertion" + ); + } + } + + fn schedule_evictions(&mut self, capacity: usize, page_states: &PageStates) { + while self.residents.len() > capacity { + if !self.schedule_eviction(page_states) { + return; + } + } + } + + fn schedule_eviction(&mut self, page_states: &PageStates) -> bool { + if let Some(victim) = self.select_victim(page_states) { + self.pending_evictions.push(victim); + true + } else { + false + } + } + + fn select_victim(&mut self, page_states: &PageStates) -> Option { + loop { + if self.hand == 0 { + return None; + } + + let index = self.hand; + let id = self.residents.id(index); + + if page(page_states, id).select(slot_offset(id)).was_visited() { + self.hand = self.residents.advance_towards_front(index); + } else { + self.hand = self.residents.hand_after_remove(index); + return Some(self.residents.remove(index)); + } + } + } +} + +type ResidentIndex = u32; + +struct Residents { + nodes: Vec, + free: Vec, + len: usize, +} + +struct Resident { + id: Option, + /// Newer resident, or the sentinel. + prev: ResidentIndex, + /// Older resident, or the sentinel. + next: ResidentIndex, +} + +impl Default for Residents { + fn default() -> Self { + Self { + nodes: vec![Resident { + id: None, + prev: 0, + next: 0, + }], + free: Vec::new(), + len: 0, + } + } +} + +impl Residents { + fn len(&self) -> usize { + self.len + } + + fn push_front(&mut self, id: Id) -> ResidentIndex { + let index = self.alloc_node(id); + let first = self.node(0).next; + + self.node_mut(index).prev = 0; + self.node_mut(index).next = first; + self.node_mut(0).next = index; + self.node_mut(first).prev = index; + + self.len += 1; + index + } + + fn remove(&mut self, index: ResidentIndex) -> Id { + debug_assert_ne!(index, 0); + + let prev = self.node(index).prev; + let next = self.node(index).next; + self.node_mut(prev).next = next; + self.node_mut(next).prev = prev; + self.len -= 1; + + let id = self + .node_mut(index) + .id + .take() + .expect("resident node should have an id"); + self.node_mut(index).prev = 0; + self.node_mut(index).next = 0; + self.free.push(index); + id + } + + fn id(&self, index: ResidentIndex) -> Id { + self.node(index) + .id + .expect("resident node should have an id") + } + + fn advance_towards_front(&self, index: ResidentIndex) -> ResidentIndex { + debug_assert_ne!(index, 0); + + let prev = self.node(index).prev; + if prev == 0 { self.node(0).prev } else { prev } + } + + fn hand_after_remove(&self, index: ResidentIndex) -> ResidentIndex { + debug_assert_ne!(index, 0); + + if self.len == 1 { + return 0; + } + + let prev = self.node(index).prev; + if prev == 0 { self.node(0).prev } else { prev } + } + + fn resident_ids(&self) -> ResidentIds<'_> { + ResidentIds { + residents: self, + next: self.node(0).next, + } + } + + fn alloc_node(&mut self, id: Id) -> ResidentIndex { + if let Some(index) = self.free.pop() { + self.node_mut(index).id = Some(id); + index + } else { + let index = ResidentIndex::try_from(self.nodes.len()) + .expect("SIEVE resident capacity should fit in u32"); + self.nodes.push(Resident { + id: Some(id), + prev: 0, + next: 0, + }); + index + } + } + + fn node(&self, index: ResidentIndex) -> &Resident { + &self.nodes[index as usize] + } + + fn node_mut(&mut self, index: ResidentIndex) -> &mut Resident { + &mut self.nodes[index as usize] + } +} + +struct ResidentIds<'a> { + residents: &'a Residents, + next: ResidentIndex, +} + +impl Iterator for ResidentIds<'_> { + type Item = Id; + + fn next(&mut self) -> Option { + if self.next == 0 { + return None; + } + + let index = self.next; + self.next = self.residents.node(index).next; + Some(self.residents.id(index)) + } +} + +#[derive(Default)] +struct PageSlot { + ptr: AtomicPtr, +} + +// SAFETY: `PageSlot`'s all-zero representation is valid because a null +// `AtomicPtr` is valid. +unsafe impl MaybeZeroable for PageSlot { + fn zeroable() -> bool { + true + } +} + +impl PageSlot { + fn get(&self) -> Option<&PageState> { + let ptr = self.ptr.load(Ordering::Acquire); + ptr::NonNull::new(ptr).map(|ptr| { + // SAFETY: A non-null pointer was allocated by `get_or_alloc` and + // remains owned by this `PageSlot` until it is dropped. + unsafe { ptr.as_ref() } + }) + } + + fn get_or_alloc(&self) -> &PageState { + if let Some(page) = self.get() { + return page; + } + + let new_page = Box::into_raw(Box::new(PageState::default())); + match self.ptr.compare_exchange( + ptr::null_mut(), + new_page, + Ordering::Release, + Ordering::Acquire, + ) { + Ok(_) => { + // SAFETY: We just installed this allocation. + unsafe { &*new_page } + } + Err(existing) => { + // SAFETY: This allocation was not published, so this thread + // still owns it. + unsafe { drop(Box::from_raw(new_page)) }; + + // SAFETY: `existing` came from a successful install by another + // thread and remains owned by this `PageSlot`. + unsafe { &*existing } + } + } + } +} + +impl Drop for PageSlot { + fn drop(&mut self) { + let ptr = *self.ptr.get_mut(); + if !ptr.is_null() { + // SAFETY: Dropping the `PageSlot` requires exclusive access, so no + // more readers can access the pointed-to page. + unsafe { drop(Box::from_raw(ptr)) }; + } + } +} + +#[derive(Default)] +struct PageState { + words: [AtomicU64; STATE_WORDS], +} + +impl PageState { + fn record_use(&self, slot: usize) -> bool { + let (word, resident, visited) = slot_state(slot); + let word = &self.words[word]; + let mut state = word.load(Ordering::Relaxed); + + loop { + if state & resident == 0 { + return false; + } + + if state & visited != 0 { + return true; + } + + match word.compare_exchange_weak( + state, + state | visited, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(new_state) => state = new_state, + } + } + } + + fn admit(&self, slot: usize) { + let (word, resident, visited) = slot_state(slot); + let word = &self.words[word]; + let mut state = word.load(Ordering::Relaxed); + + loop { + let new_state = (state | resident) & !visited; + match word.compare_exchange_weak(state, new_state, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => return, + Err(new_state) => state = new_state, + } + } + } + + fn select(&self, slot: usize) -> Selection { + let (word, resident, visited) = slot_state(slot); + let word = &self.words[word]; + let mut state = word.load(Ordering::Relaxed); + + loop { + debug_assert_ne!( + state & resident, + 0, + "resident list entry should have a resident state bit" + ); + + let (new_state, selection) = if state & visited == 0 { + (state & !(resident | visited), Selection::Evict) + } else { + (state & !visited, Selection::SecondChance) + }; + + match word.compare_exchange_weak(state, new_state, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => return selection, + Err(new_state) => state = new_state, + } + } + } + + fn is_resident(&self, slot: usize) -> bool { + let (word, resident, _) = slot_state(slot); + self.words[word].load(Ordering::Relaxed) & resident != 0 + } + + fn clear(&self, slot: usize) { + let (word, resident, visited) = slot_state(slot); + let word = &self.words[word]; + let mut state = word.load(Ordering::Relaxed); + + loop { + let new_state = state & !(resident | visited); + match word.compare_exchange_weak(state, new_state, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => return, + Err(new_state) => state = new_state, + } + } + } +} + +#[derive(Copy, Clone)] +enum Selection { + SecondChance, + Evict, +} + +impl Selection { + fn was_visited(self) -> bool { + matches!(self, Selection::SecondChance) + } +} + +fn page(page_states: &PageStates, id: Id) -> &PageState { + page_states + .get_or_alloc(page_state_index(id)) + .get_or_alloc() +} + +fn page_if_allocated(page_states: &PageStates, id: Id) -> Option<&PageState> { + page_states + .get(page_state_index(id)) + .and_then(PageSlot::get) +} + +fn page_state_index(id: Id) -> PageStateIndex { + let (page, _) = split_id(id); + PageStateIndex::new(page.as_usize()).expect("page index should fit in SIEVE page state table") +} + +fn slot_offset(id: Id) -> usize { + let (_, slot) = split_id(id); + slot.as_usize() +} + +fn slot_state(slot: usize) -> (usize, u64, u64) { + let bit = (slot % SLOTS_PER_STATE_WORD) * SLOT_STATE_BITS; + let resident = 1 << bit; + let visited = 1 << (bit + 1); + (slot / SLOTS_PER_STATE_WORD, resident, visited) +} diff --git a/src/lib.rs b/src/lib.rs index 9ba870aac..1cde4b24b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -402,7 +402,7 @@ pub mod plumbing { pub use crate::function::IngredientImpl; pub use crate::function::Memo; pub use crate::function::{ - EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction, + EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction, Sieve, }; pub use crate::table::memo::MemoEntryType; } diff --git a/src/table.rs b/src/table.rs index a047a1477..59df96df0 100644 --- a/src/table.rs +++ b/src/table.rs @@ -16,9 +16,9 @@ use crate::{Id, IngredientIndex, Revision}; pub(crate) mod memo; -const PAGE_LEN_BITS: usize = 7; +pub(crate) const PAGE_LEN_BITS: usize = 7; const PAGE_LEN_MASK: usize = PAGE_LEN - 1; -const PAGE_LEN: usize = 1 << PAGE_LEN_BITS; +pub(crate) const PAGE_LEN: usize = 1 << PAGE_LEN_BITS; const MAX_PAGES: usize = Id::MAX_USIZE / PAGE_LEN; /// A typed [`Page`] view. @@ -152,6 +152,10 @@ impl SlotIndex { debug_assert!(idx < PAGE_LEN); Self(idx) } + + pub(crate) fn as_usize(&self) -> usize { + self.0 + } } impl Default for Table { diff --git a/tests/sieve.rs b/tests/sieve.rs new file mode 100644 index 000000000..b195737bf --- /dev/null +++ b/tests/sieve.rs @@ -0,0 +1,102 @@ +#![cfg(feature = "inventory")] + +//! Tests for the SIEVE eviction policy. + +mod common; +use common::LogDatabase; + +use salsa::Database as _; +use test_log::test; + +#[salsa::input] +struct MyInput { + #[returns(copy)] + field: u32, +} + +#[salsa::tracked(returns(copy), sieve = 2)] +fn sieve_value(db: &dyn LogDatabase, input: MyInput) -> u32 { + db.push_log(format!("sieve_value({:?})", input.field(db))); + input.field(db) +} + +#[test] +fn sieve_gives_visited_values_a_second_chance() { + let mut db = common::LoggerDatabase::default(); + + let inputs: Vec = (0..3).map(|field| MyInput::new(&db, field)).collect(); + + assert_eq!(sieve_value(&db, inputs[0]), 0); + assert_eq!(sieve_value(&db, inputs[1]), 1); + + // Touch `0` again after admission. SIEVE should set its visited bit without + // moving it out of its old FIFO position. + assert_eq!(sieve_value(&db, inputs[0]), 0); + + assert_eq!(sieve_value(&db, inputs[2]), 2); + db.assert_logs_len(3); + + db.synthetic_write(salsa::Durability::HIGH); + + assert_eq!(sieve_value(&db, inputs[0]), 0); + db.assert_logs_len(0); + + assert_eq!(sieve_value(&db, inputs[1]), 1); + db.assert_logs(expect_test::expect![[r#" + [ + "sieve_value(1)", + ]"#]]); +} + +#[test] +fn sieve_readmits_evicted_values() { + let mut db = common::LoggerDatabase::default(); + + let inputs: Vec = (0..3).map(|field| MyInput::new(&db, field)).collect(); + + assert_eq!(sieve_value(&db, inputs[0]), 0); + assert_eq!(sieve_value(&db, inputs[1]), 1); + assert_eq!(sieve_value(&db, inputs[2]), 2); + db.assert_logs_len(3); + + db.synthetic_write(salsa::Durability::HIGH); + + assert_eq!(sieve_value(&db, inputs[0]), 0); + db.assert_logs(expect_test::expect![[r#" + [ + "sieve_value(0)", + ]"#]]); + + db.synthetic_write(salsa::Durability::HIGH); + + assert_eq!(sieve_value(&db, inputs[0]), 0); + db.assert_logs_len(0); + + assert_eq!(sieve_value(&db, inputs[1]), 1); + db.assert_logs(expect_test::expect![[r#" + [ + "sieve_value(1)", + ]"#]]); +} + +#[test] +fn sieve_skips_stale_pending_evictions() { + let mut db = common::LoggerDatabase::default(); + + let inputs: Vec = (0..3).map(|field| MyInput::new(&db, field)).collect(); + + assert_eq!(sieve_value(&db, inputs[0]), 0); + assert_eq!(sieve_value(&db, inputs[1]), 1); + assert_eq!(sieve_value(&db, inputs[2]), 2); + + // `0` has been selected for eviction but the value is still present until + // the next revision starts. Fetching it again must re-admit it and make the + // pending eviction stale. + assert_eq!(sieve_value(&db, inputs[0]), 0); + db.assert_logs_len(3); + + db.synthetic_write(salsa::Durability::HIGH); + + assert_eq!(sieve_value(&db, inputs[0]), 0); + db.assert_logs_len(0); +} From 417b81c1161548961587ccddd1a893c2460b38fa Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Fri, 3 Jul 2026 18:16:41 +0000 Subject: [PATCH 03/10] fix: correct and optimize SIEVE eviction --- src/function/eviction/sieve.rs | 337 +++++++++++++++++++++++++-------- 1 file changed, 263 insertions(+), 74 deletions(-) diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs index a259495a0..2f6b5d3bc 100644 --- a/src/function/eviction/sieve.rs +++ b/src/function/eviction/sieve.rs @@ -4,6 +4,21 @@ //! bit for values used after admission. At eviction time, a moving hand gives //! visited values one second chance by clearing their bit and continuing toward //! newer entries. +//! +//! Victim selection is bounded to twice the current resident count. With no +//! concurrent hits, this behaves like textbook SIEVE: if every resident starts +//! visited, the first pass clears those bits and the first resident is selected +//! when the hand reaches it again. A second pass also gives residents re-marked +//! by concurrent hits another chance. If every inspection still finds a visited +//! resident, selection force-evicts the resident at the hand so admissions +//! cannot starve without changing the scan order. +//! +//! Hits update the page-indexed visited bits without taking the state mutex. +//! Admissions, hand movement, and the pending-eviction queue are serialized by +//! that mutex. Selecting a victim removes it from the resident list immediately, +//! but its value is evicted at the next revision only if it was not re-admitted. +//! A separate page-indexed bitmap ensures each id appears in that queue at most +//! once per revision. use std::num::NonZeroUsize; use std::ptr; @@ -19,6 +34,8 @@ use boxcar::buckets::{Buckets, Index, MaybeZeroable, buckets_for_index_bits}; const SLOT_STATE_BITS: usize = 2; const SLOTS_PER_STATE_WORD: usize = u64::BITS as usize / SLOT_STATE_BITS; const STATE_WORDS: usize = PAGE_LEN.div_ceil(SLOTS_PER_STATE_WORD); +const SLOTS_PER_PENDING_WORD: usize = u64::BITS as usize; +const PENDING_WORDS: usize = PAGE_LEN.div_ceil(SLOTS_PER_PENDING_WORD); type PageStates = Buckets; type PageStateIndex = Index<{ buckets_for_index_bits(u32::BITS - PAGE_LEN_BITS as u32) }>; @@ -44,8 +61,8 @@ impl EvictionPolicy for Sieve { fn record_use(&self, id: Id) { if let Some(capacity) = self.capacity { - if let Some(page) = self.page_if_allocated(id) { - if page.record_use(slot_offset(id)) { + if let Some((page, slot)) = page_and_slot_if_allocated(&self.page_states, id) { + if page.record_use(slot) { return; } } @@ -66,8 +83,8 @@ impl EvictionPolicy for Sieve { .resident_ids() .chain(state.pending_evictions.iter().copied()) { - if let Some(page) = page_if_allocated(page_states, id) { - page.clear(slot_offset(id)); + if let Some((page, slot)) = page_and_slot_if_allocated(page_states, id) { + page.clear(slot); } } *state = State::default(); @@ -75,31 +92,29 @@ impl EvictionPolicy for Sieve { } fn start_new_revision(&mut self, context: &mut impl EvictionContext) { - if self.capacity.is_none() { + let Some(capacity) = self.capacity else { return; - } + }; let state = self.state.get_mut(); for id in state.pending_evictions.drain(..) { - if !page_if_allocated(&self.page_states, id) - .is_some_and(|page| page.is_resident(slot_offset(id))) - { + let (page, slot) = page_and_slot(&self.page_states, id); + if !page.is_resident(slot) { context.evict_value(id); } + page.clear_pending(slot); } + state + .pending_evictions + .shrink_to(capacity.get().saturating_mul(2)); } } impl HasCapacity for Sieve {} impl Sieve { - fn page_if_allocated(&self, id: Id) -> Option<&PageState> { - page_if_allocated(&self.page_states, id) - } - fn record_admission(&self, id: Id, capacity: usize) { - let page = page(&self.page_states, id); - let slot = slot_offset(id); + let (page, slot) = page_and_slot_or_alloc(&self.page_states, id); let mut state = self.state.lock(); if page.is_resident(slot) { @@ -107,7 +122,7 @@ impl Sieve { return; } - state.insert(id, capacity, &self.page_states); + state.insert(id, page, slot, capacity, &self.page_states); } } @@ -120,24 +135,42 @@ struct State { pending_evictions: Vec, } +struct SelectedVictim<'a> { + id: Id, + /// The already-resolved location avoids another page-table lookup when the + /// victim is added to the pending-eviction queue. + page: &'a PageState, + slot: usize, +} + impl State { - fn insert(&mut self, id: Id, capacity: usize, page_states: &PageStates) { + fn insert( + &mut self, + id: Id, + page: &PageState, + slot: usize, + capacity: usize, + page_states: &PageStates, + ) { + debug_assert!(self.residents.len() <= capacity); + if self.residents.len() == capacity { + assert!( + self.schedule_eviction(page_states), + "full resident list should have an eviction candidate" + ); + } + let node = self.residents.push_front(id); - page(page_states, id).admit(slot_offset(id)); + page.admit(slot); if self.hand == 0 { self.hand = node; } - - if self.residents.len() > capacity { - assert!( - self.schedule_eviction(page_states), - "resident list should have an eviction candidate after insertion" - ); - } } fn schedule_evictions(&mut self, capacity: usize, page_states: &PageStates) { + self.pending_evictions + .reserve(self.residents.len().saturating_sub(capacity)); while self.residents.len() > capacity { if !self.schedule_eviction(page_states) { return; @@ -146,30 +179,60 @@ impl State { } fn schedule_eviction(&mut self, page_states: &PageStates) -> bool { - if let Some(victim) = self.select_victim(page_states) { - self.pending_evictions.push(victim); - true - } else { - false + let Some(victim) = self.select_victim(page_states) else { + return false; + }; + + if victim.page.mark_pending(victim.slot) { + self.pending_evictions.push(victim.id); } + true } - fn select_victim(&mut self, page_states: &PageStates) -> Option { - loop { - if self.hand == 0 { - return None; - } + /// Selects and removes the next resident using a bounded SIEVE scan. + /// + /// The scan performs at most two inspections per current resident. The + /// second pass preserves second chances for concurrent re-marks; exhausting + /// both passes force-evicts at the hand to guarantee progress. + fn select_victim<'a>(&mut self, page_states: &'a PageStates) -> Option> { + if self.hand == 0 { + return None; + } + let inspection_budget = self.residents.len().saturating_mul(2); + for _ in 0..inspection_budget { let index = self.hand; let id = self.residents.id(index); + let (page, slot) = page_and_slot(page_states, id); - if page(page_states, id).select(slot_offset(id)).was_visited() { + if page.select(slot).was_visited() { self.hand = self.residents.advance_towards_front(index); } else { - self.hand = self.residents.hand_after_remove(index); - return Some(self.residents.remove(index)); + return Some(self.remove_victim(index, page, slot)); } } + + // Every inspected resident was re-marked. Evict at the hand to + // preserve SIEVE's scan order while guaranteeing progress. + let index = self.hand; + let id = self.residents.id(index); + let (page, slot) = page_and_slot(page_states, id); + page.clear_resident(slot); + Some(self.remove_victim(index, page, slot)) + } + + fn remove_victim<'a>( + &mut self, + index: ResidentIndex, + page: &'a PageState, + slot: usize, + ) -> SelectedVictim<'a> { + self.hand = self.residents.hand_after_remove(index); + SelectedVictim { + id: self.residents.remove(index), + page, + slot, + } } } @@ -177,7 +240,8 @@ type ResidentIndex = u32; struct Residents { nodes: Vec, - free: Vec, + /// Intrusive free list linked through `Resident::next`. + free_head: ResidentIndex, len: usize, } @@ -197,7 +261,7 @@ impl Default for Residents { prev: 0, next: 0, }], - free: Vec::new(), + free_head: 0, len: 0, } } @@ -230,14 +294,12 @@ impl Residents { self.node_mut(next).prev = prev; self.len -= 1; - let id = self - .node_mut(index) - .id - .take() - .expect("resident node should have an id"); - self.node_mut(index).prev = 0; - self.node_mut(index).next = 0; - self.free.push(index); + let free_head = self.free_head; + let node = self.node_mut(index); + let id = node.id.take().expect("resident node should have an id"); + node.prev = 0; + node.next = free_head; + self.free_head = index; id } @@ -273,7 +335,9 @@ impl Residents { } fn alloc_node(&mut self, id: Id) -> ResidentIndex { - if let Some(index) = self.free.pop() { + if self.free_head != 0 { + let index = self.free_head; + self.free_head = self.node(index).next; self.node_mut(index).id = Some(id); index } else { @@ -382,6 +446,12 @@ impl Drop for PageSlot { #[derive(Default)] struct PageState { words: [AtomicU64; STATE_WORDS], + /// Slots that already occur in `State::pending_evictions`. + /// + /// These words are only accessed while holding the state mutex or during + /// an exclusive revision reset. Atomics provide safe interior mutability; + /// the mutex provides synchronization. + pending: [AtomicU64; PENDING_WORDS], } impl PageState { @@ -457,19 +527,33 @@ impl PageState { self.words[word].load(Ordering::Relaxed) & resident != 0 } + fn mark_pending(&self, slot: usize) -> bool { + let (word, pending) = pending_state(slot); + let word = &self.pending[word]; + let state = word.load(Ordering::Relaxed); + if state & pending != 0 { + return false; + } + + word.store(state | pending, Ordering::Relaxed); + true + } + + fn clear_pending(&self, slot: usize) { + let (word, pending) = pending_state(slot); + let word = &self.pending[word]; + let state = word.load(Ordering::Relaxed); + word.store(state & !pending, Ordering::Relaxed); + } + fn clear(&self, slot: usize) { - let (word, resident, visited) = slot_state(slot); - let word = &self.words[word]; - let mut state = word.load(Ordering::Relaxed); + self.clear_resident(slot); + self.clear_pending(slot); + } - loop { - let new_state = state & !(resident | visited); - match word.compare_exchange_weak(state, new_state, Ordering::Relaxed, Ordering::Relaxed) - { - Ok(_) => return, - Err(new_state) => state = new_state, - } - } + fn clear_resident(&self, slot: usize) { + let (word, resident, visited) = slot_state(slot); + self.words[word].fetch_and(!(resident | visited), Ordering::Relaxed); } } @@ -485,26 +569,29 @@ impl Selection { } } -fn page(page_states: &PageStates, id: Id) -> &PageState { - page_states - .get_or_alloc(page_state_index(id)) - .get_or_alloc() +fn page_and_slot_or_alloc(page_states: &PageStates, id: Id) -> (&PageState, usize) { + let (index, slot) = page_state_index_and_slot(id); + (page_states.get_or_alloc(index).get_or_alloc(), slot) } -fn page_if_allocated(page_states: &PageStates, id: Id) -> Option<&PageState> { - page_states - .get(page_state_index(id)) - .and_then(PageSlot::get) +fn page_and_slot(page_states: &PageStates, id: Id) -> (&PageState, usize) { + // Resident and pending ids must already have allocated page state. + page_and_slot_if_allocated(page_states, id).expect("SIEVE page state should be allocated") } -fn page_state_index(id: Id) -> PageStateIndex { - let (page, _) = split_id(id); - PageStateIndex::new(page.as_usize()).expect("page index should fit in SIEVE page state table") +fn page_and_slot_if_allocated(page_states: &PageStates, id: Id) -> Option<(&PageState, usize)> { + let (index, slot) = page_state_index_and_slot(id); + page_states + .get(index) + .and_then(PageSlot::get) + .map(|page| (page, slot)) } -fn slot_offset(id: Id) -> usize { - let (_, slot) = split_id(id); - slot.as_usize() +fn page_state_index_and_slot(id: Id) -> (PageStateIndex, usize) { + let (page, slot) = split_id(id); + let index = PageStateIndex::new(page.as_usize()) + .expect("page index should fit in SIEVE page state table"); + (index, slot.as_usize()) } fn slot_state(slot: usize) -> (usize, u64, u64) { @@ -513,3 +600,105 @@ fn slot_state(slot: usize) -> (usize, u64, u64) { let visited = 1 << (bit + 1); (slot / SLOTS_PER_STATE_WORD, resident, visited) } + +fn pending_state(slot: usize) -> (usize, u64) { + let bit = slot % SLOTS_PER_PENDING_WORD; + (slot / SLOTS_PER_PENDING_WORD, 1 << bit) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Default)] + struct TestEvictionContext { + evicted: Vec, + } + + impl EvictionContext for TestEvictionContext { + fn last_verified_at(&mut self, _id: Id) -> Option { + None + } + + fn evict_value(&mut self, id: Id) { + self.evicted.push(id); + } + } + + fn id(index: u32) -> Id { + // SAFETY: Test indices are within `Id`'s valid range. + unsafe { Id::from_index(index) } + } + + #[test] + fn all_marked_residents_evict_the_initial_hand() { + let page_states = PageStates::new(); + let mut state = State::default(); + let oldest = id(0); + let middle = id(1); + let newest = id(2); + + for id in [oldest, middle, newest] { + let (page, slot) = page_and_slot_or_alloc(&page_states, id); + state.insert(id, page, slot, 3, &page_states); + assert!(page.record_use(slot)); + } + + assert_eq!( + state.select_victim(&page_states).map(|victim| victim.id), + Some(oldest) + ); + assert_eq!(state.residents.id(state.hand), middle); + assert_eq!( + state.residents.resident_ids().collect::>(), + [newest, middle] + ); + let (page, slot) = page_and_slot(&page_states, oldest); + assert!(!page.is_resident(slot)); + } + + #[test] + fn insertion_selects_a_victim_before_admitting() { + let page_states = PageStates::new(); + let mut state = State::default(); + let oldest = id(0); + let newest = id(1); + let incoming = id(2); + + for id in [oldest, newest] { + let (page, slot) = page_and_slot_or_alloc(&page_states, id); + state.insert(id, page, slot, 2, &page_states); + assert!(page.record_use(slot)); + } + + let (page, slot) = page_and_slot_or_alloc(&page_states, incoming); + state.insert(incoming, page, slot, 2, &page_states); + + assert_eq!(state.pending_evictions, [oldest]); + assert_eq!(state.residents.nodes.len(), 3); + assert!(page.is_resident(slot)); + } + + #[test] + fn pending_evictions_are_deduplicated_until_reset() { + let mut sieve = Sieve::new(1); + let first = id(0); + let second = id(1); + + for _ in 0..100 { + sieve.record_use(first); + sieve.record_use(second); + } + + assert_eq!(sieve.state.lock().pending_evictions, [first, second]); + + let mut context = TestEvictionContext::default(); + sieve.start_new_revision(&mut context); + + assert_eq!(context.evicted, [first]); + assert!(sieve.state.get_mut().pending_evictions.is_empty()); + + sieve.record_use(first); + assert_eq!(sieve.state.lock().pending_evictions, [second]); + } +} From c5d7e19e37ef6182103a3d68bcf8979540834725 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Sat, 4 Jul 2026 10:35:26 +0000 Subject: [PATCH 04/10] refactor: restore callback-based eviction interface --- src/function.rs | 37 ++++++++-------------------------- src/function/eviction.rs | 21 +++++-------------- src/function/eviction/lru.rs | 6 +++--- src/function/eviction/noop.rs | 4 ++-- src/function/eviction/sieve.rs | 27 ++++++------------------- src/function/memo.rs | 14 ------------- src/lib.rs | 4 +--- 7 files changed, 25 insertions(+), 88 deletions(-) diff --git a/src/function.rs b/src/function.rs index 64022195f..6d141f5d5 100644 --- a/src/function.rs +++ b/src/function.rs @@ -37,7 +37,7 @@ mod memo; mod specify; mod sync; -pub use eviction::{EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction, Sieve}; +pub use eviction::{EvictionPolicy, HasCapacity, Lru, NoopEviction, Sieve}; pub type Memo = memo::Memo<'static, C>; @@ -466,11 +466,13 @@ where } fn reset_for_new_revision(&mut self, table: &mut Table) { - let mut context = FunctionEvictionContext:: { - table, - memo_ingredient_indices: &self.memo_ingredient_indices, - }; - self.eviction.start_new_revision(&mut context); + self.eviction.for_each_evicted(|evict| { + let ingredient_index = table.ingredient_index(evict); + Self::evict_value_from_memo_for( + table.memos_mut(evict), + self.memo_ingredient_indices.get(ingredient_index), + ) + }); self.deleted_entries.clear(); } @@ -556,29 +558,6 @@ where } } -struct FunctionEvictionContext<'a, C: Configuration> { - table: &'a mut Table, - memo_ingredient_indices: &'a as SalsaStructInDb>::MemoIngredientMap, -} - -impl EvictionContext for FunctionEvictionContext<'_, C> { - fn last_verified_at(&mut self, id: Id) -> Option { - let ingredient_index = self.table.ingredient_index(id); - IngredientImpl::::last_verified_at_for( - self.table.memos_mut(id), - self.memo_ingredient_indices.get(ingredient_index), - ) - } - - fn evict_value(&mut self, id: Id) { - let ingredient_index = self.table.ingredient_index(id); - IngredientImpl::::evict_value_from_memo_for( - self.table.memos_mut(id), - self.memo_ingredient_indices.get(ingredient_index), - ) - } -} - impl memo::MemoHeader { fn collect_minimum_serialized_edges( &self, diff --git a/src/function/eviction.rs b/src/function/eviction.rs index 0726e2f32..360f6c698 100644 --- a/src/function/eviction.rs +++ b/src/function/eviction.rs @@ -11,7 +11,7 @@ pub use lru::Lru; pub use noop::NoopEviction; pub use sieve::Sieve; -use crate::{Id, Revision}; +use crate::Id; /// Trait for cache eviction strategies. /// @@ -33,22 +33,11 @@ pub trait EvictionPolicy: Send + Sync { /// A value of zero disables eviction. fn set_tuning(&mut self, tuning: usize); - /// Processes the start of a new revision. + /// Invokes `evict` for each value that should be evicted. /// - /// The policy may update its state, inspect memo metadata, and evict - /// resident values through `context`. - fn start_new_revision(&mut self, context: &mut impl EvictionContext); -} - -/// Memo operations available when starting a new revision. -pub trait EvictionContext { - /// Returns the revision in which `id`'s resident value was last verified. - /// - /// Returns `None` if the memo no longer contains a resident value. - fn last_verified_at(&mut self, id: Id) -> Option; - - /// Evicts `id`'s cached value while retaining its memo metadata. - fn evict_value(&mut self, id: Id); + /// Called once per revision. Implementations may also perform any + /// revision-boundary maintenance before returning. + fn for_each_evicted(&mut self, evict: impl FnMut(Id)); } /// Marker trait for eviction policies whose tuning can be changed at runtime. diff --git a/src/function/eviction/lru.rs b/src/function/eviction/lru.rs index 7225e5806..479e7da9d 100644 --- a/src/function/eviction/lru.rs +++ b/src/function/eviction/lru.rs @@ -9,7 +9,7 @@ use crate::Id; use crate::hash::FxLinkedHashSet; use crate::sync::Mutex; -use super::{EvictionContext, EvictionPolicy, HasCapacity}; +use super::{EvictionPolicy, HasCapacity}; /// Least Recently Used eviction policy. /// @@ -50,14 +50,14 @@ impl EvictionPolicy for Lru { } } - fn start_new_revision(&mut self, context: &mut impl EvictionContext) { + fn for_each_evicted(&mut self, mut evict: impl FnMut(Id)) { let Some(cap) = self.capacity else { return; }; let set = self.set.get_mut(); while set.len() > cap.get() { if let Some(id) = set.pop_front() { - context.evict_value(id); + evict(id); } } } diff --git a/src/function/eviction/noop.rs b/src/function/eviction/noop.rs index 7dc6d248d..033ffaa0d 100644 --- a/src/function/eviction/noop.rs +++ b/src/function/eviction/noop.rs @@ -2,7 +2,7 @@ //! //! This is the default eviction policy when no LRU capacity is specified. -use crate::function::{EvictionContext, EvictionPolicy}; +use crate::{Id, function::EvictionPolicy}; /// No eviction - cache grows unbounded. pub struct NoopEviction; @@ -16,5 +16,5 @@ impl EvictionPolicy for NoopEviction { fn set_tuning(&mut self, _capacity: usize) {} #[inline(always)] - fn start_new_revision(&mut self, _context: &mut impl EvictionContext) {} + fn for_each_evicted(&mut self, _evict: impl FnMut(Id)) {} } diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs index 2f6b5d3bc..c3d06fffa 100644 --- a/src/function/eviction/sieve.rs +++ b/src/function/eviction/sieve.rs @@ -28,7 +28,7 @@ use crate::sync::Mutex; use crate::sync::atomic::{AtomicPtr, AtomicU64, Ordering}; use crate::table::{PAGE_LEN, PAGE_LEN_BITS, split_id}; -use super::{EvictionContext, EvictionPolicy, HasCapacity}; +use super::{EvictionPolicy, HasCapacity}; use boxcar::buckets::{Buckets, Index, MaybeZeroable, buckets_for_index_bits}; const SLOT_STATE_BITS: usize = 2; @@ -91,7 +91,7 @@ impl EvictionPolicy for Sieve { } } - fn start_new_revision(&mut self, context: &mut impl EvictionContext) { + fn for_each_evicted(&mut self, mut evict: impl FnMut(Id)) { let Some(capacity) = self.capacity else { return; }; @@ -100,7 +100,7 @@ impl EvictionPolicy for Sieve { for id in state.pending_evictions.drain(..) { let (page, slot) = page_and_slot(&self.page_states, id); if !page.is_resident(slot) { - context.evict_value(id); + evict(id); } page.clear_pending(slot); } @@ -610,21 +610,6 @@ fn pending_state(slot: usize) -> (usize, u64) { mod tests { use super::*; - #[derive(Default)] - struct TestEvictionContext { - evicted: Vec, - } - - impl EvictionContext for TestEvictionContext { - fn last_verified_at(&mut self, _id: Id) -> Option { - None - } - - fn evict_value(&mut self, id: Id) { - self.evicted.push(id); - } - } - fn id(index: u32) -> Id { // SAFETY: Test indices are within `Id`'s valid range. unsafe { Id::from_index(index) } @@ -692,10 +677,10 @@ mod tests { assert_eq!(sieve.state.lock().pending_evictions, [first, second]); - let mut context = TestEvictionContext::default(); - sieve.start_new_revision(&mut context); + let mut evicted = Vec::new(); + sieve.for_each_evicted(|id| evicted.push(id)); - assert_eq!(context.evicted, [first]); + assert_eq!(evicted, [first]); assert!(sieve.state.get_mut().pending_evictions.is_empty()); sieve.record_use(first); diff --git a/src/function/memo.rs b/src/function/memo.rs index 40c643500..90e5fcf02 100644 --- a/src/function/memo.rs +++ b/src/function/memo.rs @@ -72,20 +72,6 @@ impl IngredientImpl { table.map_memo(memo_ingredient_index, map) } - - /// Returns when the resident value was last verified. - pub(super) fn last_verified_at_for( - table: MemoTableWithTypesMut<'_>, - memo_ingredient_index: MemoIngredientIndex, - ) -> Option { - let mut last_verified_at = None; - table.map_memo(memo_ingredient_index, |memo: &mut Memo<'static, C>| { - if memo.value.is_some() { - last_verified_at = Some(memo.header.verified_at.load()); - } - }); - last_verified_at - } } #[derive(Debug)] diff --git a/src/lib.rs b/src/lib.rs index 1cde4b24b..f0703c2be 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -401,9 +401,7 @@ pub mod plumbing { pub use crate::function::Configuration; pub use crate::function::IngredientImpl; pub use crate::function::Memo; - pub use crate::function::{ - EvictionContext, EvictionPolicy, HasCapacity, Lru, NoopEviction, Sieve, - }; + pub use crate::function::{EvictionPolicy, HasCapacity, Lru, NoopEviction, Sieve}; pub use crate::table::memo::MemoEntryType; } From 21bce209c7de5d5ac04c72b6b7097b4f0e5c4fa4 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Sat, 4 Jul 2026 15:44:54 +0000 Subject: [PATCH 05/10] perf: reduce SIEVE hit-path overhead --- src/function/eviction/sieve.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs index c3d06fffa..886d46f96 100644 --- a/src/function/eviction/sieve.rs +++ b/src/function/eviction/sieve.rs @@ -59,6 +59,7 @@ impl EvictionPolicy for Sieve { } } + #[inline] fn record_use(&self, id: Id) { if let Some(capacity) = self.capacity { if let Some((page, slot)) = page_and_slot_if_allocated(&self.page_states, id) { @@ -394,6 +395,7 @@ unsafe impl MaybeZeroable for PageSlot { } impl PageSlot { + #[inline] fn get(&self) -> Option<&PageState> { let ptr = self.ptr.load(Ordering::Acquire); ptr::NonNull::new(ptr).map(|ptr| { @@ -455,20 +457,21 @@ struct PageState { } impl PageState { + #[inline] fn record_use(&self, slot: usize) -> bool { let (word, resident, visited) = slot_state(slot); let word = &self.words[word]; let mut state = word.load(Ordering::Relaxed); loop { - if state & resident == 0 { - return false; - } - if state & visited != 0 { return true; } + if state & resident == 0 { + return false; + } + match word.compare_exchange_weak( state, state | visited, @@ -579,6 +582,7 @@ fn page_and_slot(page_states: &PageStates, id: Id) -> (&PageState, usize) { page_and_slot_if_allocated(page_states, id).expect("SIEVE page state should be allocated") } +#[inline] fn page_and_slot_if_allocated(page_states: &PageStates, id: Id) -> Option<(&PageState, usize)> { let (index, slot) = page_state_index_and_slot(id); page_states @@ -587,10 +591,12 @@ fn page_and_slot_if_allocated(page_states: &PageStates, id: Id) -> Option<(&Page .map(|page| (page, slot)) } +#[inline] fn page_state_index_and_slot(id: Id) -> (PageStateIndex, usize) { let (page, slot) = split_id(id); - let index = PageStateIndex::new(page.as_usize()) - .expect("page index should fit in SIEVE page state table"); + // SAFETY: `Id` is a `u32`, and `PageStates` has enough buckets for every + // page representable by an `Id` after removing the slot bits. + let index = unsafe { PageStateIndex::new_unchecked(page.as_usize()) }; (index, slot.as_usize()) } From 82d7230578376b1312550d3771cc823646acb820 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Sat, 4 Jul 2026 18:43:34 +0000 Subject: [PATCH 06/10] perf: reduce SIEVE state overhead --- src/function/eviction/sieve.rs | 289 ++++++++++++++++++--------------- src/table.rs | 4 - 2 files changed, 159 insertions(+), 134 deletions(-) diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs index 886d46f96..da2459627 100644 --- a/src/function/eviction/sieve.rs +++ b/src/function/eviction/sieve.rs @@ -13,101 +13,102 @@ //! resident, selection force-evicts the resident at the hand so admissions //! cannot starve without changing the scan order. //! -//! Hits update the page-indexed visited bits without taking the state mutex. +//! Hits update the block-indexed visited bits without taking the state mutex. //! Admissions, hand movement, and the pending-eviction queue are serialized by //! that mutex. Selecting a victim removes it from the resident list immediately, //! but its value is evicted at the next revision only if it was not re-admitted. -//! A separate page-indexed bitmap ensures each id appears in that queue at most +//! A separate block-indexed bitmap ensures each id appears in that queue at most //! once per revision. +//! +//! State is divided into independently allocated blocks. Within each block, +//! interleaved resident/visited bits cover 32 ids per atomic word, while the +//! pending bitmap covers 64 ids per word. This keeps SIEVE's allocation +//! granularity independent of the memo table's page size. -use std::num::NonZeroUsize; use std::ptr; use crate::Id; use crate::sync::Mutex; use crate::sync::atomic::{AtomicPtr, AtomicU64, Ordering}; -use crate::table::{PAGE_LEN, PAGE_LEN_BITS, split_id}; use super::{EvictionPolicy, HasCapacity}; use boxcar::buckets::{Buckets, Index, MaybeZeroable, buckets_for_index_bits}; const SLOT_STATE_BITS: usize = 2; const SLOTS_PER_STATE_WORD: usize = u64::BITS as usize / SLOT_STATE_BITS; -const STATE_WORDS: usize = PAGE_LEN.div_ceil(SLOTS_PER_STATE_WORD); const SLOTS_PER_PENDING_WORD: usize = u64::BITS as usize; -const PENDING_WORDS: usize = PAGE_LEN.div_ceil(SLOTS_PER_PENDING_WORD); +const BLOCK_LEN_BITS: usize = 10; +const BLOCK_LEN: usize = 1 << BLOCK_LEN_BITS; +const STATE_WORDS: usize = BLOCK_LEN / SLOTS_PER_STATE_WORD; +const PENDING_WORDS: usize = BLOCK_LEN / SLOTS_PER_PENDING_WORD; -type PageStates = Buckets; -type PageStateIndex = Index<{ buckets_for_index_bits(u32::BITS - PAGE_LEN_BITS as u32) }>; +type StateBlocks = + Buckets; +type StateBlockIndex = Index<{ buckets_for_index_bits(u32::BITS - BLOCK_LEN_BITS as u32) }>; /// SIEVE eviction policy. /// -/// Values enter at the front of a FIFO queue. Uses set a page-indexed atomic +/// Values enter at the front of a FIFO queue. Uses set a block-indexed atomic /// visited bit; they do not move the value in the queue. pub struct Sieve { - capacity: Option, + capacity: usize, state: Mutex, - page_states: PageStates, + state_blocks: StateBlocks, } impl EvictionPolicy for Sieve { fn new(capacity: usize) -> Self { Self { - capacity: NonZeroUsize::new(capacity), + capacity, state: Mutex::default(), - page_states: PageStates::new(), + state_blocks: StateBlocks::new(), } } #[inline] fn record_use(&self, id: Id) { - if let Some(capacity) = self.capacity { - if let Some((page, slot)) = page_and_slot_if_allocated(&self.page_states, id) { - if page.record_use(slot) { - return; - } + // Probe residency before capacity: resident hits do not need the capacity, + // while admissions already take the cold path below. + if let Some((block, slot)) = block_and_slot_if_allocated(&self.state_blocks, id) { + if block.record_use(slot) { + return; } + } - self.record_admission(id, capacity.get()); + if self.capacity != 0 { + self.record_admission(id, self.capacity); } } fn set_tuning(&mut self, capacity: usize) { - self.capacity = NonZeroUsize::new(capacity); - let page_states = &self.page_states; - let state = self.state.get_mut(); - if let Some(capacity) = self.capacity { - state.schedule_evictions(capacity.get(), page_states); + self.capacity = capacity; + if capacity == 0 { + *self.state.get_mut() = State::default(); + self.state_blocks = StateBlocks::new(); } else { - for id in state - .residents - .resident_ids() - .chain(state.pending_evictions.iter().copied()) - { - if let Some((page, slot)) = page_and_slot_if_allocated(page_states, id) { - page.clear(slot); - } - } - *state = State::default(); + self.state + .get_mut() + .schedule_evictions(capacity, &self.state_blocks); } } fn for_each_evicted(&mut self, mut evict: impl FnMut(Id)) { - let Some(capacity) = self.capacity else { + let capacity = self.capacity; + if capacity == 0 { return; - }; + } let state = self.state.get_mut(); for id in state.pending_evictions.drain(..) { - let (page, slot) = page_and_slot(&self.page_states, id); - if !page.is_resident(slot) { + let (block, slot) = block_and_slot(&self.state_blocks, id); + if !block.is_resident(slot) { evict(id); } - page.clear_pending(slot); + block.clear_pending(slot); } state .pending_evictions - .shrink_to(capacity.get().saturating_mul(2)); + .shrink_to(capacity.saturating_mul(2)); } } @@ -115,15 +116,15 @@ impl HasCapacity for Sieve {} impl Sieve { fn record_admission(&self, id: Id, capacity: usize) { - let (page, slot) = page_and_slot_or_alloc(&self.page_states, id); + let (block, slot) = block_and_slot_or_alloc(&self.state_blocks, id); let mut state = self.state.lock(); - if page.is_resident(slot) { - page.record_use(slot); + if block.is_resident(slot) { + block.record_use(slot); return; } - state.insert(id, page, slot, capacity, &self.page_states); + state.insert(id, block, slot, capacity, &self.state_blocks); } } @@ -138,9 +139,9 @@ struct State { struct SelectedVictim<'a> { id: Id, - /// The already-resolved location avoids another page-table lookup when the + /// The already-resolved location avoids another state-directory lookup when the /// victim is added to the pending-eviction queue. - page: &'a PageState, + block: &'a StateBlock, slot: usize, } @@ -148,43 +149,43 @@ impl State { fn insert( &mut self, id: Id, - page: &PageState, + block: &StateBlock, slot: usize, capacity: usize, - page_states: &PageStates, + state_blocks: &StateBlocks, ) { debug_assert!(self.residents.len() <= capacity); if self.residents.len() == capacity { assert!( - self.schedule_eviction(page_states), + self.schedule_eviction(state_blocks), "full resident list should have an eviction candidate" ); } let node = self.residents.push_front(id); - page.admit(slot); + block.admit(slot); if self.hand == 0 { self.hand = node; } } - fn schedule_evictions(&mut self, capacity: usize, page_states: &PageStates) { + fn schedule_evictions(&mut self, capacity: usize, state_blocks: &StateBlocks) { self.pending_evictions .reserve(self.residents.len().saturating_sub(capacity)); while self.residents.len() > capacity { - if !self.schedule_eviction(page_states) { + if !self.schedule_eviction(state_blocks) { return; } } } - fn schedule_eviction(&mut self, page_states: &PageStates) -> bool { - let Some(victim) = self.select_victim(page_states) else { + fn schedule_eviction(&mut self, state_blocks: &StateBlocks) -> bool { + let Some(victim) = self.select_victim(state_blocks) else { return false; }; - if victim.page.mark_pending(victim.slot) { + if victim.block.mark_pending(victim.slot) { self.pending_evictions.push(victim.id); } true @@ -195,7 +196,7 @@ impl State { /// The scan performs at most two inspections per current resident. The /// second pass preserves second chances for concurrent re-marks; exhausting /// both passes force-evicts at the hand to guarantee progress. - fn select_victim<'a>(&mut self, page_states: &'a PageStates) -> Option> { + fn select_victim<'a>(&mut self, state_blocks: &'a StateBlocks) -> Option> { if self.hand == 0 { return None; } @@ -204,12 +205,12 @@ impl State { for _ in 0..inspection_budget { let index = self.hand; let id = self.residents.id(index); - let (page, slot) = page_and_slot(page_states, id); + let (block, slot) = block_and_slot(state_blocks, id); - if page.select(slot).was_visited() { + if block.select(slot).was_visited() { self.hand = self.residents.advance_towards_front(index); } else { - return Some(self.remove_victim(index, page, slot)); + return Some(self.remove_victim(index, block, slot)); } } @@ -217,21 +218,21 @@ impl State { // preserve SIEVE's scan order while guaranteeing progress. let index = self.hand; let id = self.residents.id(index); - let (page, slot) = page_and_slot(page_states, id); - page.clear_resident(slot); - Some(self.remove_victim(index, page, slot)) + let (block, slot) = block_and_slot(state_blocks, id); + block.clear_resident(slot); + Some(self.remove_victim(index, block, slot)) } fn remove_victim<'a>( &mut self, index: ResidentIndex, - page: &'a PageState, + block: &'a StateBlock, slot: usize, ) -> SelectedVictim<'a> { self.hand = self.residents.hand_after_remove(index); SelectedVictim { id: self.residents.remove(index), - page, + block, slot, } } @@ -328,6 +329,7 @@ impl Residents { if prev == 0 { self.node(0).prev } else { prev } } + #[cfg(test)] fn resident_ids(&self) -> ResidentIds<'_> { ResidentIds { residents: self, @@ -362,11 +364,13 @@ impl Residents { } } +#[cfg(test)] struct ResidentIds<'a> { residents: &'a Residents, next: ResidentIndex, } +#[cfg(test)] impl Iterator for ResidentIds<'_> { type Item = Id; @@ -382,72 +386,73 @@ impl Iterator for ResidentIds<'_> { } #[derive(Default)] -struct PageSlot { - ptr: AtomicPtr, +struct BlockSlot { + ptr: AtomicPtr, } -// SAFETY: `PageSlot`'s all-zero representation is valid because a null -// `AtomicPtr` is valid. -unsafe impl MaybeZeroable for PageSlot { +// SAFETY: Outside Shuttle, `BlockSlot` contains an atomic pointer whose +// all-zero representation is a valid null pointer. Shuttle atomics require +// construction. +unsafe impl MaybeZeroable for BlockSlot { fn zeroable() -> bool { - true + cfg!(not(feature = "shuttle")) } } -impl PageSlot { +impl BlockSlot { #[inline] - fn get(&self) -> Option<&PageState> { + fn get(&self) -> Option<&StateBlock> { let ptr = self.ptr.load(Ordering::Acquire); ptr::NonNull::new(ptr).map(|ptr| { // SAFETY: A non-null pointer was allocated by `get_or_alloc` and - // remains owned by this `PageSlot` until it is dropped. + // remains owned by this slot until the directory is dropped. unsafe { ptr.as_ref() } }) } - fn get_or_alloc(&self) -> &PageState { - if let Some(page) = self.get() { - return page; + fn get_or_alloc(&self) -> &StateBlock { + if let Some(block) = self.get() { + return block; } - let new_page = Box::into_raw(Box::new(PageState::default())); + let new_block = Box::into_raw(Box::new(StateBlock::default())); match self.ptr.compare_exchange( ptr::null_mut(), - new_page, + new_block, Ordering::Release, Ordering::Acquire, ) { Ok(_) => { // SAFETY: We just installed this allocation. - unsafe { &*new_page } + unsafe { &*new_block } } Err(existing) => { // SAFETY: This allocation was not published, so this thread // still owns it. - unsafe { drop(Box::from_raw(new_page)) }; + unsafe { drop(Box::from_raw(new_block)) }; - // SAFETY: `existing` came from a successful install by another - // thread and remains owned by this `PageSlot`. + // SAFETY: `existing` was installed by another thread and + // remains owned by this slot. unsafe { &*existing } } } } } -impl Drop for PageSlot { +impl Drop for BlockSlot { fn drop(&mut self) { let ptr = *self.ptr.get_mut(); if !ptr.is_null() { - // SAFETY: Dropping the `PageSlot` requires exclusive access, so no - // more readers can access the pointed-to page. + // SAFETY: Dropping the slot requires exclusive access, so no reader + // can still access the allocation. unsafe { drop(Box::from_raw(ptr)) }; } } } -#[derive(Default)] -struct PageState { - words: [AtomicU64; STATE_WORDS], +struct StateBlock { + /// Interleaved resident and visited bits for all slots in the block. + state: [AtomicU64; STATE_WORDS], /// Slots that already occur in `State::pending_evictions`. /// /// These words are only accessed while holding the state mutex or during @@ -456,11 +461,20 @@ struct PageState { pending: [AtomicU64; PENDING_WORDS], } -impl PageState { +impl Default for StateBlock { + fn default() -> Self { + Self { + state: std::array::from_fn(|_| AtomicU64::default()), + pending: std::array::from_fn(|_| AtomicU64::default()), + } + } +} + +impl StateBlock { #[inline] fn record_use(&self, slot: usize) -> bool { let (word, resident, visited) = slot_state(slot); - let word = &self.words[word]; + let word = &self.state[word]; let mut state = word.load(Ordering::Relaxed); loop { @@ -486,7 +500,7 @@ impl PageState { fn admit(&self, slot: usize) { let (word, resident, visited) = slot_state(slot); - let word = &self.words[word]; + let word = &self.state[word]; let mut state = word.load(Ordering::Relaxed); loop { @@ -501,7 +515,7 @@ impl PageState { fn select(&self, slot: usize) -> Selection { let (word, resident, visited) = slot_state(slot); - let word = &self.words[word]; + let word = &self.state[word]; let mut state = word.load(Ordering::Relaxed); loop { @@ -527,7 +541,7 @@ impl PageState { fn is_resident(&self, slot: usize) -> bool { let (word, resident, _) = slot_state(slot); - self.words[word].load(Ordering::Relaxed) & resident != 0 + self.state[word].load(Ordering::Relaxed) & resident != 0 } fn mark_pending(&self, slot: usize) -> bool { @@ -549,14 +563,9 @@ impl PageState { word.store(state & !pending, Ordering::Relaxed); } - fn clear(&self, slot: usize) { - self.clear_resident(slot); - self.clear_pending(slot); - } - fn clear_resident(&self, slot: usize) { let (word, resident, visited) = slot_state(slot); - self.words[word].fetch_and(!(resident | visited), Ordering::Relaxed); + self.state[word].fetch_and(!(resident | visited), Ordering::Relaxed); } } @@ -572,35 +581,38 @@ impl Selection { } } -fn page_and_slot_or_alloc(page_states: &PageStates, id: Id) -> (&PageState, usize) { - let (index, slot) = page_state_index_and_slot(id); - (page_states.get_or_alloc(index).get_or_alloc(), slot) +fn block_and_slot_or_alloc(state_blocks: &StateBlocks, id: Id) -> (&StateBlock, usize) { + let (index, slot) = state_block_index_and_slot(id); + (state_blocks.get_or_alloc(index).get_or_alloc(), slot) } -fn page_and_slot(page_states: &PageStates, id: Id) -> (&PageState, usize) { - // Resident and pending ids must already have allocated page state. - page_and_slot_if_allocated(page_states, id).expect("SIEVE page state should be allocated") +fn block_and_slot(state_blocks: &StateBlocks, id: Id) -> (&StateBlock, usize) { + // Resident and pending ids must already have allocated state blocks. + block_and_slot_if_allocated(state_blocks, id).expect("SIEVE state block should be allocated") } #[inline] -fn page_and_slot_if_allocated(page_states: &PageStates, id: Id) -> Option<(&PageState, usize)> { - let (index, slot) = page_state_index_and_slot(id); - page_states +fn block_and_slot_if_allocated(state_blocks: &StateBlocks, id: Id) -> Option<(&StateBlock, usize)> { + let (index, slot) = state_block_index_and_slot(id); + state_blocks .get(index) - .and_then(PageSlot::get) - .map(|page| (page, slot)) + .and_then(BlockSlot::get) + .map(|block| (block, slot)) } #[inline] -fn page_state_index_and_slot(id: Id) -> (PageStateIndex, usize) { - let (page, slot) = split_id(id); - // SAFETY: `Id` is a `u32`, and `PageStates` has enough buckets for every - // page representable by an `Id` after removing the slot bits. - let index = unsafe { PageStateIndex::new_unchecked(page.as_usize()) }; - (index, slot.as_usize()) +fn state_block_index_and_slot(id: Id) -> (StateBlockIndex, usize) { + let index = id.index() as usize; + let block = index >> BLOCK_LEN_BITS; + let slot = index & (BLOCK_LEN - 1); + // SAFETY: `Id` is a `u32`, and `StateBlocks` has enough buckets for every + // block representable by an `Id` after removing the slot bits. + let index = unsafe { StateBlockIndex::new_unchecked(block) }; + (index, slot) } fn slot_state(slot: usize) -> (usize, u64, u64) { + debug_assert!(slot < BLOCK_LEN); let bit = (slot % SLOTS_PER_STATE_WORD) * SLOT_STATE_BITS; let resident = 1 << bit; let visited = 1 << (bit + 1); @@ -608,6 +620,7 @@ fn slot_state(slot: usize) -> (usize, u64, u64) { } fn pending_state(slot: usize) -> (usize, u64) { + debug_assert!(slot < BLOCK_LEN); let bit = slot % SLOTS_PER_PENDING_WORD; (slot / SLOTS_PER_PENDING_WORD, 1 << bit) } @@ -623,20 +636,20 @@ mod tests { #[test] fn all_marked_residents_evict_the_initial_hand() { - let page_states = PageStates::new(); + let state_blocks = StateBlocks::new(); let mut state = State::default(); let oldest = id(0); let middle = id(1); let newest = id(2); for id in [oldest, middle, newest] { - let (page, slot) = page_and_slot_or_alloc(&page_states, id); - state.insert(id, page, slot, 3, &page_states); - assert!(page.record_use(slot)); + let (block, slot) = block_and_slot_or_alloc(&state_blocks, id); + state.insert(id, block, slot, 3, &state_blocks); + assert!(block.record_use(slot)); } assert_eq!( - state.select_victim(&page_states).map(|victim| victim.id), + state.select_victim(&state_blocks).map(|victim| victim.id), Some(oldest) ); assert_eq!(state.residents.id(state.hand), middle); @@ -644,30 +657,46 @@ mod tests { state.residents.resident_ids().collect::>(), [newest, middle] ); - let (page, slot) = page_and_slot(&page_states, oldest); - assert!(!page.is_resident(slot)); + let (block, slot) = block_and_slot(&state_blocks, oldest); + assert!(!block.is_resident(slot)); } #[test] fn insertion_selects_a_victim_before_admitting() { - let page_states = PageStates::new(); + let state_blocks = StateBlocks::new(); let mut state = State::default(); let oldest = id(0); let newest = id(1); let incoming = id(2); for id in [oldest, newest] { - let (page, slot) = page_and_slot_or_alloc(&page_states, id); - state.insert(id, page, slot, 2, &page_states); - assert!(page.record_use(slot)); + let (block, slot) = block_and_slot_or_alloc(&state_blocks, id); + state.insert(id, block, slot, 2, &state_blocks); + assert!(block.record_use(slot)); } - let (page, slot) = page_and_slot_or_alloc(&page_states, incoming); - state.insert(incoming, page, slot, 2, &page_states); + let (block, slot) = block_and_slot_or_alloc(&state_blocks, incoming); + state.insert(incoming, block, slot, 2, &state_blocks); assert_eq!(state.pending_evictions, [oldest]); assert_eq!(state.residents.nodes.len(), 3); - assert!(page.is_resident(slot)); + assert!(block.is_resident(slot)); + } + + #[test] + fn disabling_discards_state_blocks() { + let mut sieve = Sieve::new(1); + let resident = id(0); + + sieve.record_use(resident); + assert!(block_and_slot_if_allocated(&sieve.state_blocks, resident).is_some()); + + sieve.set_tuning(0); + assert!(block_and_slot_if_allocated(&sieve.state_blocks, resident).is_none()); + assert_eq!(sieve.state.get_mut().residents.len(), 0); + + sieve.record_use(resident); + assert!(block_and_slot_if_allocated(&sieve.state_blocks, resident).is_none()); } #[test] diff --git a/src/table.rs b/src/table.rs index 59df96df0..8321772dd 100644 --- a/src/table.rs +++ b/src/table.rs @@ -152,10 +152,6 @@ impl SlotIndex { debug_assert!(idx < PAGE_LEN); Self(idx) } - - pub(crate) fn as_usize(&self) -> usize { - self.0 - } } impl Default for Table { From 2fec194582c222a88d23ac9ed3eb29b2d49b4a51 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Sat, 4 Jul 2026 21:07:53 +0000 Subject: [PATCH 07/10] perf: batch SIEVE evictions by state block --- src/function.rs | 4 +- src/function/eviction/sieve.rs | 198 ++++++++++++++++++++++++--------- src/table.rs | 12 +- 3 files changed, 157 insertions(+), 57 deletions(-) diff --git a/src/function.rs b/src/function.rs index 6d141f5d5..b096a2157 100644 --- a/src/function.rs +++ b/src/function.rs @@ -467,9 +467,9 @@ where fn reset_for_new_revision(&mut self, table: &mut Table) { self.eviction.for_each_evicted(|evict| { - let ingredient_index = table.ingredient_index(evict); + let (ingredient_index, memos) = table.ingredient_and_memos_mut(evict); Self::evict_value_from_memo_for( - table.memos_mut(evict), + memos, self.memo_ingredient_indices.get(ingredient_index), ) }); diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs index da2459627..4be75d0fe 100644 --- a/src/function/eviction/sieve.rs +++ b/src/function/eviction/sieve.rs @@ -14,22 +14,22 @@ //! cannot starve without changing the scan order. //! //! Hits update the block-indexed visited bits without taking the state mutex. -//! Admissions, hand movement, and the pending-eviction queue are serialized by -//! that mutex. Selecting a victim removes it from the resident list immediately, -//! but its value is evicted at the next revision only if it was not re-admitted. -//! A separate block-indexed bitmap ensures each id appears in that queue at most -//! once per revision. +//! Admissions, hand movement, and the pending-block queue are serialized by that +//! mutex. Selecting a victim removes it from the resident list immediately, but +//! its value is evicted at the next revision only if it was not re-admitted. Each +//! queued block owns a transient bitmap identifying its pending ids. //! //! State is divided into independently allocated blocks. Within each block, -//! interleaved resident/visited bits cover 32 ids per atomic word, while the -//! pending bitmap covers 64 ids per word. This keeps SIEVE's allocation -//! granularity independent of the memo table's page size. +//! interleaved resident/visited bits cover 32 ids per atomic word. Pending +//! bitmaps cover 64 ids per word and exist only while their block is queued. +//! This keeps SIEVE's allocation granularity independent of the memo table's +//! page size without reserving pending state for every id. use std::ptr; use crate::Id; use crate::sync::Mutex; -use crate::sync::atomic::{AtomicPtr, AtomicU64, Ordering}; +use crate::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; use super::{EvictionPolicy, HasCapacity}; use boxcar::buckets::{Buckets, Index, MaybeZeroable, buckets_for_index_bits}; @@ -99,16 +99,18 @@ impl EvictionPolicy for Sieve { } let state = self.state.get_mut(); - for id in state.pending_evictions.drain(..) { - let (block, slot) = block_and_slot(&self.state_blocks, id); - if !block.is_resident(slot) { - evict(id); - } - block.clear_pending(slot); + for (queue_index, pending_block) in state.pending_blocks.drain(..).enumerate() { + let block = state_block(&self.state_blocks, pending_block.state_block); + block.clear_pending_block_index(queue_index); + pending_block.for_each_pending_slot(|slot| { + if !block.is_resident(slot) { + evict(state_block_id(pending_block.state_block, slot)); + } + }); } state - .pending_evictions - .shrink_to(capacity.saturating_mul(2)); + .pending_blocks + .shrink_to(capacity.div_ceil(BLOCK_LEN).saturating_mul(2)); } } @@ -133,8 +135,13 @@ struct State { residents: Residents, /// Index of the next resident candidate to inspect. `0` is the sentinel. hand: ResidentIndex, - /// Victims selected by SIEVE admission, waiting for an eviction context. - pending_evictions: Vec, + /// State blocks containing victims waiting for an eviction context. + pending_blocks: Vec, +} + +struct PendingBlock { + state_block: StateBlockIndex, + pending: [u64; PENDING_WORDS], } struct SelectedVictim<'a> { @@ -171,8 +178,6 @@ impl State { } fn schedule_evictions(&mut self, capacity: usize, state_blocks: &StateBlocks) { - self.pending_evictions - .reserve(self.residents.len().saturating_sub(capacity)); while self.residents.len() > capacity { if !self.schedule_eviction(state_blocks) { return; @@ -185,12 +190,27 @@ impl State { return false; }; - if victim.block.mark_pending(victim.slot) { - self.pending_evictions.push(victim.id); - } + self.mark_pending(victim); true } + fn mark_pending(&mut self, victim: SelectedVictim<'_>) { + let queue_index = victim.block.pending_block_index().unwrap_or_else(|| { + let state_block = state_block_index_and_slot(victim.id).0; + let queue_index = self.pending_blocks.len(); + self.pending_blocks.push(PendingBlock::new(state_block)); + victim.block.set_pending_block_index(queue_index); + queue_index + }); + + let pending_block = &mut self.pending_blocks[queue_index]; + debug_assert_eq!( + pending_block.state_block, + state_block_index_and_slot(victim.id).0 + ); + pending_block.mark(victim.slot); + } + /// Selects and removes the next resident using a bounded SIEVE scan. /// /// The scan performs at most two inspections per current resident. The @@ -238,6 +258,39 @@ impl State { } } +impl PendingBlock { + #[inline] + fn new(state_block: StateBlockIndex) -> Self { + Self { + state_block, + pending: [0; PENDING_WORDS], + } + } + + #[inline] + fn mark(&mut self, slot: usize) { + let (word, pending) = pending_state(slot); + self.pending[word] |= pending; + } + + fn for_each_pending_slot(&self, mut f: impl FnMut(usize)) { + for (word_index, &pending) in self.pending.iter().enumerate() { + let mut pending = pending; + while pending != 0 { + let bit = pending.trailing_zeros() as usize; + f(word_index * SLOTS_PER_PENDING_WORD + bit); + pending &= pending - 1; + } + } + } + + #[cfg(test)] + fn contains(&self, slot: usize) -> bool { + let (word, pending) = pending_state(slot); + self.pending[word] & pending != 0 + } +} + type ResidentIndex = u32; struct Residents { @@ -453,19 +506,17 @@ impl Drop for BlockSlot { struct StateBlock { /// Interleaved resident and visited bits for all slots in the block. state: [AtomicU64; STATE_WORDS], - /// Slots that already occur in `State::pending_evictions`. - /// - /// These words are only accessed while holding the state mutex or during - /// an exclusive revision reset. Atomics provide safe interior mutability; - /// the mutex provides synchronization. - pending: [AtomicU64; PENDING_WORDS], + /// One-based index into `State::pending_blocks`, or zero when not queued. + /// Access is serialized by the state mutex; the atomic preserves interior + /// mutability without making blocks unavailable to concurrent hit readers. + pending_block_index: AtomicU32, } impl Default for StateBlock { fn default() -> Self { Self { state: std::array::from_fn(|_| AtomicU64::default()), - pending: std::array::from_fn(|_| AtomicU64::default()), + pending_block_index: AtomicU32::default(), } } } @@ -544,23 +595,24 @@ impl StateBlock { self.state[word].load(Ordering::Relaxed) & resident != 0 } - fn mark_pending(&self, slot: usize) -> bool { - let (word, pending) = pending_state(slot); - let word = &self.pending[word]; - let state = word.load(Ordering::Relaxed); - if state & pending != 0 { - return false; + #[inline] + fn pending_block_index(&self) -> Option { + match self.pending_block_index.load(Ordering::Relaxed) { + 0 => None, + index => Some((index - 1) as usize), } + } - word.store(state | pending, Ordering::Relaxed); - true + #[inline] + fn set_pending_block_index(&self, index: usize) { + debug_assert!(self.pending_block_index().is_none()); + let index = u32::try_from(index + 1).expect("pending block index should fit in u32"); + self.pending_block_index.store(index, Ordering::Relaxed); } - fn clear_pending(&self, slot: usize) { - let (word, pending) = pending_state(slot); - let word = &self.pending[word]; - let state = word.load(Ordering::Relaxed); - word.store(state & !pending, Ordering::Relaxed); + fn clear_pending_block_index(&self, index: usize) { + debug_assert_eq!(self.pending_block_index(), Some(index)); + self.pending_block_index.store(0, Ordering::Relaxed); } fn clear_resident(&self, slot: usize) { @@ -588,7 +640,16 @@ fn block_and_slot_or_alloc(state_blocks: &StateBlocks, id: Id) -> (&StateBlock, fn block_and_slot(state_blocks: &StateBlocks, id: Id) -> (&StateBlock, usize) { // Resident and pending ids must already have allocated state blocks. - block_and_slot_if_allocated(state_blocks, id).expect("SIEVE state block should be allocated") + let (index, slot) = state_block_index_and_slot(id); + (state_block(state_blocks, index), slot) +} + +#[inline] +fn state_block(state_blocks: &StateBlocks, index: StateBlockIndex) -> &StateBlock { + state_blocks + .get(index) + .and_then(BlockSlot::get) + .expect("SIEVE state block should be allocated") } #[inline] @@ -611,6 +672,14 @@ fn state_block_index_and_slot(id: Id) -> (StateBlockIndex, usize) { (index, slot) } +fn state_block_id(index: StateBlockIndex, slot: usize) -> Id { + debug_assert!(slot < BLOCK_LEN); + let id = (index.get() << BLOCK_LEN_BITS) | slot; + // SAFETY: `StateBlockIndex` covers exactly the block bits of a `u32` id and + // `slot` is restricted to the remaining low bits. + unsafe { Id::from_index(id as u32) } +} + fn slot_state(slot: usize) -> (usize, u64, u64) { debug_assert!(slot < BLOCK_LEN); let bit = (slot % SLOTS_PER_STATE_WORD) * SLOT_STATE_BITS; @@ -675,12 +744,15 @@ mod tests { assert!(block.record_use(slot)); } - let (block, slot) = block_and_slot_or_alloc(&state_blocks, incoming); - state.insert(incoming, block, slot, 2, &state_blocks); + let (block, incoming_slot) = block_and_slot_or_alloc(&state_blocks, incoming); + state.insert(incoming, block, incoming_slot, 2, &state_blocks); - assert_eq!(state.pending_evictions, [oldest]); + let (state_block, oldest_slot) = state_block_index_and_slot(oldest); + assert_eq!(state.pending_blocks.len(), 1); + assert_eq!(state.pending_blocks[0].state_block, state_block); + assert!(state.pending_blocks[0].contains(oldest_slot)); assert_eq!(state.residents.nodes.len(), 3); - assert!(block.is_resident(slot)); + assert!(block.is_resident(incoming_slot)); } #[test] @@ -710,15 +782,37 @@ mod tests { sieve.record_use(second); } - assert_eq!(sieve.state.lock().pending_evictions, [first, second]); + assert_eq!(sieve.state.lock().pending_blocks.len(), 1); + let (block, _) = block_and_slot(&sieve.state_blocks, first); + assert_eq!(block.pending_block_index(), Some(0)); let mut evicted = Vec::new(); sieve.for_each_evicted(|id| evicted.push(id)); assert_eq!(evicted, [first]); - assert!(sieve.state.get_mut().pending_evictions.is_empty()); + assert!(sieve.state.get_mut().pending_blocks.is_empty()); + let (block, _) = block_and_slot(&sieve.state_blocks, first); + assert_eq!(block.pending_block_index(), None); sieve.record_use(first); - assert_eq!(sieve.state.lock().pending_evictions, [second]); + assert_eq!(sieve.state.lock().pending_blocks.len(), 1); + } + + #[test] + fn pending_evictions_cross_state_blocks() { + let mut sieve = Sieve::new(1); + let first = id(0); + let second = id(BLOCK_LEN as u32); + let third = id(1); + let resident = id(BLOCK_LEN as u32 + 1); + + for id in [first, second, third, resident] { + sieve.record_use(id); + } + + let mut evicted = Vec::new(); + sieve.for_each_evicted(|id| evicted.push(id)); + + assert_eq!(evicted, [first, third, second]); } } diff --git a/src/table.rs b/src/table.rs index 8321772dd..c74a3fc9e 100644 --- a/src/table.rs +++ b/src/table.rs @@ -351,18 +351,24 @@ impl Table { unsafe { page.memo_types.attach_memos(memos) } } - /// Get the memo table associated with `id` - pub(crate) fn memos_mut(&mut self, id: Id) -> MemoTableWithTypesMut<'_> { + /// Get the ingredient index and memo table associated with `id`. + pub(crate) fn ingredient_and_memos_mut( + &mut self, + id: Id, + ) -> (IngredientIndex, MemoTableWithTypesMut<'_>) { let (page, slot) = split_id(id); let page_index = page.0; let page = self .pages .get_mut(page_index) .unwrap_or_else(|| panic!("index `{page_index}` is uninitialized")); + let ingredient = page.ingredient; // SAFETY: We supply a proper slot pointer and the caller is required to pass the `current_revision`. let memos = unsafe { &mut *(page.slot_vtable.memos_mut)(page.get(slot)) }; // SAFETY: The `Page` keeps the correct memo types. - unsafe { page.memo_types.attach_memos_mut(memos) } + (ingredient, unsafe { + page.memo_types.attach_memos_mut(memos) + }) } pub(crate) fn slots_of(&self) -> impl Iterator + '_ { From 394124f76a5f5a6b2d37cc70fc31aedc7ad2de9f Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Sat, 4 Jul 2026 22:08:40 +0000 Subject: [PATCH 08/10] refactor: simplify SIEVE implementation --- components/salsa-macros/src/accumulator.rs | 1 - components/salsa-macros/src/input.rs | 2 -- components/salsa-macros/src/interned.rs | 2 -- components/salsa-macros/src/options.rs | 2 +- components/salsa-macros/src/tracked_fn.rs | 7 +++-- components/salsa-macros/src/tracked_struct.rs | 2 -- src/function.rs | 5 ++-- src/function/eviction/sieve.rs | 28 +++++-------------- src/table.rs | 4 +-- 9 files changed, 17 insertions(+), 36 deletions(-) diff --git a/components/salsa-macros/src/accumulator.rs b/components/salsa-macros/src/accumulator.rs index 58a722988..abc694809 100644 --- a/components/salsa-macros/src/accumulator.rs +++ b/components/salsa-macros/src/accumulator.rs @@ -42,7 +42,6 @@ impl AllowedOptions for Accumulator { const CYCLE_INITIAL: bool = false; const CYCLE_RESULT: bool = false; const LRU: bool = false; - const SIEVE: bool = false; const CONSTRUCTOR_NAME: bool = false; const ID: bool = false; const REVISIONS: bool = false; diff --git a/components/salsa-macros/src/input.rs b/components/salsa-macros/src/input.rs index 545352eea..6449f0316 100644 --- a/components/salsa-macros/src/input.rs +++ b/components/salsa-macros/src/input.rs @@ -59,8 +59,6 @@ impl AllowedOptions for InputStruct { const LRU: bool = false; - const SIEVE: bool = false; - const CONSTRUCTOR_NAME: bool = true; const ID: bool = false; diff --git a/components/salsa-macros/src/interned.rs b/components/salsa-macros/src/interned.rs index 24e742ad6..30c5f2d73 100644 --- a/components/salsa-macros/src/interned.rs +++ b/components/salsa-macros/src/interned.rs @@ -59,8 +59,6 @@ impl AllowedOptions for InternedStruct { const LRU: bool = false; - const SIEVE: bool = false; - const CONSTRUCTOR_NAME: bool = true; const ID: bool = true; diff --git a/components/salsa-macros/src/options.rs b/components/salsa-macros/src/options.rs index 5804504a1..179a93e5e 100644 --- a/components/salsa-macros/src/options.rs +++ b/components/salsa-macros/src/options.rs @@ -184,7 +184,7 @@ pub(crate) trait AllowedOptions { const CYCLE_INITIAL: bool; const CYCLE_RESULT: bool; const LRU: bool; - const SIEVE: bool; + const SIEVE: bool = false; const CONSTRUCTOR_NAME: bool; const ID: bool; const REVISIONS: bool; diff --git a/components/salsa-macros/src/tracked_fn.rs b/components/salsa-macros/src/tracked_fn.rs index 49c87a365..f037e22cc 100644 --- a/components/salsa-macros/src/tracked_fn.rs +++ b/components/salsa-macros/src/tracked_fn.rs @@ -182,9 +182,10 @@ impl Macro { FunctionType::SalsaStruct => false, }; - let lru = Literal::usize_unsuffixed(self.args.lru.or(self.args.sieve).unwrap_or(0)); + let eviction_tuning = + Literal::usize_unsuffixed(self.args.lru.or(self.args.sieve).unwrap_or(0)); - // Determine the eviction policy type based on whether LRU capacity is specified + // Determine the eviction policy type from the configured option. let eviction_type = if self.args.lru.is_some() { quote!(::salsa::plumbing::function::Lru) } else if self.args.sieve.is_some() { @@ -248,7 +249,7 @@ impl Macro { needs_interner: #needs_interner, heap_size_fn: #(#heap_size_fn)*, eviction: #eviction_type, - lru: #lru, + lru: #eviction_tuning, return_mode: #return_mode, persist: #persist, assert_types_are_update: { #assert_types_are_update }, diff --git a/components/salsa-macros/src/tracked_struct.rs b/components/salsa-macros/src/tracked_struct.rs index 79d6c97c4..fb645cb5e 100644 --- a/components/salsa-macros/src/tracked_struct.rs +++ b/components/salsa-macros/src/tracked_struct.rs @@ -55,8 +55,6 @@ impl AllowedOptions for TrackedStruct { const LRU: bool = false; - const SIEVE: bool = false; - const CONSTRUCTOR_NAME: bool = true; const ID: bool = false; diff --git a/src/function.rs b/src/function.rs index b096a2157..024e40d79 100644 --- a/src/function.rs +++ b/src/function.rs @@ -280,8 +280,9 @@ where // FIXME: Use `Box::into_non_null` once stable let memo = NonNull::from(Box::leak(Box::new(memo))); - let old_value = self.insert_memo_into_table_for(zalsa, id, memo, memo_ingredient_index); - if let Some(old_value) = old_value { + if let Some(old_value) = + self.insert_memo_into_table_for(zalsa, id, memo, memo_ingredient_index) + { // In case there is a reference to the old memo out there, we have to store it // in the deleted entries. This will get cleared when a new revision starts. // diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs index 4be75d0fe..2a19f4426 100644 --- a/src/function/eviction/sieve.rs +++ b/src/function/eviction/sieve.rs @@ -163,10 +163,7 @@ impl State { ) { debug_assert!(self.residents.len() <= capacity); if self.residents.len() == capacity { - assert!( - self.schedule_eviction(state_blocks), - "full resident list should have an eviction candidate" - ); + self.schedule_eviction(state_blocks); } let node = self.residents.push_front(id); @@ -179,19 +176,16 @@ impl State { fn schedule_evictions(&mut self, capacity: usize, state_blocks: &StateBlocks) { while self.residents.len() > capacity { - if !self.schedule_eviction(state_blocks) { - return; - } + self.schedule_eviction(state_blocks); } } - fn schedule_eviction(&mut self, state_blocks: &StateBlocks) -> bool { - let Some(victim) = self.select_victim(state_blocks) else { - return false; - }; + fn schedule_eviction(&mut self, state_blocks: &StateBlocks) { + let victim = self + .select_victim(state_blocks) + .expect("non-empty resident list should have an eviction candidate"); self.mark_pending(victim); - true } fn mark_pending(&mut self, victim: SelectedVictim<'_>) { @@ -503,6 +497,7 @@ impl Drop for BlockSlot { } } +#[derive(Default)] struct StateBlock { /// Interleaved resident and visited bits for all slots in the block. state: [AtomicU64; STATE_WORDS], @@ -512,15 +507,6 @@ struct StateBlock { pending_block_index: AtomicU32, } -impl Default for StateBlock { - fn default() -> Self { - Self { - state: std::array::from_fn(|_| AtomicU64::default()), - pending_block_index: AtomicU32::default(), - } - } -} - impl StateBlock { #[inline] fn record_use(&self, slot: usize) -> bool { diff --git a/src/table.rs b/src/table.rs index c74a3fc9e..430dde475 100644 --- a/src/table.rs +++ b/src/table.rs @@ -16,9 +16,9 @@ use crate::{Id, IngredientIndex, Revision}; pub(crate) mod memo; -pub(crate) const PAGE_LEN_BITS: usize = 7; +const PAGE_LEN_BITS: usize = 7; const PAGE_LEN_MASK: usize = PAGE_LEN - 1; -pub(crate) const PAGE_LEN: usize = 1 << PAGE_LEN_BITS; +const PAGE_LEN: usize = 1 << PAGE_LEN_BITS; const MAX_PAGES: usize = Id::MAX_USIZE / PAGE_LEN; /// A typed [`Page`] view. From 96b31c96ed4067ad941ca13c4ce2b483626de988 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Sun, 5 Jul 2026 13:17:51 +0000 Subject: [PATCH 09/10] feat: add SIEVE eviction policy --- benches/eviction.rs | 4 +- benches/eviction/support.rs | 8 +- book/src/SUMMARY.md | 2 +- book/src/plumbing/maybe_changed_after.md | 1 - book/src/plumbing/terminology/LRU.md | 6 - book/src/plumbing/terminology/eviction.md | 31 + book/src/plumbing/terminology/memo.md | 4 +- book/src/tuning.md | 51 +- .../salsa-macro-rules/src/setup_tracked_fn.rs | 12 +- components/salsa-macros/src/accumulator.rs | 2 +- components/salsa-macros/src/input.rs | 2 +- components/salsa-macros/src/interned.rs | 2 +- components/salsa-macros/src/lib.rs | 9 +- components/salsa-macros/src/options.rs | 147 ++- components/salsa-macros/src/tracked_fn.rs | 46 +- components/salsa-macros/src/tracked_struct.rs | 2 +- src/database.rs | 12 +- src/function.rs | 8 +- src/function/eviction.rs | 22 +- src/function/eviction/lru.rs | 16 +- src/function/eviction/noop.rs | 5 +- src/function/eviction/sieve.rs | 1165 +++++++++++------ src/lib.rs | 10 +- src/zalsa.rs | 4 +- .../eviction_can_not_be_used_with_specify.rs | 12 + ...iction_can_not_be_used_with_specify.stderr | 5 + .../compile-fail/invalid_eviction_options.rs | 7 + .../invalid_eviction_options.stderr | 11 + tests/compile-fail/legacy_eviction_options.rs | 4 + .../legacy_eviction_options.stderr | 5 + .../lru_can_not_be_used_with_specify.rs | 12 - .../lru_can_not_be_used_with_specify.stderr | 5 - tests/compile-pass/tracked-methods.rs | 2 +- tests/lru.rs | 15 +- tests/memory-usage.rs | 2 +- tests/never_change.rs | 4 +- tests/parallel/lru_eviction_cancels_cycle.rs | 20 +- tests/sieve.rs | 2 +- 38 files changed, 1094 insertions(+), 583 deletions(-) delete mode 100644 book/src/plumbing/terminology/LRU.md create mode 100644 book/src/plumbing/terminology/eviction.md create mode 100644 tests/compile-fail/eviction_can_not_be_used_with_specify.rs create mode 100644 tests/compile-fail/eviction_can_not_be_used_with_specify.stderr create mode 100644 tests/compile-fail/invalid_eviction_options.rs create mode 100644 tests/compile-fail/invalid_eviction_options.stderr create mode 100644 tests/compile-fail/legacy_eviction_options.rs create mode 100644 tests/compile-fail/legacy_eviction_options.stderr delete mode 100644 tests/compile-fail/lru_can_not_be_used_with_specify.rs delete mode 100644 tests/compile-fail/lru_can_not_be_used_with_specify.stderr diff --git a/benches/eviction.rs b/benches/eviction.rs index a8d1abdda..c4fc807fe 100644 --- a/benches/eviction.rs +++ b/benches/eviction.rs @@ -99,7 +99,7 @@ fn fast_path_and_sweep(bencher: divan::Bencher, policy: Policy) { }) .bench_local_refs(|(db, items)| { let result = access_all(policy, &*db, items.iter().copied()); - db.trigger_lru_eviction(); + db.trigger_eviction(); black_box(result) }); } @@ -133,7 +133,7 @@ fn fill_and_evict(bencher: divan::Bencher, policy: Policy) { }) .bench_local_refs(|(db, items)| { let result = access_all(policy, &*db, items.iter().copied()); - db.trigger_lru_eviction(); + db.trigger_eviction(); black_box(result) }); } diff --git a/benches/eviction/support.rs b/benches/eviction/support.rs index 2a0526eae..5b4fe2501 100644 --- a/benches/eviction/support.rs +++ b/benches/eviction/support.rs @@ -14,12 +14,12 @@ pub(crate) fn no_eviction_value(db: &dyn salsa::Database, item: Item) -> Value { compute_value(item.value(db)) } -#[salsa::tracked(returns(copy), lru = 4096)] +#[salsa::tracked(returns(copy), eviction(policy = lru, capacity = 4096))] pub(crate) fn lru_value(db: &dyn salsa::Database, item: Item) -> Value { compute_value(item.value(db)) } -#[salsa::tracked(returns(copy), sieve = 4096)] +#[salsa::tracked(returns(copy), eviction(policy = sieve, capacity = 4096))] pub(crate) fn sieve_value(db: &dyn salsa::Database, item: Item) -> Value { compute_value(item.value(db)) } @@ -35,8 +35,8 @@ impl Policy { pub(crate) fn set_capacity(self, db: &mut salsa::DatabaseImpl, capacity: usize) { match self { Self::NoEviction => {} - Self::Lru => lru_value::set_lru_capacity(db, capacity), - Self::Sieve => sieve_value::set_lru_capacity(db, capacity), + Self::Lru => lru_value::set_eviction_capacity(db, capacity), + Self::Sieve => sieve_value::set_eviction_capacity(db, capacity), } } } diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index b7d4edc80..f2c937150 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -43,7 +43,7 @@ - [Durability](./plumbing/terminology/durability.md) - [Input query](./plumbing/terminology/input_query.md) - [Ingredient](./plumbing/terminology/ingredient.md) - - [LRU](./plumbing/terminology/LRU.md) + - [Eviction](./plumbing/terminology/eviction.md) - [Memo](./plumbing/terminology/memo.md) - [Query](./plumbing/terminology/query.md) - [Query function](./plumbing/terminology/query_function.md) diff --git a/book/src/plumbing/maybe_changed_after.md b/book/src/plumbing/maybe_changed_after.md index 4252b6c51..f2bedd562 100644 --- a/book/src/plumbing/maybe_changed_after.md +++ b/book/src/plumbing/maybe_changed_after.md @@ -33,4 +33,3 @@ The logic for derived queries is more complex. We summarize the high-level ideas [revision]: ./terminology/revision.md [verified]: ./terminology/verified.md [fetch]: ./fetch.md -[LRU]: ./terminology/LRU.md \ No newline at end of file diff --git a/book/src/plumbing/terminology/LRU.md b/book/src/plumbing/terminology/LRU.md deleted file mode 100644 index 6c71c5ca4..000000000 --- a/book/src/plumbing/terminology/LRU.md +++ /dev/null @@ -1,6 +0,0 @@ -# LRU - -The `lru` option on `#[salsa::tracked]` fixes the maximum number of values retained for that function. Functions with `lru` also generate [`function_name::set_lru_capacity(&mut db, capacity)`](https://docs.rs/salsa/latest/salsa/attr.tracked.html) to adjust the capacity. Salsa drops values from older [memos] to conserve memory, but retains their [dependency] information so that it can still compute whether values may have changed. See [cache eviction](../../tuning.md#cache-eviction-lru) for examples. - -[memos]: ./memo.md -[dependency]: ./dependency.md diff --git a/book/src/plumbing/terminology/eviction.md b/book/src/plumbing/terminology/eviction.md new file mode 100644 index 000000000..5e16e1449 --- /dev/null +++ b/book/src/plumbing/terminology/eviction.md @@ -0,0 +1,31 @@ +# Eviction + +The `eviction` option on `#[salsa::tracked]` bounds the number of memoized values +retained for that function. The policy defaults to [SIEVE]. To use LRU instead, +set `policy = lru`. + +SIEVE is recommended because its lock-free cache-hit path supports higher +throughput under concurrency without serializing on the eviction policy. The +[SIEVE paper] also found miss ratios comparable to or better than more complex +eviction policies. SIEVE can favor repeatedly accessed entries over one-hit +wonders, but tracks approximate rather than exact recency and may inspect +several residents when choosing a victim. Results remain workload-dependent. + +LRU maintains exact recency. Salsa's current implementation guards that order +with an exclusive mutex. Every query access, including a cache hit, takes the +mutex and updates the order, so concurrent hits to otherwise independent entries +serialize. + +A configured function generates +[`function_name::set_eviction_capacity(&mut db, capacity)`](https://docs.rs/salsa/latest/salsa/attr.tracked.html) +to adjust its capacity at runtime. + +Eviction drops values from older [memos] to conserve memory, but retains their +[dependency] information so Salsa can still determine whether dependent values +may have changed. See [cache eviction] for configuration examples. + +[cache eviction]: ../../tuning.md#cache-eviction +[dependency]: ./dependency.md +[memos]: ./memo.md +[SIEVE]: https://cachemon.github.io/SIEVE-website/ +[SIEVE paper]: https://www.usenix.org/conference/nsdi24/presentation/zhang-yazhuo diff --git a/book/src/plumbing/terminology/memo.md b/book/src/plumbing/terminology/memo.md index dce9d9179..7e31181d8 100644 --- a/book/src/plumbing/terminology/memo.md +++ b/book/src/plumbing/terminology/memo.md @@ -3,7 +3,7 @@ A *memo* stores information about the last time that a [query function] for some [query] Q was executed: * Typically, it contains the value that was returned from that function, so that we don't have to execute it again. - * However, this is not always true: some queries don't cache their result values, and values can also be dropped as a result of [LRU] collection. In those cases, the memo just stores [dependency] information, which can still be useful to determine if other queries that have Q as a [dependency] may have changed. + * However, this is not always true: some queries don't cache their result values, and an [eviction policy] can also drop values. In those cases, the memo just stores [dependency] information, which can still be useful to determine if other queries that have Q as a [dependency] may have changed. * The revision in which the memo last [verified]. * The [changed at] revision in which the memo's value last changed. (Note that it may be [backdated].) * The minimum durability of the memo's [dependencies]. @@ -18,4 +18,4 @@ A *memo* stores information about the last time that a [query function] for some [query]: ./query.md [query function]: ./query_function.md [changed at]: ./changed_at.md -[LRU]: ./LRU.md \ No newline at end of file +[eviction policy]: ./eviction.md diff --git a/book/src/tuning.md b/book/src/tuning.md index 0353e2c33..aa582bfa3 100644 --- a/book/src/tuning.md +++ b/book/src/tuning.md @@ -1,50 +1,63 @@ # Tuning Salsa -## Cache Eviction (LRU) +## Cache eviction -Salsa supports Least Recently Used (LRU) cache eviction for tracked functions. -By default, memoized values are never evicted (unbounded cache). You can enable -LRU eviction by specifying a capacity at compile time: +By default, tracked functions retain every memoized value for as long as its key +remains in the database. Configure an eviction policy to bound the number of +values retained by a function: ```rust -#[salsa::tracked(lru = 128)] +#[salsa::tracked(eviction(policy = sieve, capacity = 128))] fn parse(db: &dyn Db, input: SourceFile) -> Ast { // ... } ``` -With `lru = 128`, Salsa will keep at most 128 memoized values for this function. -When the cache exceeds this capacity, the least recently used values are evicted -at the start of each new revision. +The `capacity` option is required. The optional `policy` defaults to `sieve`. +Salsa supports two policies: + +- `sieve` is recommended. Its lock-free cache-hit path avoids serializing + concurrent accesses, and it often achieves better cache efficiency (a lower + miss ratio) than LRU. +- `lru` maintains exact least-recently-used order, but takes an exclusive lock on + every access. + +See [eviction policies] for their behavior and tradeoffs. + +Eviction drops memoized values while retaining query keys and dependency +metadata. A later access can therefore recompute an evicted value, while queries +that depend on it can still determine whether their own values may have changed. + +[eviction policies]: ./plumbing/terminology/eviction.md ### Zero-Cost When Disabled -When no `lru` capacity is specified (the default), Salsa uses a no-op eviction -policy that is completely optimized away by the compiler. This means there is -zero runtime overhead for functions that don't need cache eviction. +When `eviction` is absent, Salsa uses a no-op policy that is optimized away by +the compiler. Functions without cache eviction therefore have no policy +bookkeeping on cache hits. ### Runtime Capacity Adjustment -For functions with LRU enabled, you can adjust the capacity at runtime: +For functions with eviction configured, you can adjust the capacity at runtime: ```rust -#[salsa::tracked(lru = 128)] +#[salsa::tracked(eviction(capacity = 128))] fn my_query(db: &dyn Db, input: MyInput) -> Output { // ... } // Later, adjust the capacity: -my_query::set_lru_capacity(&mut db, 256); +my_query::set_eviction_capacity(&mut db, 256); ``` -**Note:** The `set_lru_capacity` method is only generated for functions that have -an `lru` attribute. Functions without LRU enabled do not have this method. +The `set_eviction_capacity` method is only generated for functions with an +`eviction` option. Setting the capacity to zero disables eviction by that policy. ### Memory Management -LRU evicts memoized values, not query keys or dependency metadata. Salsa also -reclaims stale tracked outputs and unused low-durability interned values. Input -identities remain until the database is dropped. +Eviction removes memoized values, not query keys or dependency metadata. Salsa +also reclaims stale tracked outputs and unused low-durability interned values. +Input identities remain until the database is dropped. ## Intern Queries diff --git a/components/salsa-macro-rules/src/setup_tracked_fn.rs b/components/salsa-macro-rules/src/setup_tracked_fn.rs index 6cdcb8870..2994d0ce3 100644 --- a/components/salsa-macro-rules/src/setup_tracked_fn.rs +++ b/components/salsa-macro-rules/src/setup_tracked_fn.rs @@ -61,8 +61,8 @@ macro_rules! setup_tracked_fn { // The eviction policy type for this function eviction: $Eviction:ty, - // LRU capacity (a literal, maybe 0) - lru: $lru:tt, + // Eviction capacity (a literal, or 0 when eviction is not configured) + eviction_capacity: $eviction_capacity:tt, // The return mode for the function, see `salsa_macros::options::Option::returns` return_mode: $return_mode:tt, @@ -428,7 +428,7 @@ macro_rules! setup_tracked_fn { let fn_ingredient = <$zalsa::function::IngredientImpl<$Configuration>>::new( first_index, memo_ingredient_indices, - $lru, + $eviction_capacity, ); $zalsa::macro_if! { if $needs_interner { @@ -484,14 +484,14 @@ macro_rules! setup_tracked_fn { } } - /// Sets the eviction policy's tuning value. + /// Sets the eviction policy's capacity. /// /// **WARNING:** Just like an ordinary write, this method triggers /// cancellation. If you invoke it while a snapshot exists, it /// will block until that snapshot is dropped -- if that snapshot /// is owned by the current thread, this could trigger deadlock. - fn set_lru_capacity(db: &mut dyn $Db, value: usize) where for<'trivial_bounds> $Eviction: $zalsa::function::HasCapacity { - $Configuration::fn_ingredient_mut(db).set_tuning(value); + fn set_eviction_capacity(db: &mut dyn $Db, capacity: usize) where for<'trivial_bounds> $Eviction: $zalsa::function::HasCapacity { + $Configuration::fn_ingredient_mut(db).set_capacity(capacity); } $zalsa::macro_if! { $needs_interner => diff --git a/components/salsa-macros/src/accumulator.rs b/components/salsa-macros/src/accumulator.rs index abc694809..49e4d6aba 100644 --- a/components/salsa-macros/src/accumulator.rs +++ b/components/salsa-macros/src/accumulator.rs @@ -41,7 +41,7 @@ impl AllowedOptions for Accumulator { const CYCLE_FN: bool = false; const CYCLE_INITIAL: bool = false; const CYCLE_RESULT: bool = false; - const LRU: bool = false; + const EVICTION: bool = false; const CONSTRUCTOR_NAME: bool = false; const ID: bool = false; const REVISIONS: bool = false; diff --git a/components/salsa-macros/src/input.rs b/components/salsa-macros/src/input.rs index 6449f0316..5c2f4fda2 100644 --- a/components/salsa-macros/src/input.rs +++ b/components/salsa-macros/src/input.rs @@ -57,7 +57,7 @@ impl AllowedOptions for InputStruct { const CYCLE_RESULT: bool = false; - const LRU: bool = false; + const EVICTION: bool = false; const CONSTRUCTOR_NAME: bool = true; diff --git a/components/salsa-macros/src/interned.rs b/components/salsa-macros/src/interned.rs index 30c5f2d73..e4ca4b8b0 100644 --- a/components/salsa-macros/src/interned.rs +++ b/components/salsa-macros/src/interned.rs @@ -57,7 +57,7 @@ impl AllowedOptions for InternedStruct { const CYCLE_RESULT: bool = false; - const LRU: bool = false; + const EVICTION: bool = false; const CONSTRUCTOR_NAME: bool = true; diff --git a/components/salsa-macros/src/lib.rs b/components/salsa-macros/src/lib.rs index 0af605b7f..a53b3662e 100644 --- a/components/salsa-macros/src/lib.rs +++ b/components/salsa-macros/src/lib.rs @@ -423,10 +423,11 @@ pub fn input(args: TokenStream, input: TokenStream) -> TokenStream { /// per-key incremental implementation and a batch implementation that computes many results at /// once. The function must take exactly one key argument, and it must be a tracked struct, not an /// input or interned struct. `specify` must be called during the same tracked query invocation -/// that created the key. It cannot be combined with `lru`. See [specifying query results in the -/// Salsa book] for an example. -/// - `lru = INTEGER` bounds the number of memoized values retained by the function and sets the -/// initial capacity used by `FUNCTION::set_lru_capacity`. +/// that created the key. It cannot be combined with `eviction`. See [specifying query results in +/// the Salsa book] for an example. +/// - `eviction(capacity = INTEGER)` bounds the number of memoized values retained by the function +/// using SIEVE. Specify `policy = lru` to use LRU instead. `INTEGER` is also the initial capacity +/// used by `FUNCTION::set_eviction_capacity`. /// - `cycle_initial = EXPR` enables fixed-point cycle recovery and computes the initial value. The /// expression is called as `(db, cycle_head_id, query_arguments...)`. /// - `cycle_fn = EXPR` combines successive fixed-point values. It must be accompanied by diff --git a/components/salsa-macros/src/options.rs b/components/salsa-macros/src/options.rs index 179a93e5e..3b8c9608d 100644 --- a/components/salsa-macros/src/options.rs +++ b/components/salsa-macros/src/options.rs @@ -84,15 +84,8 @@ pub(crate) struct Options { /// If this is `Some`, the value is the ``. pub data: Option, - /// The `lru = ` option is used to set the lru capacity for a tracked function. - /// - /// If this is `Some`, the value is the ``. - pub lru: Option, - - /// The `sieve = ` option is used to set the SIEVE capacity for a tracked function. - /// - /// If this is `Some`, the value is the ``. - pub sieve: Option, + /// Configures cache eviction for a tracked function. + pub eviction: Option, /// The `constructor = ` option lets the user specify the name of /// the constructor of a salsa struct. @@ -132,6 +125,86 @@ impl Options { } } +/// Cache-eviction configuration for a tracked function. +#[derive(Debug)] +pub(crate) struct EvictionOptions { + /// Replacement policy used to choose retained values. Defaults to SIEVE. + pub policy: EvictionPolicy, + + /// Initial maximum number of resident values. + pub capacity: usize, +} + +impl syn::parse::Parse for EvictionOptions { + fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result { + let mut policy = None; + let mut capacity = None; + + while !input.is_empty() { + let option = syn::Ident::parse_any(input)?; + let _eq = Equals::parse(input)?; + + if option == "policy" { + let ident = syn::Ident::parse_any(input)?; + let value = EvictionPolicy::parse(&ident)?; + if policy.replace(value).is_some() { + return Err(syn::Error::new( + option.span(), + "option `policy` provided twice", + )); + } + } else if option == "capacity" { + let lit = syn::LitInt::parse(input)?; + let value = lit.base10_parse::()?; + if capacity.replace(value).is_some() { + return Err(syn::Error::new( + option.span(), + "option `capacity` provided twice", + )); + } + } else { + return Err(syn::Error::new( + option.span(), + format!("unrecognized eviction option `{option}`"), + )); + } + + if input.is_empty() { + break; + } + + let _comma = Comma::parse(input)?; + } + + Ok(Self { + policy: policy.unwrap_or(EvictionPolicy::Sieve), + capacity: capacity.ok_or_else(|| input.error("missing required `capacity` option"))?, + }) + } +} + +/// Cache-eviction policy selected for a tracked function. +#[derive(Copy, Clone, Debug)] +pub(crate) enum EvictionPolicy { + Lru, + Sieve, +} + +impl EvictionPolicy { + fn parse(ident: &syn::Ident) -> syn::Result { + if ident == "lru" { + Ok(Self::Lru) + } else if ident == "sieve" { + Ok(Self::Sieve) + } else { + Err(syn::Error::new( + ident.span(), + "unknown eviction policy; expected `sieve` or `lru`", + )) + } + } +} + #[derive(Debug, Default, Clone)] pub struct PersistOptions { /// Path to a custom serialize function. @@ -157,8 +230,7 @@ impl Default for Options { data: Default::default(), constructor_name: Default::default(), phantom: Default::default(), - lru: Default::default(), - sieve: Default::default(), + eviction: Default::default(), singleton: Default::default(), id: Default::default(), revisions: Default::default(), @@ -183,8 +255,7 @@ pub(crate) trait AllowedOptions { const CYCLE_FN: bool; const CYCLE_INITIAL: bool; const CYCLE_RESULT: bool; - const LRU: bool; - const SIEVE: bool = false; + const EVICTION: bool; const CONSTRUCTOR_NAME: bool; const ID: bool; const REVISIONS: bool; @@ -453,32 +524,33 @@ impl syn::parse::Parse for Options { "`data` option not allowed here", )); } - } else if ident == "lru" { - if A::LRU { - let _eq = Equals::parse(input)?; - let lit = syn::LitInt::parse(input)?; - let value = lit.base10_parse::()?; - if let Some(old) = options.lru.replace(value) { - return Err(syn::Error::new(old.span(), "option `lru` provided twice")); + } else if ident == "eviction" { + if A::EVICTION { + let content; + parenthesized!(content in input); + let value = content.parse()?; + if options.eviction.replace(value).is_some() { + return Err(syn::Error::new( + ident.span(), + "option `eviction` provided twice", + )); } } else { return Err(syn::Error::new( ident.span(), - "`lru` option not allowed here", + "`eviction` option not allowed here", )); } - } else if ident == "sieve" { - if A::SIEVE { - let _eq = Equals::parse(input)?; - let lit = syn::LitInt::parse(input)?; - let value = lit.base10_parse::()?; - if let Some(old) = options.sieve.replace(value) { - return Err(syn::Error::new(old.span(), "option `sieve` provided twice")); - } + } else if ident == "lru" { + if A::EVICTION { + return Err(syn::Error::new( + ident.span(), + "`lru = ...` has been removed; use `eviction(capacity = ...)` for SIEVE (recommended) or `eviction(policy = lru, capacity = ...)`", + )); } else { return Err(syn::Error::new( ident.span(), - "`sieve` option not allowed here", + "`lru` option not allowed here", )); } } else if ident == "constructor" { @@ -588,8 +660,7 @@ impl quote::ToTokens for Options { cycle_initial, cycle_result, data, - lru, - sieve, + eviction, constructor_name, id, revisions, @@ -634,11 +705,13 @@ impl quote::ToTokens for Options { if let Some(data) = data { tokens.extend(quote::quote! { data = #data, }); } - if let Some(lru) = lru { - tokens.extend(quote::quote! { lru = #lru, }); - } - if let Some(sieve) = sieve { - tokens.extend(quote::quote! { sieve = #sieve, }); + if let Some(eviction) = eviction { + let capacity = eviction.capacity; + let policy = match eviction.policy { + EvictionPolicy::Lru => quote::quote!(lru), + EvictionPolicy::Sieve => quote::quote!(sieve), + }; + tokens.extend(quote::quote! { eviction(policy = #policy, capacity = #capacity), }); } if let Some(constructor_name) = constructor_name { tokens.extend(quote::quote! { constructor = #constructor_name, }); diff --git a/components/salsa-macros/src/tracked_fn.rs b/components/salsa-macros/src/tracked_fn.rs index f037e22cc..b93fb7826 100644 --- a/components/salsa-macros/src/tracked_fn.rs +++ b/components/salsa-macros/src/tracked_fn.rs @@ -4,7 +4,7 @@ use syn::spanned::Spanned; use syn::{Ident, ItemFn}; use crate::hygiene::Hygiene; -use crate::options::{AllowedOptions, AllowedPersistOptions, Options}; +use crate::options::{AllowedOptions, AllowedPersistOptions, EvictionPolicy, Options}; use crate::{db_lifetime, fn_util}; // Source: @@ -50,9 +50,7 @@ impl AllowedOptions for TrackedFn { const CYCLE_RESULT: bool = true; - const LRU: bool = true; - - const SIEVE: bool = true; + const EVICTION: bool = true; const CONSTRUCTOR_NAME: bool = false; @@ -156,24 +154,10 @@ impl Macro { } } - if let (Some(_), Some(token)) = (&self.args.lru, &self.args.specify) { - return Err(syn::Error::new_spanned( - token, - "the `specify` and `lru` options cannot be used together", - )); - } - - if let (Some(_), Some(token)) = (&self.args.sieve, &self.args.specify) { + if let (Some(_), Some(token)) = (&self.args.eviction, &self.args.specify) { return Err(syn::Error::new_spanned( token, - "the `specify` and `sieve` options cannot be used together", - )); - } - - if let (Some(_), Some(token)) = (&self.args.lru, &self.args.sieve) { - return Err(syn::Error::new_spanned( - token, - "the `lru` and `sieve` options cannot be used together", + "the `specify` and `eviction` options cannot be used together", )); } @@ -182,17 +166,17 @@ impl Macro { FunctionType::SalsaStruct => false, }; - let eviction_tuning = - Literal::usize_unsuffixed(self.args.lru.or(self.args.sieve).unwrap_or(0)); - - // Determine the eviction policy type from the configured option. - let eviction_type = if self.args.lru.is_some() { - quote!(::salsa::plumbing::function::Lru) - } else if self.args.sieve.is_some() { - quote!(::salsa::plumbing::function::Sieve) - } else { - quote!(::salsa::plumbing::function::NoopEviction) + let (eviction_type, eviction_capacity) = match &self.args.eviction { + Some(eviction) => { + let policy = match eviction.policy { + EvictionPolicy::Lru => quote!(::salsa::plumbing::function::Lru), + EvictionPolicy::Sieve => quote!(::salsa::plumbing::function::Sieve), + }; + (policy, eviction.capacity) + } + None => (quote!(::salsa::plumbing::function::NoopEviction), 0), }; + let eviction_capacity = Literal::usize_unsuffixed(eviction_capacity); let return_mode = self .args @@ -249,7 +233,7 @@ impl Macro { needs_interner: #needs_interner, heap_size_fn: #(#heap_size_fn)*, eviction: #eviction_type, - lru: #eviction_tuning, + eviction_capacity: #eviction_capacity, return_mode: #return_mode, persist: #persist, assert_types_are_update: { #assert_types_are_update }, diff --git a/components/salsa-macros/src/tracked_struct.rs b/components/salsa-macros/src/tracked_struct.rs index fb645cb5e..68ca8fe5d 100644 --- a/components/salsa-macros/src/tracked_struct.rs +++ b/components/salsa-macros/src/tracked_struct.rs @@ -53,7 +53,7 @@ impl AllowedOptions for TrackedStruct { const CYCLE_RESULT: bool = false; - const LRU: bool = false; + const EVICTION: bool = false; const CONSTRUCTOR_NAME: bool = true; diff --git a/src/database.rs b/src/database.rs index 100665104..af5fe29a2 100644 --- a/src/database.rs +++ b/src/database.rs @@ -36,15 +36,21 @@ impl<'db, Db: Database + ?Sized> From<&'db mut Db> for RawDatabase<'db> { /// The trait implemented by all Salsa databases. /// You can create your own subtraits of this trait using the `#[salsa::db]`(`crate::db`) procedural macro. pub trait Database: Send + ZalsaDatabase + AsDynDatabase { - /// Enforces current LRU limits, evicting entries if necessary. + /// Applies configured eviction policies, evicting entries if necessary. /// /// **WARNING:** Just like an ordinary write, this method triggers /// cancellation. If you invoke it while a snapshot exists, it /// will block until that snapshot is dropped -- if that snapshot /// is owned by the current thread, this could trigger deadlock. - fn trigger_lru_eviction(&mut self) { + fn trigger_eviction(&mut self) { let zalsa_mut = self.zalsa_mut(); - zalsa_mut.evict_lru(); + zalsa_mut.apply_eviction(); + } + + /// Deprecated alias for [`Database::trigger_eviction`]. + #[deprecated(note = "use `Database::trigger_eviction` instead")] + fn trigger_lru_eviction(&mut self) { + self.trigger_eviction(); } /// A "synthetic write" causes the system to act *as though* some diff --git a/src/function.rs b/src/function.rs index 024e40d79..06644e352 100644 --- a/src/function.rs +++ b/src/function.rs @@ -239,14 +239,14 @@ where DatabaseKeyIndex::new(self.index, key) } - /// Sets the eviction policy's tuning value. + /// Sets the eviction policy's capacity. /// - /// Only available when the eviction policy supports runtime tuning. - pub fn set_tuning(&mut self, tuning: usize) + /// Only available when the eviction policy supports runtime capacity changes. + pub fn set_capacity(&mut self, capacity: usize) where C::Eviction: HasCapacity, { - self.eviction.set_tuning(tuning); + self.eviction.set_capacity(capacity); } /// Returns a reference to the memo value that lives as long as self. diff --git a/src/function/eviction.rs b/src/function/eviction.rs index 360f6c698..1968520ad 100644 --- a/src/function/eviction.rs +++ b/src/function/eviction.rs @@ -18,21 +18,16 @@ use crate::Id; /// Implementations control when memoized values are evicted from the cache. /// The eviction policy is selected at compile time via the `Configuration` trait. pub trait EvictionPolicy: Send + Sync { - /// Creates a policy from its configured tuning value. + /// Creates a policy with the configured capacity. /// /// A value of zero disables eviction. - fn new(tuning: usize) -> Self; + fn new(capacity: usize) -> Self; /// Records that a value was used. /// /// Implementations may treat this as a best-effort hint. fn record_use(&self, _id: Id) {} - /// Changes the policy's tuning value. - /// - /// A value of zero disables eviction. - fn set_tuning(&mut self, tuning: usize); - /// Invokes `evict` for each value that should be evicted. /// /// Called once per revision. Implementations may also perform any @@ -40,9 +35,14 @@ pub trait EvictionPolicy: Send + Sync { fn for_each_evicted(&mut self, evict: impl FnMut(Id)); } -/// Marker trait for eviction policies whose tuning can be changed at runtime. +/// An eviction policy whose capacity can be changed at runtime. /// -/// This trait is used to conditionally generate the `set_lru_capacity` method +/// This trait is used to conditionally generate the `set_eviction_capacity` method /// on tracked functions. Only policies that implement this trait will expose -/// runtime tuning. -pub trait HasCapacity: EvictionPolicy {} +/// runtime capacity changes. +pub trait HasCapacity: EvictionPolicy { + /// Changes the policy's capacity. + /// + /// A value of zero disables eviction. + fn set_capacity(&mut self, capacity: usize); +} diff --git a/src/function/eviction/lru.rs b/src/function/eviction/lru.rs index 479e7da9d..b797898ee 100644 --- a/src/function/eviction/lru.rs +++ b/src/function/eviction/lru.rs @@ -43,13 +43,6 @@ impl EvictionPolicy for Lru { } } - fn set_tuning(&mut self, capacity: usize) { - self.capacity = NonZeroUsize::new(capacity); - if self.capacity.is_none() { - self.set.get_mut().clear(); - } - } - fn for_each_evicted(&mut self, mut evict: impl FnMut(Id)) { let Some(cap) = self.capacity else { return; @@ -63,4 +56,11 @@ impl EvictionPolicy for Lru { } } -impl HasCapacity for Lru {} +impl HasCapacity for Lru { + fn set_capacity(&mut self, capacity: usize) { + self.capacity = NonZeroUsize::new(capacity); + if self.capacity.is_none() { + self.set.get_mut().clear(); + } + } +} diff --git a/src/function/eviction/noop.rs b/src/function/eviction/noop.rs index 033ffaa0d..83c4dbd36 100644 --- a/src/function/eviction/noop.rs +++ b/src/function/eviction/noop.rs @@ -1,6 +1,6 @@ //! No-op eviction policy - cache grows unbounded. //! -//! This is the default eviction policy when no LRU capacity is specified. +//! This is the default eviction policy when eviction is not configured. use crate::{Id, function::EvictionPolicy}; @@ -12,9 +12,6 @@ impl EvictionPolicy for NoopEviction { Self } - #[inline(always)] - fn set_tuning(&mut self, _capacity: usize) {} - #[inline(always)] fn for_each_evicted(&mut self, _evict: impl FnMut(Id)) {} } diff --git a/src/function/eviction/sieve.rs b/src/function/eviction/sieve.rs index 2a19f4426..78aec8f54 100644 --- a/src/function/eviction/sieve.rs +++ b/src/function/eviction/sieve.rs @@ -1,29 +1,181 @@ -//! SIEVE eviction policy. -//! -//! This policy keeps resident values in FIFO order and records a single visited -//! bit for values used after admission. At eviction time, a moving hand gives -//! visited values one second chance by clearing their bit and continuing toward -//! newer entries. -//! -//! Victim selection is bounded to twice the current resident count. With no -//! concurrent hits, this behaves like textbook SIEVE: if every resident starts -//! visited, the first pass clears those bits and the first resident is selected -//! when the hand reaches it again. A second pass also gives residents re-marked -//! by concurrent hits another chance. If every inspection still finds a visited -//! resident, selection force-evicts the resident at the hand so admissions -//! cannot starve without changing the scan order. -//! -//! Hits update the block-indexed visited bits without taking the state mutex. -//! Admissions, hand movement, and the pending-block queue are serialized by that -//! mutex. Selecting a victim removes it from the resident list immediately, but -//! its value is evicted at the next revision only if it was not re-admitted. Each -//! queued block owns a transient bitmap identifying its pending ids. -//! -//! State is divided into independently allocated blocks. Within each block, -//! interleaved resident/visited bits cover 32 ids per atomic word. Pending -//! bitmaps cover 64 ids per word and exist only while their block is queued. -//! This keeps SIEVE's allocation granularity independent of the memo table's -//! page size without reserving pending state for every id. +//! An implementation of the SIEVE cache eviction algorithm described in the +//! [NSDI '24 paper]. The [SIEVE project website] provides a visual explanation. +//! +//! # How SIEVE works +//! +//! SIEVE keeps residents in admission order, from newest to oldest, and keeps a +//! moving hand that starts at the oldest resident. Every resident also has one +//! `visited` bit: +//! +//! - New residents enter at the newest end with `visited = false`. +//! - A cache hit sets `visited = true`, but does not move the resident. +//! - When space is needed, the hand starts at its current position and inspects +//! residents toward the newest end. It stops as soon as it evicts an +//! unvisited resident. A visited resident instead gets a second chance: its +//! visited bit is cleared and the hand advances to the next newer resident. +//! - The next time space is needed, the hand resumes at the resident after the +//! previous victim. Across eviction requests, it therefore moves toward newer +//! residents, wrapping to the oldest after passing the newest. +//! +//! For example, `*` marks a visited resident: +//! +//! ```text +//! newest oldest +//! [D] [C*] [B] [A*] +//! ^ hand +//! +//! inspect A*: clear its visited bit, advance to B +//! inspect B: B is unvisited, so evict B +//! next hand: C* +//! ``` +//! +//! SIEVE therefore does not track exact recency. Its visited bit only answers +//! whether a resident was used since the hand last gave it a chance. This is +//! enough to protect reused values from one-shot scans while making a hit much +//! cheaper than moving a node in an LRU list. +//! +//! ## Bounded selection +//! +//! The textbook scan has no bound when concurrent hits can continually re-mark +//! entries while the hand scans. In Salsa, that could keep granting second +//! chances forever while holding the admission mutex. +//! +//! Victim selection therefore performs at most two inspections per current +//! resident. With no concurrent re-marks this is equivalent to textbook SIEVE: +//! one pass can clear every visited bit and the next inspection finds an +//! unvisited victim. The second pass also honors entries re-marked during the +//! first pass. If every inspection still observes a visited entry, Salsa evicts +//! the resident currently at the hand to guarantee progress while preserving +//! the scan order. +//! +//! # Implementation +//! +//! The main design goal is to keep cache hits lock-free. Salsa therefore splits +//! SIEVE's state into two parts: +//! +//! - `state_pages` contains the atomic resident and visited bits used by cache +//! hits. A hit accesses only this state and does not acquire a mutex. +//! - `state` contains the admission-order list, the hand, and pending evictions. +//! Admissions and victim selection access this state through a mutex. +//! +//! The resulting layout is: +//! +//! ```text +//! Sieve +//! ├── capacity +//! ├── state_pages: StatePages lock-free hit state +//! │ └── StatePageCell lazily owns one StatePage allocation +//! │ └── StatePage state for 1,024 table slots +//! │ ├── resident/visited bits +//! │ └── pending-page queue index +//! └── state: Mutex admission and eviction state +//! ├── residents: Residents intrusive admission-order list +//! ├── hand: Option next candidate in that list +//! └── pending_pages: Vec +//! └── 1,024-bit bitmap selected slots awaiting validation +//! ``` +//! +//! The hit path looks up the [`StatePage`], atomically marks a resident as +//! visited, and returns. It falls back to [`State`] only when the id is not +//! resident or selection won a race with the hit. Supporting this lock-free +//! path drives the implementation complexity: ids map through a sparse, +//! atomically published page directory, state transitions must tolerate +//! races, and admissions and victim selection must coordinate the atomic state +//! with the mutex-protected list. +//! +//! ## State-page mapping without a hash map +//! +//! A conventional standalone SIEVE cache stores the visited bit on each node in +//! the admission-order list and uses a hash map to find that node from a cache +//! key. Every hit therefore performs a hash-table lookup before it can set the +//! visited bit. +//! +//! Salsa can instead address replacement state from [`Id::index`], whose +//! consecutive values identify consecutive table slots. SIEVE partitions that +//! index into a state-page number and an offset within the state page: +//! +//! ```text +//! high bits -> StatePageNumber +//! low 10 bits -> StatePageOffset +//! ``` +//! +//! [`StatePages`] implements this mapping as a sparse geometric bucket +//! directory. The page number locates a [`StatePageCell`] without hashing, and +//! the cell lazily allocates one [`StatePage`] containing the bits selected by +//! the page offset. A hit can therefore find the resident and visited bits and +//! set `visited` without finding the corresponding [`Resident`] node; those +//! nodes are used only while admitting or selecting residents under the state +//! mutex. +//! +//! Allocating the corresponding memo-table pages does not by itself allocate +//! SIEVE state. A state page is allocated only when the enabled policy first +//! sees a use from its id range, and remains available for lock-free reads +//! until SIEVE is disabled or the policy is dropped. +//! +//! A state page deliberately covers 1,024 table slots, eight times the current +//! 128-slot memo-table page (as of July 5th 2026). +//! State pages and memo-table pages are independent: +//! memo slots contain comparatively large values, so smaller memo pages limit +//! unused table storage. SIEVE needs only two persistent bits per slot, making +//! a 1,024-slot resident/visited bitmap 256 bytes. The larger state page +//! amortizes the directory pointer, heap allocation, pending-queue index, and +//! pending-victim batching across more ids while keeping the excess bitmap +//! memory for a sparsely used page small. +//! +//! ## Deferred eviction +//! +//! Selecting a victim removes it from [`Residents`] and clears its resident bit +//! immediately, but Salsa cannot drop its memo value while queries may still +//! hold references from the current revision. The victim must remain pending +//! until the next policy drain, normally at a revision boundary. +//! +//! Pending victims are grouped using the same page-number-and-offset mapping. A +//! [`PendingPage`] stores one state-page number and a 1,024-bit bitmap; each set +//! bit identifies one selected offset from that page. A state page's +//! `pending_page_index` points back into `State::pending_pages`, so selecting +//! another victim from the same page finds the existing bitmap directly. Each +//! state page therefore appears at most once in the pending queue, without a +//! hash set or one vector entry per victim. +//! +//! During a drain, Salsa resolves each state page once and visits the set bits +//! in its pending bitmap. An access between selection and draining can re-admit +//! a victim and reuse its still-present memo, so the drain checks the resident +//! bit again. It deletes the memo only if the slot is still non-resident. +//! +//! ## Synchronization +//! +//! Admissions, hand movement, the resident list, and the pending queue are +//! serialized by the state mutex. The resident/visited words deliberately sit +//! outside that protection so the common hit path does not acquire the mutex. +//! Victim selection holds the mutex, but that provides no mutual exclusion with +//! hits because they never acquire it. Both paths may access the same word at +//! the same time, so the words must be atomic. +//! The resident and visited bits for a slot are packed into the same atomic +//! word. This improves locality and lets the hit path observe both bits with a +//! single atomic load. It also prevents a race between checking residency and +//! setting `visited`: either the hit marks the still-resident slot and selection +//! observes that visit, or selection clears `resident` and the hit retries +//! through admission. Separate fields would require another atomic load and +//! permit a hit to observe `resident`, lose the race to selection, and then set +//! `visited` without re-admitting the selected slot. +//! +//! These atomics use relaxed ordering: the bits carry replacement-policy state +//! only and do not publish or protect memo data. In contrast, [`StatePageCell`] +//! uses acquire/release ordering because it publishes a newly allocated page +//! to concurrent readers. +//! +//! The per-slot state machine is: +//! +//! ```text +//! absent (00) --admit--> resident, unvisited (01) +//! resident, unvisited (01) --hit--> resident, visited (11) +//! resident, visited (11) --select--> resident, unvisited (01) +//! resident, unvisited (01) --select--> absent (00) + pending eviction +//! absent + pending eviction --readmit--> resident, unvisited (01) +//! ``` +//! +//! [NSDI '24 paper]: https://www.usenix.org/conference/nsdi24/presentation/zhang-yazhuo +//! [SIEVE project website]: https://cachemon.github.io/SIEVE-website/ use std::ptr; @@ -34,34 +186,35 @@ use crate::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; use super::{EvictionPolicy, HasCapacity}; use boxcar::buckets::{Buckets, Index, MaybeZeroable, buckets_for_index_bits}; -const SLOT_STATE_BITS: usize = 2; -const SLOTS_PER_STATE_WORD: usize = u64::BITS as usize / SLOT_STATE_BITS; -const SLOTS_PER_PENDING_WORD: usize = u64::BITS as usize; -const BLOCK_LEN_BITS: usize = 10; -const BLOCK_LEN: usize = 1 << BLOCK_LEN_BITS; -const STATE_WORDS: usize = BLOCK_LEN / SLOTS_PER_STATE_WORD; -const PENDING_WORDS: usize = BLOCK_LEN / SLOTS_PER_PENDING_WORD; - -type StateBlocks = - Buckets; -type StateBlockIndex = Index<{ buckets_for_index_bits(u32::BITS - BLOCK_LEN_BITS as u32) }>; - /// SIEVE eviction policy. /// -/// Values enter at the front of a FIFO queue. Uses set a block-indexed atomic -/// visited bit; they do not move the value in the queue. +/// Values enter at the front of a FIFO queue. Uses atomically set a visited bit +/// in direct-addressed state pages; they do not move the value in the queue. pub struct Sieve { + /// Maximum number of residents. Zero disables SIEVE entirely. + /// + /// Mutation requires `&mut self`, so hit paths can read this without an + /// atomic or the state mutex. capacity: usize, + + /// Lock-free lookup for the resident/visited bits used by cache hits. + /// + /// Allocated pages and their addresses remain valid until this directory + /// is replaced while disabling SIEVE or the policy is dropped. + state_pages: StatePages, + + /// Admission-order list, SIEVE hand, and pending-eviction pages. + /// + /// The mutex serializes admissions and all mutations to this state. state: Mutex, - state_blocks: StateBlocks, } impl EvictionPolicy for Sieve { fn new(capacity: usize) -> Self { Self { capacity, - state: Mutex::default(), - state_blocks: StateBlocks::new(), + state_pages: StatePages::new(), + state: Mutex::new(State::with_capacity(capacity)), } } @@ -69,26 +222,14 @@ impl EvictionPolicy for Sieve { fn record_use(&self, id: Id) { // Probe residency before capacity: resident hits do not need the capacity, // while admissions already take the cold path below. - if let Some((block, slot)) = block_and_slot_if_allocated(&self.state_blocks, id) { - if block.record_use(slot) { + if let Some((page, offset)) = self.state_pages.try_page_and_offset(id) { + if page.record_use(offset) { return; } } if self.capacity != 0 { - self.record_admission(id, self.capacity); - } - } - - fn set_tuning(&mut self, capacity: usize) { - self.capacity = capacity; - if capacity == 0 { - *self.state.get_mut() = State::default(); - self.state_blocks = StateBlocks::new(); - } else { - self.state - .get_mut() - .schedule_evictions(capacity, &self.state_blocks); + self.record_admission(id); } } @@ -99,110 +240,143 @@ impl EvictionPolicy for Sieve { } let state = self.state.get_mut(); - for (queue_index, pending_block) in state.pending_blocks.drain(..).enumerate() { - let block = state_block(&self.state_blocks, pending_block.state_block); - block.clear_pending_block_index(queue_index); - pending_block.for_each_pending_slot(|slot| { - if !block.is_resident(slot) { - evict(state_block_id(pending_block.state_block, slot)); + for (queue_index, pending_page) in state.pending_pages.drain(..).enumerate() { + let page = &self.state_pages[pending_page.page_number]; + + // Queue indexes refer to positions before `drain`. Enumerating in + // order lets each page validate and clear its reverse mapping. + page.clear_pending_page_index(queue_index); + + pending_page.for_each_pending_offset(|offset| { + // A victim may have been re-admitted since selection. Its stale + // pending bit must not delete the memo it now reuses. + if !page.is_resident(offset) { + evict(slot_id(pending_page.page_number, offset)); } }); } + + // Sparse ids can temporarily queue one page per resident. Once empty, + // retain only a small amount of storage relative to a dense cache. state - .pending_blocks - .shrink_to(capacity.div_ceil(BLOCK_LEN).saturating_mul(2)); + .pending_pages + .shrink_to(capacity.div_ceil(STATE_PAGE_LEN).saturating_mul(2)); } } -impl HasCapacity for Sieve {} +impl HasCapacity for Sieve { + fn set_capacity(&mut self, capacity: usize) { + self.capacity = capacity; + if capacity == 0 { + *self.state.get_mut() = State::default(); + self.state_pages = StatePages::new(); + } else { + let state = self.state.get_mut(); + state.residents.reserve_capacity(capacity); + state.schedule_excess_evictions(capacity, &self.state_pages); + } + } +} impl Sieve { - fn record_admission(&self, id: Id, capacity: usize) { - let (block, slot) = block_and_slot_or_alloc(&self.state_blocks, id); + fn record_admission(&self, id: Id) { + let (page, offset) = self.state_pages.page_and_offset_or_alloc(id); let mut state = self.state.lock(); - if block.is_resident(slot) { - block.record_use(slot); + if page.is_resident(offset) { + page.record_use(offset); return; } - state.insert(id, block, slot, capacity, &self.state_blocks); + state.insert(id, page, offset, self.capacity, &self.state_pages); } } +/// Mutable replacement-policy state protected by [`Sieve::state`]. +/// +/// Invariants outside an in-progress mutation: +/// +/// - `hand` is `None` exactly when `residents` is empty; otherwise it names a +/// resident node. +/// - Every list node has its resident bit set in `StatePage::state`. +/// - Every `pending_pages[i]` is paired with a state page whose +/// `pending_page_index` stores `i + 1`, and no state page appears twice. #[derive(Default)] struct State { + /// Residents in admission order, newest at the front and oldest at the back. residents: Residents, - /// Index of the next resident candidate to inspect. `0` is the sentinel. - hand: ResidentIndex, - /// State blocks containing victims waiting for an eviction context. - pending_blocks: Vec, -} - -struct PendingBlock { - state_block: StateBlockIndex, - pending: [u64; PENDING_WORDS], -} - -struct SelectedVictim<'a> { - id: Id, - /// The already-resolved location avoids another state-directory lookup when the - /// victim is added to the pending-eviction queue. - block: &'a StateBlock, - slot: usize, + /// Next resident candidate to inspect, or `None` when the list is empty. + hand: Option, + /// State pages containing victims whose memo values await deletion. + /// + /// Grouping victims by state page avoids one vector entry per victim and + /// lets a drain reuse the resolved page for all of its pending slots. + pending_pages: Vec, } impl State { + fn with_capacity(capacity: usize) -> Self { + Self { + residents: Residents::with_capacity(capacity), + hand: None, + pending_pages: Vec::new(), + } + } + fn insert( &mut self, id: Id, - block: &StateBlock, - slot: usize, + page: &StatePage, + offset: StatePageOffset, capacity: usize, - state_blocks: &StateBlocks, + state_pages: &StatePages, ) { debug_assert!(self.residents.len() <= capacity); if self.residents.len() == capacity { - self.schedule_eviction(state_blocks); + self.schedule_eviction(state_pages); } let node = self.residents.push_front(id); - block.admit(slot); + page.admit(offset); - if self.hand == 0 { - self.hand = node; + // The first resident starts the scan. Later admissions join at the + // front without disturbing the hand's current position. + if self.hand.is_none() { + self.hand = Some(node); } } - fn schedule_evictions(&mut self, capacity: usize, state_blocks: &StateBlocks) { + /// Selects residents above `capacity` for deferred eviction. + fn schedule_excess_evictions(&mut self, capacity: usize, state_pages: &StatePages) { while self.residents.len() > capacity { - self.schedule_eviction(state_blocks); + self.schedule_eviction(state_pages); } } - fn schedule_eviction(&mut self, state_blocks: &StateBlocks) { - let victim = self - .select_victim(state_blocks) - .expect("non-empty resident list should have an eviction candidate"); - + fn schedule_eviction(&mut self, state_pages: &StatePages) { + let victim = self.select_victim(state_pages); self.mark_pending(victim); } + /// Adds a victim to its page batch, queuing that page at most once. + /// + /// `StatePage::pending_page_index` is the reverse mapping that turns page + /// deduplication into a direct lookup instead of a queue scan. fn mark_pending(&mut self, victim: SelectedVictim<'_>) { - let queue_index = victim.block.pending_block_index().unwrap_or_else(|| { - let state_block = state_block_index_and_slot(victim.id).0; - let queue_index = self.pending_blocks.len(); - self.pending_blocks.push(PendingBlock::new(state_block)); - victim.block.set_pending_block_index(queue_index); + let queue_index = victim.page.pending_page_index().unwrap_or_else(|| { + let page_number = state_page_number_and_offset(victim.id).0; + let queue_index = self.pending_pages.len(); + self.pending_pages.push(PendingPage::new(page_number)); + victim.page.set_pending_page_index(queue_index); queue_index }); - let pending_block = &mut self.pending_blocks[queue_index]; + let pending_page = &mut self.pending_pages[queue_index]; debug_assert_eq!( - pending_block.state_block, - state_block_index_and_slot(victim.id).0 + pending_page.page_number, + state_page_number_and_offset(victim.id).0 ); - pending_block.mark(victim.slot); + pending_page.mark(victim.offset); } /// Selects and removes the next resident using a bounded SIEVE scan. @@ -210,245 +384,396 @@ impl State { /// The scan performs at most two inspections per current resident. The /// second pass preserves second chances for concurrent re-marks; exhausting /// both passes force-evicts at the hand to guarantee progress. - fn select_victim<'a>(&mut self, state_blocks: &'a StateBlocks) -> Option> { - if self.hand == 0 { - return None; - } + /// + /// # Panics + /// + /// Panics if `self.hand` is `None`, which indicates that the resident list + /// is empty. + fn select_victim<'a>(&mut self, state_pages: &'a StatePages) -> SelectedVictim<'a> { + let mut hand = self + .hand + .expect("non-empty resident list should have a hand"); let inspection_budget = self.residents.len().saturating_mul(2); for _ in 0..inspection_budget { - let index = self.hand; - let id = self.residents.id(index); - let (block, slot) = block_and_slot(state_blocks, id); - - if block.select(slot).was_visited() { - self.hand = self.residents.advance_towards_front(index); + let id = self.residents.id(hand); + let (page, offset) = state_pages.page_and_offset(id); + + if page.select(offset).was_visited() { + // Give a visited resident a second chance. Selection cleared + // its visited bit but left it resident, so inspect the next + // newer candidate. + hand = self.residents.wrapping_prev(hand); } else { - return Some(self.remove_victim(index, block, slot)); + // Selection made this unvisited candidate non-resident. Remove + // it from admission order and return it for pending eviction. + return self.remove_victim(hand, page, offset); } } // Every inspected resident was re-marked. Evict at the hand to // preserve SIEVE's scan order while guaranteeing progress. - let index = self.hand; - let id = self.residents.id(index); - let (block, slot) = block_and_slot(state_blocks, id); - block.clear_resident(slot); - Some(self.remove_victim(index, block, slot)) + let id = self.residents.id(hand); + let (page, offset) = state_pages.page_and_offset(id); + page.clear_resident(offset); + self.remove_victim(hand, page, offset) } fn remove_victim<'a>( &mut self, index: ResidentIndex, - block: &'a StateBlock, - slot: usize, + page: &'a StatePage, + offset: StatePageOffset, ) -> SelectedVictim<'a> { - self.hand = self.residents.hand_after_remove(index); - SelectedVictim { - id: self.residents.remove(index), - block, - slot, - } + let next_hand = if self.residents.len() == 1 { + None + } else { + Some(self.residents.wrapping_prev(index)) + }; + let id = self.residents.remove(index); + self.hand = next_hand; + + SelectedVictim { id, page, offset } } } -impl PendingBlock { +/// A victim removed from [`Residents`] and atomically marked non-resident. +/// +/// The page reference and offset are carried out of selection so queuing the +/// victim does not repeat the sparse-directory lookup. +struct SelectedVictim<'a> { + /// Full id retained by the resident list. + id: Id, + + /// The already-resolved location avoids another state-directory lookup when the + /// victim is added to the pending-eviction queue. + page: &'a StatePage, + + /// Offset of `id.index()` within `page`. + offset: StatePageOffset, +} + +/// Number of ids represented by one pending-eviction word. +const SLOTS_PER_PENDING_WORD: usize = u64::BITS as usize; +const PENDING_WORDS: usize = STATE_PAGE_LEN / SLOTS_PER_PENDING_WORD; + +/// A transient batch of selected victims from one [`StatePage`]. +/// +/// `page_number` identifies both the state page and the high bits of every +/// pending id. `pending` stores the low page-offset bits. An id can be resident +/// and pending simultaneously after re-admission; the drain checks the +/// resident bitmap and skips that stale pending entry. +struct PendingPage { + /// State-page number shared by every slot in `pending`. + page_number: StatePageNumber, + /// One bit per slot selected from this state page. + pending: [u64; PENDING_WORDS], +} + +impl PendingPage { #[inline] - fn new(state_block: StateBlockIndex) -> Self { + fn new(page_number: StatePageNumber) -> Self { Self { - state_block, + page_number, pending: [0; PENDING_WORDS], } } #[inline] - fn mark(&mut self, slot: usize) { - let (word, pending) = pending_state(slot); - self.pending[word] |= pending; + fn mark(&mut self, offset: StatePageOffset) { + let (word, pending_mask) = offset.pending_word(); + self.pending[word] |= pending_mask; } - fn for_each_pending_slot(&self, mut f: impl FnMut(usize)) { + /// Visits only set bits, in ascending page-offset order. + /// + /// This intentionally uses a callback instead of an iterator. An iterator + /// must preserve the current word and word index across calls to `next`; + /// measured iterator versions increased `fill_and_evict` instructions. + fn for_each_pending_offset(&self, mut f: impl FnMut(StatePageOffset)) { for (word_index, &pending) in self.pending.iter().enumerate() { let mut pending = pending; while pending != 0 { let bit = pending.trailing_zeros() as usize; - f(word_index * SLOTS_PER_PENDING_WORD + bit); + f(StatePageOffset::new( + word_index * SLOTS_PER_PENDING_WORD + bit, + )); pending &= pending - 1; } } } - - #[cfg(test)] - fn contains(&self, slot: usize) -> bool { - let (word, pending) = pending_state(slot); - self.pending[word] & pending != 0 - } } +/// Index into [`Residents::nodes`]; zero is always the sentinel. type ResidentIndex = u32; +/// Intrusive circular list that owns the full ids of current residents. +/// +/// `nodes[0]` is a permanent sentinel. Its `next` points to the newest +/// resident (the front) and its `prev` points to the oldest (the back). For a +/// resident node, `prev` points toward newer entries and `next` toward older +/// entries. SIEVE's hand walks through `prev`, wrapping from the newest entry +/// back to the oldest. +/// +/// Removing a node clears its id and links it into an intrusive free list via +/// `next`. This avoids a second allocation-prone vector and lets later +/// admissions reuse storage. Consequently `nodes` grows to the peak number of +/// simultaneous residents plus the sentinel, rather than with total admissions. +/// Its backing allocation is eagerly reserved for the configured capacity and +/// retained across nonzero capacity changes. +/// +/// There is deliberately no map from id to resident node. Hits never move list +/// nodes, and victim selection already has the node index stored in the hand; +/// direct id lookup is needed only for the bits in [`StatePage`]. struct Residents { nodes: Vec, /// Intrusive free list linked through `Resident::next`. - free_head: ResidentIndex, + free_list_head: ResidentIndex, len: usize, } -struct Resident { - id: Option, - /// Newer resident, or the sentinel. - prev: ResidentIndex, - /// Older resident, or the sentinel. - next: ResidentIndex, -} - impl Default for Residents { fn default() -> Self { + Self::with_capacity(0) + } +} + +impl Residents { + /// Permanent list sentinel and empty free-list marker. + const SENTINEL: ResidentIndex = 0; + + fn with_capacity(capacity: usize) -> Self { + let mut nodes = Vec::with_capacity(Self::capacity_with_sentinel(capacity)); + nodes.push(Resident { + id: None, + prev: Self::SENTINEL, + next: Self::SENTINEL, + }); + Self { - nodes: vec![Resident { - id: None, - prev: 0, - next: 0, - }], - free_head: 0, + nodes, + free_list_head: Self::SENTINEL, len: 0, } } -} -impl Residents { fn len(&self) -> usize { self.len } fn push_front(&mut self, id: Id) -> ResidentIndex { let index = self.alloc_node(id); - let first = self.node(0).next; + let first = self[Self::SENTINEL].next; - self.node_mut(index).prev = 0; - self.node_mut(index).next = first; - self.node_mut(0).next = index; - self.node_mut(first).prev = index; + let node = &mut self[index]; + node.prev = Self::SENTINEL; + node.next = first; + self[Self::SENTINEL].next = index; + self[first].prev = index; self.len += 1; index } fn remove(&mut self, index: ResidentIndex) -> Id { - debug_assert_ne!(index, 0); + debug_assert_ne!(index, Self::SENTINEL); - let prev = self.node(index).prev; - let next = self.node(index).next; - self.node_mut(prev).next = next; - self.node_mut(next).prev = prev; - self.len -= 1; - - let free_head = self.free_head; - let node = self.node_mut(index); + let free_list_head = self.free_list_head; + let node = &mut self[index]; + let prev = node.prev; + let next = node.next; let id = node.id.take().expect("resident node should have an id"); - node.prev = 0; - node.next = free_head; - self.free_head = index; + node.prev = Self::SENTINEL; + node.next = free_list_head; + + self[prev].next = next; + self[next].prev = prev; + self.len -= 1; + self.free_list_head = index; id } fn id(&self, index: ResidentIndex) -> Id { - self.node(index) - .id - .expect("resident node should have an id") + self[index].id.expect("resident node should have an id") } - fn advance_towards_front(&self, index: ResidentIndex) -> ResidentIndex { - debug_assert_ne!(index, 0); + /// Returns `index.prev`, wrapping past the sentinel to the list's tail. + fn wrapping_prev(&self, index: ResidentIndex) -> ResidentIndex { + debug_assert_ne!(index, Self::SENTINEL); - let prev = self.node(index).prev; - if prev == 0 { self.node(0).prev } else { prev } - } - - fn hand_after_remove(&self, index: ResidentIndex) -> ResidentIndex { - debug_assert_ne!(index, 0); - - if self.len == 1 { - return 0; - } - - let prev = self.node(index).prev; - if prev == 0 { self.node(0).prev } else { prev } - } - - #[cfg(test)] - fn resident_ids(&self) -> ResidentIds<'_> { - ResidentIds { - residents: self, - next: self.node(0).next, + let prev = self[index].prev; + if prev == Self::SENTINEL { + self[Self::SENTINEL].prev + } else { + prev } } fn alloc_node(&mut self, id: Id) -> ResidentIndex { - if self.free_head != 0 { - let index = self.free_head; - self.free_head = self.node(index).next; - self.node_mut(index).id = Some(id); + // Reuse a tombstoned node from the intrusive free list. + if self.free_list_head != Self::SENTINEL { + let index = self.free_list_head; + let node = &mut self[index]; + let next = node.next; + node.id = Some(id); + self.free_list_head = next; index } else { + // Append storage for a new resident node. let index = ResidentIndex::try_from(self.nodes.len()) .expect("SIEVE resident capacity should fit in u32"); self.nodes.push(Resident { id: Some(id), - prev: 0, - next: 0, + prev: Self::SENTINEL, + next: Self::SENTINEL, }); index } } - fn node(&self, index: ResidentIndex) -> &Resident { + fn reserve_capacity(&mut self, capacity: usize) { + let node_capacity = Self::capacity_with_sentinel(capacity); + if self.nodes.capacity() < node_capacity { + self.nodes.reserve_exact(node_capacity - self.nodes.len()); + } + } + + fn capacity_with_sentinel(resident_capacity: usize) -> usize { + resident_capacity + .checked_add(1) + .expect("SIEVE resident capacity should leave room for the sentinel") + } +} + +impl std::ops::Index for Residents { + type Output = Resident; + + fn index(&self, index: ResidentIndex) -> &Self::Output { &self.nodes[index as usize] } +} - fn node_mut(&mut self, index: ResidentIndex) -> &mut Resident { +impl std::ops::IndexMut for Residents { + fn index_mut(&mut self, index: ResidentIndex) -> &mut Self::Output { &mut self.nodes[index as usize] } } -#[cfg(test)] -struct ResidentIds<'a> { - residents: &'a Residents, +/// One allocated slot in the resident list or its free list. +struct Resident { + /// Full id while resident; `None` for the sentinel and free nodes. + id: Option, + /// Newer resident, or the sentinel. + prev: ResidentIndex, + /// Older resident, or the sentinel. next: ResidentIndex, } -#[cfg(test)] -impl Iterator for ResidentIds<'_> { - type Item = Id; +/// Number of low id-index bits used as the offset within a state page. +const STATE_PAGE_LEN_BITS: usize = 10; +/// Number of table slots represented by one independently allocated state page. +const STATE_PAGE_LEN: usize = 1 << STATE_PAGE_LEN_BITS; - fn next(&mut self) -> Option { - if self.next == 0 { - return None; - } +/// Number of geometric buckets needed to cover every possible state-page number. +/// +/// Boxcar skips its first small buckets, so [`buckets_for_index_bits`] may not +/// accept every index within the requested bit width. The additional bucket +/// covers the otherwise missing final indices. +const STATE_PAGE_BUCKETS: usize = + buckets_for_index_bits(u32::BITS - STATE_PAGE_LEN_BITS as u32) + 1; + +/// State-page number formed from the high bits of [`Id::index`]. +type StatePageNumber = Index; - let index = self.next; - self.next = self.residents.node(index).next; - Some(self.residents.id(index)) +/// Sparse, direct-addressed directory from [`StatePageNumber`] to +/// [`StatePageCell`]. +/// +/// The directory uses geometrically growing buckets. It allocates those +/// buckets and the [`StatePage`] owned by each cell lazily. Once published, an +/// allocation is neither moved nor freed until the directory is dropped, so +/// lock-free readers can safely retain references to it. +struct StatePages(Buckets); + +impl StatePages { + fn new() -> Self { + Self(Buckets::new()) + } + + /// Returns the allocated state page and offset for `id` without allocating. + /// + /// Returns `None` if no state page has been allocated for the range + /// containing `id`. + #[inline] + fn try_page_and_offset(&self, id: Id) -> Option<(&StatePage, StatePageOffset)> { + let (page_number, offset) = state_page_number_and_offset(id); + self.get(page_number).map(|page| (page, offset)) + } + + /// Returns the state page and offset for `id`, allocating the page if needed. + /// + /// Concurrent callers may race to allocate the same page; all receive the + /// single allocation published by its [`StatePageCell`]. + /// + /// # Panics + /// + /// May panic if allocating the directory bucket or state page fails. + fn page_and_offset_or_alloc(&self, id: Id) -> (&StatePage, StatePageOffset) { + let (page_number, offset) = state_page_number_and_offset(id); + (self.0.get_or_alloc(page_number).get_or_alloc(), offset) + } + + /// Returns the already-allocated state page and offset for `id`. + /// + /// # Panics + /// + /// Panics if no state page has been allocated for the range containing + /// `id`. Resident and pending ids always satisfy this requirement. + fn page_and_offset(&self, id: Id) -> (&StatePage, StatePageOffset) { + let (page_number, offset) = state_page_number_and_offset(id); + (&self[page_number], offset) + } + + #[inline] + fn get(&self, page_number: StatePageNumber) -> Option<&StatePage> { + self.0.get(page_number).and_then(StatePageCell::get) + } +} + +impl std::ops::Index for StatePages { + type Output = StatePage; + + fn index(&self, page_number: StatePageNumber) -> &Self::Output { + self.get(page_number) + .expect("SIEVE state page should be allocated") } } +/// Lazily initialized owner of one [`StatePage`] allocation. +/// +/// The surrounding [`StatePages`] directory allocates cells in buckets, but +/// each comparatively large state page is allocated only when an id in its +/// range first reaches the admission path. The pointer is installed once and +/// remains valid until the directory is exclusively dropped. A sparse, high id +/// can therefore reserve pointer slots for its directory bucket, but does not +/// allocate `StatePage`s for untouched entries in that bucket. #[derive(Default)] -struct BlockSlot { - ptr: AtomicPtr, +struct StatePageCell { + /// Null before initialization; otherwise owns the published allocation. + ptr: AtomicPtr, } -// SAFETY: Outside Shuttle, `BlockSlot` contains an atomic pointer whose +// SAFETY: Outside Shuttle, `StatePageCell` contains an atomic pointer whose // all-zero representation is a valid null pointer. Shuttle atomics require // construction. -unsafe impl MaybeZeroable for BlockSlot { +unsafe impl MaybeZeroable for StatePageCell { fn zeroable() -> bool { cfg!(not(feature = "shuttle")) } } -impl BlockSlot { +impl StatePageCell { #[inline] - fn get(&self) -> Option<&StateBlock> { + fn get(&self) -> Option<&StatePage> { let ptr = self.ptr.load(Ordering::Acquire); ptr::NonNull::new(ptr).map(|ptr| { // SAFETY: A non-null pointer was allocated by `get_or_alloc` and @@ -457,26 +782,26 @@ impl BlockSlot { }) } - fn get_or_alloc(&self) -> &StateBlock { - if let Some(block) = self.get() { - return block; + fn get_or_alloc(&self) -> &StatePage { + if let Some(page) = self.get() { + return page; } - let new_block = Box::into_raw(Box::new(StateBlock::default())); + let new_page = Box::into_raw(Box::new(StatePage::default())); match self.ptr.compare_exchange( ptr::null_mut(), - new_block, + new_page, Ordering::Release, Ordering::Acquire, ) { Ok(_) => { // SAFETY: We just installed this allocation. - unsafe { &*new_block } + unsafe { &*new_page } } Err(existing) => { // SAFETY: This allocation was not published, so this thread // still owns it. - unsafe { drop(Box::from_raw(new_block)) }; + unsafe { drop(Box::from_raw(new_page)) }; // SAFETY: `existing` was installed by another thread and // remains owned by this slot. @@ -486,7 +811,7 @@ impl BlockSlot { } } -impl Drop for BlockSlot { +impl Drop for StatePageCell { fn drop(&mut self) { let ptr = *self.ptr.get_mut(); if !ptr.is_null() { @@ -497,35 +822,62 @@ impl Drop for BlockSlot { } } +/// Resident and visited bits stored for each id in a [`StatePage`]. +const SLOT_STATE_BITS: usize = 2; +/// Number of ids represented by one resident/visited word. +const SLOTS_PER_STATE_WORD: usize = u64::BITS as usize / SLOT_STATE_BITS; +const STATE_WORDS: usize = STATE_PAGE_LEN / SLOTS_PER_STATE_WORD; + +/// Compact replacement-policy state for [`STATE_PAGE_LEN`] consecutive table slots. +/// +/// Each slot has adjacent `resident` and `visited` bits. Keeping both bits in +/// one atomic word makes every transition indivisible with respect to a racing +/// hit: either the hit marks the still-resident value, or it observes that +/// selection removed the value and falls back to admission. #[derive(Default)] -struct StateBlock { - /// Interleaved resident and visited bits for all slots in the block. +struct StatePage { + /// Interleaved `[resident, visited]` bit pairs for all slots in the page. + /// + /// The only valid stable states are absent (`00`), resident/unvisited + /// (`01`), and resident/visited (`11`). state: [AtomicU64; STATE_WORDS], - /// One-based index into `State::pending_blocks`, or zero when not queued. - /// Access is serialized by the state mutex; the atomic preserves interior - /// mutability without making blocks unavailable to concurrent hit readers. - pending_block_index: AtomicU32, + /// One-based index into `State::pending_pages`, or zero when not queued. + /// Mutation is serialized by the state mutex or exclusive access to the + /// policy; the atomic preserves interior mutability without making pages + /// unavailable to concurrent hit readers. + pending_page_index: AtomicU32, } -impl StateBlock { +impl StatePage { + /// Marks a resident as visited. + /// + /// Returns `true` if the slot was resident (including when it was already + /// visited), allowing the hit path to return without locking. Returns + /// `false` if selection won the race and the caller must attempt admission. #[inline] - fn record_use(&self, slot: usize) -> bool { - let (word, resident, visited) = slot_state(slot); + fn record_use(&self, offset: StatePageOffset) -> bool { + let (word, resident_mask, visited_mask) = offset.state_word(); let word = &self.state[word]; let mut state = word.load(Ordering::Relaxed); loop { - if state & visited != 0 { + // Already marked visited. `visited` implies `resident`, so the hit + // has been recorded. + if state & visited_mask != 0 { return true; } - if state & resident == 0 { + // Not resident. The caller must return to the admission path. + if state & resident_mask == 0 { return false; } + // Mark the resident visited if the word still matches this + // snapshot. Otherwise, recheck whether another hit marked it or + // selection removed it. match word.compare_exchange_weak( state, - state | visited, + state | visited_mask, Ordering::Relaxed, Ordering::Relaxed, ) { @@ -535,13 +887,15 @@ impl StateBlock { } } - fn admit(&self, slot: usize) { - let (word, resident, visited) = slot_state(slot); + /// Marks a slot resident and unvisited, including a pending victim being + /// re-admitted before its memo value is deleted. + fn admit(&self, offset: StatePageOffset) { + let (word, resident_mask, visited_mask) = offset.state_word(); let word = &self.state[word]; let mut state = word.load(Ordering::Relaxed); loop { - let new_state = (state | resident) & !visited; + let new_state = (state | resident_mask) & !visited_mask; match word.compare_exchange_weak(state, new_state, Ordering::Relaxed, Ordering::Relaxed) { Ok(_) => return, @@ -550,22 +904,27 @@ impl StateBlock { } } - fn select(&self, slot: usize) -> Selection { - let (word, resident, visited) = slot_state(slot); + /// Atomically grants a second chance or makes an unvisited slot + /// non-resident and eligible for pending eviction. + fn select(&self, offset: StatePageOffset) -> Selection { + let (word, resident_mask, visited_mask) = offset.state_word(); let word = &self.state[word]; let mut state = word.load(Ordering::Relaxed); loop { debug_assert_ne!( - state & resident, + state & resident_mask, 0, "resident list entry should have a resident state bit" ); - let (new_state, selection) = if state & visited == 0 { - (state & !(resident | visited), Selection::Evict) + let (new_state, selection) = if state & visited_mask == 0 { + // Unvisited: remove it from the resident set and evict it. + (state & !(resident_mask | visited_mask), Selection::Evict) } else { - (state & !visited, Selection::SecondChance) + // Visited: keep it resident, clear the mark, and grant a + // second chance. + (state & !visited_mask, Selection::SecondChance) }; match word.compare_exchange_weak(state, new_state, Ordering::Relaxed, Ordering::Relaxed) @@ -576,37 +935,77 @@ impl StateBlock { } } - fn is_resident(&self, slot: usize) -> bool { - let (word, resident, _) = slot_state(slot); - self.state[word].load(Ordering::Relaxed) & resident != 0 + /// Tests whether a pending victim has since been re-admitted. + fn is_resident(&self, offset: StatePageOffset) -> bool { + let (word, resident_mask, _) = offset.state_word(); + self.state[word].load(Ordering::Relaxed) & resident_mask != 0 } #[inline] - fn pending_block_index(&self) -> Option { - match self.pending_block_index.load(Ordering::Relaxed) { + fn pending_page_index(&self) -> Option { + match self.pending_page_index.load(Ordering::Relaxed) { 0 => None, index => Some((index - 1) as usize), } } #[inline] - fn set_pending_block_index(&self, index: usize) { - debug_assert!(self.pending_block_index().is_none()); - let index = u32::try_from(index + 1).expect("pending block index should fit in u32"); - self.pending_block_index.store(index, Ordering::Relaxed); + fn set_pending_page_index(&self, index: usize) { + debug_assert!(self.pending_page_index().is_none()); + let index = u32::try_from(index + 1).expect("pending page index should fit in u32"); + self.pending_page_index.store(index, Ordering::Relaxed); } - fn clear_pending_block_index(&self, index: usize) { - debug_assert_eq!(self.pending_block_index(), Some(index)); - self.pending_block_index.store(0, Ordering::Relaxed); + fn clear_pending_page_index(&self, index: usize) { + debug_assert_eq!(self.pending_page_index(), Some(index)); + self.pending_page_index.store(0, Ordering::Relaxed); } - fn clear_resident(&self, slot: usize) { - let (word, resident, visited) = slot_state(slot); - self.state[word].fetch_and(!(resident | visited), Ordering::Relaxed); + /// Atomically marks the slot at `offset` absent by clearing its resident and + /// visited bits. + fn clear_resident(&self, offset: StatePageOffset) { + let (word, resident_mask, visited_mask) = offset.state_word(); + self.state[word].fetch_and(!(resident_mask | visited_mask), Ordering::Relaxed); } } +/// Offset of one table slot within a [`StatePage`]. +/// +/// Values are in `0..STATE_PAGE_LEN`. Keeping the offset distinct from other +/// indexes prevents accidentally using a resident, pending-queue, or table-page +/// index to access the state-page bitmaps. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +struct StatePageOffset(usize); + +impl StatePageOffset { + const fn new(offset: usize) -> Self { + debug_assert!(offset < STATE_PAGE_LEN); + Self(offset) + } + + const fn get(self) -> usize { + self.0 + } + + /// Returns the word index, resident mask, and visited mask in + /// [`StatePage::state`]. + const fn state_word(self) -> (usize, u64, u64) { + let offset = self.get(); + let bit = (offset % SLOTS_PER_STATE_WORD) * SLOT_STATE_BITS; + let resident_mask = 1 << bit; + let visited_mask = 1 << (bit + 1); + (offset / SLOTS_PER_STATE_WORD, resident_mask, visited_mask) + } + + /// Returns the word index and one-bit mask in [`PendingPage::pending`]. + const fn pending_word(self) -> (usize, u64) { + let offset = self.get(); + let bit = offset % SLOTS_PER_PENDING_WORD; + (offset / SLOTS_PER_PENDING_WORD, 1 << bit) + } +} + +/// Result of inspecting one resident at the SIEVE hand. #[derive(Copy, Clone)] enum Selection { SecondChance, @@ -614,191 +1013,171 @@ enum Selection { } impl Selection { - fn was_visited(self) -> bool { + const fn was_visited(self) -> bool { matches!(self, Selection::SecondChance) } } -fn block_and_slot_or_alloc(state_blocks: &StateBlocks, id: Id) -> (&StateBlock, usize) { - let (index, slot) = state_block_index_and_slot(id); - (state_blocks.get_or_alloc(index).get_or_alloc(), slot) -} - -fn block_and_slot(state_blocks: &StateBlocks, id: Id) -> (&StateBlock, usize) { - // Resident and pending ids must already have allocated state blocks. - let (index, slot) = state_block_index_and_slot(id); - (state_block(state_blocks, index), slot) -} - -#[inline] -fn state_block(state_blocks: &StateBlocks, index: StateBlockIndex) -> &StateBlock { - state_blocks - .get(index) - .and_then(BlockSlot::get) - .expect("SIEVE state block should be allocated") -} - -#[inline] -fn block_and_slot_if_allocated(state_blocks: &StateBlocks, id: Id) -> Option<(&StateBlock, usize)> { - let (index, slot) = state_block_index_and_slot(id); - state_blocks - .get(index) - .and_then(BlockSlot::get) - .map(|block| (block, slot)) -} - +/// Splits the table-slot index of an id into a state-page number and offset. +/// +/// SIEVE state follows the physical table slot, so the generation component of +/// the id is intentionally not part of this mapping. #[inline] -fn state_block_index_and_slot(id: Id) -> (StateBlockIndex, usize) { +const fn state_page_number_and_offset(id: Id) -> (StatePageNumber, StatePageOffset) { let index = id.index() as usize; - let block = index >> BLOCK_LEN_BITS; - let slot = index & (BLOCK_LEN - 1); - // SAFETY: `Id` is a `u32`, and `StateBlocks` has enough buckets for every - // block representable by an `Id` after removing the slot bits. - let index = unsafe { StateBlockIndex::new_unchecked(block) }; - (index, slot) + let page_number = index >> STATE_PAGE_LEN_BITS; + let offset = StatePageOffset::new(index & (STATE_PAGE_LEN - 1)); + let page_number = StatePageNumber::new(page_number) + .expect("state-page directory should cover every page in a u32 id"); + (page_number, offset) } -fn state_block_id(index: StateBlockIndex, slot: usize) -> Id { - debug_assert!(slot < BLOCK_LEN); - let id = (index.get() << BLOCK_LEN_BITS) | slot; - // SAFETY: `StateBlockIndex` covers exactly the block bits of a `u32` id and - // `slot` is restricted to the remaining low bits. +/// Reconstructs the physical table-slot id used by memo eviction. +/// +/// Memo lookup uses only [`Id::index`]; the generation is irrelevant here. +const fn slot_id(page_number: StatePageNumber, offset: StatePageOffset) -> Id { + let id = (page_number.get() << STATE_PAGE_LEN_BITS) | offset.get(); + // SAFETY: Callers recombine a page number and offset originally produced by + // `state_page_number_and_offset` for an existing id. unsafe { Id::from_index(id as u32) } } -fn slot_state(slot: usize) -> (usize, u64, u64) { - debug_assert!(slot < BLOCK_LEN); - let bit = (slot % SLOTS_PER_STATE_WORD) * SLOT_STATE_BITS; - let resident = 1 << bit; - let visited = 1 << (bit + 1); - (slot / SLOTS_PER_STATE_WORD, resident, visited) -} - -fn pending_state(slot: usize) -> (usize, u64) { - debug_assert!(slot < BLOCK_LEN); - let bit = slot % SLOTS_PER_PENDING_WORD; - (slot / SLOTS_PER_PENDING_WORD, 1 << bit) -} - #[cfg(test)] mod tests { use super::*; fn id(index: u32) -> Id { - // SAFETY: Test indices are within `Id`'s valid range. - unsafe { Id::from_index(index) } + assert!(index < Id::MAX_U32); + Id::from_bits(u64::from(index) + 1) + } + + fn drain_evicted(sieve: &mut Sieve) -> Vec { + let mut evicted = Vec::new(); + sieve.for_each_evicted(|id| evicted.push(id)); + evicted } #[test] fn all_marked_residents_evict_the_initial_hand() { - let state_blocks = StateBlocks::new(); - let mut state = State::default(); + let mut sieve = Sieve::new(3); let oldest = id(0); let middle = id(1); let newest = id(2); for id in [oldest, middle, newest] { - let (block, slot) = block_and_slot_or_alloc(&state_blocks, id); - state.insert(id, block, slot, 3, &state_blocks); - assert!(block.record_use(slot)); + // Admission starts unvisited; the second use marks the resident. + sieve.record_use(id); + sieve.record_use(id); } - assert_eq!( - state.select_victim(&state_blocks).map(|victim| victim.id), - Some(oldest) - ); - assert_eq!(state.residents.id(state.hand), middle); - assert_eq!( - state.residents.resident_ids().collect::>(), - [newest, middle] - ); - let (block, slot) = block_and_slot(&state_blocks, oldest); - assert!(!block.is_resident(slot)); + sieve.set_capacity(2); + assert_eq!(drain_evicted(&mut sieve), [oldest]); + + // The hand resumes with the resident after the first victim. + sieve.set_capacity(1); + assert_eq!(drain_evicted(&mut sieve), [middle]); } #[test] fn insertion_selects_a_victim_before_admitting() { - let state_blocks = StateBlocks::new(); - let mut state = State::default(); + let mut sieve = Sieve::new(2); let oldest = id(0); let newest = id(1); let incoming = id(2); for id in [oldest, newest] { - let (block, slot) = block_and_slot_or_alloc(&state_blocks, id); - state.insert(id, block, slot, 2, &state_blocks); - assert!(block.record_use(slot)); + // Ensure the existing residents receive a second chance while the + // incoming, initially unvisited resident does not participate. + sieve.record_use(id); + sieve.record_use(id); } - let (block, incoming_slot) = block_and_slot_or_alloc(&state_blocks, incoming); - state.insert(incoming, block, incoming_slot, 2, &state_blocks); + sieve.record_use(incoming); + assert_eq!(drain_evicted(&mut sieve), [oldest]); + } - let (state_block, oldest_slot) = state_block_index_and_slot(oldest); - assert_eq!(state.pending_blocks.len(), 1); - assert_eq!(state.pending_blocks[0].state_block, state_block); - assert!(state.pending_blocks[0].contains(oldest_slot)); - assert_eq!(state.residents.nodes.len(), 3); - assert!(block.is_resident(incoming_slot)); + #[test] + fn capacity_changes_preserve_policy_state_and_evict_all_excess() { + let mut sieve = Sieve::new(5); + for index in 0..5 { + sieve.record_use(id(index)); + } + + sieve.set_capacity(2); + sieve.set_capacity(8); + assert_eq!(drain_evicted(&mut sieve), [id(0), id(1), id(2)]); + + for index in 5..11 { + sieve.record_use(id(index)); + } + assert!(drain_evicted(&mut sieve).is_empty()); + + sieve.record_use(id(11)); + assert_eq!(drain_evicted(&mut sieve), [id(3)]); } #[test] - fn disabling_discards_state_blocks() { + fn disabling_discards_residents_and_ignores_uses() { let mut sieve = Sieve::new(1); let resident = id(0); + let ignored_while_disabled = id(1); + let incoming = id(2); sieve.record_use(resident); - assert!(block_and_slot_if_allocated(&sieve.state_blocks, resident).is_some()); - - sieve.set_tuning(0); - assert!(block_and_slot_if_allocated(&sieve.state_blocks, resident).is_none()); - assert_eq!(sieve.state.get_mut().residents.len(), 0); + sieve.set_capacity(0); + sieve.record_use(ignored_while_disabled); + assert!(drain_evicted(&mut sieve).is_empty()); + sieve.set_capacity(1); sieve.record_use(resident); - assert!(block_and_slot_if_allocated(&sieve.state_blocks, resident).is_none()); + sieve.record_use(incoming); + assert_eq!(drain_evicted(&mut sieve), [resident]); } #[test] - fn pending_evictions_are_deduplicated_until_reset() { + fn repeated_readmission_reports_each_nonresident_once() { let mut sieve = Sieve::new(1); let first = id(0); let second = id(1); - for _ in 0..100 { + for _ in 0..2 { sieve.record_use(first); sieve.record_use(second); } - assert_eq!(sieve.state.lock().pending_blocks.len(), 1); - let (block, _) = block_and_slot(&sieve.state_blocks, first); - assert_eq!(block.pending_block_index(), Some(0)); - - let mut evicted = Vec::new(); - sieve.for_each_evicted(|id| evicted.push(id)); - - assert_eq!(evicted, [first]); - assert!(sieve.state.get_mut().pending_blocks.is_empty()); - let (block, _) = block_and_slot(&sieve.state_blocks, first); - assert_eq!(block.pending_block_index(), None); + assert_eq!(drain_evicted(&mut sieve), [first]); sieve.record_use(first); - assert_eq!(sieve.state.lock().pending_blocks.len(), 1); + assert_eq!(drain_evicted(&mut sieve), [second]); } #[test] - fn pending_evictions_cross_state_blocks() { + fn evictions_across_state_pages_are_all_reported() { let mut sieve = Sieve::new(1); let first = id(0); - let second = id(BLOCK_LEN as u32); + let second = id(STATE_PAGE_LEN as u32); let third = id(1); - let resident = id(BLOCK_LEN as u32 + 1); + let resident = id(STATE_PAGE_LEN as u32 + 1); for id in [first, second, third, resident] { sieve.record_use(id); } - let mut evicted = Vec::new(); - sieve.for_each_evicted(|id| evicted.push(id)); - + let mut evicted = drain_evicted(&mut sieve); + evicted.sort(); assert_eq!(evicted, [first, third, second]); } + + #[test] + fn state_page_number_covers_largest_id() { + let id = id(Id::MAX_U32 - 1); + let (page_number, offset) = state_page_number_and_offset(id); + + assert_eq!( + page_number.get(), + id.index() as usize >> STATE_PAGE_LEN_BITS + ); + assert_eq!(offset.get(), id.index() as usize & (STATE_PAGE_LEN - 1)); + assert_eq!(slot_id(page_number, offset), id); + } } diff --git a/src/lib.rs b/src/lib.rs index f0703c2be..d1ef7ef3e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -160,9 +160,9 @@ //! //! A memo stores one current result, not a history. By default, each key that remains in the //! database retains its result. Re-execution may update or replace the value; reclaiming the key or -//! dropping the database removes it. The `lru` option additionally evicts least-recently-used -//! results at the start of a new revision, but retains their memo entries so a later call can -//! recompute the value. +//! dropping the database removes it. The `eviction` option bounds retained results using either +//! the recommended SIEVE policy or LRU. Eviction drops result values but retains their memo +//! entries so a later call can recompute the value. //! //! The [return mode](#return-modes) controls whether callers receive an owned result or borrow it //! from the memo. @@ -183,7 +183,7 @@ //! tracked struct, not an input or interned struct. `specify` must be called during the same tracked //! query invocation that created the key. Salsa records the specified memo as an output of that //! creating query, so validating or re-executing the creator also validates or replaces the -//! specified result. The `specify` and `lru` options cannot currently be combined. +//! specified result. The `specify` and `eviction` options cannot currently be combined. //! //! See [specifying query results in the Salsa book] for an example. //! @@ -227,7 +227,7 @@ //! [`'db` database lifetime]: https://salsa-rs.github.io/salsa/plumbing/db_lifetime.html //! [accumulators in the Salsa book]: https://salsa-rs.github.io/salsa/tutorial/accumulators.html //! [backdating]: https://salsa-rs.github.io/salsa/reference/algorithm.html#backdating-sometimes-we-can-be-smarter -//! [cache tuning]: https://salsa-rs.github.io/salsa/tuning.html#cache-eviction-lru +//! [cache tuning]: https://salsa-rs.github.io/salsa/tuning.html#cache-eviction //! [durability reference]: https://salsa-rs.github.io/salsa/reference/durability.html //! [input structs in the Salsa book]: https://salsa-rs.github.io/salsa/overview.html#inputs //! [interned structs in the Salsa book]: https://salsa-rs.github.io/salsa/overview.html#interned-structs diff --git a/src/zalsa.rs b/src/zalsa.rs index 4a79dcab4..53e137f55 100644 --- a/src/zalsa.rs +++ b/src/zalsa.rs @@ -466,9 +466,9 @@ impl Zalsa { /// **NOT SEMVER STABLE** #[doc(hidden)] - pub fn evict_lru(&mut self) { + pub fn apply_eviction(&mut self) { #[cfg(feature = "detailed-trace")] - let _span = crate::tracing::debug_span!("evict_lru").entered(); + let _span = crate::tracing::debug_span!("apply_eviction").entered(); for ingredient in &self.ingredients_requiring_reset { self.ingredients_vec[ingredient.as_u32() as usize] .reset_for_new_revision(self.runtime.table_mut()); diff --git a/tests/compile-fail/eviction_can_not_be_used_with_specify.rs b/tests/compile-fail/eviction_can_not_be_used_with_specify.rs new file mode 100644 index 000000000..2cfbf8c08 --- /dev/null +++ b/tests/compile-fail/eviction_can_not_be_used_with_specify.rs @@ -0,0 +1,12 @@ +#[salsa::input] +struct MyInput { + #[returns(copy)] + field: u32, +} + +#[salsa::tracked(eviction(policy = lru, capacity = 3), specify)] +fn eviction_can_not_be_used_with_specify(db: &dyn salsa::Database, input: MyInput) -> u32 { + input.field(db) +} + +fn main() {} diff --git a/tests/compile-fail/eviction_can_not_be_used_with_specify.stderr b/tests/compile-fail/eviction_can_not_be_used_with_specify.stderr new file mode 100644 index 000000000..aa0658c56 --- /dev/null +++ b/tests/compile-fail/eviction_can_not_be_used_with_specify.stderr @@ -0,0 +1,5 @@ +error: the `specify` and `eviction` options cannot be used together + --> tests/compile-fail/eviction_can_not_be_used_with_specify.rs:7:56 + | +7 | #[salsa::tracked(eviction(policy = lru, capacity = 3), specify)] + | ^^^^^^^ diff --git a/tests/compile-fail/invalid_eviction_options.rs b/tests/compile-fail/invalid_eviction_options.rs new file mode 100644 index 000000000..34814bc75 --- /dev/null +++ b/tests/compile-fail/invalid_eviction_options.rs @@ -0,0 +1,7 @@ +#[salsa::tracked(eviction(policy = sieve))] +fn missing_capacity(_db: &dyn salsa::Database) {} + +#[salsa::tracked(eviction(policy = random, capacity = 128))] +fn unknown_policy(_db: &dyn salsa::Database) {} + +fn main() {} diff --git a/tests/compile-fail/invalid_eviction_options.stderr b/tests/compile-fail/invalid_eviction_options.stderr new file mode 100644 index 000000000..e7f20071b --- /dev/null +++ b/tests/compile-fail/invalid_eviction_options.stderr @@ -0,0 +1,11 @@ +error: unexpected end of input, missing required `capacity` option + --> tests/compile-fail/invalid_eviction_options.rs:1:41 + | +1 | #[salsa::tracked(eviction(policy = sieve))] + | ^ + +error: unknown eviction policy; expected `sieve` or `lru` + --> tests/compile-fail/invalid_eviction_options.rs:4:36 + | +4 | #[salsa::tracked(eviction(policy = random, capacity = 128))] + | ^^^^^^ diff --git a/tests/compile-fail/legacy_eviction_options.rs b/tests/compile-fail/legacy_eviction_options.rs new file mode 100644 index 000000000..fcc8ecaee --- /dev/null +++ b/tests/compile-fail/legacy_eviction_options.rs @@ -0,0 +1,4 @@ +#[salsa::tracked(lru = 4000)] +fn legacy_lru(_db: &dyn salsa::Database) {} + +fn main() {} diff --git a/tests/compile-fail/legacy_eviction_options.stderr b/tests/compile-fail/legacy_eviction_options.stderr new file mode 100644 index 000000000..7191b4a8a --- /dev/null +++ b/tests/compile-fail/legacy_eviction_options.stderr @@ -0,0 +1,5 @@ +error: `lru = ...` has been removed; use `eviction(capacity = ...)` for SIEVE (recommended) or `eviction(policy = lru, capacity = ...)` + --> tests/compile-fail/legacy_eviction_options.rs:1:18 + | +1 | #[salsa::tracked(lru = 4000)] + | ^^^ diff --git a/tests/compile-fail/lru_can_not_be_used_with_specify.rs b/tests/compile-fail/lru_can_not_be_used_with_specify.rs deleted file mode 100644 index 4587f9496..000000000 --- a/tests/compile-fail/lru_can_not_be_used_with_specify.rs +++ /dev/null @@ -1,12 +0,0 @@ -#[salsa::input] -struct MyInput { - #[returns(copy)] - field: u32, -} - -#[salsa::tracked(lru = 3, specify)] -fn lru_can_not_be_used_with_specify(db: &dyn salsa::Database, input: MyInput) -> u32 { - input.field(db) -} - -fn main() {} diff --git a/tests/compile-fail/lru_can_not_be_used_with_specify.stderr b/tests/compile-fail/lru_can_not_be_used_with_specify.stderr deleted file mode 100644 index f6e0d1f23..000000000 --- a/tests/compile-fail/lru_can_not_be_used_with_specify.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: the `specify` and `lru` options cannot be used together - --> tests/compile-fail/lru_can_not_be_used_with_specify.rs:7:27 - | -7 | #[salsa::tracked(lru = 3, specify)] - | ^^^^^^^ diff --git a/tests/compile-pass/tracked-methods.rs b/tests/compile-pass/tracked-methods.rs index 081f80f34..f891df7f9 100644 --- a/tests/compile-pass/tracked-methods.rs +++ b/tests/compile-pass/tracked-methods.rs @@ -17,7 +17,7 @@ impl Things<'_> { todo!() } - #[salsa::tracked(returns(copy))] + #[salsa::tracked(returns(copy), eviction(capacity = 128))] pub fn implicit2(db: &dyn salsa::Database, _: Other<'_>) -> Things<'_> { _ = db; todo!() diff --git a/tests/lru.rs b/tests/lru.rs index 2f9fc0ce6..a5d4f485a 100644 --- a/tests/lru.rs +++ b/tests/lru.rs @@ -1,6 +1,6 @@ #![cfg(feature = "inventory")] -//! Test that a `tracked` fn with lru options +//! Test that a `tracked` fn with the LRU eviction policy //! compiles and executes successfully. use std::sync::Arc; @@ -38,7 +38,7 @@ struct MyInput { field: u32, } -#[salsa::tracked(returns(clone), lru = 8)] +#[salsa::tracked(returns(clone), eviction(policy = lru, capacity = 8))] fn get_hot_potato(db: &dyn LogDatabase, input: MyInput) -> Arc { db.push_log(format!("get_hot_potato({:?})", input.field(db))); Arc::new(HotPotato::new(input.field(db))) @@ -88,7 +88,7 @@ fn lru_can_be_changed_at_runtime() { db.synthetic_write(salsa::Durability::HIGH); assert_eq!(load_n_potatoes(), 8); - get_hot_potato::set_lru_capacity(&mut db, 16); + get_hot_potato::set_eviction_capacity(&mut db, 16); assert_eq!(load_n_potatoes(), 8); for &(i, input) in inputs.iter() { let p = get_hot_potato(&db, input); @@ -101,7 +101,7 @@ fn lru_can_be_changed_at_runtime() { assert_eq!(load_n_potatoes(), 16); // Special case: setting capacity to zero disables LRU - get_hot_potato::set_lru_capacity(&mut db, 0); + get_hot_potato::set_eviction_capacity(&mut db, 0); assert_eq!(load_n_potatoes(), 16); for &(i, input) in inputs.iter() { let p = get_hot_potato(&db, input); @@ -117,6 +117,13 @@ fn lru_can_be_changed_at_runtime() { assert_eq!(load_n_potatoes(), 0); } +#[test] +#[allow(deprecated)] +fn trigger_lru_eviction_is_a_compatible_alias() { + let mut db = common::LoggerDatabase::default(); + db.trigger_lru_eviction(); +} + #[test] fn lru_keeps_dependency_info() { let mut db = common::LoggerDatabase::default(); diff --git a/tests/memory-usage.rs b/tests/memory-usage.rs index 4f9235f56..53719f2bb 100644 --- a/tests/memory-usage.rs +++ b/tests/memory-usage.rs @@ -320,7 +320,7 @@ fn cancellation_does_not_allocate_extra_for_ordinary_memos() { let before = &before.queries["input_to_length"]; assert_eq!(before.count(), 1); - db.trigger_lru_eviction(); + db.trigger_eviction(); assert_eq!(input_to_length(&db, input2), 150); let after = ::memory_usage(&db); diff --git a/tests/never_change.rs b/tests/never_change.rs index 8a08f5079..07f97b903 100644 --- a/tests/never_change.rs +++ b/tests/never_change.rs @@ -30,7 +30,7 @@ fn mixed_input_value(db: &dyn Database, immutable_input: MyInput, mutable_input: immutable_input.value(db) + mutable_input.value(db) } -#[salsa::tracked(returns(copy), lru = 1)] +#[salsa::tracked(returns(copy), eviction(policy = lru, capacity = 1))] #[cfg(not(feature = "persistence"))] fn immutable_value_with_lru(db: &dyn LogDatabase, input: MyInput) -> u32 { db.push_log(format!("immutable_value_with_lru({})", input.value(db))); @@ -57,7 +57,7 @@ fn output(db: &dyn Database, input: MyInput) -> Output<'_> { output } -#[salsa::tracked(returns(copy), lru = 1)] +#[salsa::tracked(returns(copy), eviction(policy = lru, capacity = 1))] fn output_with_lru(db: &dyn Database, input: MyInput) -> Output<'_> { let output = Output::new(db, input.value(db)); specified::specify(db, output, input.value(db) + 1); diff --git a/tests/parallel/lru_eviction_cancels_cycle.rs b/tests/parallel/lru_eviction_cancels_cycle.rs index fdd234f56..f3d5369b3 100644 --- a/tests/parallel/lru_eviction_cancels_cycle.rs +++ b/tests/parallel/lru_eviction_cancels_cycle.rs @@ -36,9 +36,9 @@ fn cycle_initial(_db: &dyn KnobsDatabase, _id: salsa::Id, _input: Input) -> u32 0 } -/// Test that `trigger_lru_eviction` during cycle iteration causes the query to recompute. +/// Test that `trigger_eviction` during cycle iteration causes the query to recompute. #[test] -fn lru_eviction_recomputes_cycle_query() { +fn eviction_recomputes_cycle_query() { let db = Knobs::default(); // Create clones BEFORE setting up signal handlers @@ -52,7 +52,7 @@ fn lru_eviction_recomputes_cycle_query() { // Set up: when cancellation flag is set, signal stage 2 so thread 1 can continue db.signal_on_did_cancel(2); - // Drop the original db so trigger_lru_eviction can complete + // Drop the original db so trigger_eviction can complete // (it waits for all snapshots to be dropped) drop(db); @@ -62,17 +62,17 @@ fn lru_eviction_recomputes_cycle_query() { // Wait for thread 1 to enter the cycle query and signal stage 1 db_waiter.wait_for(1); - // Drop waiter so trigger_lru_eviction can proceed after t1 drops its handle + // Drop waiter so trigger_eviction can proceed after t1 drops its handle drop(db_waiter); - // Thread 2: Trigger LRU eviction, which will: + // Thread 2: Trigger eviction, which will: // 1. Set the cancellation flag (this signals stage 2, letting thread 1 continue) // 2. Wait for thread 1 to drop its snapshot (thread 1 will check cancellation and panic) // 3. NOT bump the revision let t2 = thread::spawn({ let mut db_writer = db_writer; move || { - db_writer.trigger_lru_eviction(); + db_writer.trigger_eviction(); db_writer } }); @@ -159,7 +159,7 @@ fn revision_bump_after_cancellation_recomputes_cycle_query() { // Set up: when cancellation flag is set, signal stage 2 so thread 1 can continue db.signal_on_did_cancel(2); - // Drop the original db so trigger_lru_eviction can complete + // Drop the original db so trigger_eviction can complete drop(db); // Thread 1: Start a cycle query that will be interrupted @@ -168,14 +168,14 @@ fn revision_bump_after_cancellation_recomputes_cycle_query() { // Wait for thread 1 to enter the cycle query db_waiter.wait_for(1); - // Drop waiter so trigger_lru_eviction can proceed + // Drop waiter so trigger_eviction can proceed drop(db_waiter); - // Thread 2: Trigger LRU eviction + // Thread 2: Trigger eviction let t2 = thread::spawn({ let mut db_writer = db_writer; move || { - db_writer.trigger_lru_eviction(); + db_writer.trigger_eviction(); db_writer } }); diff --git a/tests/sieve.rs b/tests/sieve.rs index b195737bf..9ebeb3542 100644 --- a/tests/sieve.rs +++ b/tests/sieve.rs @@ -14,7 +14,7 @@ struct MyInput { field: u32, } -#[salsa::tracked(returns(copy), sieve = 2)] +#[salsa::tracked(returns(copy), eviction(capacity = 2))] fn sieve_value(db: &dyn LogDatabase, input: MyInput) -> u32 { db.push_log(format!("sieve_value({:?})", input.field(db))); input.field(db) From 16465a0c5c111949885935bcf04511eb921db105 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Sun, 5 Jul 2026 13:32:55 +0000 Subject: [PATCH 10/10] docs: clarify eviction policy configuration --- book/src/tuning.md | 10 ++++++---- components/salsa-macros/src/lib.rs | 7 ++++--- src/function/eviction.rs | 20 +++++++++----------- src/function/eviction/noop.rs | 3 +++ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/book/src/tuning.md b/book/src/tuning.md index aa582bfa3..1aa3a0756 100644 --- a/book/src/tuning.md +++ b/book/src/tuning.md @@ -13,8 +13,10 @@ fn parse(db: &dyn Db, input: SourceFile) -> Ast { } ``` -The `capacity` option is required. The optional `policy` defaults to `sieve`. -Salsa supports two policies: +With `capacity = 128`, Salsa will keep at most 128 memoized values for this +function. When the cache exceeds this capacity, Salsa uses the configured policy +to choose which values to evict at the start of the next revision. The optional +`policy` defaults to `sieve`. Salsa supports two policies: - `sieve` is recommended. Its lock-free cache-hit path avoids serializing concurrent accesses, and it often achieves better cache efficiency (a lower @@ -33,8 +35,8 @@ that depend on it can still determine whether their own values may have changed. ### Zero-Cost When Disabled When `eviction` is absent, Salsa uses a no-op policy that is optimized away by -the compiler. Functions without cache eviction therefore have no policy -bookkeeping on cache hits. +the compiler. Functions without cache eviction therefore have no eviction-policy +bookkeeping. ### Runtime Capacity Adjustment diff --git a/components/salsa-macros/src/lib.rs b/components/salsa-macros/src/lib.rs index a53b3662e..380a77794 100644 --- a/components/salsa-macros/src/lib.rs +++ b/components/salsa-macros/src/lib.rs @@ -425,9 +425,10 @@ pub fn input(args: TokenStream, input: TokenStream) -> TokenStream { /// input or interned struct. `specify` must be called during the same tracked query invocation /// that created the key. It cannot be combined with `eviction`. See [specifying query results in /// the Salsa book] for an example. -/// - `eviction(capacity = INTEGER)` bounds the number of memoized values retained by the function -/// using SIEVE. Specify `policy = lru` to use LRU instead. `INTEGER` is also the initial capacity -/// used by `FUNCTION::set_eviction_capacity`. +/// - `eviction(capacity = INTEGER)` enables cache eviction. `capacity` is required; `INTEGER` +/// bounds the number of retained memoized values and is the initial capacity used by +/// `FUNCTION::set_eviction_capacity`. The policy defaults to SIEVE; specify `policy = lru` to use +/// LRU instead. /// - `cycle_initial = EXPR` enables fixed-point cycle recovery and computes the initial value. The /// expression is called as `(db, cycle_head_id, query_arguments...)`. /// - `cycle_fn = EXPR` combines successive fixed-point values. It must be accompanied by diff --git a/src/function/eviction.rs b/src/function/eviction.rs index 1968520ad..abea146a4 100644 --- a/src/function/eviction.rs +++ b/src/function/eviction.rs @@ -13,25 +13,23 @@ pub use sieve::Sieve; use crate::Id; -/// Trait for cache eviction strategies. +/// Cache eviction policy for memoized function values. /// -/// Implementations control when memoized values are evicted from the cache. -/// The eviction policy is selected at compile time via the `Configuration` trait. +/// Implementations control which memoized values are evicted from the cache. +/// The eviction policy is selected at compile time by the `Configuration` +/// trait. pub trait EvictionPolicy: Send + Sync { - /// Creates a policy with the configured capacity. + /// Creates an eviction policy with the given capacity. /// /// A value of zero disables eviction. fn new(capacity: usize) -> Self; - /// Records that a value was used. - /// - /// Implementations may treat this as a best-effort hint. + /// Records that a memoized value was accessed. fn record_use(&self, _id: Id) {} - /// Invokes `evict` for each value that should be evicted. + /// Invokes `evict` for each value selected for eviction. /// - /// Called once per revision. Implementations may also perform any - /// revision-boundary maintenance before returning. + /// Salsa calls this once per revision during `reset_for_new_revision`. fn for_each_evicted(&mut self, evict: impl FnMut(Id)); } @@ -41,7 +39,7 @@ pub trait EvictionPolicy: Send + Sync { /// on tracked functions. Only policies that implement this trait will expose /// runtime capacity changes. pub trait HasCapacity: EvictionPolicy { - /// Changes the policy's capacity. + /// Sets the maximum capacity. /// /// A value of zero disables eviction. fn set_capacity(&mut self, capacity: usize); diff --git a/src/function/eviction/noop.rs b/src/function/eviction/noop.rs index 83c4dbd36..ebd3d052a 100644 --- a/src/function/eviction/noop.rs +++ b/src/function/eviction/noop.rs @@ -12,6 +12,9 @@ impl EvictionPolicy for NoopEviction { Self } + #[inline(always)] + fn record_use(&self, _id: Id) {} + #[inline(always)] fn for_each_evicted(&mut self, _evict: impl FnMut(Id)) {} }