perf: avoid type checks in memo and field lookups

components/salsa-macro-rules/src/setup_input_struct.rs

@@ -152,6 +152,16 @@ macro_rules! setup_input_struct {
                     Self::ingredient_(db.zalsa())
                 }
 
+                #[inline]
+                fn ingredient_in_db<'db, $Db>(db: &'db $Db) -> ($zalsa::IngredientInDb<'db, $Db, $zalsa_struct::IngredientImpl<Self>>, &'db $zalsa::ZalsaLocal)
+                where
+                    $Db: ?Sized + $zalsa::Database,
+                {
+                    // SAFETY: `ingredient_` looks up the input ingredient in the `Zalsa`
+                    // supplied by `IngredientInDb`.
+                    unsafe { $zalsa::IngredientInDb::new_unchecked_with_zalsa_local(db, Self::ingredient_) }
+                }
+
                 fn ingredient_(zalsa: &$zalsa::Zalsa) -> &$zalsa_struct::IngredientImpl<Self> {
                     static CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Configuration>> =
                         $zalsa::IngredientCache::new();
@@ -280,13 +290,8 @@ macro_rules! setup_input_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let (zalsa, zalsa_local) = db.zalsas();
-                        let fields = $Configuration::ingredient_(zalsa).field(
-                            zalsa,
-                            zalsa_local,
-                            self,
-                            $field_index,
-                        );
+                        let (ingredient, zalsa_local) = $Configuration::ingredient_in_db(db);
+                        let fields = ingredient.field(zalsa_local, self, $field_index);
                         $zalsa::return_mode_expression!(
                             $field_option,
                             $field_ty,

components/salsa-macro-rules/src/setup_interned_struct.rs

@@ -207,6 +207,16 @@ macro_rules! setup_interned_struct {
                         CACHE.get_or_create::<$zalsa_struct::JarImpl<$Configuration>, 0>(zalsa)
                     }
                 }
+
+                #[inline]
+                fn ingredient_in_db<'db, $Db>(db: &'db $Db) -> $zalsa::IngredientInDb<'db, $Db, $zalsa_struct::IngredientImpl<Self>>
+                where
+                    $Db: ?Sized + $zalsa::Database,
+                {
+                    // SAFETY: `ingredient` looks up the interned ingredient in the `Zalsa`
+                    // supplied by `IngredientInDb`.
+                    unsafe { $zalsa::IngredientInDb::new_unchecked(db, Self::ingredient) }
+                }
             }
 
             impl< $($db_lt_arg)? > $zalsa::AsId for $Struct< $($db_lt_arg)? > {
@@ -324,8 +334,7 @@ macro_rules! setup_interned_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let zalsa = db.zalsa();
-                        let fields = $Configuration::ingredient(zalsa).fields(zalsa, self);
+                        let fields = $Configuration::ingredient_in_db(db).fields(self);
                         $zalsa::return_mode_expression!(
                             $field_option,
                             $field_ty,

components/salsa-macro-rules/src/setup_tracked_fn.rs

@@ -96,16 +96,17 @@ macro_rules! setup_tracked_fn {
             use ::salsa::plumbing as $zalsa;
 
             $zalsa::attach($db, || {
-                let (zalsa, zalsa_local) = $db.zalsas();
+                let (fn_ingredient, zalsa_local) = $fn_name::fn_ingredient_($db);
                 let result = $zalsa::macro_if! {
                     if $needs_interner {
                         {
+                            let zalsa = fn_ingredient.zalsa();
                             let key = $fn_name::intern_ingredient_(zalsa).intern_id(zalsa, zalsa_local, ($($input_id),*), |_, data| data);
-                            $fn_name::fn_ingredient_($db, zalsa).fetch($db, zalsa, zalsa_local, key)
+                            fn_ingredient.fetch(zalsa_local, key)
                         }
                     } else {
                         {
-                            $fn_name::fn_ingredient_($db, zalsa).fetch($db, zalsa, zalsa_local, $zalsa::AsId::as_id(&($($input_id),*)))
+                            fn_ingredient.fetch(zalsa_local, $zalsa::AsId::as_id(&($($input_id),*)))
                         }
                     }
                 };
@@ -255,19 +256,18 @@ macro_rules! setup_tracked_fn {
             }
 
             impl $Configuration {
-                fn fn_ingredient(db: &dyn $Db) -> &$zalsa::function::IngredientImpl<$Configuration> {
-                    let zalsa = db.zalsa();
-                    Self::fn_ingredient_(db, zalsa)
-                }
-
                 #[inline]
-                fn fn_ingredient_<'z>(db: &dyn $Db, zalsa: &'z $zalsa::Zalsa) -> &'z $zalsa::function::IngredientImpl<$Configuration> {
+                fn fn_ingredient<'db>(db: &'db dyn $Db) -> ($zalsa::IngredientInDb<'db, dyn $Db, $zalsa::function::IngredientImpl<$Configuration>>, &'db $zalsa::ZalsaLocal) {
                     // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the first
-                    // ingredient created by our jar is the function ingredient.
+                    // ingredient created by our jar is the function ingredient in the provided
+                    // `zalsa`.
                     unsafe {
-                        $FN_CACHE.get_or_create::<$fn_name, 0>(zalsa)
+                        $zalsa::IngredientInDb::new_unchecked_with_zalsa_local(db, |zalsa| {
+                            $FN_CACHE
+                                .get_or_create::<$fn_name, 0>(zalsa)
+                                .get_or_init(|| *<dyn $Db as $Db>::zalsa_register_downcaster(db))
+                        })
                     }
-                    .get_or_init(|| *<dyn $Db as $Db>::zalsa_register_downcaster(db))
                 }
 
                 pub fn fn_ingredient_mut(db: &mut dyn $Db) -> &mut $zalsa::function::IngredientImpl<Self> {
@@ -456,16 +456,17 @@ macro_rules! setup_tracked_fn {
                         $($input_id: $interned_input_ty,)*
                     ) -> Vec<&$db_lt A> {
                         use ::salsa::plumbing as $zalsa;
+                        let (fn_ingredient, zalsa_local) = $Configuration::fn_ingredient($db);
                         let key = $zalsa::macro_if! {
                             if $needs_interner {{
-                                let (zalsa, zalsa_local) = $db.zalsas();
+                                let zalsa = fn_ingredient.zalsa();
                                 $Configuration::intern_ingredient(zalsa).intern_id(zalsa, zalsa_local, ($($input_id),*), |_, data| data)
                             }} else {
                                 $zalsa::AsId::as_id(&($($input_id),*))
                             }
                         };
 
-                        $Configuration::fn_ingredient($db).accumulated_by::<A>($db, key)
+                        fn_ingredient.accumulated_by::<A>(zalsa_local, key)
                     }
                 }
 
@@ -476,8 +477,9 @@ macro_rules! setup_tracked_fn {
                         value: $output_ty,
                     ) {
                         let key = $zalsa::AsId::as_id(&($($input_id),*));
-                        $Configuration::fn_ingredient($db).specify_and_record(
-                            $db,
+                        let (fn_ingredient, zalsa_local) = $Configuration::fn_ingredient($db);
+                        fn_ingredient.specify_and_record(
+                            zalsa_local,
                             key,
                             value,
                         )
@@ -504,8 +506,8 @@ macro_rules! setup_tracked_fn {
                 }
 
                 #[inline]
-                fn fn_ingredient_<'z>(db: &dyn $Db, zalsa: &'z $zalsa::Zalsa) -> &'z $zalsa::function::IngredientImpl<$Configuration> {
-                    $Configuration::fn_ingredient_(db, zalsa)
+                fn fn_ingredient_<'db>(db: &'db dyn $Db) -> ($zalsa::IngredientInDb<'db, dyn $Db, $zalsa::function::IngredientImpl<$Configuration>>, &'db $zalsa::ZalsaLocal) {
+                    $Configuration::fn_ingredient(db)
                 }
             }
         };

components/salsa-macro-rules/src/setup_tracked_struct.rs

@@ -224,6 +224,26 @@ macro_rules! setup_tracked_struct {
                     Self::ingredient_(db.zalsa())
                 }
 
+                #[inline]
+                fn ingredient_in_db<'db, $Db>(db: &'db $Db) -> $zalsa::IngredientInDb<'db, $Db, $zalsa_struct::IngredientImpl<Self>>
+                where
+                    $Db: ?Sized + $zalsa::Database,
+                {
+                    // SAFETY: `ingredient_` looks up the tracked-struct ingredient in the `Zalsa`
+                    // supplied by `IngredientInDb`.
+                    unsafe { $zalsa::IngredientInDb::new_unchecked(db, Self::ingredient_) }
+                }
+
+                #[inline]
+                fn ingredient_in_db_with_zalsa_local<'db, $Db>(db: &'db $Db) -> ($zalsa::IngredientInDb<'db, $Db, $zalsa_struct::IngredientImpl<Self>>, &'db $zalsa::ZalsaLocal)
+                where
+                    $Db: ?Sized + $zalsa::Database,
+                {
+                    // SAFETY: `ingredient_` looks up the tracked-struct ingredient in the `Zalsa`
+                    // supplied by `IngredientInDb`.
+                    unsafe { $zalsa::IngredientInDb::new_unchecked_with_zalsa_local(db, Self::ingredient_) }
+                }
+
                 fn ingredient_(zalsa: &$zalsa::Zalsa) -> &$zalsa_struct::IngredientImpl<Self> {
                     static CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Configuration>> =
                         $zalsa::IngredientCache::new();
@@ -356,8 +376,8 @@ macro_rules! setup_tracked_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let (zalsa, zalsa_local) = db.zalsas();
-                        let fields = $Configuration::ingredient_(zalsa).tracked_field(zalsa, zalsa_local, self, $relative_tracked_index);
+                        let (ingredient, zalsa_local) = $Configuration::ingredient_in_db_with_zalsa_local(db);
+                        let fields = ingredient.tracked_field(zalsa_local, self, $relative_tracked_index);
                         $crate::return_mode_expression!(
                             $tracked_option,
                             $tracked_ty,
@@ -373,8 +393,7 @@ macro_rules! setup_tracked_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let zalsa = db.zalsa();
-                        let fields = $Configuration::ingredient_(zalsa).untracked_field(zalsa, self);
+                        let fields = $Configuration::ingredient_in_db(db).untracked_field(self);
                         $crate::return_mode_expression!(
                             $untracked_option,
                             $untracked_ty,

src/function.rs

@@ -503,7 +503,11 @@ where
     ) {
         // SAFETY: The `db` belongs to the ingredient as per caller invariant
         let db = unsafe { self.view_caster().downcast_unchecked(db) };
-        self.accumulated_map(db, key_index)
+        // SAFETY: The caller guarantees that `self` is registered in the `Zalsa` owned by `db`.
+        let (ingredient, zalsa_local) = unsafe {
+            crate::ingredient::IngredientInDb::new_unchecked_with_zalsa_local(db, |_| self)
+        };
+        ingredient.accumulated_map(zalsa_local, key_index)
     }
 
     fn is_persistable(&self) -> bool {

src/function/accumulated.rs

@@ -2,21 +2,22 @@ use crate::accumulator::accumulated_map::{AccumulatedMap, InputAccumulatedValues
 use crate::accumulator::{self};
 use crate::function::{Configuration, IngredientImpl};
 use crate::hash::FxHashSet;
-use crate::zalsa::ZalsaDatabase;
-use crate::zalsa_local::QueryOriginRef;
+use crate::ingredient::IngredientInDb;
+use crate::zalsa_local::{QueryOriginRef, ZalsaLocal};
 use crate::{DatabaseKeyIndex, Id};
 
-impl<C> IngredientImpl<C>
+impl<'db, C> IngredientInDb<'db, C::DbView, IngredientImpl<C>>
 where
     C: Configuration,
 {
-    /// Helper used by `accumulate` functions. Computes the results accumulated by `database_key_index`
-    /// and its inputs.
-    pub fn accumulated_by<'db, A>(&self, db: &'db C::DbView, key: Id) -> Vec<&'db A>
+    /// Helper used by `accumulate` functions. Computes the results accumulated by
+    /// `database_key_index` and its inputs.
+    pub fn accumulated_by<A>(&self, zalsa_local: &'db ZalsaLocal, key: Id) -> Vec<&'db A>
     where
         A: accumulator::Accumulator,
     {
-        let (zalsa, zalsa_local) = db.zalsas();
+        let zalsa = self.zalsa();
+        let db = self.db();
 
         // NOTE: We don't have a precise way to track accumulated values at present,
         // so we report any read of them as an untracked read.
@@ -37,9 +38,9 @@ where
         let mut output = vec![];
 
         // First ensure the result is up to date
-        self.fetch(db, zalsa, zalsa_local, key);
+        self.fetch(zalsa_local, key);
 
-        let db_key = self.database_key_index(key);
+        let db_key = self.ingredient().database_key_index(key);
         let mut visited: FxHashSet<DatabaseKeyIndex> = FxHashSet::default();
         let mut stack: Vec<DatabaseKeyIndex> = vec![db_key];
 
@@ -53,7 +54,7 @@ where
 
             let ingredient = zalsa.lookup_ingredient(k.ingredient_index());
             // Extend `output` with any values accumulated by `k`.
-            // SAFETY: `db` owns the `ingredient`
+            // SAFETY: `ingredient` and `k` belong to the `Zalsa` obtained from `db`.
             let (accumulated_map, input) =
                 unsafe { ingredient.accumulated(db.into(), k.key_index()) };
             if let Some(accumulated_map) = accumulated_map {
@@ -87,14 +88,13 @@ where
         output
     }
 
-    pub(super) fn accumulated_map<'db>(
-        &'db self,
-        db: &'db C::DbView,
+    pub(super) fn accumulated_map(
+        &self,
+        zalsa_local: &'db ZalsaLocal,
         key: Id,
     ) -> (Option<&'db AccumulatedMap>, InputAccumulatedValues) {
-        let (zalsa, zalsa_local) = db.zalsas();
         // NEXT STEP: stash and refactor `fetch` to return an `&Memo` so we can make this work
-        let memo = self.refresh_memo(db, zalsa, zalsa_local, key);
+        let memo = self.refresh_memo(zalsa_local, key);
         (
             memo.header.revisions.accumulated(),
             memo.header.revisions.accumulated_inputs.load(),