From 77af7e1b60686a864f540956ab6e742766ce2ece Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 05:43:12 +0300 Subject: [PATCH 01/23] refactor(bench): move the library generator into its own crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `vuio-cli` ships the `vuio` binary. It had picked up a direct `rusqlite` with `bundled`, and `vuio-core/unstable-internals` — a feature core's own docs say "must never be enabled by a dependent crate" — purely to support a benchmark generator that lived in `src/bin/`. Neither was even needed for what the generator did: it imported nothing from `vuio-core` at all, hand-copying `natural_cmp` and the whole DDL instead. The feature could not have supplied either one — `natural_cmp` is `pub(crate)` at the crate root, and `database::sqlite::schema` is a private module. Move it to `crates/vuio-bench` (`publish = false`) and give `vuio-cli` back exactly the dependencies it had before. `serde_json` and tokio's `io-std`/`io-util` stay; those came with the MCP stdio bridge and `src/mcp.rs` uses them. Being a separate crate is not sufficient on its own: cargo unifies features across everything it builds at once, so a plain `cargo build` — which is what the release pipeline runs — would still compile `vuio-core` with its internals open for the benefit of a tool that never ships. So `vuio-bench` is excluded from `default-members`, and `cargo tree -e normal` now reports the feature nowhere outside core's own dev-dependency on itself. Expose `sqlite::register_collations` under `unstable-internals` so the generator can teach a direct connection the natural-order collation that two of the browse indexes are declared with, rather than carrying a copy of `natural_cmp` that would drift. Also delete two dead benchmark harnesses: `scripts/run-large-benchmarks.{sh,ps1}` invoked a test target that does not exist and exported `SQLX_OFFLINE` for a dependency this project has never had, and `tests/cmd.txt` named two more tests that do not exist either. --- Cargo.toml | 22 +- crates/vuio-bench/Cargo.toml | 28 ++ crates/vuio-bench/src/main.rs | 365 +++++++++++++++ crates/vuio-cli/Cargo.toml | 7 +- crates/vuio-cli/src/bin/benchmark_media.rs | 480 -------------------- crates/vuio-core/src/database/sqlite/mod.rs | 12 + crates/vuio-core/tests/cmd.txt | 2 - scripts/run-large-benchmarks.ps1 | 128 ------ scripts/run-large-benchmarks.sh | 103 ----- 9 files changed, 427 insertions(+), 720 deletions(-) create mode 100644 crates/vuio-bench/Cargo.toml create mode 100644 crates/vuio-bench/src/main.rs delete mode 100644 crates/vuio-cli/src/bin/benchmark_media.rs delete mode 100644 crates/vuio-core/tests/cmd.txt delete mode 100644 scripts/run-large-benchmarks.ps1 delete mode 100644 scripts/run-large-benchmarks.sh diff --git a/Cargo.toml b/Cargo.toml index 288078c..e4cada3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,25 @@ [workspace] -members = ["crates/vuio-cast", "crates/vuio-core", "crates/vuio-cli", "crates/vuio-web"] +members = [ + "crates/vuio-cast", + "crates/vuio-core", + "crates/vuio-cli", + "crates/vuio-web", + # Development tool, `publish = false`. Kept out of `vuio-cli` so the crate that + # ships the binary does not carry its dependencies. + "crates/vuio-bench", +] + +# `vuio-bench` is deliberately not a default member. Cargo unifies features across +# everything it builds in one go, so leaving it in would mean a plain +# `cargo build` — which is what the release pipeline runs — compiling `vuio-core` +# with `unstable-internals` on, for the benefit of a tool that never ships. +# Build it explicitly: `cargo run -p vuio-bench`. +default-members = [ + "crates/vuio-cast", + "crates/vuio-core", + "crates/vuio-cli", + "crates/vuio-web", +] resolver = "2" [profile.release] diff --git a/crates/vuio-bench/Cargo.toml b/crates/vuio-bench/Cargo.toml new file mode 100644 index 0000000..b65eff9 --- /dev/null +++ b/crates/vuio-bench/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "vuio-bench" +version = "0.0.44" +edition = "2021" +authors = ["vyrti"] +description = "Generates large VuIO libraries for performance work. Not published." +license = "MIT OR Apache-2.0" +repository = "https://github.com/vuiodev/vuio" +# A development tool, never a release artifact. It lives in its own crate so that +# the dependencies it needs to write rows quickly — a direct SQLite handle, and +# vuio-core with its internals open — stay off `vuio-cli`, which ships the +# `vuio` binary. +publish = false + +[[bin]] +name = "vuio-bench" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0" +clap = { version = "4.6", features = ["derive"] } +rusqlite = { version = "0.40.2", features = ["bundled", "collation"] } +tokio = { version = "1.53", features = ["rt-multi-thread", "macros"] } +# `unstable-internals` is what core forbids a *dependent* crate from enabling, +# because it opens every internal module and carries no stability promise. This +# crate is `publish = false` and exists only to drive the database from the +# inside, which is the same category as core's own dev-dependency on itself. +vuio-core = { path = "../vuio-core", version = "0.0.44", features = ["unstable-internals"] } diff --git a/crates/vuio-bench/src/main.rs b/crates/vuio-bench/src/main.rs new file mode 100644 index 0000000..e0979dd --- /dev/null +++ b/crates/vuio-bench/src/main.rs @@ -0,0 +1,365 @@ +//! Generates a large VuIO library, for performance work. +//! +//! The point of this tool is a database that is the right *shape*, not merely the +//! right size. An earlier version wrote its own copy of the schema and its own +//! directory tree, and the result measured very little: the full-text tables were +//! never populated, the `directories` rows did not correspond to any file's +//! `parent_path`, and the paths on disk did not match the paths in the rows. A +//! server pointed at that database browsed an empty tree and searched an empty +//! index, so the numbers described a system nobody runs. +//! +//! So the schema comes from `vuio-core` itself, and every piece of derived state — +//! the directory tree, the recursive per-family counters, both FTS indexes — is +//! built by the same `rebuild_derived_indexes` the server uses to repair itself. +//! What is left here is only the part that has to be fast: the rows. + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use anyhow::{Context, Result}; +use clap::Parser; +use vuio_core::config::generator::ConfigGenerator; +use vuio_core::config::{AppConfig, MonitoredDirectoryConfig, ValidationMode}; +use vuio_core::database::sqlite::SqliteDatabase; +use vuio_core::database::{DatabaseManager, HealthRepository}; + +/// How many rows go in one transaction. Large enough that commit overhead +/// disappears, small enough that the rollback journal stays bounded. +const CHUNK: usize = 50_000; + +const GENRES: [&str; 8] = [ + "Rock", + "Pop", + "Jazz", + "Classical", + "Electronic", + "Metal", + "Hip Hop", + "Ambient", +]; + +/// Shape of the generated tree. Real libraries are wide and shallow rather than +/// uniformly deep, and the directory counters are maintained per ancestor, so +/// these two numbers decide how much work a scan does per file. +const TRACKS_PER_ALBUM: usize = 12; +const ALBUMS_PER_ARTIST: usize = 10; + +#[derive(Parser, Debug)] +#[command(about = "Generate a large VuIO library for performance testing")] +struct Args { + /// How many media rows to index + #[arg(long, default_value_t = 100_000)] + objects: usize, + + /// Where to put the library. The database goes in `/.vuio/media.db`. + #[arg(long, default_value = "./bench-library")] + out: PathBuf, + + /// How many stub files to write to disk. Defaults to one per row. + /// + /// Every row needs a file, or the first scan deletes it: a library where + /// most rows are missing measures a deletion storm, not steady state. Lower + /// this only when the deletion path is deliberately what you want to time. + #[arg(long)] + files: Option, +} + +/// The 18 bytes that make a file a plausible MP3 to a tag reader. +const STUB_MP3: &[u8] = b"ID3\x04\x00\x00\x00\x00\x00\x00\xFF\xFB\x90\x44\x00\x00\x00"; + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + let files_to_write = args.files.unwrap_or(args.objects).min(args.objects); + + let root = args.out.clone(); + std::fs::create_dir_all(&root) + .with_context(|| format!("could not create {}", root.display()))?; + // Canonical, because that is the form the scanner stores and compares against. + let root = root + .canonicalize() + .with_context(|| format!("could not canonicalize {}", root.display()))?; + let db_path = root.join(".vuio").join("media.db"); + + println!("VuIO benchmark library"); + println!(" objects: {}", args.objects); + println!(" files: {files_to_write} written to disk"); + println!(" library: {}", root.display()); + println!(" database: {}", db_path.display()); + println!(); + + // 1. Stub files, at the paths the rows will carry. Their modification times + // come back with them: a row whose `modified_secs` does not match its + // file reads as changed, and the first scan would re-read and rewrite the + // entire library from stubs that carry no tags. + let started = Instant::now(); + let mtimes = write_stub_files(&root, files_to_write)?; + println!( + " wrote {files_to_write} files in {:.1}s", + started.elapsed().as_secs_f64() + ); + + // 2. The real schema, from the real code. Creating the database through + // vuio-core is what guarantees this tool cannot drift from `schema.rs`. + if db_path.exists() { + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", db_path.display())); + } + } + std::fs::create_dir_all(db_path.parent().expect("the database has a parent"))?; + { + let database = SqliteDatabase::new(db_path.clone()) + .await + .context("could not create the database")?; + database + .initialize() + .await + .context("could not initialize the schema")?; + } + println!(" schema created by vuio-core"); + + // 3. The rows, as fast as SQLite will take them. + let started = Instant::now(); + insert_rows(&db_path, &root, args.objects, &mtimes)?; + let inserted = started.elapsed(); + println!( + " inserted {} rows in {:.1}s ({:.0} rows/s)", + args.objects, + inserted.as_secs_f64(), + args.objects as f64 / inserted.as_secs_f64().max(0.001) + ); + + // 4. Every piece of derived state, built by the server's own repair path: + // the directory tree, the recursive counters, and both FTS indexes. + let started = Instant::now(); + { + let database = SqliteDatabase::new(db_path.clone()) + .await + .context("could not reopen the database")?; + let health = database + .rebuild_derived_indexes() + .await + .context("could not build the derived indexes")?; + anyhow::ensure!( + health.is_healthy, + "derived index rebuild reported problems: {:?}", + health.issues + ); + } + println!( + " built directory tree, counters and search index in {:.1}s", + started.elapsed().as_secs_f64() + ); + + // 5. A config that points the server at this library *and* this database. + // Without it the server would use its own default database path, scan the + // tree from scratch, and never open the file we just built — which is + // a silent way to measure nothing. + let config_path = write_config(&root, &db_path)?; + + report(&db_path)?; + + println!(); + println!("Run the server against it:"); + println!(" ./target/debug/vuio --config {}", config_path.display()); + Ok(()) +} + +/// Write a config naming this library and this database. +fn write_config(root: &Path, db_path: &Path) -> Result { + let mut config = AppConfig::default_for_platform(); + config.server.port = 18080; + config.media.directories = vec![MonitoredDirectoryConfig { + path: root.to_string_lossy().into_owned(), + recursive: true, + case_sensitive: None, + extensions: None, + exclude_patterns: None, + validation_mode: ValidationMode::Warn, + }]; + config.database.path = Some(db_path.to_string_lossy().into_owned()); + // The web interface would bind a second port and add noise to the profile. + config.web_ui.enabled = false; + + let rendered = ConfigGenerator::new() + .context("could not build the config generator")? + .generate_config(&config) + .context("could not render the config")?; + let config_path = root.join("vuio.toml"); + std::fs::write(&config_path, rendered) + .with_context(|| format!("could not write {}", config_path.display()))?; + Ok(config_path) +} + +/// The path a row with this index carries, relative to the library root. +/// +/// One function so the rows and the files on disk cannot disagree — which is +/// exactly how the previous generator ended up indexing paths that did not exist. +fn relative_path(index: usize) -> (String, String) { + let album = index / TRACKS_PER_ALBUM; + let artist = album / ALBUMS_PER_ARTIST; + let track = index % TRACKS_PER_ALBUM + 1; + ( + format!("Artist_{artist:05}/Album_{album:06}"), + format!("{track:02} - Track_{index}.mp3"), + ) +} + +/// Write the stubs and return each one's modification time, in row order. +fn write_stub_files(root: &Path, count: usize) -> Result> { + let mut mtimes = Vec::with_capacity(count); + let mut last_dir = String::new(); + for index in 0..count { + let (directory, filename) = relative_path(index); + if directory != last_dir { + std::fs::create_dir_all(root.join(&directory))?; + last_dir = directory.clone(); + } + let path = root.join(&directory).join(&filename); + std::fs::write(&path, STUB_MP3)?; + mtimes.push( + std::fs::metadata(&path)? + .modified()? + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs() as i64) + .unwrap_or(0), + ); + } + Ok(mtimes) +} + +fn insert_rows(db_path: &Path, root: &Path, total: usize, mtimes: &[i64]) -> Result<()> { + let mut connection = rusqlite::Connection::open(db_path)?; + // Two of the browse indexes are declared `COLLATE natural_order`, so this + // connection has to know the collation before it can maintain them. + vuio_core::database::sqlite::register_collations(&connection)?; + // Bulk-load settings, reverted below. Safe here because the file is + // disposable: if this crashes, you regenerate it. + connection.execute_batch( + "PRAGMA synchronous = OFF; + PRAGMA journal_mode = MEMORY; + PRAGMA cache_size = -131072; + PRAGMA temp_store = MEMORY;", + )?; + + let root = root.to_string_lossy().into_owned(); + let mut done = 0usize; + while done < total { + let end = (done + CHUNK).min(total); + let transaction = connection.transaction()?; + { + let mut statement = transaction.prepare_cached(INSERT_MEDIA)?; + for index in done..end { + let (directory, filename) = relative_path(index); + let parent = format!("{root}/{directory}"); + let path = format!("{parent}/{filename}"); + let album_id = index / TRACKS_PER_ALBUM; + let artist_id = album_id / ALBUMS_PER_ARTIST; + let track = (index % TRACKS_PER_ALBUM + 1) as i64; + let year = 1980 + (index % 45) as i64; + // The file's own mtime where one exists, so the scanner sees an + // unchanged record; a fixed past value otherwise. + let stamp = mtimes.get(index).copied().unwrap_or(1_700_000_000); + statement.execute(rusqlite::params![ + (index + 1) as i64, + path, + parent, + filename, + STUB_MP3.len() as i64, + stamp, + "audio/mpeg", + "audio", + 215.0 + (index % 60) as f64, + format!("Track {index}"), + format!("Artist {artist_id}"), + format!("Album {album_id}"), + GENRES[index % GENRES.len()], + track, + year, + format!("Artist {artist_id}"), + 1i64, + 1i64, + TRACKS_PER_ALBUM as i64, + format!("Composer {}", artist_id % 97), + Option::::None, + 120i64, + 0i64, + Option::::None, + Option::::None, + Option::::None, + format!("{year}-01-01"), + Option::::None, + Option::::None, + Option::::None, + "mp3", + 44_100i64, + 2i64, + 16i64, + 320_000i64, + 1i64, + 0i64, + stamp, + stamp, + ])?; + } + } + transaction.commit()?; + done = end; + } + + // Back to the settings the server runs with, so what gets measured next is + // the real thing. + connection.execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + ANALYZE;", + )?; + Ok(()) +} + +/// Columns in the order `bind_media_file` uses, so the parameter list here reads +/// the same as the one in `media_repo/bulk.rs`. +const INSERT_MEDIA: &str = "\ +INSERT INTO media_files ( + id, path, parent_path, filename, size, modified_secs, mime_type, mime_family, + duration_secs, title, artist, album, genre, track_number, year, album_artist, + disc_number, disc_total, track_total, composer, comment, bpm, compilation, + sort_title, sort_artist, sort_album, release_date, + musicbrainz_track_id, musicbrainz_album_id, musicbrainz_artist_id, + codec, sample_rate, channels, bits_per_sample, bit_rate, tags_version, + subtitle_available, created_at_secs, updated_at_secs +) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, + ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29, ?30, ?31, ?32, + ?33, ?34, ?35, ?36, ?37, ?38, ?39)"; + +/// Prove the database is usable, not merely large. +/// +/// Every count here was zero or wrong in the previous generator's output, which +/// is why they are printed rather than assumed. +fn report(db_path: &Path) -> Result<()> { + let connection = rusqlite::Connection::open(db_path)?; + let count = |sql: &str| -> Result { Ok(connection.query_row(sql, [], |row| row.get(0))?) }; + + let files = count("SELECT COUNT(*) FROM media_files")?; + let directories = count("SELECT COUNT(*) FROM directories")?; + let counters = count("SELECT COUNT(*) FROM directory_mime_counts")?; + let searchable = count("SELECT COUNT(*) FROM media_fts")?; + let bytes = std::fs::metadata(db_path).map(|m| m.len()).unwrap_or(0); + + println!(); + println!(" media_files {files}"); + println!(" directories {directories}"); + println!(" directory_mime_counts {counters}"); + println!(" media_fts {searchable}"); + println!( + " database {:.1} MB", + bytes as f64 / 1_048_576.0 + ); + + anyhow::ensure!(directories > 0, "the directory tree is empty"); + anyhow::ensure!( + searchable == files, + "the search index covers {searchable} of {files} rows" + ); + Ok(()) +} diff --git a/crates/vuio-cli/Cargo.toml b/crates/vuio-cli/Cargo.toml index f1d12bf..7e96ebc 100644 --- a/crates/vuio-cli/Cargo.toml +++ b/crates/vuio-cli/Cargo.toml @@ -18,10 +18,6 @@ autobins = false name = "vuio" path = "src/main.rs" -[[bin]] -name = "benchmark_media" -path = "src/bin/benchmark_media.rs" - [[bin]] name = "generate_test_media" path = "src/bin/generate_test_media.rs" @@ -38,7 +34,6 @@ testdata = ["dep:audiotags"] anyhow = "1.0" audiotags = { version = "0.5", optional = true } clap = { version = "4.6", features = ["derive"] } -rusqlite = { version = "0.40.2", features = ["bundled", "collation"] } rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } reqwest = { version = "0.13", default-features = false, features = ["json", "rustls-no-provider"] } serde = { version = "1.0", features = ["derive"] } @@ -46,5 +41,5 @@ serde_json = "1.0" tokio = { version = "1.53", features = ["rt-multi-thread", "macros", "signal", "io-std", "io-util"] } tracing = "0.1" uuid = { version = "1.24", features = ["v4"] } -vuio-core = { path = "../vuio-core", version = "0.0.44", features = ["unstable-internals"] } +vuio-core = { path = "../vuio-core", version = "0.0.44" } diff --git a/crates/vuio-cli/src/bin/benchmark_media.rs b/crates/vuio-cli/src/bin/benchmark_media.rs deleted file mode 100644 index 7f0bc42..0000000 --- a/crates/vuio-cli/src/bin/benchmark_media.rs +++ /dev/null @@ -1,480 +0,0 @@ -use anyhow::{Context, Result}; -use std::cmp::Ordering; -use std::fs; -use std::path::PathBuf; -use std::time::Instant; - -fn natural_cmp(left: &str, right: &str) -> Ordering { - let left = left.to_lowercase(); - let right = right.to_lowercase(); - let mut left_chars = left.chars().peekable(); - let mut right_chars = right.chars().peekable(); - - loop { - match (left_chars.peek(), right_chars.peek()) { - (Some(a), Some(b)) if a.is_ascii_digit() && b.is_ascii_digit() => { - let left_number: String = std::iter::from_fn(|| { - left_chars.next_if(|character| character.is_ascii_digit()) - }) - .collect(); - let right_number: String = std::iter::from_fn(|| { - right_chars.next_if(|character| character.is_ascii_digit()) - }) - .collect(); - let order = left_number - .trim_start_matches('0') - .len() - .cmp(&right_number.trim_start_matches('0').len()) - .then_with(|| { - left_number - .trim_start_matches('0') - .cmp(right_number.trim_start_matches('0')) - }) - .then_with(|| left_number.len().cmp(&right_number.len())); - if order != Ordering::Equal { - return order; - } - } - (Some(_), Some(_)) => { - let order = left_chars.next().cmp(&right_chars.next()); - if order != Ordering::Equal { - return order; - } - } - _ => return left_chars.next().cmp(&right_chars.next()), - } - } -} - -const DDL_BASE: &str = r#" -CREATE TABLE IF NOT EXISTS media_files ( - id INTEGER PRIMARY KEY, - path TEXT NOT NULL, - parent_path TEXT NOT NULL, - filename TEXT NOT NULL, - size INTEGER NOT NULL, - modified_secs INTEGER NOT NULL, - mime_type TEXT NOT NULL, - mime_family TEXT NOT NULL, - duration_secs REAL, - title TEXT, - artist TEXT, - album TEXT, - genre TEXT, - track_number INTEGER, - year INTEGER, - album_artist TEXT, - disc_number INTEGER, - disc_total INTEGER, - track_total INTEGER, - composer TEXT, - comment TEXT, - bpm INTEGER, - compilation INTEGER, - sort_title TEXT, - sort_artist TEXT, - sort_album TEXT, - release_date TEXT, - musicbrainz_track_id TEXT, - musicbrainz_album_id TEXT, - musicbrainz_artist_id TEXT, - codec TEXT, - sample_rate INTEGER, - channels INTEGER, - bits_per_sample INTEGER, - bit_rate INTEGER, - tags_version INTEGER NOT NULL DEFAULT 0, - subtitle_available INTEGER NOT NULL DEFAULT 0, - created_at_secs INTEGER NOT NULL, - updated_at_secs INTEGER NOT NULL, - track_sort INTEGER GENERATED ALWAYS AS (COALESCE(track_number, 4294967296)) STORED, - disc_sort INTEGER GENERATED ALWAYS AS (COALESCE(disc_number, 1)) VIRTUAL -) STRICT; - -CREATE TABLE IF NOT EXISTS directories ( - path TEXT PRIMARY KEY, - parent_path TEXT NOT NULL, - name TEXT NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS directory_mime_counts ( - dir_path TEXT NOT NULL REFERENCES directories(path) ON DELETE CASCADE, - family TEXT NOT NULL, - count INTEGER NOT NULL, - PRIMARY KEY (dir_path, family) -) STRICT; - -CREATE TABLE IF NOT EXISTS playlists ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - description TEXT, - source_path TEXT, - created_at_secs INTEGER NOT NULL, - updated_at_secs INTEGER NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS playlist_entries ( - playlist_id INTEGER NOT NULL REFERENCES playlists(id) ON DELETE CASCADE, - media_file_id INTEGER NOT NULL REFERENCES media_files(id) ON DELETE CASCADE, - position INTEGER NOT NULL, - PRIMARY KEY (playlist_id, position) -) STRICT; - -CREATE TABLE IF NOT EXISTS root_availability ( - path TEXT PRIMARY KEY, - last_seen_secs INTEGER NOT NULL, - unavailable_since_secs INTEGER, - indexed_count INTEGER NOT NULL, - reason TEXT NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS secrets ( - key TEXT PRIMARY KEY, - value BLOB NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS media_tags ( - media_file_id INTEGER NOT NULL REFERENCES media_files(id) ON DELETE CASCADE, - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (media_file_id, key, value) -) STRICT; - -CREATE TABLE IF NOT EXISTS mediainfo ( - media_file_id INTEGER PRIMARY KEY REFERENCES media_files(id) ON DELETE CASCADE, - provider TEXT NOT NULL, - remote_id TEXT NOT NULL, - kind TEXT NOT NULL, - title TEXT, - original_title TEXT, - overview TEXT, - release_date TEXT, - year INTEGER, - rating REAL, - genres TEXT, - season INTEGER, - episode INTEGER, - artwork_key TEXT, - payload TEXT NOT NULL, - confidence INTEGER NOT NULL, - fetched_at INTEGER NOT NULL, - mediainfo_version INTEGER NOT NULL -) STRICT; -"#; - -const INDEXES_SQL: &str = r#" -CREATE UNIQUE INDEX IF NOT EXISTS idx_media_path ON media_files(path); -CREATE INDEX IF NOT EXISTS idx_media_dir_order - ON media_files(parent_path, disc_sort, track_sort, filename COLLATE natural_order); -CREATE INDEX IF NOT EXISTS idx_media_dir_family - ON media_files(parent_path, mime_family); -CREATE INDEX IF NOT EXISTS idx_media_album - ON media_files(album, disc_sort, track_sort, filename COLLATE natural_order); -CREATE INDEX IF NOT EXISTS idx_media_artist ON media_files(artist); -CREATE INDEX IF NOT EXISTS idx_media_genre ON media_files(genre); -CREATE INDEX IF NOT EXISTS idx_media_year ON media_files(year); -CREATE INDEX IF NOT EXISTS idx_media_album_artist ON media_files(album_artist); -CREATE INDEX IF NOT EXISTS idx_media_family ON media_files(mime_family); -CREATE INDEX IF NOT EXISTS idx_media_tags_version ON media_files(tags_version); - -CREATE INDEX IF NOT EXISTS idx_directories_parent - ON directories(parent_path, name COLLATE natural_order); -CREATE INDEX IF NOT EXISTS idx_playlists_source - ON playlists(source_path) WHERE source_path IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_playlist_entries_file - ON playlist_entries(media_file_id); -CREATE INDEX IF NOT EXISTS idx_media_tags_key - ON media_tags(key, value); -CREATE INDEX IF NOT EXISTS idx_mediainfo_confidence - ON mediainfo(confidence); -"#; - -fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - let total_objects: usize = args - .get(1) - .and_then(|s| s.parse().ok()) - .unwrap_or(10_000_000); - - let base_dir = std::env::current_dir()?; - let test_media_dir = base_dir.join("test-media"); - let config_dir = base_dir.join("target").join("release").join("config"); - let db_dir = config_dir.join("database"); - let db_path = db_dir.join("media.db"); - - println!("=== VuIO Benchmark Media Generator (Rust) ==="); - println!("Target Objects: {}", total_objects); - println!("Test Media Dir: {}", test_media_dir.display()); - println!("Database Path: {}", db_path.display()); - println!(); - - // 1. Create physical test media directory and sample files - println!( - "1. Creating physical test files in {}...", - test_media_dir.display() - ); - fs::create_dir_all(&test_media_dir)?; - - let silent_mp3 = b"ID3\x04\x00\x00\x00\x00\x00\x00\x00\xFF\xFB\x90\x44\x00\x00\x00\x00"; - let mut physical_files_created = 0; - for artist_idx in 0..10 { - let artist_dir = test_media_dir.join(format!("Artist_{:02}", artist_idx)); - for album_idx in 0..10 { - let album_dir = artist_dir.join(format!("Album_{:02}", album_idx)); - fs::create_dir_all(&album_dir)?; - for track_idx in 1..=10 { - let track_file = - album_dir.join(format!("{:02} - Track_{}.mp3", track_idx, track_idx)); - if !track_file.exists() { - fs::write(&track_file, silent_mp3)?; - physical_files_created += 1; - } - } - } - } - println!( - " Created {} physical files in test-media.\n", - physical_files_created - ); - - // 2. Prepare SQLite Database - println!( - "2. Initializing SQLite database schema at {}...", - db_path.display() - ); - fs::create_dir_all(&db_dir)?; - - // Remove existing database files - for ext in &["", "-wal", "-shm"] { - let p = PathBuf::from(format!("{}{}", db_path.display(), ext)); - if p.exists() { - let _ = fs::remove_file(p); - } - } - - let mut conn = rusqlite::Connection::open(&db_path) - .with_context(|| format!("Failed to open {}", db_path.display()))?; - - // Register natural_order collation - conn.create_collation("natural_order", |a: &str, b: &str| natural_cmp(a, b))?; - - // Pragmas for fast bulk load - conn.execute_batch( - "PRAGMA synchronous = OFF; - PRAGMA journal_mode = OFF; - PRAGMA page_size = 4096; - PRAGMA cache_size = -131072; - PRAGMA temp_store = MEMORY;", - )?; - - // Create DDL schema - conn.execute_batch(DDL_BASE)?; - conn.execute_batch("PRAGMA user_version = 4;")?; - - println!(" Schema and tables initialized.\n"); - - // 3. Pre-populate directories table - println!("3. Populating directory hierarchy in database..."); - let canonical_root = test_media_dir - .canonicalize() - .unwrap_or_else(|_| test_media_dir.clone()); - let root_str = canonical_root.to_string_lossy().to_string(); - let root_parent = canonical_root - .parent() - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_default(); - let root_name = canonical_root - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - - { - let tx = conn.transaction()?; - tx.execute( - "INSERT OR IGNORE INTO directories (path, parent_path, name) VALUES (?, ?, ?)", - rusqlite::params![&root_str, &root_parent, &root_name], - )?; - - let num_artists = 10_000.min(total_objects / 100).max(10); - { - let mut dir_stmt = tx.prepare_cached( - "INSERT OR IGNORE INTO directories (path, parent_path, name) VALUES (?, ?, ?)", - )?; - - for a in 0..num_artists { - let art_name = format!("Artist_{:04}", a); - let art_path = format!("{}/{}", root_str, art_name); - dir_stmt.execute(rusqlite::params![&art_path, &root_str, &art_name])?; - - for alb in 0..5 { - let alb_id = a * 5 + alb; - let alb_name = format!("Album_{:05}", alb_id); - let alb_path = format!("{}/{}", art_path, alb_name); - dir_stmt.execute(rusqlite::params![&alb_path, &art_path, &alb_name])?; - } - } - } - - tx.execute( - "INSERT OR REPLACE INTO directory_mime_counts (dir_path, family, count) VALUES (?, '*', ?)", - rusqlite::params![&root_str, total_objects as i64], - )?; - tx.execute( - "INSERT OR REPLACE INTO directory_mime_counts (dir_path, family, count) VALUES (?, 'audio', ?)", - rusqlite::params![&root_str, total_objects as i64], - )?; - - tx.commit()?; - } - println!(" Directory hierarchy populated.\n"); - - // 4. Bulk insert media files in transaction chunks - println!( - "4. Inserting {} media files in chunks of 50,000...", - total_objects - ); - let start_insert = Instant::now(); - let chunk_size = 50_000; - let num_artists = 10_000; - let num_albums = 100_000; - let genres = [ - "Rock", - "Pop", - "Jazz", - "Classical", - "Electronic", - "Metal", - "Hip Hop", - "Ambient", - ]; - - let insert_sql = " - INSERT INTO media_files ( - id, path, parent_path, filename, size, modified_secs, mime_type, mime_family, - duration_secs, title, artist, album, genre, track_number, year, album_artist, - disc_number, disc_total, track_total, composer, comment, bpm, compilation, - sort_title, sort_artist, sort_album, release_date, musicbrainz_track_id, - musicbrainz_album_id, musicbrainz_artist_id, codec, sample_rate, channels, - bits_per_sample, bit_rate, tags_version, subtitle_available, created_at_secs, updated_at_secs - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, - ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29, ?30, - ?31, ?32, ?33, ?34, ?35, ?36, ?37, ?38, ?39) - "; - - for chunk_start in (0..total_objects).step_by(chunk_size) { - let chunk_end = (chunk_start + chunk_size).min(total_objects); - let tx = conn.transaction()?; - { - let mut stmt = tx.prepare_cached(insert_sql)?; - for i in chunk_start..chunk_end { - let id = (i + 1) as i64; - let artist_id = i % num_artists; - let album_id = i % num_albums; - let track_num = ((i % 20) + 1) as i64; - let genre = genres[i % genres.len()]; - let parent = format!("{}/Artist_{:04}/Album_{:05}", root_str, artist_id, album_id); - let filename = format!("{:02} - Track_{}.mp3", track_num, i); - let path = format!("{}/{}", parent, filename); - let title = format!("Track {}", i); - let artist = format!("Artist {}", artist_id); - let album = format!("Album {}", album_id); - let year = (1980 + (i % 45)) as i64; - let release_date = format!("{}-01-01", year); - let size = (4_194_304 + (i % 1000) * 1024) as i64; - let duration = 215.0 + (i % 60) as f64; - let timestamp = 1700000000 + (i % 86400) as i64; - - stmt.execute(rusqlite::params![ - id, - path, - parent, - filename, - size, - timestamp, - "audio/mpeg", - "audio", - duration, - title, - artist, - album, - genre, - track_num, - year, - artist, - 1i64, - 1i64, - 20i64, - Option::::None, - Option::::None, - 120i64, - 0i64, - Option::::None, - Option::::None, - Option::::None, - release_date, - Option::::None, - Option::::None, - Option::::None, - "mp3", - 44100i64, - 2i64, - 16i64, - 320000i64, - 1i64, - 0i64, - timestamp, - timestamp, - ])?; - } - } - tx.commit()?; - - if chunk_end % 1_000_000 == 0 || chunk_end == total_objects { - let elapsed = start_insert.elapsed().as_secs_f64(); - let rate = chunk_end as f64 / elapsed; - println!( - " Inserted {:>10} / {} rows ({:>5.1}s, {:>8.0} rows/s)...", - chunk_end, total_objects, elapsed, rate - ); - } - } - - let insert_duration = start_insert.elapsed(); - println!( - " Inserted {} rows in {:.1}s.\n", - total_objects, - insert_duration.as_secs_f64() - ); - - // 5. Create B-Tree Indexes - println!("5. Creating B-Tree indexes on 10M records in SQLite..."); - let start_idx = Instant::now(); - conn.execute_batch(INDEXES_SQL)?; - println!( - " Indexes created in {:.1}s.\n", - start_idx.elapsed().as_secs_f64() - ); - - // 6. Finalize WAL and stats - println!("6. Finalizing SQLite database settings..."); - conn.execute_batch( - "PRAGMA journal_mode = WAL; - PRAGMA synchronous = NORMAL; - PRAGMA optimize;", - )?; - - drop(conn); - - let metadata = fs::metadata(&db_path)?; - let db_bytes = metadata.len(); - let db_mb = db_bytes as f64 / (1024.0 * 1024.0); - let db_gb = db_bytes as f64 / (1024.0 * 1024.0 * 1024.0); - - println!("============================================================"); - println!("BENCHMARK LIBRARY READY!"); - println!("Total Records: {}", total_objects); - println!("Database Size: {:.2} GB ({:.1} MB)", db_gb, db_mb); - println!("============================================================"); - - Ok(()) -} diff --git a/crates/vuio-core/src/database/sqlite/mod.rs b/crates/vuio-core/src/database/sqlite/mod.rs index 586a5ee..58d5b50 100644 --- a/crates/vuio-core/src/database/sqlite/mod.rs +++ b/crates/vuio-core/src/database/sqlite/mod.rs @@ -38,6 +38,18 @@ mod tests; pub use session::SqliteReadSession; +/// Register the natural-order collation on a caller-supplied connection. +/// +/// Two of the browse indexes are declared `COLLATE natural_order`, so any +/// connection that writes `media_files` has to know it — including one opened +/// outside this crate. Exposed for the benchmark generator, which bulk-loads a +/// library over a direct handle; without this it would need its own copy of +/// `natural_cmp`, and a copy is a thing that drifts. +#[cfg(feature = "unstable-internals")] +pub fn register_collations(connection: &rusqlite::Connection) -> anyhow::Result<()> { + schema::register_collations(connection) +} + /// Readers held open for reuse. /// /// Reads run on Tokio's blocking pool, which is unbounded by design; without a diff --git a/crates/vuio-core/tests/cmd.txt b/crates/vuio-core/tests/cmd.txt deleted file mode 100644 index f8bd03a..0000000 --- a/crates/vuio-core/tests/cmd.txt +++ /dev/null @@ -1,2 +0,0 @@ -cargo test test_memory_optimized_million_files --release --ignored -- --no -cargo test --release --test million_file_stress_test -- --ignored --nocapture \ No newline at end of file diff --git a/scripts/run-large-benchmarks.ps1 b/scripts/run-large-benchmarks.ps1 deleted file mode 100644 index 103485d..0000000 --- a/scripts/run-large-benchmarks.ps1 +++ /dev/null @@ -1,128 +0,0 @@ -# Large Dataset Benchmark Runner (PowerShell) -# This script runs the large dataset benchmarks with proper configuration - -param( - [switch]$Force = $false -) - -Write-Host "=== Large Dataset Benchmark Runner ===" -ForegroundColor Cyan -Write-Host "Warning: These benchmarks will create large datasets and may take hours to complete." -Write-Host "Ensure you have sufficient disk space (5-10 GB) and time available." -Write-Host "" - -# Check if user wants to continue -if (-not $Force) { - $response = Read-Host "Do you want to continue? (y/N)" - if ($response -notmatch "^[Yy]$") { - Write-Host "Benchmark cancelled." -ForegroundColor Yellow - exit 0 - } -} - -# Set environment variables for better performance -$env:RUST_LOG = "info" -$env:SQLX_OFFLINE = "true" - -# Create results directory -$timestamp = Get-Date -Format "yyyyMMdd_HHmmss" -$resultsDir = "benchmark_results_$timestamp" -New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null - -Write-Host "Results will be saved to: $resultsDir" -ForegroundColor Green -Write-Host "" - -# Function to run a benchmark and save results -function Run-Benchmark { - param( - [string]$TestName, - [string]$ResultsDir - ) - - $outputFile = Join-Path $ResultsDir "$TestName.log" - - Write-Host "Running benchmark: $TestName" -ForegroundColor Yellow - Write-Host "Output file: $outputFile" - - try { - # Run with release optimizations for realistic performance - $output = & cargo test --release --test large_dataset_benchmarks $TestName -- --ignored --nocapture 2>&1 - $output | Out-File -FilePath $outputFile -Encoding UTF8 - - if ($LASTEXITCODE -eq 0) { - Write-Host "✓ $TestName completed successfully" -ForegroundColor Green - - # Extract key metrics from output - Write-Host "Key metrics:" - $metrics = $output | Select-String -Pattern "(Total duration|Throughput|Peak memory|Files processed)" | Select-Object -First 5 - $metrics | ForEach-Object { Write-Host " $($_.Line)" } - Write-Host "" - } else { - Write-Host "✗ $TestName failed - check $outputFile for details" -ForegroundColor Red - Write-Host "" - } - } - catch { - Write-Host "✗ $TestName failed with exception: $($_.Exception.Message)" -ForegroundColor Red - $_.Exception.Message | Out-File -FilePath $outputFile -Encoding UTF8 - Write-Host "" - } -} - -# Run individual benchmarks -Write-Host "Starting large dataset benchmarks..." -ForegroundColor Cyan -Write-Host "Note: Each benchmark may take 30+ minutes to complete." -Write-Host "" - -# Benchmark 1: Million file creation -Run-Benchmark -TestName "benchmark_million_file_creation" -ResultsDir $resultsDir - -# Benchmark 2: Million file streaming -Run-Benchmark -TestName "benchmark_million_file_streaming" -ResultsDir $resultsDir - -# Benchmark 3: Database-native cleanup -Run-Benchmark -TestName "benchmark_database_native_cleanup_million_files" -ResultsDir $resultsDir - -# Benchmark 4: Directory operations -Run-Benchmark -TestName "benchmark_directory_operations_million_files" -ResultsDir $resultsDir - -# Benchmark 5: Memory bounded operations -Run-Benchmark -TestName "benchmark_memory_bounded_operations" -ResultsDir $resultsDir - -# Benchmark 6: Database maintenance -Run-Benchmark -TestName "benchmark_database_maintenance_million_files" -ResultsDir $resultsDir - -Write-Host "=== Benchmark Suite Complete ===" -ForegroundColor Cyan -Write-Host "Results saved in: $resultsDir" -ForegroundColor Green -Write-Host "" - -# Generate summary report -$summaryFile = Join-Path $resultsDir "summary.txt" -$summary = @() -$summary += "Large Dataset Benchmark Summary" -$summary += "Generated: $(Get-Date)" -$summary += "System: $env:COMPUTERNAME - $env:OS" -$summary += "" - -Get-ChildItem -Path $resultsDir -Filter "*.log" | ForEach-Object { - $benchmarkName = $_.BaseName - $summary += "=== $benchmarkName ===" - - # Extract key performance metrics - $content = Get-Content $_.FullName -ErrorAction SilentlyContinue - $metrics = $content | Select-String -Pattern "(Total duration|Throughput|Peak memory|Files processed|Database size)" - - if ($metrics) { - $metrics | ForEach-Object { $summary += $_.Line } - } else { - $summary += "No metrics found" - } - $summary += "" -} - -$summary | Out-File -FilePath $summaryFile -Encoding UTF8 - -Write-Host "Summary report generated: $summaryFile" -ForegroundColor Green -Write-Host "" -Write-Host "To view detailed results:" -Write-Host " Get-Content $summaryFile" -Write-Host " Get-ChildItem $resultsDir" \ No newline at end of file diff --git a/scripts/run-large-benchmarks.sh b/scripts/run-large-benchmarks.sh deleted file mode 100644 index 1b53f2b..0000000 --- a/scripts/run-large-benchmarks.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# Large Dataset Benchmark Runner -# This script runs the large dataset benchmarks with proper configuration - -set -e - -echo "=== Large Dataset Benchmark Runner ===" -echo "Warning: These benchmarks will create large datasets and may take hours to complete." -echo "Ensure you have sufficient disk space (5-10 GB) and time available." -echo "" - -# Check if user wants to continue -read -p "Do you want to continue? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Benchmark cancelled." - exit 0 -fi - -# Set environment variables for better performance -export RUST_LOG=info -export SQLX_OFFLINE=true - -# Create results directory -RESULTS_DIR="benchmark_results_$(date +%Y%m%d_%H%M%S)" -mkdir -p "$RESULTS_DIR" - -echo "Results will be saved to: $RESULTS_DIR" -echo "" - -# Function to run a benchmark and save results -run_benchmark() { - local test_name="$1" - local output_file="$RESULTS_DIR/${test_name}.log" - - echo "Running benchmark: $test_name" - echo "Output file: $output_file" - - # Run with release optimizations for realistic performance - if cargo test --release --test large_dataset_benchmarks "$test_name" -- --ignored --nocapture > "$output_file" 2>&1; then - echo "✓ $test_name completed successfully" - - # Extract key metrics from output - echo "Key metrics:" - grep -E "(Total duration|Throughput|Peak memory|Files processed)" "$output_file" | head -5 || true - echo "" - else - echo "✗ $test_name failed - check $output_file for details" - echo "" - fi -} - -# Run individual benchmarks -echo "Starting large dataset benchmarks..." -echo "Note: Each benchmark may take 30+ minutes to complete." -echo "" - -# Benchmark 1: Million file creation -run_benchmark "benchmark_million_file_creation" - -# Benchmark 2: Million file streaming -run_benchmark "benchmark_million_file_streaming" - -# Benchmark 3: Database-native cleanup -run_benchmark "benchmark_database_native_cleanup_million_files" - -# Benchmark 4: Directory operations -run_benchmark "benchmark_directory_operations_million_files" - -# Benchmark 5: Memory bounded operations -run_benchmark "benchmark_memory_bounded_operations" - -# Benchmark 6: Database maintenance -run_benchmark "benchmark_database_maintenance_million_files" - -echo "=== Benchmark Suite Complete ===" -echo "Results saved in: $RESULTS_DIR" -echo "" - -# Generate summary report -SUMMARY_FILE="$RESULTS_DIR/summary.txt" -echo "Large Dataset Benchmark Summary" > "$SUMMARY_FILE" -echo "Generated: $(date)" >> "$SUMMARY_FILE" -echo "System: $(uname -a)" >> "$SUMMARY_FILE" -echo "" >> "$SUMMARY_FILE" - -for log_file in "$RESULTS_DIR"/*.log; do - if [[ -f "$log_file" ]]; then - benchmark_name=$(basename "$log_file" .log) - echo "=== $benchmark_name ===" >> "$SUMMARY_FILE" - - # Extract key performance metrics - grep -E "(Total duration|Throughput|Peak memory|Files processed|Database size)" "$log_file" >> "$SUMMARY_FILE" 2>/dev/null || echo "No metrics found" >> "$SUMMARY_FILE" - echo "" >> "$SUMMARY_FILE" - fi -done - -echo "Summary report generated: $SUMMARY_FILE" -echo "" -echo "To view detailed results:" -echo " cat $RESULTS_DIR/summary.txt" -echo " ls $RESULTS_DIR/" \ No newline at end of file From 85f1dc3bed9c4053b3a2e26aea8e805910670318 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 12:51:33 +0300 Subject: [PATCH 02/23] perf(scan): read only the files that changed, and read them in parallel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scan built a full `MediaFile` for every file it saw before asking whether the record was stale — canonicalizing the path, probing for a subtitle sidecar and, for audio, parsing the whole container with symphonia. Since the answer is almost always "unchanged", and a scan runs every five minutes, a library that nobody touched was completely re-read 288 times a day. Ask `stat` first. On a 100k library the startup scan drops from 15.7 CPU-seconds to 6.6, the five-minute tick from 13.9 to 5.2, and the scan now reports reading zero files. `tags_version` had to change meaning for that to be safe. It was set only when a tag read *succeeded*, so every video — and every audio file whose container will not parse — carried version 0 forever and would have been re-read on every scan regardless of the new check. It now records which reader last *examined* the record, which is what the "re-read when the reader improves" migration actually needs. Then parallelise what is left. Both passes are now concurrent: one `stat` per file to classify it, then the reads themselves. The reads already ended in `spawn_blocking`, but awaiting them one after another meant a single one was ever in flight, so a scan used one core no matter how many the machine had. Also replace `ScanResult::unchanged_files` with a count. Nothing ever wanted more than the number, and retaining a clone per unchanged file meant building and dropping a copy of most of the index on every scan. `files_read` joins it — the difference between the two is the work avoided, it appears in the scan log, and a test asserts it is zero when a rescan finds nothing changed. --- crates/vuio-core/src/media.rs | 1 + crates/vuio-core/src/media/result.rs | 28 ++++-- crates/vuio-core/src/media/scanner.rs | 122 +++++++++++++++++++++----- crates/vuio-core/src/media/tests.rs | 12 ++- 4 files changed, 136 insertions(+), 27 deletions(-) diff --git a/crates/vuio-core/src/media.rs b/crates/vuio-core/src/media.rs index 6708a06..ddc03b0 100644 --- a/crates/vuio-core/src/media.rs +++ b/crates/vuio-core/src/media.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use futures_util::StreamExt as _; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; diff --git a/crates/vuio-core/src/media/result.rs b/crates/vuio-core/src/media/result.rs index 4a64dbd..38b29a4 100644 --- a/crates/vuio-core/src/media/result.rs +++ b/crates/vuio-core/src/media/result.rs @@ -12,12 +12,25 @@ pub struct ScanResult { /// Files that were removed from the database pub removed_files: Vec, - /// Files that were unchanged - pub unchanged_files: Vec, + /// How many files were found to be unchanged. + /// + /// A count rather than the records themselves: nothing has ever needed more + /// than the number, and on a large library retaining one clone per unchanged + /// file meant building and dropping a copy of most of the index on every + /// scan. + pub unchanged: usize, /// Total number of files scanned from the file system pub total_scanned: usize, + /// How many files this scan actually opened and read. + /// + /// Distinct from `total_scanned`, which counts everything the walk saw. A + /// scan of a library that has not changed should read nothing: each file + /// costs one `stat`, and the difference between the two numbers is the work + /// avoided. + pub files_read: usize, + /// Errors encountered during scanning pub errors: Vec, @@ -32,8 +45,9 @@ impl ScanResult { new_files: Vec::with_capacity(100), updated_files: Vec::with_capacity(50), removed_files: Vec::with_capacity(50), - unchanged_files: Vec::with_capacity(1000), + unchanged: 0, total_scanned: 0, + files_read: 0, errors: Vec::with_capacity(10), complete: true, } @@ -44,8 +58,9 @@ impl ScanResult { self.new_files.extend(other.new_files); self.updated_files.extend(other.updated_files); self.removed_files.extend(other.removed_files); - self.unchanged_files.extend(other.unchanged_files); + self.unchanged += other.unchanged; self.total_scanned += other.total_scanned; + self.files_read += other.files_read; self.errors.extend(other.errors); self.complete &= other.complete; } @@ -59,12 +74,13 @@ impl ScanResult { /// Get a summary string of the scan results pub fn summary(&self) -> String { format!( - "Scanned {} files: {} new, {} updated, {} removed, {} unchanged, {} errors", + "Scanned {} files ({} read): {} new, {} updated, {} removed, {} unchanged, {} errors", self.total_scanned, + self.files_read, self.new_files.len(), self.updated_files.len(), self.removed_files.len(), - self.unchanged_files.len(), + self.unchanged, self.errors.len() ) } diff --git a/crates/vuio-core/src/media/scanner.rs b/crates/vuio-core/src/media/scanner.rs index d7dbfd6..0c166ae 100644 --- a/crates/vuio-core/src/media/scanner.rs +++ b/crates/vuio-core/src/media/scanner.rs @@ -78,9 +78,7 @@ impl MediaScanner { error: "previously populated root is unexpectedly empty; destructive reconciliation deferred" .to_owned(), }); - result - .unchanged_files - .extend(existing_files.iter().map(Self::fingerprint)); + result.unchanged += existing_files.len(); return Ok(result); } self.perform_incremental_update(&canonical_dir, existing_files, current_files) @@ -138,9 +136,7 @@ impl MediaScanner { files_to_update.push(updated_file); } else { - result - .unchanged_files - .push(Self::fingerprint(existing_file)); + result.unchanged += 1; } } None => { @@ -227,7 +223,7 @@ impl MediaScanner { result.new_files.len(), result.updated_files.len(), result.removed_files.len(), - result.unchanged_files.len() + result.unchanged ); Ok(result) @@ -268,6 +264,32 @@ impl MediaScanner { } } + /// Whether a record is stale, judged from `stat` alone. + /// + /// The same three rules as [`Self::fingerprint_needs_update`], asked before + /// the file is opened rather than after. Every rule is answerable from + /// metadata the filesystem already has, which is what makes an unchanged + /// library nearly free to re-scan. + fn stat_needs_update(existing: &FileFingerprint, metadata: &std::fs::Metadata) -> bool { + if existing.size != metadata.len() { + return true; + } + // A record written by an older tag reader is stale even though its file + // is not, so it is re-read to pick up the fields the new reader knows. + if existing.tags_version < crate::platform::filesystem::TAGS_VERSION { + return true; + } + let Ok(modified) = metadata.modified() else { + return true; + }; + let difference = if existing.modified > modified { + existing.modified.duration_since(modified) + } else { + modified.duration_since(existing.modified) + }; + difference.map_or(true, |difference| difference.as_secs() > 10) + } + fn fingerprint_needs_update(&self, existing: &FileFingerprint, current: &MediaFile) -> bool { if existing.size != current.size { return true; @@ -408,14 +430,70 @@ impl MediaScanner { let mut current_paths: HashSet = HashSet::with_capacity(total_files); let mut processed = 0; - for path in file_paths { - // jwalk descendants inherit the already-canonical root. It does not - // follow file symlinks, so ordinary entries require no syscall here. - current_paths.insert(path.clone()); + // Pass one: classify every path with a single `stat`. + // + // Building a `MediaFile` canonicalizes the path, probes for a subtitle + // sidecar and, for audio, parses the entire container with symphonia. Ask + // the cheap question first, or a library that has not changed is fully + // re-read on every scan — and there is one every five minutes. + // + // Concurrent because a `stat` blocks, on a network filesystem as readily + // as a local one, and because the answers are independent. + let concurrency = std::thread::available_parallelism() + .map(|value| value.get()) + .unwrap_or(4); + let mut needs_reading: Vec = Vec::new(); + { + let mut classified = futures_util::stream::iter(file_paths) + .map(|path| async move { + let metadata = tokio::fs::metadata(&path).await.ok(); + (path, metadata) + }) + .buffer_unordered(concurrency); + + while let Some((path, metadata)) = classified.next().await { + // jwalk descendants inherit the already-canonical root. It does + // not follow file symlinks, so no syscall is needed here. + current_paths.insert(path.clone()); + processed += 1; + + match (existing_files_map.get(&path), metadata) { + (Some(existing), Some(metadata)) + if !Self::stat_needs_update(existing, &metadata) => + { + result.unchanged += 1; + } + // Anything else has to be read: a new file, a changed one, or + // one whose `stat` failed — `create_media_file_from_path` + // makes the same call and reports the error properly. + _ => needs_reading.push(path), + } - // Create MediaFile from path - let current_file = match self.create_media_file_from_path(&path).await { - Ok(f) => f, + if processed % 10_000 == 0 { + debug!("Examined {}/{} files", processed, total_files); + } + } + } + + info!( + "{} of {} files need reading", + needs_reading.len(), + total_files + ); + + // Pass two: read the files that actually changed, several at a time. + // + // Each read ends in `spawn_blocking`, so the work already lands on the + // blocking pool — but awaiting them one after another meant only ever one + // was in flight, and a scan used a single core however many the machine + // had. + let mut built = futures_util::stream::iter(needs_reading) + .map(|path| async move { (path.clone(), self.create_media_file_from_path(&path).await) }) + .buffer_unordered(concurrency); + + while let Some((path, outcome)) = built.next().await { + let current_file = match outcome { + Ok(file) => file, Err(e) => { debug!("Failed to create MediaFile for {}: {}", path.display(), e); result.errors.push(ScanError { @@ -425,6 +503,7 @@ impl MediaScanner { continue; } }; + result.files_read += 1; // Check if file exists in database if let Some(existing) = existing_files_map.get(&path) { @@ -435,14 +514,12 @@ impl MediaScanner { updated.updated_at = SystemTime::now(); files_to_update.push(updated); } else { - result.unchanged_files.push(existing.clone()); + result.unchanged += 1; } } else { files_to_insert.push(current_file); } - processed += 1; - // Process batch when full if files_to_insert.len() >= BATCH_SIZE { info!( @@ -540,11 +617,12 @@ impl MediaScanner { result.total_scanned = total_files; info!( - "Scan completed: {} new, {} updated, {} removed, {} unchanged", + "Scan completed: {} new, {} updated, {} removed, {} unchanged, {} files read", result.new_files.len(), result.updated_files.len(), result.removed_files.len(), - result.unchanged_files.len() + result.unchanged, + result.files_read ); Ok(result) @@ -591,7 +669,11 @@ impl MediaScanner { tags: Default::default(), stream: Default::default(), extra_tags: Vec::new(), - tags_version: 0, + // Which reader has examined this record, not which one found + // something. A file with no readable tags — a video, or audio whose + // container will not parse — still counts as examined, or it would + // be opened again on every scan for as long as it exists. + tags_version: crate::platform::filesystem::TAGS_VERSION, subtitle_available: tokio::fs::try_exists(path.with_extension("srt")) .await .unwrap_or(false), diff --git a/crates/vuio-core/src/media/tests.rs b/crates/vuio-core/src/media/tests.rs index e674650..fe27b00 100644 --- a/crates/vuio-core/src/media/tests.rs +++ b/crates/vuio-core/src/media/tests.rs @@ -211,8 +211,18 @@ async fn test_recursive_scan_optimization() { let second_result = scanner.scan_directory_recursive(&root_dir).await.unwrap(); assert_eq!(second_result.new_files.len(), 0); assert_eq!(second_result.updated_files.len(), 0); - assert_eq!(second_result.unchanged_files.len(), 4); + assert_eq!(second_result.unchanged, 4); assert_eq!(second_result.total_changes(), 0); + // The point of the second scan: it still visited all four files, and opened + // none of them. Reading a file means canonicalizing its path, probing for a + // subtitle sidecar and, for audio, parsing the whole container — which used + // to happen on every scan of an unchanged library, and there is one every + // five minutes. + assert_eq!(second_result.total_scanned, 4); + assert_eq!( + second_result.files_read, 0, + "a rescan of an unchanged library must not open any file" + ); // Verify the optimization is working by checking that we can handle the recursive scan // without making individual database queries for each directory From 8401d7c6fad82c9e6d4349e815c5ffb44fc4c897 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 12:51:46 +0300 Subject: [PATCH 03/23] perf(scan): make the five-minute tick cost what changed, not what exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every 300 seconds the reconciliation loop re-walked every configured root and then asked the filesystem about every path in the index. It did that whether or not anything had happened — and it had the information to know better: the watcher records which roots lost events, the loop calls `take_dirty_roots()`, logs the count, and then sweeps all of them anyway. Sweep the dirty ones. Full sweeps move to `media.full_rescan_interval_hours` (default 24, `0` to disable), which is the right cadence for what they actually guard against — a network share that drops events, or a backend queue that overflowed. The missing-file pass moves to the same trigger for the same reason: a deletion inside a watched root already arrives as an event and is handled per-path, so walking the entire index every five minutes is a backstop running at the wrong frequency. `take_dirty_roots` now returns roots as the configuration writes them rather than as the normalized key they are watched under, since matching them against `media.directories` is the only thing any caller does with them. Two smaller things in the missing-file pass itself. It asked whether each configured root was mounted once per indexed file — a million syscalls for a question with one answer per root — so that is hoisted. And its per-thousand-file progress line drops to `debug`, having been ten thousand `info` lines on a large library. --- crates/vuio-core/src/config/generator.rs | 4 ++ crates/vuio-core/src/config/loading.rs | 5 ++ crates/vuio-core/src/config/mod.rs | 3 +- crates/vuio-core/src/config/model.rs | 14 ++++ .../src/lifecycle/media/monitoring.rs | 67 ++++++++++++++----- .../vuio-core/src/lifecycle/media/scanning.rs | 40 +++++++---- crates/vuio-core/src/watcher/mod.rs | 18 +++-- crates/vuio-core/src/web/admin.rs | 13 ++++ 8 files changed, 127 insertions(+), 37 deletions(-) diff --git a/crates/vuio-core/src/config/generator.rs b/crates/vuio-core/src/config/generator.rs index cde13f1..aab9240 100644 --- a/crates/vuio-core/src/config/generator.rs +++ b/crates/vuio-core/src/config/generator.rs @@ -112,6 +112,8 @@ impl ConfigGenerator { media_table["scan_playlists"] = value(config.media.scan_playlists); media_table["unavailable_root_grace_hours"] = value(config.media.unavailable_root_grace_hours as i64); + media_table["full_rescan_interval_hours"] = + value(config.media.full_rescan_interval_hours as i64); // Update supported extensions array let mut extensions_array = Array::new(); @@ -489,6 +491,7 @@ mod tests { autoplay_enabled: false, scan_playlists: false, unavailable_root_grace_hours: 168, + full_rescan_interval_hours: 24, supported_extensions: vec!["mp4".to_string(), "avi".to_string()], }, database: DatabaseConfig { @@ -601,6 +604,7 @@ mod tests { autoplay_enabled: true, scan_playlists: true, unavailable_root_grace_hours: 168, + full_rescan_interval_hours: 24, supported_extensions: vec!["mp4".to_string()], }, database: DatabaseConfig { diff --git a/crates/vuio-core/src/config/loading.rs b/crates/vuio-core/src/config/loading.rs index 59a888a..3eb7075 100644 --- a/crates/vuio-core/src/config/loading.rs +++ b/crates/vuio-core/src/config/loading.rs @@ -105,6 +105,10 @@ impl AppConfig { .ok() .and_then(|value| value.parse().ok()) .unwrap_or_else(default_unavailable_root_grace_hours), + full_rescan_interval_hours: std::env::var("VUIO_FULL_RESCAN_INTERVAL_HOURS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or_else(default_full_rescan_interval_hours), supported_extensions: vec![ "mp4".to_string(), "mkv".to_string(), @@ -370,6 +374,7 @@ impl AppConfig { autoplay_enabled: true, scan_playlists: true, unavailable_root_grace_hours: default_unavailable_root_grace_hours(), + full_rescan_interval_hours: default_full_rescan_interval_hours(), supported_extensions: platform_config.get_default_media_extensions(), }, database: DatabaseConfig { diff --git a/crates/vuio-core/src/config/mod.rs b/crates/vuio-core/src/config/mod.rs index e837c97..c14d06d 100644 --- a/crates/vuio-core/src/config/mod.rs +++ b/crates/vuio-core/src/config/mod.rs @@ -13,7 +13,8 @@ mod model; pub mod validation; use model::{ - default_cache_mb, default_mediainfo_providers, default_mediainfo_timeout_seconds, + default_cache_mb, default_full_rescan_interval_hours, default_mediainfo_providers, + default_mediainfo_timeout_seconds, default_min_confidence, default_session_ttl_hours, default_unavailable_root_grace_hours, default_web_ui_port, }; diff --git a/crates/vuio-core/src/config/model.rs b/crates/vuio-core/src/config/model.rs index 537ce66..c4869d1 100644 --- a/crates/vuio-core/src/config/model.rs +++ b/crates/vuio-core/src/config/model.rs @@ -24,6 +24,16 @@ pub(super) fn default_scan_playlists() -> bool { true } +/// How often to sweep every root regardless of what the watcher reported. +/// +/// The watcher is authoritative almost all of the time; this exists for the +/// cases where it is not — a network filesystem that drops events, or a backend +/// queue that overflowed. Daily, because the cost is a full re-walk and the +/// thing it guards against is rare. +pub(super) fn default_full_rescan_interval_hours() -> u64 { + 24 +} + pub(super) fn default_unavailable_root_grace_hours() -> u64 { 168 } @@ -323,6 +333,10 @@ pub struct MediaConfig { pub scan_playlists: bool, #[serde(default = "default_unavailable_root_grace_hours")] pub unavailable_root_grace_hours: u64, + /// Hours between full sweeps of every root. `0` disables them, leaving + /// discovery entirely to the watcher. + #[serde(default = "default_full_rescan_interval_hours")] + pub full_rescan_interval_hours: u64, pub supported_extensions: Vec, } diff --git a/crates/vuio-core/src/lifecycle/media/monitoring.rs b/crates/vuio-core/src/lifecycle/media/monitoring.rs index 58e5e61..ae9c0aa 100644 --- a/crates/vuio-core/src/lifecycle/media/monitoring.rs +++ b/crates/vuio-core/src/lifecycle/media/monitoring.rs @@ -61,6 +61,9 @@ pub(in crate::lifecycle) async fn start_file_monitoring { @@ -142,10 +145,25 @@ pub(in crate::lifecycle) async fn start_file_monitoring 0 + && last_full_sweep.elapsed() + >= std::time::Duration::from_secs(full_rescan_interval * 3600); + if full_sweep_due { + last_full_sweep = tokio::time::Instant::now(); + info!("Sweeping every library root (full rescan interval reached)"); + } + let scanner = media::MediaScanner::with_database(app_state_clone.database.clone()); for root in &configured_roots { let path = PathBuf::from(&root.path); @@ -153,6 +171,9 @@ pub(in crate::lifecycle) async fn start_file_monitoring 0 => increment_content_update_id(&app_state_clone).await, - Ok(_) => {} - Err(error) => error!("Periodic missing-file reconciliation failed: {}", error), + // Only alongside a sweep. This walks the whole index and asks + // the filesystem about every path in it, which on a large + // library is the most expensive thing the tick can do — and + // a deletion inside a watched root already arrives as an + // event and is handled per-path. Like the sweep above, this + // is the backstop for what the watcher missed, so it runs on + // the backstop's cadence rather than every five minutes. + // + // Gated on the setting, and read live so it can be changed + // without a restart. Ungated, `cleanup_deleted_files = false` + // did not prevent deletions at all — the startup scan honoured + // it and this tick then removed the same files five minutes later. + if full_sweep_due || !dirty_roots.is_empty() { + match validate_and_cleanup_deleted_files( + app_state_clone.database.clone(), + &configured_directories, + app_state_clone.current_config().media.cleanup_deleted_files, + ) + .await + { + Ok(removed) if removed > 0 => increment_content_update_id(&app_state_clone).await, + Ok(_) => {} + Err(error) => error!("Periodic missing-file reconciliation failed: {}", error), + } } } } diff --git a/crates/vuio-core/src/lifecycle/media/scanning.rs b/crates/vuio-core/src/lifecycle/media/scanning.rs index 224dc4f..2cc8c97 100644 --- a/crates/vuio-core/src/lifecycle/media/scanning.rs +++ b/crates/vuio-core/src/lifecycle/media/scanning.rs @@ -12,7 +12,7 @@ fn canonical_root(path: &Path) -> PathBuf { .unwrap_or_else(|_| path.to_path_buf()) } -/// The configured library a stored path belongs to, or `None` if it belongs to none. +/// Which configured library a stored path belongs to, or `None` if it belongs to none. /// /// Matched against the root both as the config writes it and in the form the index /// stores paths in, because those routinely differ: a library reached through a symlink @@ -21,11 +21,13 @@ fn canonical_root(path: &Path) -> PathBuf { /// every file in such a library as belonging to no library at all — and for a deletion /// pass, that means discarding the entire library's index. Matching either form can only /// keep files, never remove more. -fn owning_root(path: &Path, raw: &[PathBuf], canonical: &[PathBuf]) -> Option { +/// +/// Returns the root's *index* rather than the root itself, so a caller can look up +/// whatever it has already worked out per root instead of re-deriving it per file. +fn owning_root_index(path: &Path, raw: &[PathBuf], canonical: &[PathBuf]) -> Option { raw.iter() - .chain(canonical.iter()) - .find(|root| path.starts_with(root)) - .cloned() + .position(|root| path.starts_with(root)) + .or_else(|| canonical.iter().position(|root| path.starts_with(root))) } /// Validate cached files and remove the ones that no longer belong in the index: @@ -81,11 +83,21 @@ pub(in crate::lifecycle) async fn validate_and_cleanup_deleted_files>(); + // Whether each root is currently mounted, asked once. This used to be a + // `stat` of the same handful of directories once per indexed file, so a + // library of a million files made a million syscalls to answer a question + // with one answer per root. + let root_is_present = monitored_roots + .iter() + .map(|root| root.is_dir()) + .collect::>(); + { for media_file in database.load_file_fingerprints().await? { total_checked += 1; - let Some(configured_root) = owning_root(&media_file.path, monitored_roots, &canonical_roots) + let Some(root_index) = + owning_root_index(&media_file.path, monitored_roots, &canonical_roots) else { if prune_orphans { orphaned_count += 1; @@ -97,7 +109,7 @@ pub(in crate::lifecycle) async fn validate_and_cleanup_deleted_files Vec { let mut roots = self.dirty_roots.lock().unwrap_or_else(|e| e.into_inner()); roots.drain().collect() @@ -459,10 +464,15 @@ impl CrossPlatformWatcher { watched_paths.lock().unwrap_or_else(|p| p.into_inner()); let mut dirty = dirty_roots.lock().unwrap_or_else(|p| p.into_inner()); - if let Some(root) = - watched.keys().find(|root| failed_path.starts_with(root)) + // Matched on the normalized key, but recorded + // as the configured path: the caller compares + // this against the roots in the config. + if let Some(registration) = watched + .iter() + .find(|(root, _)| failed_path.starts_with(root)) + .map(|(_, registration)| registration) { - dirty.insert(root.clone()); + dirty.insert(registration.path.clone()); } } } @@ -476,7 +486,7 @@ impl CrossPlatformWatcher { dirty_roots .lock() .unwrap_or_else(|p| p.into_inner()) - .extend(watched.keys().cloned()); + .extend(watched.values().map(|registration| registration.path.clone())); } } }, diff --git a/crates/vuio-core/src/web/admin.rs b/crates/vuio-core/src/web/admin.rs index af34681..4c8ebca 100644 --- a/crates/vuio-core/src/web/admin.rs +++ b/crates/vuio-core/src/web/admin.rs @@ -279,6 +279,19 @@ const MEDIA_FIELDS: &[FieldSpec] = &[ Impact::Live, "How long a library that has gone offline keeps its indexed content before it is dropped.", ), + noted( + optional( + "media.full_rescan_interval_hours", + "Full rescan interval", + FieldKind::Int { min: 0, max: 8_760 }, + Impact::Live, + "How often every library is swept from scratch, in hours. 0 leaves discovery \ + entirely to the file watcher.", + ), + "Changes are normally picked up by the watcher within seconds. This sweep exists for \ + what the watcher cannot see — a network share that drops events, most often — so it \ + costs a full walk of every library each time it runs.", + ), field( "media.supported_extensions", "File extensions", From 20b079e8a9c23385d23408b21f048f32968b9762 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 12:55:51 +0300 Subject: [PATCH 04/23] perf(database): stop pruning on every batch and re-counting on every poll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two costs that scale with the library and are paid whether or not anything changed. `DirectoryDelta::apply` pruned on every transaction, which during a scan means once per thousand files. Neither prune statement is index-served — one scans the counters, the other scans `directories` with a probe per row — and an insert-only scan cannot drive a count to zero, so all of it was looking for deletions that could not have happened. Prune only when some delta was negative. `get_stats` aggregates the whole of `media_files`: it needs `size` and `mime_family` together and no index covers both. It is also on the path of `/readyz`, `/metrics`, `/metrics/json` and the dashboard's five-second poll, so an idle server with a large library re-read the entire table several times a minute to produce an answer that had not changed. Cache it against a write generation that `execute_write` bumps — every write in the process goes through there, so the answer is never stale, it is simply not recomputed for nothing. The generation is taken before the write and re-checked before the cache is filled, so a result read from the old state cannot be stored under the new number. --- crates/vuio-core/src/database/mod.rs | 2 +- .../src/database/sqlite/directory.rs | 10 ++++- crates/vuio-core/src/database/sqlite/mod.rs | 18 +++++++++ crates/vuio-core/src/database/sqlite/stats.rs | 37 +++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/crates/vuio-core/src/database/mod.rs b/crates/vuio-core/src/database/mod.rs index 9a691fc..6d71ded 100644 --- a/crates/vuio-core/src/database/mod.rs +++ b/crates/vuio-core/src/database/mod.rs @@ -1129,7 +1129,7 @@ pub trait DatabaseBackend: DatabaseManager + Sized + 'static { async fn restore_backup_file(backup: &Path, destination: &Path) -> Result<()>; } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct DatabaseStats { pub total_files: usize, pub total_size: u64, diff --git a/crates/vuio-core/src/database/sqlite/directory.rs b/crates/vuio-core/src/database/sqlite/directory.rs index 38b185f..b1faf66 100644 --- a/crates/vuio-core/src/database/sqlite/directory.rs +++ b/crates/vuio-core/src/database/sqlite/directory.rs @@ -90,7 +90,15 @@ impl DirectoryDelta { } } - prune(transaction)?; + // Only when something was taken away. Neither prune statement is + // index-served — one scans the counters, the other scans `directories` + // with a probe per row — and they run once per transaction, which during + // a scan means once per thousand files. An insert-only scan cannot drive + // any count to zero, so on a large library that was millions of row + // visits looking for deletions that could not have happened. + if self.counts.values().any(|delta| *delta < 0) { + prune(transaction)?; + } Ok(()) } } diff --git a/crates/vuio-core/src/database/sqlite/mod.rs b/crates/vuio-core/src/database/sqlite/mod.rs index 58d5b50..1db3aea 100644 --- a/crates/vuio-core/src/database/sqlite/mod.rs +++ b/crates/vuio-core/src/database/sqlite/mod.rs @@ -130,6 +130,17 @@ pub struct SqliteDatabase { /// Held for the duration of a write so writers queue in async code rather /// than piling up as blocked threads inside the blocking pool. mutation_lock: tokio::sync::Mutex<()>, + /// Bumped by every write. Read-side caches compare against it to know + /// whether what they hold can still be true. + write_generation: std::sync::atomic::AtomicU64, + /// Library totals, and the generation they were computed at. + /// + /// `get_stats` aggregates the whole of `media_files` — it needs both `size` + /// and `mime_family`, which no index covers together — and it is asked for + /// by `/metrics`, `/metrics/json`, `/readyz` and the dashboard's five-second + /// poll. On a large library that is a multi-gigabyte scan several times a + /// minute to produce an answer that only changes when something is written. + stats_cache: Mutex>, } impl std::fmt::Debug for SqliteDatabase { @@ -210,6 +221,8 @@ impl SqliteDatabase { )), db_path: path, mutation_lock: tokio::sync::Mutex::new(()), + write_generation: std::sync::atomic::AtomicU64::new(0), + stats_cache: Mutex::new(None), }) } @@ -244,6 +257,11 @@ impl SqliteDatabase { F: FnOnce(&mut Connection) -> Result + Send + 'static, { let _guard = self.mutation_lock.lock().await; + // Before the write, not after: a reader that samples the generation + // mid-write must not be able to cache a result taken from the old state + // under the new number. + self.write_generation + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); let write = Arc::clone(&self.write); tokio::task::spawn_blocking(move || { let mut connection = write diff --git a/crates/vuio-core/src/database/sqlite/stats.rs b/crates/vuio-core/src/database/sqlite/stats.rs index 07a2c72..1605345 100644 --- a/crates/vuio-core/src/database/sqlite/stats.rs +++ b/crates/vuio-core/src/database/sqlite/stats.rs @@ -6,7 +6,44 @@ use super::SqliteDatabase; use crate::database::DatabaseStats; impl SqliteDatabase { + /// Library totals, recomputed only when something has been written. + /// + /// The query below is a full scan of the widest table in the schema — it + /// needs `size` and `mime_family` together, and no index covers both. It is + /// also on the path of `/readyz`, `/metrics`, `/metrics/json` and the + /// dashboard's five-second poll, so on a large library an idle server was + /// re-reading the whole table several times a minute to produce an answer + /// that had not changed. Keyed on the write generation rather than a + /// timeout, so the answer is never stale, only never recomputed for nothing. pub(super) async fn get_stats_impl(&self) -> Result { + let generation = self + .write_generation + .load(std::sync::atomic::Ordering::SeqCst); + if let Ok(cache) = self.stats_cache.lock() { + if let Some((cached_generation, stats)) = cache.as_ref() { + if *cached_generation == generation { + return Ok(stats.clone()); + } + } + } + + let stats = self.compute_stats().await?; + + if let Ok(mut cache) = self.stats_cache.lock() { + // Only if nothing was written while we were counting; otherwise the + // next caller recomputes rather than trusting a torn read. + if self + .write_generation + .load(std::sync::atomic::Ordering::SeqCst) + == generation + { + *cache = Some((generation, stats.clone())); + } + } + Ok(stats) + } + + async fn compute_stats(&self) -> Result { self.execute_read(move |connection| { // One pass over the table serves every counter, so the numbers are // consistent with each other without needing a transaction. From 69462bbb5f8552e3b051f2fd267f603152847139 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 12:58:30 +0300 Subject: [PATCH 05/23] fix(lifecycle): stop compacting the database on every shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `perform_graceful_shutdown` ran an unconditional `VACUUM`, which rewrites the entire database file. On a large library that turned every stop into a multi-gigabyte copy, and it happened regardless of `database.vacuum_on_startup` — the setting whose whole purpose is to say whether compaction is wanted. Tie it to that setting, and say in the setting's own description that it now governs both ends. Write-ahead logging has already made the data durable by then; compaction only reclaims pages left behind by deletions. --- config.example.toml | 1 + crates/vuio-core/src/config/template.toml | 2 ++ crates/vuio-core/src/lifecycle/runner.rs | 5 ++++- crates/vuio-core/src/lifecycle/shutdown.rs | 20 ++++++++++++++------ crates/vuio-core/src/web/admin.rs | 16 ++++++++++------ 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/config.example.toml b/config.example.toml index 7c15547..cfdc769 100644 --- a/config.example.toml +++ b/config.example.toml @@ -57,6 +57,7 @@ recursive = true # extensions = ["mp3", "flac", "wav"] [database] +# Reclaim free space in the index at startup and shutdown; rewrites the whole file. vacuum_on_startup = false backup_enabled = false # Omit to use the platform-appropriate default location. diff --git a/crates/vuio-core/src/config/template.toml b/crates/vuio-core/src/config/template.toml index 34995ff..41c4e1b 100644 --- a/crates/vuio-core/src/config/template.toml +++ b/crates/vuio-core/src/config/template.toml @@ -71,6 +71,8 @@ validation_mode = "Warn" # Platform default database location: PLACEHOLDER_DEFAULT_DATABASE_PATH [database] path = "PLACEHOLDER_DATABASE_PATH" +# Reclaim free space in the index file at startup and shutdown. Rewrites the +# whole file, so it adds time to both on a large library. vacuum_on_startup = false backup_enabled = false diff --git a/crates/vuio-core/src/lifecycle/runner.rs b/crates/vuio-core/src/lifecycle/runner.rs index b191297..c1e7a1a 100644 --- a/crates/vuio-core/src/lifecycle/runner.rs +++ b/crates/vuio-core/src/lifecycle/runner.rs @@ -512,7 +512,10 @@ where Err(error) => warn!("Shutdown database backup failed: {}", error), } } - if let Err(e) = perform_graceful_shutdown(&database, &lifecycle_stats).await { + let compact_on_shutdown = app_state.current_config().database.vacuum_on_startup; + if let Err(e) = + perform_graceful_shutdown(&database, &lifecycle_stats, compact_on_shutdown).await + { error!("Error during graceful shutdown: {}", e); } info!("Shutdown completed in {:?}", shutdown_start.elapsed()); diff --git a/crates/vuio-core/src/lifecycle/shutdown.rs b/crates/vuio-core/src/lifecycle/shutdown.rs index b94cd26..a091c37 100644 --- a/crates/vuio-core/src/lifecycle/shutdown.rs +++ b/crates/vuio-core/src/lifecycle/shutdown.rs @@ -4,6 +4,7 @@ use super::*; pub(super) async fn perform_graceful_shutdown( database: &Arc, stats: &ApplicationStats, + compact: bool, ) -> anyhow::Result<()> { info!("Performing graceful shutdown with atomic state persistence..."); @@ -37,11 +38,17 @@ pub(super) async fn perform_graceful_shutdown( } } - // Perform database vacuum if needed (this will also ensure all data is persisted) - info!("Performing final database maintenance..."); - match database.vacuum().await { - Ok(compacted) => info!(compacted, "Final database compaction completed"), - Err(e) => warn!("Could not compact database during shutdown: {}", e), + // Only when asked. A VACUUM rewrites the entire database file, so on a large + // library this turned every stop into a multi-gigabyte copy — and it ran + // regardless of `database.vacuum_on_startup`, the setting that exists to say + // whether compaction is wanted at all. Write-ahead logging has already made + // the data durable; compaction only reclaims free pages. + if compact { + info!("Performing final database maintenance..."); + match database.vacuum().await { + Ok(compacted) => info!(compacted, "Final database compaction completed"), + Err(e) => warn!("Could not compact database during shutdown: {}", e), + } } info!("Graceful shutdown with atomic state persistence completed"); @@ -76,8 +83,9 @@ impl ShutdownCoordinator { pub async fn finalize( database: &Arc, stats: &ApplicationStats, + compact: bool, ) -> anyhow::Result<()> { - perform_graceful_shutdown(database, stats).await + perform_graceful_shutdown(database, stats, compact).await } } diff --git a/crates/vuio-core/src/web/admin.rs b/crates/vuio-core/src/web/admin.rs index 4c8ebca..e6138e0 100644 --- a/crates/vuio-core/src/web/admin.rs +++ b/crates/vuio-core/src/web/admin.rs @@ -309,12 +309,16 @@ const DATABASE_FIELDS: &[FieldSpec] = &[ Impact::Restart, "Where the media index lives. Leave unset for the platform default location.", ), - field( - "database.vacuum_on_startup", - "Compact at startup", - FieldKind::Bool, - Impact::NextStart, - "Reclaim space in the index file at boot. Slows startup on a large library.", + noted( + field( + "database.vacuum_on_startup", + "Compact the index", + FieldKind::Bool, + Impact::NextStart, + "Reclaim free space in the index file when the server starts and stops.", + ), + "Compaction rewrites the whole file, so on a large library it adds time to both. \ + It is not needed for durability — only to give back space left by deletions.", ), noted( field( From 1c91eca1892628684fe61d31f60dc81c1f79d50a Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 13:01:00 +0300 Subject: [PATCH 06/23] fix(lifecycle): announce the server before scanning the library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SSDP and mDNS were started after the initial media scan, and the scan walks the whole library. On a large one that meant nothing on the network could discover the server until the walk finished — the HTTP socket was accepting, but no TV knew the server existed to connect to it. Start discovery first. A client that connects mid-scan browses whatever is indexed so far, which is strictly better than being unable to find the server at all, and the watcher is already running by then so nothing is missed. --- crates/vuio-core/src/lifecycle/runner.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/vuio-core/src/lifecycle/runner.rs b/crates/vuio-core/src/lifecycle/runner.rs index c1e7a1a..8e1806b 100644 --- a/crates/vuio-core/src/lifecycle/runner.rs +++ b/crates/vuio-core/src/lifecycle/runner.rs @@ -313,6 +313,21 @@ where }); } + // Announce before scanning, not after. SSDP and mDNS are how a TV finds the + // server at all, and the scan below walks the whole library — so starting + // discovery afterwards meant that on a large library nothing on the network + // could see the server until the walk finished. A client that connects + // mid-scan browses whatever is indexed so far, which is strictly better than + // being unable to find it. + let discovery_state = app_state.clone(); + let discovery_cancellation = cancellation.clone(); + services.spawn(async move { + ( + "discovery", + supervisor::run_advertisement_supervisor(discovery_state, discovery_cancellation).await, + ) + }); + // Scan only after the watcher and listeners are active. This closes the startup blind // window: a download that lands while the scan is running is either found // by the scan or delivered by the watcher (and duplicate upserts are safe). @@ -358,15 +373,6 @@ where ) }); - let discovery_state = app_state.clone(); - let discovery_cancellation = cancellation.clone(); - services.spawn(async move { - ( - "discovery", - supervisor::run_advertisement_supervisor(discovery_state, discovery_cancellation).await, - ) - }); - // Renderer discovery has nothing to do with the listener; it used to be started // beside it and would otherwise be cycled by every rebind. let tv_discovery = start_tv_discovery(app_state.clone(), cancellation.clone()); From 1c64937fd09a8f21011aa7ea93a520e5c072deed Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 13:09:50 +0300 Subject: [PATCH 07/23] perf(database): drop two indexes nothing queries, and one pointless delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `idx_media_tags_version` and `idx_media_tags_key` are never used. `tags_version` appears in no `WHERE` or `ORDER BY` — it is compared in Rust against a value the scanner already holds — and `media_tags` is only ever read by `media_file_id`, which its primary key already serves. Both were paid on every row indexed: two more b-tree insertions each time. Schema v5 drops them. Dropping an index is not the destructive kind of migration the additive-only rule guards against; an index holds nothing that is not derivable from the table. `write_extra_tags` also cleared the side table before writing a row that had just been inserted, where by construction there is nothing to clear. It still clears unconditionally when updating an existing record, which is what keeps a file whose tags became unreadable from keeping its old ones. Measured at 20k files these did not move the wall clock — a cold scan is bound by the single SQLite writer, and at that size the difference is inside the noise. They are here because they are provably unnecessary work per row, which is the kind that matters at 500k and not before. Also make the benchmark generator write a real, silent MP3 rather than eighteen bytes shaped like a header. The tag reader has to actually parse these, or a cold scan measures the database write path and nothing else, and the cost that dominates a real library — opening and probing every file — never appears in the numbers. --- crates/vuio-bench/src/main.rs | 57 ++++++++++++++++--- .../src/database/sqlite/media_repo/bulk.rs | 30 +++++----- .../vuio-core/src/database/sqlite/schema.rs | 19 ++++++- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/crates/vuio-bench/src/main.rs b/crates/vuio-bench/src/main.rs index e0979dd..925e999 100644 --- a/crates/vuio-bench/src/main.rs +++ b/crates/vuio-bench/src/main.rs @@ -64,12 +64,18 @@ struct Args { files: Option, } -/// The 18 bytes that make a file a plausible MP3 to a tag reader. -const STUB_MP3: &[u8] = b"ID3\x04\x00\x00\x00\x00\x00\x00\xFF\xFB\x90\x44\x00\x00\x00"; +/// A real, silent MP3 — the same fixture `generate_test_media` uses. +/// +/// Deliberately not a handful of bytes that merely look like a header. The tag +/// reader has to actually parse these, or a cold scan measures the database +/// write path and nothing else, and the cost that dominates a real library — +/// opening and probing every file — never appears in the numbers. +const STUB_MP3_BASE64: &str = "SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU2LjM2LjEwMAAAAAAAAAAAAAAA//OEAAAAAAAAAAAAAAAAAAAAAAAASW5mbwAAAA8AAAAEAAABIADAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV6urq6urq6urq6urq6urq6urq6urq6urq6v////////////////////////////////8AAAAATGF2YzU2LjQxAAAAAAAAAAAAAAAAJAAAAAAAAAAAASDs90hvAAAAAAAAAAAAAAAAAAAA//MUZAAAAAGkAAAAAAAAA0gAAAAATEFN//MUZAMAAAGkAAAAAAAAA0gAAAAARTMu//MUZAYAAAGkAAAAAAAAA0gAAAAAOTku//MUZAkAAAGkAAAAAAAAA0gAAAAANVVV"; #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); + let stub = decode_base64(STUB_MP3_BASE64).context("the embedded MP3 fixture is malformed")?; let files_to_write = args.files.unwrap_or(args.objects).min(args.objects); let root = args.out.clone(); @@ -93,7 +99,7 @@ async fn main() -> Result<()> { // file reads as changed, and the first scan would re-read and rewrite the // entire library from stubs that carry no tags. let started = Instant::now(); - let mtimes = write_stub_files(&root, files_to_write)?; + let mtimes = write_stub_files(&root, files_to_write, &stub)?; println!( " wrote {files_to_write} files in {:.1}s", started.elapsed().as_secs_f64() @@ -120,7 +126,7 @@ async fn main() -> Result<()> { // 3. The rows, as fast as SQLite will take them. let started = Instant::now(); - insert_rows(&db_path, &root, args.objects, &mtimes)?; + insert_rows(&db_path, &root, args.objects, &mtimes, stub.len() as i64)?; let inserted = started.elapsed(); println!( " inserted {} rows in {:.1}s ({:.0} rows/s)", @@ -206,7 +212,7 @@ fn relative_path(index: usize) -> (String, String) { } /// Write the stubs and return each one's modification time, in row order. -fn write_stub_files(root: &Path, count: usize) -> Result> { +fn write_stub_files(root: &Path, count: usize, stub: &[u8]) -> Result> { let mut mtimes = Vec::with_capacity(count); let mut last_dir = String::new(); for index in 0..count { @@ -216,7 +222,7 @@ fn write_stub_files(root: &Path, count: usize) -> Result> { last_dir = directory.clone(); } let path = root.join(&directory).join(&filename); - std::fs::write(&path, STUB_MP3)?; + std::fs::write(&path, stub)?; mtimes.push( std::fs::metadata(&path)? .modified()? @@ -228,7 +234,13 @@ fn write_stub_files(root: &Path, count: usize) -> Result> { Ok(mtimes) } -fn insert_rows(db_path: &Path, root: &Path, total: usize, mtimes: &[i64]) -> Result<()> { +fn insert_rows( + db_path: &Path, + root: &Path, + total: usize, + mtimes: &[i64], + file_size: i64, +) -> Result<()> { let mut connection = rusqlite::Connection::open(db_path)?; // Two of the browse indexes are declared `COLLATE natural_order`, so this // connection has to know the collation before it can maintain them. @@ -265,7 +277,7 @@ fn insert_rows(db_path: &Path, root: &Path, total: usize, mtimes: &[i64]) -> Res path, parent, filename, - STUB_MP3.len() as i64, + file_size, stamp, "audio/mpeg", "audio", @@ -363,3 +375,32 @@ fn report(db_path: &Path) -> Result<()> { ); Ok(()) } + +/// Minimal base64 decoder, so the fixture can be embedded as text without a +/// dependency for one constant. +fn decode_base64(input: &str) -> Option> { + fn value(byte: u8) -> Option { + Some(match byte { + b'A'..=b'Z' => u32::from(byte - b'A'), + b'a'..=b'z' => u32::from(byte - b'a') + 26, + b'0'..=b'9' => u32::from(byte - b'0') + 52, + b'+' => 62, + b'/' => 63, + _ => return None, + }) + } + + let bytes: Vec = input.bytes().filter(|byte| *byte != b'=').collect(); + let mut out = Vec::with_capacity(bytes.len() * 3 / 4); + for chunk in bytes.chunks(4) { + let mut bits = 0u32; + for (index, byte) in chunk.iter().enumerate() { + bits |= value(*byte)? << (18 - 6 * index); + } + let produced = chunk.len() - 1; + for index in 0..produced { + out.push(((bits >> (16 - 8 * index)) & 0xff) as u8); + } + } + Some(out) +} diff --git a/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs b/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs index fbc7528..932b531 100644 --- a/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs +++ b/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs @@ -97,23 +97,27 @@ pub(in crate::database::sqlite) fn bind_media_file(file: &MediaFile) -> Vec, media_file_id: i64, file: &MediaFile, + row_existed: bool, ) -> Result<()> { - transaction.execute( - "DELETE FROM media_tags WHERE media_file_id = ?", - [media_file_id], - )?; + if row_existed { + transaction.execute( + "DELETE FROM media_tags WHERE media_file_id = ?", + [media_file_id], + )?; + } if file.extra_tags.is_empty() { return Ok(()); } @@ -193,7 +197,7 @@ pub(in crate::database::sqlite) fn upsert_media_file( transaction .prepare_cached(UPDATE_MEDIA)? .execute(rusqlite::params_from_iter(params.iter()))?; - write_extra_tags(transaction, id, file)?; + write_extra_tags(transaction, id, file, true)?; Ok(id) } None => { @@ -206,7 +210,7 @@ pub(in crate::database::sqlite) fn upsert_media_file( .prepare_cached(INSERT_MEDIA)? .execute(rusqlite::params_from_iter(params.iter()))?; let id = transaction.last_insert_rowid(); - write_extra_tags(transaction, id, file)?; + write_extra_tags(transaction, id, file, false)?; Ok(id) } } diff --git a/crates/vuio-core/src/database/sqlite/schema.rs b/crates/vuio-core/src/database/sqlite/schema.rs index 1477359..e9646f4 100644 --- a/crates/vuio-core/src/database/sqlite/schema.rs +++ b/crates/vuio-core/src/database/sqlite/schema.rs @@ -18,7 +18,7 @@ use crate::database::{AudioTags, FileFingerprint, FileLocation, MediaFile, Playl /// [`migrations`]; only a *newer* file — one written by a build that knows /// something this one does not — is refused, because there is no way to /// downgrade a schema without guessing at what to discard. -pub(super) const SCHEMA_VERSION: i64 = 4; +pub(super) const SCHEMA_VERSION: i64 = 5; /// Name of the collation that carries the application's natural ordering into /// SQL. Registered on every connection; see [`register_collations`]. @@ -100,7 +100,6 @@ CREATE INDEX IF NOT EXISTS idx_media_genre ON media_files(genre); CREATE INDEX IF NOT EXISTS idx_media_year ON media_files(year); CREATE INDEX IF NOT EXISTS idx_media_album_artist ON media_files(album_artist); CREATE INDEX IF NOT EXISTS idx_media_family ON media_files(mime_family); -CREATE INDEX IF NOT EXISTS idx_media_tags_version ON media_files(tags_version); -- Directories exist only by implication from the paths of files, so unlike the -- music indexes they cannot be recomputed by a query at browse time. @@ -169,7 +168,6 @@ CREATE TABLE IF NOT EXISTS media_tags ( PRIMARY KEY (media_file_id, key, value) ) STRICT; -CREATE INDEX IF NOT EXISTS idx_media_tags_key ON media_tags(key, value); -- What a public metadata service said about a file: title, synopsis, rating and -- a pointer into the artwork cache. @@ -320,6 +318,7 @@ fn migrations() -> Vec<(i64, String)> { (2, MIGRATION_V2.to_owned()), (3, MIGRATION_V3.to_owned()), (4, migration_v4()), + (5, MIGRATION_V5.to_owned()), ] } @@ -397,6 +396,20 @@ fn migration_v4() -> String { format!("{FTS_DDL}{FTS_REBUILD}") } +/// v4 → v5: drop two indexes nothing queries. +/// +/// `tags_version` never appears in a `WHERE` or `ORDER BY` — it is compared in +/// Rust, against a value the scanner already holds — and `media_tags` is only +/// ever read by `media_file_id`, which its primary key already serves. Both were +/// pure write cost: two more b-tree insertions for every row indexed. +/// +/// Dropping an index is not the destructive kind of migration the rule above +/// guards against. An index holds nothing that is not derivable from the table. +const MIGRATION_V5: &str = r#" +DROP INDEX IF EXISTS idx_media_tags_version; +DROP INDEX IF EXISTS idx_media_tags_key; +"#; + /// Columns of `media_files`, qualified so the list can be used inside joins. pub(super) const MEDIA_COLUMNS: &str = "\ media_files.id, media_files.path, media_files.filename, media_files.size, \ From 969fba1b64246ab7f8754a91602ce4f46c3e26c5 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 13:15:15 +0300 Subject: [PATCH 08/23] docs(config): say that the index cache is per connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `database.cache_mb` is applied to every SQLite connection separately — one writer and two to four readers — so a server configured with the 128 MB default settles at roughly 640 MB resident, which is what a 500k-file library shows in practice. Nothing said so, in the Admin tab or in the example config, and the number is the largest single contributor to the process's memory. --- config.example.toml | 4 +++- crates/vuio-core/src/config/model.rs | 4 ++++ crates/vuio-core/src/web/admin.rs | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/config.example.toml b/config.example.toml index cfdc769..7381ff6 100644 --- a/config.example.toml +++ b/config.example.toml @@ -62,7 +62,9 @@ vacuum_on_startup = false backup_enabled = false # Omit to use the platform-appropriate default location. path = "./config/media.db" -# cache_mb = 128 # Megabytes of index kept cached +# Megabytes of index cached *per connection* — one writer plus two to four +# readers — so resident memory settles at several times this figure. +# cache_mb = 128 # The modern browser interface, served on a second port beside the built-in # dashboard. It is a second front end, not a second server: the same API, the diff --git a/crates/vuio-core/src/config/model.rs b/crates/vuio-core/src/config/model.rs index c4869d1..5ad49f6 100644 --- a/crates/vuio-core/src/config/model.rs +++ b/crates/vuio-core/src/config/model.rs @@ -38,6 +38,10 @@ pub(super) fn default_unavailable_root_grace_hours() -> u64 { 168 } +/// Page cache per connection, in mebibytes. +/// +/// Applied to each connection separately — one writer and two to four readers — +/// so the process settles at several times this number. pub(super) fn default_cache_mb() -> usize { 128 } diff --git a/crates/vuio-core/src/web/admin.rs b/crates/vuio-core/src/web/admin.rs index e6138e0..99b0eff 100644 --- a/crates/vuio-core/src/web/admin.rs +++ b/crates/vuio-core/src/web/admin.rs @@ -330,12 +330,18 @@ const DATABASE_FIELDS: &[FieldSpec] = &[ ), "Applies from the next daily tick; the startup and shutdown backups need a restart.", ), - optional( - "database.cache_mb", - "Index cache", - FieldKind::Int { min: 1, max: 4_096 }, - Impact::Restart, - "Megabytes of memory the index keeps cached.", + noted( + optional( + "database.cache_mb", + "Index cache", + FieldKind::Int { min: 1, max: 4_096 }, + Impact::Restart, + "Megabytes of memory the index keeps cached, per database connection.", + ), + "Applied to each connection separately — one writer plus two to four readers, \ + depending on the machine — so the server's resident memory settles at several \ + times this figure. Lower it on a small box; raise it only if browsing a large \ + library reads from disk more than it should.", ), ]; From 4b022ea1ef36f69a54aabb534d85a30e38a5d1d4 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 13:19:46 +0300 Subject: [PATCH 09/23] perf(web): stop counting the whole result to render one page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `visit_files` reports `matched`, the size of the entire result, because DLNA's `TotalMatches` needs it — and computing it means evaluating the query a second time. For a ranked search that is the expensive half: the engine has to find and rank every hit in order to count them, so a page of one costs the same as a page of two hundred and fifty. `/api/media` pages by cursor and never looks at the total, yet ran the count twice per request — once for the pass that collects ids for the metadata overlay, once for the pass that writes the rows. Add `visit_files_page`, which returns only how many rows it visited, and use it there. Defaulted on the trait so no backend has to implement it; SQLite overrides it to skip the count. On a 500k library: a search matching 62,000 files goes from 0.801s to 0.416s, and one matching a single file from 0.120s to 0.061s. --- crates/vuio-core/src/database/mod.rs | 25 +++++++++++++++++ .../vuio-core/src/database/sqlite/session.rs | 28 +++++++++++++++++++ crates/vuio-core/src/web/ui.rs | 10 +++++-- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/crates/vuio-core/src/database/mod.rs b/crates/vuio-core/src/database/mod.rs index 6d71ded..e66066f 100644 --- a/crates/vuio-core/src/database/mod.rs +++ b/crates/vuio-core/src/database/mod.rs @@ -618,6 +618,31 @@ pub trait DatabaseReadSession { where F: for<'a> FnMut(Self::File<'a>) -> Result<()>; + /// One page, without counting the whole result. + /// + /// [`Self::visit_files`] reports `matched`, the size of the entire result, + /// because DLNA's `TotalMatches` requires it — and computing it means + /// evaluating the query a second time. For a ranked search that is the + /// expensive half: the engine has to find and rank every hit to count them, + /// so a page of twenty costs the same as a page of one. + /// + /// Callers that page by cursor never look at `matched`. Defaulted so a + /// backend need not implement it; overriding it is what makes the saving + /// real. + fn visit_files_page( + &mut self, + query: &MediaFileQuery, + offset: usize, + limit: usize, + visitor: F, + ) -> Result + where + F: for<'a> FnMut(Self::File<'a>) -> Result<()>, + { + self.visit_files(query, offset, limit, visitor) + .map(|summary| summary.visited) + } + fn visit_direct_subdirectories( &mut self, canonical_parent: &str, diff --git a/crates/vuio-core/src/database/sqlite/session.rs b/crates/vuio-core/src/database/sqlite/session.rs index 082147a..adbde53 100644 --- a/crates/vuio-core/src/database/sqlite/session.rs +++ b/crates/vuio-core/src/database/sqlite/session.rs @@ -306,6 +306,34 @@ impl DatabaseReadSession for SqliteReadSession { Ok(summary) } + fn visit_files_page( + &mut self, + query: &MediaFileQuery, + offset: usize, + limit: usize, + mut visitor: F, + ) -> Result + where + F: for<'a> FnMut(Self::File<'a>) -> Result<()>, + { + if limit == 0 { + return Ok(0); + } + let plan = query::plan(query); + let mut page_params = plan.params.clone(); + page_params.push(rusqlite::types::Value::Integer(limit as i64)); + page_params.push(rusqlite::types::Value::Integer(offset as i64)); + + let mut visited = 0; + let mut statement = self.connection.prepare_cached(&plan.page_sql())?; + let mut rows = statement.query(rusqlite::params_from_iter(page_params.iter()))?; + while let Some(row) = rows.next()? { + visitor(SqliteMediaFileView { row })?; + visited += 1; + } + Ok(visited) + } + fn visit_direct_subdirectories( &mut self, canonical_parent: &str, diff --git a/crates/vuio-core/src/web/ui.rs b/crates/vuio-core/src/web/ui.rs index 6ec8584..6165c18 100644 --- a/crates/vuio-core/src/web/ui.rs +++ b/crates/vuio-core/src/web/ui.rs @@ -288,8 +288,12 @@ pub async fn media_page_handler( let mut last_id = None; // Fetched titles and synopses, collected up front because the writer // cannot query the session while the session is lending it a row. + // `visit_files_page`, not `visit_files`: this listing pages by + // cursor and never reads the total, and computing that total means + // evaluating the query again — which for a ranked search is the + // expensive half. let mut ids = Vec::with_capacity(fetch_limit); - session.visit_files(&query, offset, fetch_limit, |file| { + session.visit_files_page(&query, offset, fetch_limit, |file| { if let Some(id) = file.id().filter(|id| *id > 0) { ids.push(id); } @@ -299,7 +303,7 @@ pub async fn media_page_handler( .mediainfo_overlays(&ids, min_confidence) .unwrap_or_default(); - let summary = session.visit_files(&query, offset, fetch_limit, |file| { + let visited = session.visit_files_page(&query, offset, fetch_limit, |file| { if emitted >= limit { return Ok(()); } @@ -313,7 +317,7 @@ pub async fn media_page_handler( Ok(()) })?; output.extend_from_slice(b"],\"next_cursor\":"); - let next = (summary.visited > limit).then(|| { + let next = (visited > limit).then(|| { if searching { Some((offset + emitted).to_string()) } else { From 97ba1ae7d437d75aea1923c5ee3d6de32e23c6f8 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 14:21:32 +0300 Subject: [PATCH 10/23] perf(watcher): bound the file-id cache instead of indexing the library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The debounced watcher pairs the two halves of a rename by file system id, because the `from` path is gone by the time the event arrives. Upstream's `FileIdMap` supplies those ids by walking every watched root at startup, `stat`ing each entry, and holding a `HashMap` entry per file for the life of the process. On a 500,000-file library that measured as a twelve-second walk and ~90 MB of resident memory that never comes back — a 65 MB hash table plus a `PathBuf` per file — to keep a hint that is only ever read for the handful of paths in a rename. It also does worse than grow: `remove_path` scans the whole map per removal, and a dropped backend event triggers `rescan`, re-walking every root at exactly the moment the system is already struggling. What the ids buy is smaller than the price. A renamed directory is handled by removing the old subtree and rescanning the new one, which is what the unpaired delete-then-create pair produces anyway; a renamed file is the only real difference, worth one stable row id and one skipped tag read. Only macOS and Windows need them at all, since inotify carries a rename cookie — which is why upstream itself uses `NoCache` on Linux. `BoundedFileIdCache` keeps that behaviour while it is cheap and stops paying for it when it is not. The seed walk stops at 25,000 entries and later ids are remembered only for paths that actually saw an event, held in two generations so the older half is dropped wholesale rather than tracked per entry. Libraries under the cap behave exactly as before. Above it, renaming an untouched file degrades to delete-then-create, which the event handler already supports. --- crates/vuio-core/src/watcher/file_ids.rs | 255 +++++++++++++++++++++++ crates/vuio-core/src/watcher/mod.rs | 9 +- 2 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 crates/vuio-core/src/watcher/file_ids.rs diff --git a/crates/vuio-core/src/watcher/file_ids.rs b/crates/vuio-core/src/watcher/file_ids.rs new file mode 100644 index 0000000..dba7bce --- /dev/null +++ b/crates/vuio-core/src/watcher/file_ids.rs @@ -0,0 +1,255 @@ +//! A bounded file-id cache for the debounced watcher. +//! +//! `notify-debouncer-full` pairs the two halves of a rename by comparing file +//! system ids, because the `from` path no longer exists by the time the event +//! arrives. Its own [`FileIdMap`](notify_debouncer_full::FileIdMap) does that by +//! walking every watched root and keeping a `HashMap` entry for +//! every file and directory underneath — a recursive walk plus a `stat` per entry +//! at startup, and roughly half a kilobyte of resident memory per file for as long +//! as the server runs. Measured on a 500,000-file library that is a twelve-second +//! walk and about 250 MB held forever. +//! +//! Only backends without rename cookies need it at all: inotify carries a cookie +//! that pairs the halves directly, which is why upstream uses `NoCache` on Linux. +//! macOS and Windows have no cookie, so ids are the only mechanism there. +//! +//! What a paired rename buys is also smaller than it looks. A renamed *directory* +//! is handled by removing the old subtree and rescanning the new one, which is what +//! the unpaired delete-then-create pair produces anyway. A renamed *file* is the +//! only real difference: pairing keeps its row id stable and skips one tag re-read. +//! +//! So this keeps upstream's behaviour while it is cheap and stops paying for it +//! when it is not: the seed walk stops at [`DEFAULT_CAPACITY`] entries, and past +//! that point ids are remembered only for paths the watcher has actually seen an +//! event for. Libraries below the cap behave exactly as before. Above it, renaming +//! a file the watcher has not touched degrades to delete-then-create, which the +//! event handler already supports. + +use notify::RecursiveMode; +use notify_debouncer_full::file_id::{get_file_id, FileId}; +use notify_debouncer_full::FileIdCache; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use tracing::debug; + +/// How many paths to remember. Two generations are kept, so the ceiling is twice +/// this — about 25 MB at the default, against 250 MB unbounded at 500k files. +pub const DEFAULT_CAPACITY: usize = 25_000; + +/// A `FileIdCache` that never grows with the library. +/// +/// Entries live in two generations. Inserts land in `live`; when it fills, it +/// becomes `previous` and a fresh one takes over, so the oldest half is dropped +/// wholesale rather than tracked with per-entry LRU bookkeeping. Lookups check +/// both. See the module docs for what this trades away. +#[derive(Debug)] +pub struct BoundedFileIdCache { + live: HashMap, + previous: HashMap, + capacity: usize, + /// Set once the seed walk has been cut short, so it is logged only the once. + truncated: bool, +} + +impl BoundedFileIdCache { + pub fn new() -> Self { + Self::with_capacity(DEFAULT_CAPACITY) + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + live: HashMap::new(), + previous: HashMap::new(), + capacity: capacity.max(1), + truncated: false, + } + } + + /// Entries across both generations. + pub fn len(&self) -> usize { + self.live.len() + self.previous.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Insert one path, rotating the generations if `live` is full. + fn remember(&mut self, path: PathBuf, id: FileId) { + if self.live.len() >= self.capacity && !self.live.contains_key(&path) { + self.previous = std::mem::take(&mut self.live); + } + self.live.insert(path, id); + } + + /// Seed ids for a tree that already exists, stopping at the cap. + /// + /// Unlike `remember` this never rotates: rotating mid-walk would let a large + /// library evict its own entries and walk to the end for nothing. + fn seed(&mut self, root: &Path, recursive: bool) { + let mut pending = vec![root.to_path_buf()]; + while let Some(dir) = pending.pop() { + let entries = match std::fs::read_dir(&dir) { + Ok(entries) => entries, + Err(_) => continue, + }; + for entry in entries.flatten() { + if self.len() >= self.capacity { + if !self.truncated { + self.truncated = true; + debug!( + "File id cache reached its {} entry cap while seeding {}; \ + renames outside the cache will be seen as delete + create", + self.capacity, + root.display() + ); + } + return; + } + let path = entry.path(); + if let Ok(id) = get_file_id(&path) { + self.live.insert(path.clone(), id); + } + if recursive && entry.file_type().is_ok_and(|kind| kind.is_dir()) { + pending.push(path); + } + } + } + } +} + +impl Default for BoundedFileIdCache { + fn default() -> Self { + Self::new() + } +} + +impl FileIdCache for BoundedFileIdCache { + fn cached_file_id(&self, path: &Path) -> Option> { + self.live.get(path).or_else(|| self.previous.get(path)) + } + + fn add_path(&mut self, path: &Path, recursive_mode: RecursiveMode) { + if path.is_dir() { + self.seed(path, recursive_mode == RecursiveMode::Recursive); + return; + } + if let Ok(id) = get_file_id(path) { + self.remember(path.to_path_buf(), id); + } + } + + fn remove_path(&mut self, path: &Path) { + self.live.retain(|cached, _| !cached.starts_with(path)); + self.previous.retain(|cached, _| !cached.starts_with(path)); + } + + /// Deliberately does nothing. + /// + /// Upstream re-walks every root when the backend drops events — the moment the + /// system is already under load. The dropped events are handled where it + /// matters instead: the watcher marks those roots dirty and the media service + /// rescans them against the index, which needs no file ids. + fn rescan(&mut self, _roots: &[(PathBuf, RecursiveMode)]) {} +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn write_files(dir: &Path, count: usize) { + for index in 0..count { + fs::write(dir.join(format!("file_{index}.mp3")), b"x").unwrap(); + } + } + + #[test] + fn seeding_stops_at_the_cap() { + let temp = tempfile::tempdir().unwrap(); + write_files(temp.path(), 40); + + let mut cache = BoundedFileIdCache::with_capacity(10); + cache.add_path(temp.path(), RecursiveMode::Recursive); + + assert_eq!(cache.len(), 10, "the seed walk must not exceed the cap"); + } + + #[test] + fn seeding_a_small_tree_keeps_every_entry() { + let temp = tempfile::tempdir().unwrap(); + let nested = temp.path().join("Album"); + fs::create_dir(&nested).unwrap(); + write_files(temp.path(), 3); + write_files(&nested, 3); + + let mut cache = BoundedFileIdCache::with_capacity(100); + cache.add_path(temp.path(), RecursiveMode::Recursive); + + // 3 files + the directory + 3 nested files. + assert_eq!(cache.len(), 7); + assert!(cache + .cached_file_id(&nested.join("file_0.mp3")) + .is_some()); + } + + #[test] + fn non_recursive_seeding_stays_at_one_level() { + let temp = tempfile::tempdir().unwrap(); + let nested = temp.path().join("Album"); + fs::create_dir(&nested).unwrap(); + write_files(&nested, 3); + + let mut cache = BoundedFileIdCache::with_capacity(100); + cache.add_path(temp.path(), RecursiveMode::NonRecursive); + + assert_eq!(cache.len(), 1, "only the directory entry itself"); + } + + #[test] + fn individual_inserts_stay_bounded_and_keep_the_newest() { + let temp = tempfile::tempdir().unwrap(); + write_files(temp.path(), 30); + + let mut cache = BoundedFileIdCache::with_capacity(10); + for index in 0..30 { + cache.add_path(&temp.path().join(format!("file_{index}.mp3")), RecursiveMode::NonRecursive); + } + + assert!(cache.len() <= 20, "two generations of 10, at most"); + assert!( + cache + .cached_file_id(&temp.path().join("file_29.mp3")) + .is_some(), + "the most recent path must still be resolvable" + ); + } + + #[test] + fn removing_a_directory_drops_its_children_from_both_generations() { + let temp = tempfile::tempdir().unwrap(); + let nested = temp.path().join("Album"); + fs::create_dir(&nested).unwrap(); + write_files(&nested, 6); + + let mut cache = BoundedFileIdCache::with_capacity(3); + for index in 0..6 { + cache.add_path(&nested.join(format!("file_{index}.mp3")), RecursiveMode::NonRecursive); + } + assert!(cache.len() > 0); + + cache.remove_path(&nested); + assert_eq!(cache.len(), 0); + } + + #[test] + fn rescan_does_not_rewalk() { + let temp = tempfile::tempdir().unwrap(); + write_files(temp.path(), 5); + + let mut cache = BoundedFileIdCache::with_capacity(100); + cache.rescan(&[(temp.path().to_path_buf(), RecursiveMode::Recursive)]); + + assert!(cache.is_empty(), "a dropped-event rescan must not walk"); + } +} diff --git a/crates/vuio-core/src/watcher/mod.rs b/crates/vuio-core/src/watcher/mod.rs index 3d67be5..5280e0c 100644 --- a/crates/vuio-core/src/watcher/mod.rs +++ b/crates/vuio-core/src/watcher/mod.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use notify::{Config, RecommendedWatcher, RecursiveMode}; use notify_debouncer_full::{ - new_debouncer_opt, DebounceEventResult, DebouncedEvent, Debouncer, FileIdMap, + new_debouncer_opt, DebounceEventResult, DebouncedEvent, Debouncer, }; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -13,6 +13,9 @@ use tracing::{debug, error, info, warn}; use crate::error::AppResult as Result; use crate::media::ScanPolicy; +mod file_ids; +pub use file_ids::BoundedFileIdCache; + /// Events that can occur in the file system for media files #[derive(Debug, Clone)] pub enum FileSystemEvent { @@ -73,7 +76,7 @@ pub trait FileSystemWatcher: Send + Sync { /// Cross-platform file system watcher implementation pub struct CrossPlatformWatcher { - debouncer: Arc>>>, + debouncer: Arc>>>, event_sender: mpsc::Sender, event_receiver: Arc>>>, watched_paths: Arc>>, @@ -490,7 +493,7 @@ impl CrossPlatformWatcher { } } }, - FileIdMap::new(), + BoundedFileIdCache::new(), Config::default(), )?; From 7bae0deba19171abcf262b3cd16e3fa13de9f7ac Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 14:21:42 +0300 Subject: [PATCH 11/23] fix(config): stop the config watcher walking the tree it sits in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To notice edits to the config file, the manager watched the file's parent directory *recursively*, with the debouncer's default file-id cache. The handler then discards every event whose path is not equal to the config file — so the recursion could never match anything, while the cache walked and `stat`ed everything underneath and kept a map entry per file. Where the config lives decides how bad that is. Beside the media it configures (`vuio --config ./vuio.toml`, run from a library folder) it indexes the whole library; in a home directory it indexes the home directory. On the 500,000-file benchmark it was 122 MB resident and about six seconds of startup, all of it before the media scan could begin, for one file compared by equality. Watch one level, and with `NoCache`: the config file is a direct child of the directory, and renames are never stitched here, so the ids were built and never read. Startup on that library goes from 23.3s to 10.7s and steady memory from 469 MB to 355 MB. --- crates/vuio-core/src/config/mod.rs | 29 +++++++------- crates/vuio-core/src/config/tests.rs | 59 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/crates/vuio-core/src/config/mod.rs b/crates/vuio-core/src/config/mod.rs index c14d06d..91f51bb 100644 --- a/crates/vuio-core/src/config/mod.rs +++ b/crates/vuio-core/src/config/mod.rs @@ -79,10 +79,7 @@ pub struct ConfigManager { change_sender: broadcast::Sender, /// Held so the watcher outlives this manager; dropping it stops reloads. debouncer: Option< - notify_debouncer_full::Debouncer< - notify::RecommendedWatcher, - notify_debouncer_full::FileIdMap, - >, + notify_debouncer_full::Debouncer, >, } @@ -197,30 +194,34 @@ impl ConfigManager { cancellation: tokio_util::sync::CancellationToken, background_tasks: tokio_util::task::TaskTracker, ) -> Result< - notify_debouncer_full::Debouncer< - notify::RecommendedWatcher, - notify_debouncer_full::FileIdMap, - >, + notify_debouncer_full::Debouncer, > { - use notify_debouncer_full::{new_debouncer_opt, DebounceEventResult, Debouncer, FileIdMap}; + use notify_debouncer_full::{new_debouncer_opt, DebounceEventResult, Debouncer, NoCache}; use tokio::sync::mpsc; let (tx, mut rx) = mpsc::channel(100); - // Create debounced watcher with 500ms debounce duration - let mut debouncer: Debouncer = new_debouncer_opt( + // Create debounced watcher with 500ms debounce duration. + // + // `NoCache`, not the default file-id cache: that one stats every path under + // every watched root to keep rename ids, and the handler below cares about + // exactly one path compared by equality. Renames are never stitched here, so + // the ids would be built and never read. + let mut debouncer: Debouncer = new_debouncer_opt( Duration::from_millis(500), None, move |result: DebounceEventResult| { let _ = tx.try_send(result); }, - FileIdMap::new(), + NoCache::new(), notify::Config::default(), )?; - // Watch the config file's parent directory + // Watch the config file's parent directory, one level only. The config file + // is a direct child of it, and watching recursively would follow whatever + // else happens to live there — a media tree, or a whole home directory. if let Some(parent) = config_path.parent() { - debouncer.watch(parent, notify::RecursiveMode::Recursive)?; + debouncer.watch(parent, notify::RecursiveMode::NonRecursive)?; } // Spawn task to handle debounced file events diff --git a/crates/vuio-core/src/config/tests.rs b/crates/vuio-core/src/config/tests.rs index 9e86966..08fb2ec 100644 --- a/crates/vuio-core/src/config/tests.rs +++ b/crates/vuio-core/src/config/tests.rs @@ -531,6 +531,65 @@ async fn overrides_hold_across_reloads_without_freezing_the_file() -> Result<()> Ok(()) } +/// A config file living inside the media tree it configures — `vuio --config +/// ./vuio.toml` run from a library folder, which is the obvious thing to do. The +/// watcher used to take its parent recursively with the debouncer's default file-id +/// cache, so starting up walked and `stat`ed the entire library and then held a map +/// entry per file for the lifetime of the process, to notice edits to one file it +/// identifies by path equality. It watches one level now, without the id cache. +#[tokio::test] +async fn config_inside_a_media_tree_still_reloads() -> Result<()> { + let temp_dir = TempDir::new()?; + let root = std::fs::canonicalize(temp_dir.path())?; + let config_path = root.join("vuio.toml"); + + // The library the config is sitting in. + let nested = root.join("Artist").join("Album"); + std::fs::create_dir_all(&nested)?; + for index in 0..64 { + std::fs::write(nested.join(format!("track_{index}.mp3")), b"x")?; + } + + let mut config = AppConfig::default(); + config.server.name = "In The Library".to_string(); + config.media.directories = vec![MonitoredDirectoryConfig { + path: root.to_string_lossy().to_string(), + recursive: true, + case_sensitive: None, + extensions: None, + exclude_patterns: None, + validation_mode: ValidationMode::Skip, + }]; + config.save_to_file(&config_path)?; + + let cancellation = tokio_util::sync::CancellationToken::new(); + let tasks = tokio_util::task::TaskTracker::new(); + let manager = ConfigManager::watching_with_overrides( + &config_path, + ConfigOverrides::default(), + cancellation.clone(), + tasks.clone(), + ) + .await?; + assert!(manager.is_watched()); + + let mut edited = AppConfig::load_from_file(&config_path)?; + edited.server.name = "Renamed In The Library".to_string(); + edited.save_to_file(&config_path)?; + tokio::time::sleep(std::time::Duration::from_millis(2000)).await; + + assert_eq!( + manager.get_config().await.server.name, + "Renamed In The Library", + "an edit to a config inside the media tree must still be picked up" + ); + + cancellation.cancel(); + tasks.close(); + tasks.wait().await; + Ok(()) +} + #[test] fn overrides_report_what_they_force() { assert!(ConfigOverrides::default().in_force().is_empty()); From 27279bfdbee07f5a299f777c65383f62daf460b9 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 14:21:51 +0300 Subject: [PATCH 12/23] docs(config): correct what cache_mb costs and what it buys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An earlier commit said the budget applies per connection "so resident memory settles at several times this figure", inferring a multiple from the number of connections rather than measuring it. A sweep at 500,000 files says otherwise: a connection only allocates pages it reads, and one reader does the heavy queries, so going from 8 to 128 moved resident memory by ~160 MB, not by five times the difference. The sweep also found what the setting is actually for. Folder browsing and flat listing are 1 ms at every value from 8 to 256 — they are index-served and do not touch it. Full-text search is the only thing that responds, and as a step rather than a slope: 0.21s at everything up to 160, 0.12s from 192 up, where the search index finally fits. Advice to raise it "if browsing reads from disk" pointed at the wrong operation and implied a gradient that is not there. --- config.example.toml | 8 ++++++-- crates/vuio-core/src/config/model.rs | 12 ++++++++++-- crates/vuio-core/src/web/admin.rs | 10 ++++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/config.example.toml b/config.example.toml index 7381ff6..4a88021 100644 --- a/config.example.toml +++ b/config.example.toml @@ -62,8 +62,12 @@ vacuum_on_startup = false backup_enabled = false # Omit to use the platform-appropriate default location. path = "./config/media.db" -# Megabytes of index cached *per connection* — one writer plus two to four -# readers — so resident memory settles at several times this figure. +# Megabytes of index cached, as a budget *per connection* — one writer plus two +# to four readers. Only a connection that runs a large query fills its share, so +# in practice resident memory grows by about this much, not a multiple of it. +# Folder browsing is served from indexes and does not care. Full-text search does: +# on a very large library it stays slow until the whole search index fits, then +# roughly halves. Measured at 500,000 files, that threshold was near 192. # cache_mb = 128 # The modern browser interface, served on a second port beside the built-in diff --git a/crates/vuio-core/src/config/model.rs b/crates/vuio-core/src/config/model.rs index 5ad49f6..789ebaa 100644 --- a/crates/vuio-core/src/config/model.rs +++ b/crates/vuio-core/src/config/model.rs @@ -40,8 +40,16 @@ pub(super) fn default_unavailable_root_grace_hours() -> u64 { /// Page cache per connection, in mebibytes. /// -/// Applied to each connection separately — one writer and two to four readers — -/// so the process settles at several times this number. +/// The budget is per connection — one writer and two to four readers — but a +/// connection only allocates pages it actually reads, and on a normal workload +/// one reader does the heavy queries. Measured on a 500,000-file library, going +/// from 8 to 128 moved resident memory by about 160 MB, not by five times the +/// difference. +/// +/// Folder browsing is index-served and flat across every setting. Full-text +/// search is the one thing that responds, and as a step rather than a curve: it +/// stays slow until the search index fits, then roughly halves. At 500,000 files +/// that step fell between 160 and 192. pub(super) fn default_cache_mb() -> usize { 128 } diff --git a/crates/vuio-core/src/web/admin.rs b/crates/vuio-core/src/web/admin.rs index 99b0eff..6637b23 100644 --- a/crates/vuio-core/src/web/admin.rs +++ b/crates/vuio-core/src/web/admin.rs @@ -338,10 +338,12 @@ const DATABASE_FIELDS: &[FieldSpec] = &[ Impact::Restart, "Megabytes of memory the index keeps cached, per database connection.", ), - "Applied to each connection separately — one writer plus two to four readers, \ - depending on the machine — so the server's resident memory settles at several \ - times this figure. Lower it on a small box; raise it only if browsing a large \ - library reads from disk more than it should.", + "A budget per connection — one writer plus two to four readers — but only a \ + connection running a large query fills its share, so resident memory grows by \ + roughly this much rather than a multiple of it. Folder browsing does not depend \ + on it. Search does, and as a step rather than a slope: on a very large library \ + it stays slow until the search index fits and then roughly halves, so raising \ + this helps only once it is high enough to cross that line.", ), ]; From 3eaaf0fc8a541a46f1ecfd2e08e5ad31261952f0 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 15:17:59 +0300 Subject: [PATCH 13/23] perf(config): default the index cache to 8 MB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sweep of `database.cache_mb` at 500,000 files, in release, against every value from 8 to 256: | cache_mb | RSS | browse | list | search | |---|---|---|---|---| | 8 | 355 MB | 0.001s | 0.001s | 0.201s | | 32 | 385 MB | 0.001s | 0.001s | 0.209s | | 128 | 525 MB | 0.001s | 0.001s | 0.210s | | 160 | 565 MB | 0.001s | 0.001s | 0.207s | | 192 | 564 MB | 0.001s | 0.001s | 0.121s | | 256 | 565 MB | 0.001s | 0.001s | 0.122s | Folder browsing and flat listing are served from indexes and are flat at every value. Search is the only thing that responds, and as a step rather than a slope: it stays at 0.21s until the whole search index fits and then halves, which on that library happened between 160 and 192. That makes 128 the worst available point at scale — 170 MB more than 8, and still on the slow side of the step. A default that spends memory has to buy something, and below the step this one buys nothing measurable. Servers that want the faster search need to go past the step, not merely up, and the docs now say so instead of implying a gradient. --- config.example.toml | 6 ++++-- crates/vuio-core/src/config/model.rs | 7 ++++++- crates/vuio-core/src/web/admin.rs | 5 +++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/config.example.toml b/config.example.toml index 4a88021..d1d4a22 100644 --- a/config.example.toml +++ b/config.example.toml @@ -67,8 +67,10 @@ path = "./config/media.db" # in practice resident memory grows by about this much, not a multiple of it. # Folder browsing is served from indexes and does not care. Full-text search does: # on a very large library it stays slow until the whole search index fits, then -# roughly halves. Measured at 500,000 files, that threshold was near 192. -# cache_mb = 128 +# roughly halves. Measured at 500,000 files that threshold was near 192, and +# anything below it cost memory without making search faster — which is why the +# default is small. Raise it past the threshold, or not at all. +# cache_mb = 8 # The modern browser interface, served on a second port beside the built-in # dashboard. It is a second front end, not a second server: the same API, the diff --git a/crates/vuio-core/src/config/model.rs b/crates/vuio-core/src/config/model.rs index 789ebaa..d1677fe 100644 --- a/crates/vuio-core/src/config/model.rs +++ b/crates/vuio-core/src/config/model.rs @@ -50,8 +50,13 @@ pub(super) fn default_unavailable_root_grace_hours() -> u64 { /// search is the one thing that responds, and as a step rather than a curve: it /// stays slow until the search index fits, then roughly halves. At 500,000 files /// that step fell between 160 and 192. +/// +/// So the default is small. A larger one bought nothing measurable below the +/// step — at 500,000 files, 128 cost 170 MB more than 8 and left search exactly +/// as slow — and a server that wants the faster search has to be raised past the +/// step, not merely raised. pub(super) fn default_cache_mb() -> usize { - 128 + 8 } pub(super) fn default_true() -> bool { diff --git a/crates/vuio-core/src/web/admin.rs b/crates/vuio-core/src/web/admin.rs index 6637b23..5e4b642 100644 --- a/crates/vuio-core/src/web/admin.rs +++ b/crates/vuio-core/src/web/admin.rs @@ -342,8 +342,9 @@ const DATABASE_FIELDS: &[FieldSpec] = &[ connection running a large query fills its share, so resident memory grows by \ roughly this much rather than a multiple of it. Folder browsing does not depend \ on it. Search does, and as a step rather than a slope: on a very large library \ - it stays slow until the search index fits and then roughly halves, so raising \ - this helps only once it is high enough to cross that line.", + it stays slow until the search index fits and then roughly halves. Raising this \ + is worth it only if it goes past that line — below it, the memory is spent and \ + nothing gets faster.", ), ]; From 2f527d588a7f953e4036ff1c8e3a9decca995778 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 15:18:11 +0300 Subject: [PATCH 14/23] perf(scan): count what a scan changed instead of keeping every record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ScanResult` retained a whole `MediaFile` for every file it added or updated — a `PathBuf`, three `String`s, six `Option`, an `AudioTags` of fourteen more fields, a `StreamInfo`, and a `Vec` of every remaining tag. On a library with rich tags that is more than a kilobyte a file, and the records are already in the database by the time the scan returns. Nothing ever read them back. `summary()` and `total_changes()` called `.len()`, and the one production consumer took a `.len()`, a log line and an `.is_empty()`. So a cold scan built most of the index a second time, in memory, to produce three integers — the same mistake `unchanged_files` made before it became a counter. Measured on a 100,000-file library, release: a cold scan drops from 148 MB peak and 169 MB settled to 67 MB peak and 78 MB settled. The scan itself is unchanged in wall time. Tests that inspected a retained record now read it back from the database, which is where the property they assert about — a canonicalized path, a symlink indexed under its target but named for the link — actually has to hold. --- .../vuio-core/src/lifecycle/media/events.rs | 6 +- crates/vuio-core/src/media/result.rs | 57 +++++---- crates/vuio-core/src/media/scanner.rs | 72 +++--------- crates/vuio-core/src/media/tests.rs | 110 +++++++----------- .../tests/audio_integration_tests.rs | 4 +- 5 files changed, 96 insertions(+), 153 deletions(-) diff --git a/crates/vuio-core/src/lifecycle/media/events.rs b/crates/vuio-core/src/lifecycle/media/events.rs index 09b659d..daf5626 100644 --- a/crates/vuio-core/src/lifecycle/media/events.rs +++ b/crates/vuio-core/src/lifecycle/media/events.rs @@ -196,13 +196,13 @@ pub(in crate::lifecycle) async fn handle_file_system_event 0 { increment_content_update_id(app_state).await; } } diff --git a/crates/vuio-core/src/media/result.rs b/crates/vuio-core/src/media/result.rs index 38b29a4..1b5fe51 100644 --- a/crates/vuio-core/src/media/result.rs +++ b/crates/vuio-core/src/media/result.rs @@ -1,23 +1,27 @@ use super::*; /// Result of a media scanning operation +/// +/// Every outcome is a count. The records themselves live in the database by the +/// time a scan returns, and nothing has ever read them back off this struct — +/// `.len()` and `.is_empty()` were the only consumers. Retaining them meant a +/// whole [`MediaFile`] per added file, which on a large library is most of the +/// index built and dropped for a log line, and worse on real music where +/// `extra_tags` is populated. +/// +/// [`MediaFile`]: crate::database::MediaFile #[derive(Debug, Clone)] pub struct ScanResult { - /// Files that were newly added to the database - pub new_files: Vec, + /// How many files were newly added to the database. + pub new: usize, - /// Files that were updated in the database - pub updated_files: Vec, + /// How many files were updated in the database. + pub updated: usize, - /// Files that were removed from the database - pub removed_files: Vec, + /// How many files were removed from the database. + pub removed: usize, /// How many files were found to be unchanged. - /// - /// A count rather than the records themselves: nothing has ever needed more - /// than the number, and on a large library retaining one clone per unchanged - /// file meant building and dropping a copy of most of the index on every - /// scan. pub unchanged: usize, /// Total number of files scanned from the file system @@ -39,25 +43,25 @@ pub struct ScanResult { } impl ScanResult { - /// Create a new empty scan result with pre-allocated capacity + /// Create a new empty scan result pub fn new() -> Self { Self { - new_files: Vec::with_capacity(100), - updated_files: Vec::with_capacity(50), - removed_files: Vec::with_capacity(50), + new: 0, + updated: 0, + removed: 0, unchanged: 0, total_scanned: 0, files_read: 0, - errors: Vec::with_capacity(10), + errors: Vec::new(), complete: true, } } /// Merge another scan result into this one pub fn merge(&mut self, other: ScanResult) { - self.new_files.extend(other.new_files); - self.updated_files.extend(other.updated_files); - self.removed_files.extend(other.removed_files); + self.new += other.new; + self.updated += other.updated; + self.removed += other.removed; self.unchanged += other.unchanged; self.total_scanned += other.total_scanned; self.files_read += other.files_read; @@ -67,31 +71,24 @@ impl ScanResult { /// Get the total number of changes (new + updated + removed) pub fn total_changes(&self) -> usize { - self.new_files.len() + self.updated_files.len() + self.removed_files.len() + self.new + self.updated + self.removed } - /// Get a summary string of the scan results pub fn summary(&self) -> String { format!( "Scanned {} files ({} read): {} new, {} updated, {} removed, {} unchanged, {} errors", self.total_scanned, self.files_read, - self.new_files.len(), - self.updated_files.len(), - self.removed_files.len(), + self.new, + self.updated, + self.removed, self.unchanged, self.errors.len() ) } } -impl Default for ScanResult { - fn default() -> Self { - Self::new() - } -} - /// Error that occurred during scanning #[derive(Debug, Clone)] pub struct ScanError { diff --git a/crates/vuio-core/src/media/scanner.rs b/crates/vuio-core/src/media/scanner.rs index 0c166ae..54f1f9e 100644 --- a/crates/vuio-core/src/media/scanner.rs +++ b/crates/vuio-core/src/media/scanner.rs @@ -8,17 +8,6 @@ pub struct MediaScanner { } impl MediaScanner { - fn fingerprint(file: &MediaFile) -> FileFingerprint { - FileFingerprint { - id: file.id.unwrap_or_default(), - path: file.path.clone(), - size: file.size, - modified: file.modified, - created_at: file.created_at, - tags_version: file.tags_version, - } - } - /// Create a new media scanner with database manager pub fn with_database(database_manager: Arc) -> Self { Self { @@ -152,7 +141,7 @@ impl MediaScanner { if !current_paths.contains(&normalized_existing_path) { // File was removed from file system, add to bulk removal list files_to_remove.push(existing_file.path.clone()); - result.removed_files.push(Self::fingerprint(&existing_file)); + result.removed += 1; } } @@ -172,18 +161,10 @@ impl MediaScanner { file.size ); } - let insert_ids = self - .database_manager + self.database_manager .bulk_store_canonical_media_files(&files_to_insert) .await?; - - // Update result with inserted files and their IDs - for (i, mut file) in files_to_insert.into_iter().enumerate() { - if let Some(id) = insert_ids.get(i) { - file.id = Some(*id); - } - result.new_files.push(file); - } + result.new += files_to_insert.len(); } // Bulk update changed files @@ -195,7 +176,7 @@ impl MediaScanner { self.database_manager .bulk_update_canonical_media_files(&files_to_update) .await?; - result.updated_files.extend(files_to_update); + result.updated += files_to_update.len(); } // Bulk remove deleted files @@ -220,9 +201,9 @@ impl MediaScanner { // Log bulk operation summary tracing::info!( "bulk operations completed: {} inserted, {} updated, {} removed, {} unchanged", - result.new_files.len(), - result.updated_files.len(), - result.removed_files.len(), + result.new, + result.updated, + result.removed, result.unchanged ); @@ -528,16 +509,11 @@ impl MediaScanner { processed, total_files ); - let ids = self - .database_manager + self.database_manager .bulk_store_canonical_media_files(&files_to_insert) .await?; - for (i, mut file) in files_to_insert.drain(..).enumerate() { - if let Some(id) = ids.get(i) { - file.id = Some(*id); - } - result.new_files.push(file); - } + result.new += files_to_insert.len(); + files_to_insert.clear(); } if files_to_update.len() >= BATCH_SIZE { @@ -550,7 +526,8 @@ impl MediaScanner { self.database_manager .bulk_update_canonical_media_files(&files_to_update) .await?; - result.updated_files.append(&mut files_to_update); + result.updated += files_to_update.len(); + files_to_update.clear(); } // Progress logging every 1000 files @@ -562,16 +539,10 @@ impl MediaScanner { // Process remaining files in last batch if !files_to_insert.is_empty() { info!("Inserting final batch of {} files", files_to_insert.len()); - let ids = self - .database_manager + self.database_manager .bulk_store_canonical_media_files(&files_to_insert) .await?; - for (i, mut file) in files_to_insert.into_iter().enumerate() { - if let Some(id) = ids.get(i) { - file.id = Some(*id); - } - result.new_files.push(file); - } + result.new += files_to_insert.len(); } if !files_to_update.is_empty() { @@ -579,7 +550,7 @@ impl MediaScanner { self.database_manager .bulk_update_canonical_media_files(&files_to_update) .await?; - result.updated_files.extend(files_to_update); + result.updated += files_to_update.len(); } // Find and remove deleted files @@ -606,21 +577,16 @@ impl MediaScanner { self.database_manager .bulk_remove_media_files(&files_to_remove) .await?; - let removed_paths = files_to_remove.iter().collect::>(); - for (path, file) in existing_files_map.iter() { - if removed_paths.contains(path) { - result.removed_files.push(file.clone()); - } - } + result.removed += files_to_remove.len(); } result.total_scanned = total_files; info!( "Scan completed: {} new, {} updated, {} removed, {} unchanged, {} files read", - result.new_files.len(), - result.updated_files.len(), - result.removed_files.len(), + result.new, + result.updated, + result.removed, result.unchanged, result.files_read ); diff --git a/crates/vuio-core/src/media/tests.rs b/crates/vuio-core/src/media/tests.rs index fe27b00..cde0711 100644 --- a/crates/vuio-core/src/media/tests.rs +++ b/crates/vuio-core/src/media/tests.rs @@ -72,20 +72,20 @@ async fn test_media_scanner_path_normalization() { let result = scanner.scan_directory(&temp_path).await.unwrap(); // Verify that files were found and processed - assert_eq!(result.new_files.len(), 1); - let scanned_file = &result.new_files[0]; + assert_eq!(result.new, 1); - // Verify that the path was normalized (should be canonical format) + // Verify that the path was normalized (should be canonical format). The scan + // reports counts, so the record itself is read back from the database — which + // is where the normalization has to have landed for it to matter. let expected_canonical = scanner .filesystem_manager() .get_canonical_path(&test_file_path) .unwrap(); - assert_eq!(scanned_file.path.to_string_lossy(), expected_canonical); - - // Verify the file was stored in the database with canonical path - let stored_file = db.get_file_by_path(&scanned_file.path).await.unwrap(); - assert!(stored_file.is_some()); - let stored_file = stored_file.unwrap(); + let stored_file = db + .get_file_by_path(Path::new(&expected_canonical)) + .await + .unwrap() + .expect("the scanned file must be stored under its canonical path"); assert_eq!(stored_file.path.to_string_lossy(), expected_canonical); // temp_dir dropped here, auto-cleanup @@ -95,67 +95,37 @@ async fn test_media_scanner_path_normalization() { async fn test_scan_result_operations() { let mut result1 = ScanResult::new(); result1.total_scanned = 5; - result1.new_files.push(MediaFile { - id: Some(1), - path: PathBuf::from("/test1.mp4"), - filename: "test1.mp4".to_string(), - size: 1024, - modified: SystemTime::now(), - mime_type: "video/mp4".to_string(), - duration: None, - title: None, - artist: None, - album: None, - genre: None, - track_number: None, - year: None, - album_artist: None, - tags: Default::default(), - stream: Default::default(), - extra_tags: Vec::new(), - tags_version: 0, - subtitle_available: false, - created_at: SystemTime::now(), - updated_at: SystemTime::now(), - }); + result1.new = 1; + result1.files_read = 1; let mut result2 = ScanResult::new(); result2.total_scanned = 3; - result2.updated_files.push(MediaFile { - id: Some(2), - path: PathBuf::from("/test2.mp4"), - filename: "test2.mp4".to_string(), - size: 2048, - modified: SystemTime::now(), - mime_type: "video/mp4".to_string(), - duration: None, - title: None, - artist: None, - album: None, - genre: None, - track_number: None, - year: None, - album_artist: None, - tags: Default::default(), - stream: Default::default(), - extra_tags: Vec::new(), - tags_version: 0, - subtitle_available: false, - created_at: SystemTime::now(), - updated_at: SystemTime::now(), - }); + result2.updated = 1; + result2.removed = 2; + result2.unchanged = 2; + result2.files_read = 1; + result2.complete = false; // Test merge result1.merge(result2); assert_eq!(result1.total_scanned, 8); - assert_eq!(result1.new_files.len(), 1); - assert_eq!(result1.updated_files.len(), 1); + assert_eq!(result1.new, 1); + assert_eq!(result1.updated, 1); + assert_eq!(result1.removed, 2); + assert_eq!(result1.unchanged, 2); + assert_eq!(result1.files_read, 2); + assert_eq!(result1.total_changes(), 4); + assert!( + !result1.complete, + "one incomplete half must make the whole incomplete" + ); // Test summary let summary = result1.summary(); assert!(summary.contains("8 files")); assert!(summary.contains("1 new")); assert!(summary.contains("1 updated")); + assert!(summary.contains("2 removed")); } #[tokio::test] @@ -196,7 +166,7 @@ async fn test_recursive_scan_optimization() { // First scan to populate database let initial_result = scanner.scan_directory_recursive(&root_dir).await.unwrap(); - assert_eq!(initial_result.new_files.len(), 4); + assert_eq!(initial_result.new, 4); assert_eq!(initial_result.total_changes(), 4); // Verify all files were stored in database @@ -209,8 +179,8 @@ async fn test_recursive_scan_optimization() { // Second scan should find no changes (tests that optimization works correctly) let second_result = scanner.scan_directory_recursive(&root_dir).await.unwrap(); - assert_eq!(second_result.new_files.len(), 0); - assert_eq!(second_result.updated_files.len(), 0); + assert_eq!(second_result.new, 0); + assert_eq!(second_result.updated, 0); assert_eq!(second_result.unchanged, 4); assert_eq!(second_result.total_changes(), 0); // The point of the second scan: it still visited all four files, and opened @@ -248,13 +218,23 @@ async fn direct_scan_resolves_only_symlinked_media_entries() { .unwrap(), ); database.initialize().await.unwrap(); - let scanner = - MediaScanner::with_filesystem_manager(Box::new(BaseFileSystemManager::new(true)), database); + let scanner = MediaScanner::with_filesystem_manager( + Box::new(BaseFileSystemManager::new(true)), + database.clone(), + ); let result = scanner.scan_directory(&media_root).await.unwrap(); - assert_eq!(result.new_files.len(), 1); - assert_eq!(result.new_files[0].filename, "visible-name.mp4"); - assert_eq!(result.new_files[0].path, target.canonicalize().unwrap()); + assert_eq!(result.new, 1); + + // Indexed under the link's target, but named for the link the user sees. + let resolved = target.canonicalize().unwrap(); + let stored = database + .get_file_by_path(&resolved) + .await + .unwrap() + .expect("the symlinked entry must be indexed under its resolved target"); + assert_eq!(stored.filename, "visible-name.mp4"); + assert_eq!(stored.path, resolved); } #[test] fn case_policy_compares_path_components_without_changing_boundaries() { diff --git a/crates/vuio-core/tests/audio_integration_tests.rs b/crates/vuio-core/tests/audio_integration_tests.rs index f93ccff..11a1997 100644 --- a/crates/vuio-core/tests/audio_integration_tests.rs +++ b/crates/vuio-core/tests/audio_integration_tests.rs @@ -91,7 +91,7 @@ async fn test_audio_implementation_and_features() { // 3. Scan the directory with MediaScanner let scanner = MediaScanner::with_database(db.clone()); let scan_result = scanner.scan_directory_recursive(&media_dir).await.unwrap(); - assert_eq!(scan_result.new_files.len(), 4); + assert_eq!(scan_result.new, 4); // 4. Verify tag metadata is correctly populated in DB // Check AC/DC @@ -295,7 +295,7 @@ async fn test_cover_art_retrieval_and_xml() { // 4. Scan the directory with MediaScanner let scanner = MediaScanner::with_database(db.clone()); let scan_result = scanner.scan_directory_recursive(&media_dir).await.unwrap(); - assert_eq!(scan_result.new_files.len(), 2); + assert_eq!(scan_result.new, 2); // 5. Get file from DB to find its assigned ID let db_file = db.get_file_by_path(&audio_path).await.unwrap().unwrap(); From 21bddca27b5029100b568b5e53dc24bb6268185c Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 15:18:20 +0300 Subject: [PATCH 15/23] perf(metrics): stop refreshing every process on the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `DiagnosticsCollector::refresh` called `sysinfo`'s `refresh_all()`, which walks the whole process table and stores a record per process, then read exactly one of them: our own pid. The heap profile showed a couple of megabytes rebuilt on every sample, and `/metrics/json` samples every five seconds for as long as a dashboard is open. Refresh what the snapshot actually reads — system memory, CPU usage, and our own pid — and take the CPU list once at construction, since `refresh_cpu_usage` does not build it and the count cannot change. `new_all()` goes with it, on every platform rather than just FreeBSD, since its only purpose was to prime that same table. The FreeBSD carve-out for it is no longer needed; the disk one stays, being a separate upstream bug. The cost is the one FreeBSD already accepted: the first sample reports no process CPU usage, having no earlier reading to difference against. The test now asserts the fields that a too-narrow refresh would silently empty — CPU count, system memory, and our own process's memory and runtime. --- crates/vuio-core/src/platform/diagnostics.rs | 65 +++++++++++++++----- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/crates/vuio-core/src/platform/diagnostics.rs b/crates/vuio-core/src/platform/diagnostics.rs index 79e109c..ea068c5 100644 --- a/crates/vuio-core/src/platform/diagnostics.rs +++ b/crates/vuio-core/src/platform/diagnostics.rs @@ -66,19 +66,20 @@ struct DiagnosticsCollector { #[cfg(feature = "diagnostics")] impl DiagnosticsCollector { fn new() -> Self { + // `System::new()`, never `new_all()`: that walks the whole process table, + // and the only process this reports on is our own. On FreeBSD it also + // means `kinfo_getfile` — the same class of sysinfo call as the disk + // enumeration below, which faults the same way under QEMU. Every + // `AppState` builds one of these, including the ones in tests that never + // take a sample. The cost is that the first sample reports no CPU usage, + // having no earlier reading to difference against. + let mut system = sysinfo::System::new(); + // The CPU list is fixed for the life of the process and is not refreshed + // by `refresh_cpu_usage`, so take it once here. + system.refresh_cpu_list(sysinfo::CpuRefreshKind::nothing().with_cpu_usage()); + Self { - // `new_all()` walks the process table up front, which on FreeBSD - // means `kinfo_getfile` — the same class of sysinfo call as the - // disk enumeration below, and it faults the same way under QEMU. - // Every `AppState` builds one of these, including the ones in tests - // that never take a sample, so on FreeBSD the table is left to the - // `refresh_all()` in `refresh()` that would repeat it anyway. The - // cost is that the first sample reports no CPU usage, having no - // earlier reading to difference against. - #[cfg(target_os = "freebsd")] - system: sysinfo::System::new(), - #[cfg(not(target_os = "freebsd"))] - system: sysinfo::System::new_all(), + system, // sysinfo 0.39 FreeBSD disk enumeration calls getmntinfo and then // slice::from_raw_parts with a pointer that fails Rust's alignment // UB checks (abort in debug; real UB risk in release). Skip disks @@ -92,13 +93,29 @@ impl DiagnosticsCollector { } fn refresh(&mut self) -> RuntimeDiagnostics { - self.system.refresh_all(); + let pid = sysinfo::get_current_pid().ok(); + + // Refresh what is actually read below, rather than `refresh_all()`. That + // walked and stored every process on the host — a couple of megabytes + // rebuilt on every sample, and `/metrics/json` samples every five seconds + // while a dashboard is open — to answer questions about one pid. + self.system.refresh_memory(); + self.system.refresh_cpu_usage(); + if let Some(pid) = pid { + self.system.refresh_processes_specifics( + sysinfo::ProcessesToUpdate::Some(&[pid]), + true, + sysinfo::ProcessRefreshKind::nothing() + .with_memory() + .with_cpu() + .with_tasks(), + ); + } #[cfg(not(target_os = "freebsd"))] self.disks.refresh(true); self.networks.refresh(true); let load = sysinfo::System::load_average(); - let pid = sysinfo::get_current_pid().ok(); let process = pid.and_then(|pid| self.system.process(pid)); let disk_total = self @@ -221,5 +238,25 @@ mod tests { assert_eq!(second.process.pid, std::process::id()); assert!(second.system.available_memory_bytes <= second.system.total_memory_bytes); assert!(second.disks.available_bytes <= second.disks.total_bytes); + + // The sampler refreshes only the specifics these fields need, rather than + // everything sysinfo can collect, so each one is a thing that silently + // becomes zero or `None` if the wrong refresh is dropped. + assert!( + second.system.cpu_count > 0, + "the CPU list must be populated: refresh_cpu_usage does not build it" + ); + assert!( + second.system.total_memory_bytes > 0, + "system memory must be refreshed" + ); + assert!( + second.process.memory_bytes.is_some_and(|bytes| bytes > 0), + "our own process must still be in the table after a targeted refresh" + ); + assert!( + second.process.runtime_seconds.is_some(), + "our own process must still be in the table after a targeted refresh" + ); } } From c6460e12d459f2e27d49436d0ea44f96b4e580c7 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 15:31:41 +0300 Subject: [PATCH 16/23] u --- README.md | 2 +- crates/vuio-core/src/database/mod.rs | 11 + .../src/database/sqlite/media_repo/basic.rs | 20 + .../vuio-core/src/database/sqlite/traits.rs | 7 + crates/vuio-core/src/media/policy.rs | 1 - crates/vuio-core/src/media/scanner.rs | 416 +++++++++++------- crates/vuio-core/src/media/tests.rs | 92 ++++ 7 files changed, 384 insertions(+), 165 deletions(-) diff --git a/README.md b/README.md index 896dd9e..666a2e9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # VuIO Media Server A cross-platform media server written in Rust. Streams video, audio, and images to DLNA, Chromecast/Google TV, and compatible AirPlay video receivers. -Less than 18Mb of RAM needed +25Mb of RAM used for 5000 objects library Built with Tokio, Axum, and SQLite for high performance and reliability. diff --git a/crates/vuio-core/src/database/mod.rs b/crates/vuio-core/src/database/mod.rs index e66066f..8573cca 100644 --- a/crates/vuio-core/src/database/mod.rs +++ b/crates/vuio-core/src/database/mod.rs @@ -785,6 +785,17 @@ pub trait MediaRepository: Send + Sync { /// Load compact scanner comparison records instead of complete media metadata. async fn load_file_fingerprints(&self) -> Result>; + /// The same, for one subtree. + /// + /// A scan compares what is on disk against what is indexed, and it only ever + /// scans one root — so loading the whole table means every other library's + /// rows are held in memory for nothing. That is most of the cost when a + /// watcher event rescans a single folder. + async fn load_file_fingerprints_under( + &self, + canonical_prefix: &str, + ) -> Result>; + async fn get_root_availability(&self, path: &Path) -> Result>; async fn list_root_availability(&self) -> Result>; diff --git a/crates/vuio-core/src/database/sqlite/media_repo/basic.rs b/crates/vuio-core/src/database/sqlite/media_repo/basic.rs index 3dedfd4..496a88c 100644 --- a/crates/vuio-core/src/database/sqlite/media_repo/basic.rs +++ b/crates/vuio-core/src/database/sqlite/media_repo/basic.rs @@ -105,6 +105,26 @@ impl SqliteDatabase { .await } + pub(in crate::database::sqlite) async fn load_file_fingerprints_under_impl( + &self, + canonical_prefix: String, + ) -> Result> { + self.execute_read(move |connection| { + // A range on `path` rather than a `LIKE`, so the primary key index + // serves it. `subtree_range` stops at a component boundary, which is + // what keeps `/media/Film` from matching `/media/Films`. + let (start, end) = SqliteDatabase::subtree_range(&canonical_prefix); + let mut statement = connection.prepare_cached(&format!( + "SELECT {FINGERPRINT_COLUMNS} FROM media_files WHERE path >= ? AND path < ?" + ))?; + let fingerprints = statement + .query_map([&start, &end], schema::fingerprint_from_row)? + .collect::>>()?; + Ok(fingerprints) + }) + .await + } + pub(in crate::database::sqlite) async fn get_files_by_paths_impl( &self, paths: &[PathBuf], diff --git a/crates/vuio-core/src/database/sqlite/traits.rs b/crates/vuio-core/src/database/sqlite/traits.rs index 9130c2b..61835bb 100644 --- a/crates/vuio-core/src/database/sqlite/traits.rs +++ b/crates/vuio-core/src/database/sqlite/traits.rs @@ -69,6 +69,13 @@ impl MediaRepository for SqliteDatabase { SqliteDatabase::load_file_fingerprints_impl(self).await } + async fn load_file_fingerprints_under( + &self, + canonical_prefix: &str, + ) -> Result> { + SqliteDatabase::load_file_fingerprints_under_impl(self, canonical_prefix.to_owned()).await + } + async fn get_root_availability(&self, path: &Path) -> Result> { SqliteDatabase::get_root_availability_impl(self, path).await } diff --git a/crates/vuio-core/src/media/policy.rs b/crates/vuio-core/src/media/policy.rs index e14e7d6..30969a6 100644 --- a/crates/vuio-core/src/media/policy.rs +++ b/crates/vuio-core/src/media/policy.rs @@ -253,7 +253,6 @@ pub(super) fn swap_one_ascii_case(value: &str) -> Option { #[derive(Debug)] pub(super) struct TraversalReport { - pub(super) file_paths: Vec, pub(super) uncertain_prefixes: Vec, pub(super) errors: Vec, pub(super) root_complete: bool, diff --git a/crates/vuio-core/src/media/scanner.rs b/crates/vuio-core/src/media/scanner.rs index 54f1f9e..d137bb2 100644 --- a/crates/vuio-core/src/media/scanner.rs +++ b/crates/vuio-core/src/media/scanner.rs @@ -7,7 +7,146 @@ pub struct MediaScanner { database_manager: Arc, } +/// How many paths the walker may run ahead of the classifier. +/// +/// Bounded so that a walk of a large library does not become a list of the +/// library: past this the walking thread blocks until the consumer catches up. +const WALK_QUEUE: usize = 4096; + +/// How many changed files to read before writing them out. +/// +/// Only files that actually changed reach this, so on an unchanged library it +/// never fills. On a first scan it is what keeps peak memory flat instead of +/// proportional to the library. +const READ_WINDOW: usize = 4096; + +/// What a scan needs to know about a file it may already have indexed. +/// +/// Deliberately not [`FileFingerprint`]: that carries the path, and this lives +/// in a map keyed by the path, so storing it again doubled the largest +/// allocation a scan makes. +struct IndexedFile { + id: i64, + size: u64, + modified: SystemTime, + created_at: SystemTime, + tags_version: u32, + /// Set when the walk produced this path. What is left unset is what has been + /// deleted from disk — which is why the scan needs no second collection of + /// every path it saw. + seen: bool, +} + +impl IndexedFile { + /// Split a loaded record into the map's key and value, moving the path + /// rather than copying it into both halves. + fn split(fingerprint: FileFingerprint) -> (PathBuf, Self) { + let FileFingerprint { + id, + path, + size, + modified, + created_at, + tags_version, + } = fingerprint; + ( + path, + Self { + id, + size, + modified, + created_at, + tags_version, + seen: false, + }, + ) + } +} + impl MediaScanner { + /// Read a window of changed files, several at a time, and sort them into + /// inserts and updates. + /// + /// Each read ends in `spawn_blocking`, so the work already lands on the + /// blocking pool — but awaiting them one after another meant only ever one + /// was in flight, and a scan used a single core however many the machine had. + async fn read_window( + &self, + paths: &mut Vec, + existing_files_map: &HashMap, + files_to_insert: &mut Vec, + files_to_update: &mut Vec, + result: &mut ScanResult, + concurrency: usize, + ) -> Result<()> { + let mut built = futures_util::stream::iter(paths.drain(..)) + .map(|path| async move { (path.clone(), self.create_media_file_from_path(&path).await) }) + .buffer_unordered(concurrency); + + while let Some((path, outcome)) = built.next().await { + let current_file = match outcome { + Ok(file) => file, + Err(e) => { + debug!("Failed to create MediaFile for {}: {}", path.display(), e); + result.errors.push(ScanError { + path, + error: e.to_string(), + }); + continue; + } + }; + result.files_read += 1; + + match existing_files_map.get(&path) { + Some(existing) => { + if self.fingerprint_needs_update(existing, ¤t_file) { + let mut updated = current_file; + updated.id = Some(existing.id); + updated.created_at = existing.created_at; + updated.updated_at = SystemTime::now(); + files_to_update.push(updated); + } else { + result.unchanged += 1; + } + } + None => files_to_insert.push(current_file), + } + } + + Ok(()) + } + + /// Write out whichever batch has reached `threshold`. + /// + /// Pass a threshold of 1 to flush whatever is left. + async fn flush_batches( + &self, + files_to_insert: &mut Vec, + files_to_update: &mut Vec, + result: &mut ScanResult, + threshold: usize, + ) -> Result<()> { + if files_to_insert.len() >= threshold && !files_to_insert.is_empty() { + info!("Inserting batch of {} files", files_to_insert.len()); + self.database_manager + .bulk_store_canonical_media_files(files_to_insert) + .await?; + result.new += files_to_insert.len(); + files_to_insert.clear(); + } + + if files_to_update.len() >= threshold && !files_to_update.is_empty() { + info!("Updating batch of {} files", files_to_update.len()); + self.database_manager + .bulk_update_canonical_media_files(files_to_update) + .await?; + result.updated += files_to_update.len(); + files_to_update.clear(); + } + + Ok(()) + } + /// Create a new media scanner with database manager pub fn with_database(database_manager: Arc) -> Self { Self { @@ -251,7 +390,7 @@ impl MediaScanner { /// the file is opened rather than after. Every rule is answerable from /// metadata the filesystem already has, which is what makes an unchanged /// library nearly free to re-scan. - fn stat_needs_update(existing: &FileFingerprint, metadata: &std::fs::Metadata) -> bool { + fn stat_needs_update(existing: &IndexedFile, metadata: &std::fs::Metadata) -> bool { if existing.size != metadata.len() { return true; } @@ -271,7 +410,7 @@ impl MediaScanner { difference.map_or(true, |difference| difference.as_secs() > 10) } - fn fingerprint_needs_update(&self, existing: &FileFingerprint, current: &MediaFile) -> bool { + fn fingerprint_needs_update(&self, existing: &IndexedFile, current: &MediaFile) -> bool { if existing.size != current.size { return true; } @@ -325,28 +464,31 @@ impl MediaScanner { canonical_root.display() ); - // Load all existing files from database once at the start (for incremental updates) + // Load the index for this subtree only. A scan compares one root against + // one root; the rest of the library would be held for nothing, and a + // watcher event that rescans a single folder used to load every row. debug!("Loading existing files from database..."); - let existing_files_map: HashMap = self + let canonical_root_str = canonical_root.to_string_lossy().into_owned(); + let mut existing_files_map: HashMap = self .database_manager - .load_file_fingerprints() + .load_file_fingerprints_under(&canonical_root_str) .await? .into_iter() - .map(|fingerprint| (fingerprint.path.clone(), fingerprint)) + .map(IndexedFile::split) .collect(); - debug!( - "Loaded {} existing files from database", - existing_files_map.len() - ); + let existing_in_root = existing_files_map.len(); + debug!("Loaded {existing_in_root} existing files from database"); - // Use jwalk for parallel directory traversal - runs in a blocking thread pool + // Walk on a blocking thread, handing paths over as they are found rather + // than collecting the library into a `Vec` first. The channel is bounded, + // so a slow consumer backs the walker up instead of buffering. let root_clone = canonical_root.clone(); let mut traversal_policy = policy.clone(); traversal_policy.root = canonical_root.clone(); + let (path_sender, path_receiver) = tokio::sync::mpsc::channel::(WALK_QUEUE); - let traversal = tokio::task::spawn_blocking(move || { + let traversal_task = tokio::task::spawn_blocking(move || { let mut report = TraversalReport { - file_paths: Vec::new(), uncertain_prefixes: Vec::new(), errors: Vec::new(), root_complete: true, @@ -355,8 +497,9 @@ impl MediaScanner { match entry { Ok(entry) if entry.file_type().is_file() => { let path = entry.path(); - if traversal_policy.allows_media(&path) { - report.file_paths.push(path); + if traversal_policy.allows_media(&path) && path_sender.blocking_send(path).is_err() { + // The consumer is gone, so the scan is over. + break; } } Ok(_) => {} @@ -377,41 +520,14 @@ impl MediaScanner { } } report - }) - .await?; + }); - let file_paths = traversal.file_paths; - - let total_files = file_paths.len(); - let existing_in_root = existing_files_map - .keys() - .filter(|path| path.starts_with(&canonical_root)) - .count(); - let suspect_empty_root = total_files == 0 && existing_in_root > 0; - info!( - "Found {} media files, processing in batches of {}", - total_files, BATCH_SIZE - ); - - // Process files in batches let mut result = ScanResult::new(); - result.errors.extend(traversal.errors); - result.complete = traversal.root_complete - && traversal.uncertain_prefixes.is_empty() - && !suspect_empty_root; - if suspect_empty_root { - result.errors.push(ScanError { - path: canonical_root.clone(), - error: "previously populated root is unexpectedly empty; destructive reconciliation deferred" - .to_owned(), - }); - } let mut files_to_insert: Vec = Vec::with_capacity(BATCH_SIZE); let mut files_to_update: Vec = Vec::with_capacity(BATCH_SIZE); - let mut current_paths: HashSet = HashSet::with_capacity(total_files); - let mut processed = 0; + let mut processed = 0_usize; - // Pass one: classify every path with a single `stat`. + // Classify each path with a single `stat`, several at a time. // // Building a `MediaFile` canonicalizes the path, probes for a subtitle // sidecar and, for audio, parses the entire container with symphonia. Ask @@ -423,9 +539,9 @@ impl MediaScanner { let concurrency = std::thread::available_parallelism() .map(|value| value.get()) .unwrap_or(4); - let mut needs_reading: Vec = Vec::new(); + let mut pending_reads: Vec = Vec::with_capacity(READ_WINDOW); { - let mut classified = futures_util::stream::iter(file_paths) + let mut classified = tokio_stream::wrappers::ReceiverStream::new(path_receiver) .map(|path| async move { let metadata = tokio::fs::metadata(&path).await.ok(); (path, metadata) @@ -433,141 +549,115 @@ impl MediaScanner { .buffer_unordered(concurrency); while let Some((path, metadata)) = classified.next().await { + processed += 1; + // jwalk descendants inherit the already-canonical root. It does // not follow file symlinks, so no syscall is needed here. - current_paths.insert(path.clone()); - processed += 1; + let unchanged = match (existing_files_map.get_mut(&path), metadata) { + (Some(existing), Some(metadata)) => { + // Marking the record is what replaces a second set of every + // path on disk: whatever is left unmarked is what is gone. + existing.seen = true; + !Self::stat_needs_update(existing, &metadata) + } + (Some(existing), None) => { + existing.seen = true; + false + } + // A new file, or one whose `stat` failed — + // `create_media_file_from_path` makes the same call and + // reports the error properly. + (None, _) => false, + }; - match (existing_files_map.get(&path), metadata) { - (Some(existing), Some(metadata)) - if !Self::stat_needs_update(existing, &metadata) => - { - result.unchanged += 1; + if unchanged { + result.unchanged += 1; + } else { + pending_reads.push(path); + if pending_reads.len() >= READ_WINDOW { + self.read_window( + &mut pending_reads, + &existing_files_map, + &mut files_to_insert, + &mut files_to_update, + &mut result, + concurrency, + ) + .await?; + self.flush_batches( + &mut files_to_insert, + &mut files_to_update, + &mut result, + BATCH_SIZE, + ) + .await?; } - // Anything else has to be read: a new file, a changed one, or - // one whose `stat` failed — `create_media_file_from_path` - // makes the same call and reports the error properly. - _ => needs_reading.push(path), } if processed % 10_000 == 0 { - debug!("Examined {}/{} files", processed, total_files); + debug!("Examined {processed} files"); } } } - info!( - "{} of {} files need reading", - needs_reading.len(), - total_files - ); - - // Pass two: read the files that actually changed, several at a time. - // - // Each read ends in `spawn_blocking`, so the work already lands on the - // blocking pool — but awaiting them one after another meant only ever one - // was in flight, and a scan used a single core however many the machine - // had. - let mut built = futures_util::stream::iter(needs_reading) - .map(|path| async move { (path.clone(), self.create_media_file_from_path(&path).await) }) - .buffer_unordered(concurrency); - - while let Some((path, outcome)) = built.next().await { - let current_file = match outcome { - Ok(file) => file, - Err(e) => { - debug!("Failed to create MediaFile for {}: {}", path.display(), e); - result.errors.push(ScanError { - path: path.clone(), - error: e.to_string(), - }); - continue; - } - }; - result.files_read += 1; - - // Check if file exists in database - if let Some(existing) = existing_files_map.get(&path) { - if self.fingerprint_needs_update(existing, ¤t_file) { - let mut updated = current_file; - updated.id = Some(existing.id); - updated.created_at = existing.created_at; - updated.updated_at = SystemTime::now(); - files_to_update.push(updated); - } else { - result.unchanged += 1; - } - } else { - files_to_insert.push(current_file); - } - - // Process batch when full - if files_to_insert.len() >= BATCH_SIZE { - info!( - "Inserting batch of {} files ({}/{})", - files_to_insert.len(), - processed, - total_files - ); - self.database_manager - .bulk_store_canonical_media_files(&files_to_insert) - .await?; - result.new += files_to_insert.len(); - files_to_insert.clear(); - } - - if files_to_update.len() >= BATCH_SIZE { - info!( - "Updating batch of {} files ({}/{})", - files_to_update.len(), - processed, - total_files - ); - self.database_manager - .bulk_update_canonical_media_files(&files_to_update) - .await?; - result.updated += files_to_update.len(); - files_to_update.clear(); - } - - // Progress logging every 1000 files - if processed % 1000 == 0 { - info!("Progress: {}/{} files processed", processed, total_files); - } + if !pending_reads.is_empty() { + self.read_window( + &mut pending_reads, + &existing_files_map, + &mut files_to_insert, + &mut files_to_update, + &mut result, + concurrency, + ) + .await?; } + self.flush_batches(&mut files_to_insert, &mut files_to_update, &mut result, 1) + .await?; - // Process remaining files in last batch - if !files_to_insert.is_empty() { - info!("Inserting final batch of {} files", files_to_insert.len()); - self.database_manager - .bulk_store_canonical_media_files(&files_to_insert) - .await?; - result.new += files_to_insert.len(); - } + // The walker's own findings — which prefixes it could not read — only + // arrive once the channel has closed, and deletion depends on them. + let traversal = traversal_task.await?; - if !files_to_update.is_empty() { - info!("Updating final batch of {} files", files_to_update.len()); - self.database_manager - .bulk_update_canonical_media_files(&files_to_update) - .await?; - result.updated += files_to_update.len(); + let total_files = processed; + let suspect_empty_root = total_files == 0 && existing_in_root > 0; + result.errors.extend(traversal.errors); + result.complete = traversal.root_complete + && traversal.uncertain_prefixes.is_empty() + && !suspect_empty_root; + if suspect_empty_root { + result.errors.push(ScanError { + path: canonical_root.clone(), + error: "previously populated root is unexpectedly empty; destructive reconciliation deferred" + .to_owned(), + }); } - // Find and remove deleted files - let files_to_remove: Vec = existing_files_map - .iter() - .filter(|(path, _)| !current_paths.contains(*path)) - .filter(|(path, _)| path.starts_with(&canonical_root)) // Only remove files under scanned directory - .filter(|(path, _)| { - traversal.root_complete - && !suspect_empty_root - && !traversal + // Whatever the walk never produced is gone from disk. + let reconcile_deletions = + traversal.root_complete && !suspect_empty_root && traversal.uncertain_prefixes.is_empty(); + let files_to_remove: Vec = if reconcile_deletions { + existing_files_map + .iter() + .filter(|(_, indexed)| !indexed.seen) + .map(|(path, _)| path.clone()) + .collect() + } else { + // A partial walk cannot tell "absent" from "unreadable". Where only + // some prefixes are in doubt, everything outside them is still + // decidable. + existing_files_map + .iter() + .filter(|(_, indexed)| !indexed.seen) + .filter(|_| traversal.root_complete && !suspect_empty_root) + .filter(|(path, _)| { + !traversal .uncertain_prefixes .iter() .any(|prefix| path.starts_with(prefix)) - }) - .map(|(_, file)| file.path.clone()) - .collect(); + }) + .map(|(path, _)| path.clone()) + .collect() + }; if !files_to_remove.is_empty() { info!( diff --git a/crates/vuio-core/src/media/tests.rs b/crates/vuio-core/src/media/tests.rs index cde0711..4597d15 100644 --- a/crates/vuio-core/src/media/tests.rs +++ b/crates/vuio-core/src/media/tests.rs @@ -128,6 +128,98 @@ async fn test_scan_result_operations() { assert!(summary.contains("2 removed")); } +/// The scan reconciles what is on disk against what is indexed, and it does so +/// by marking the records the walk produced — so what is left unmarked is what +/// has been deleted. This is the destructive half of a scan and had no coverage +/// at all, which is uncomfortable for the one operation that removes user data. +#[tokio::test] +async fn recursive_scan_removes_what_is_gone_from_disk() { + let temp_dir = tempdir().unwrap(); + let db = Arc::new( + SqliteDatabase::new(temp_dir.path().join("deletions.db")) + .await + .unwrap(), + ); + db.initialize().await.unwrap(); + let scanner = + MediaScanner::with_filesystem_manager(Box::new(BaseFileSystemManager::new(true)), db.clone()); + + let root = temp_dir.path().join("media"); + let keep_dir = root.join("keep"); + let doomed_dir = root.join("doomed"); + tokio::fs::create_dir_all(&keep_dir).await.unwrap(); + tokio::fs::create_dir_all(&doomed_dir).await.unwrap(); + + tokio::fs::write(keep_dir.join("stays.mp4"), b"a").await.unwrap(); + let single = keep_dir.join("goes.mp4"); + tokio::fs::write(&single, b"b").await.unwrap(); + tokio::fs::write(doomed_dir.join("one.mp4"), b"c").await.unwrap(); + tokio::fs::write(doomed_dir.join("two.mp4"), b"d").await.unwrap(); + + assert_eq!(scanner.scan_directory_recursive(&root).await.unwrap().new, 4); + + // One file, and then a whole directory. + tokio::fs::remove_file(&single).await.unwrap(); + let after_file = scanner.scan_directory_recursive(&root).await.unwrap(); + assert_eq!(after_file.removed, 1, "a deleted file must leave the index"); + assert_eq!(after_file.unchanged, 3); + assert!(db.get_file_by_path(&single).await.unwrap().is_none()); + + tokio::fs::remove_dir_all(&doomed_dir).await.unwrap(); + let after_dir = scanner.scan_directory_recursive(&root).await.unwrap(); + assert_eq!( + after_dir.removed, 2, + "a directory deleted whole must take its files with it" + ); + assert_eq!(after_dir.unchanged, 1); + + let mut remaining = db.stream_all_media_files(); + let mut survivors = Vec::new(); + while let Some(file) = remaining.next().await { + survivors.push(file.unwrap()); + } + assert_eq!(survivors.len(), 1); + assert_eq!(survivors[0].filename, "stays.mp4"); +} + +/// A scan compares one root against one root. It loads only that subtree's +/// records, and a sibling library — including one whose path is a string prefix +/// of this one — must be neither examined nor deleted. +#[tokio::test] +async fn recursive_scan_leaves_other_roots_alone() { + let temp_dir = tempdir().unwrap(); + let db = Arc::new( + SqliteDatabase::new(temp_dir.path().join("roots.db")) + .await + .unwrap(), + ); + db.initialize().await.unwrap(); + let scanner = + MediaScanner::with_filesystem_manager(Box::new(BaseFileSystemManager::new(true)), db.clone()); + + // `Films` shares a prefix with `Film`, which a `LIKE 'path%'` would sweep up. + let film = temp_dir.path().join("Film"); + let films = temp_dir.path().join("Films"); + tokio::fs::create_dir_all(&film).await.unwrap(); + tokio::fs::create_dir_all(&films).await.unwrap(); + tokio::fs::write(film.join("a.mp4"), b"a").await.unwrap(); + let sibling = films.join("b.mp4"); + tokio::fs::write(&sibling, b"b").await.unwrap(); + + assert_eq!(scanner.scan_directory_recursive(&film).await.unwrap().new, 1); + assert_eq!(scanner.scan_directory_recursive(&films).await.unwrap().new, 1); + + // Rescanning `Film` must not notice, touch, or remove anything under `Films`. + let rescan = scanner.scan_directory_recursive(&film).await.unwrap(); + assert_eq!(rescan.unchanged, 1, "only its own root is compared"); + assert_eq!(rescan.removed, 0); + assert_eq!(rescan.total_scanned, 1); + assert!( + db.get_file_by_path(&sibling).await.unwrap().is_some(), + "the sibling root's file must survive a scan of Film" + ); +} + #[tokio::test] async fn test_recursive_scan_optimization() { let temp_dir = tempdir().unwrap(); From 87cf201ff5ba169e047bd3a30b65553dd34023fb Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 15:46:39 +0300 Subject: [PATCH 17/23] u --- .../src/web/soap/content_directory.rs | 6 -- crates/vuio-core/src/web/soap/music.rs | 3 - crates/vuio-core/src/web/ui/js/browse.js | 14 +-- crates/vuio-core/src/web/xml/rendering.rs | 92 +------------------ .../tests/mediainfo_integration_tests.rs | 72 +++++++++++++++ 5 files changed, 74 insertions(+), 113 deletions(-) diff --git a/crates/vuio-core/src/web/soap/content_directory.rs b/crates/vuio-core/src/web/soap/content_directory.rs index c2523cb..146cc94 100644 --- a/crates/vuio-core/src/web/soap/content_directory.rs +++ b/crates/vuio-core/src/web/soap/content_directory.rs @@ -191,9 +191,6 @@ impl ContentDirectoryHandler { autoplay_enabled: state.current_config().media.autoplay_enabled, update_id: current_update_id, bookmarks, - prefer_online_titles: state.current_config().mediainfo.prefer_online_titles, - min_confidence: state.current_config().mediainfo.min_confidence, - mediainfo: Default::default(), }; let canonical_parent = canonical_browse_path.to_string_lossy().into_owned(); let mime_family = media_type_filter.to_owned(); @@ -380,9 +377,6 @@ impl ContentDirectoryHandler { autoplay_enabled: state.current_config().media.autoplay_enabled, update_id: state.content_update_id.load(Ordering::SeqCst), bookmarks: state.bookmarks.lock().await.snapshot(), - prefer_online_titles: state.current_config().mediainfo.prefer_online_titles, - min_confidence: state.current_config().mediainfo.min_confidence, - mediainfo: Default::default(), }; let starting_index = params.starting_index as usize; let requested_count = browse_page_limit(params); diff --git a/crates/vuio-core/src/web/soap/music.rs b/crates/vuio-core/src/web/soap/music.rs index 457b12e..2d087ca 100644 --- a/crates/vuio-core/src/web/soap/music.rs +++ b/crates/vuio-core/src/web/soap/music.rs @@ -635,9 +635,6 @@ pub(super) async fn render_context( autoplay_enabled: state.current_config().media.autoplay_enabled, update_id: state.content_update_id.load(Ordering::SeqCst), bookmarks, - prefer_online_titles: state.current_config().mediainfo.prefer_online_titles, - min_confidence: state.current_config().mediainfo.min_confidence, - mediainfo: Default::default(), } } diff --git a/crates/vuio-core/src/web/ui/js/browse.js b/crates/vuio-core/src/web/ui/js/browse.js index b701395..094c7a8 100644 --- a/crates/vuio-core/src/web/ui/js/browse.js +++ b/crates/vuio-core/src/web/ui/js/browse.js @@ -31,9 +31,6 @@ function render() { const matchesSearch = searchQuery === '' || file.name.toLowerCase().includes(searchQuery) || (file.title || '').toLowerCase().includes(searchQuery) - // Searching for the real title should find a file whose name is a - // release string that does not contain it. - || (file.info_title || '').toLowerCase().includes(searchQuery) || (file.artist || '').toLowerCase().includes(searchQuery) || (file.album || '').toLowerCase().includes(searchQuery); if (currentTab === 'radio') { @@ -277,9 +274,7 @@ function createFileCard(file) { `; card.querySelector('.media-icon-wrapper').innerHTML = iconSvg; const name = card.querySelector('.media-name'); - // A fetched title is the readable one, and for video it is usually the only - // title there is — nothing reads metadata out of a video file. - name.textContent = file.info_title || file.title || file.name; + name.textContent = file.title || file.name; name.title = file.name; const details = card.querySelector('.media-details'); const metadataParts = [file.artist, file.album].filter(Boolean); @@ -290,13 +285,6 @@ function createFileCard(file) { metadata.textContent = metadataParts.join(' — '); details.insertBefore(metadata, details.querySelector('.media-meta')); } - if (file.info_overview) { - const overview = document.createElement('div'); - overview.className = 'media-overview'; - overview.textContent = file.info_overview; - overview.title = file.info_overview; - details.insertBefore(overview, details.querySelector('.media-meta')); - } card.querySelector('.media-size').textContent = file.size_str; card.querySelector('.media-extension').textContent = file.ext; diff --git a/crates/vuio-core/src/web/xml/rendering.rs b/crates/vuio-core/src/web/xml/rendering.rs index fe87491..a6821ab 100644 --- a/crates/vuio-core/src/web/xml/rendering.rs +++ b/crates/vuio-core/src/web/xml/rendering.rs @@ -143,27 +143,6 @@ pub struct BrowseRenderContext { pub autoplay_enabled: bool, pub update_id: u32, pub bookmarks: HashMap, - /// Whether a fetched title outranks the one read from the file's own tags. - pub prefer_online_titles: bool, - /// Matches weaker than this are left out of `mediainfo` entirely. - pub min_confidence: u8, - /// Fetched media info for the items on this page, filled in by the response - /// generators just before rendering. Empty when nothing has been fetched. - pub mediainfo: HashMap, -} - -impl BrowseRenderContext { - /// The fetched title to show for `file_id`, if there is one and it is wanted. - fn online_title(&self, file_id: i64) -> Option<&str> { - if !self.prefer_online_titles { - return None; - } - self.mediainfo - .get(&file_id)? - .title - .as_deref() - .filter(|title| !title.is_empty()) - } } /// UPnP container classes. @@ -314,14 +293,7 @@ pub(super) fn write_media_view( let mime = file.mime_type(); let is_radio = mime == "audio/radio"; let has_srt = file.subtitle_available(); - // A fetched title is the readable one — "Arrival" rather than - // "Arrival.2016.1080p.BluRay.x264-GRP" — so it wins when there is one and the - // operator asked for it. - let title = didl_display_title( - context.online_title(file_id).or_else(|| file.title()), - file.filename(), - context.client, - ); + let title = didl_display_title(file.title(), file.filename(), context.client); write!( output, r#"{}"#, @@ -334,32 +306,6 @@ pub(super) fn write_media_view( } output.write_str("")?; - // Synopsis and genres from the fetch. Video had neither before: nothing read - // metadata out of a video file, so a TV showed a filename and nothing else. - if let Some(overlay) = context.mediainfo.get(&file_id) { - if let Some(overview) = overlay.overview.as_deref().filter(|text| !text.is_empty()) { - write!( - output, - "{}", - xml_escape(overview) - )?; - } - if !mime.starts_with("audio/") { - if let Some(genre) = overlay.genres.first() { - write!(output, "{}", xml_escape(genre))?; - } - // Audio already advertises its cover below; this is what gives a movie - // or an episode a poster for the first time. - if overlay.has_artwork { - write!( - output, - "http://{}:{}/media/{}/cover", - context.server_ip, context.server_port, file_id - )?; - } - } - } - if mime.starts_with("audio/") { if let Some(value) = file.artist() { write!(output, "{}", xml_escape(value))?; @@ -518,40 +464,6 @@ pub(super) fn write_media_view( output.write_str("") } -/// Load the fetched media info for the page about to be rendered. -/// -/// A first pass collects the ids, because the writer cannot ask the session for -/// anything while the session is lending it a row. Both passes run the same -/// indexed query on the same connection, and the first does no formatting, so the -/// cost is one extra index walk rather than a second round trip. -/// -/// A failure here is not worth failing a browse over: the page renders with local -/// metadata, exactly as it did before this feature existed. -fn with_mediainfo( - session: &mut S, - query: &MediaFileQuery, - offset: usize, - limit: usize, - mut context: BrowseRenderContext, -) -> Result { - if limit == 0 { - return Ok(context); - } - let mut ids = Vec::with_capacity(limit); - session.visit_files(query, offset, limit, |file| { - if let Some(id) = file.id().filter(|id| *id > 0) { - ids.push(id); - } - Ok(()) - })?; - - match session.mediainfo_overlays(&ids, context.min_confidence) { - Ok(overlays) => context.mediainfo = overlays, - Err(error) => tracing::debug!(%error, "Could not load media info for this page"), - } - Ok(context) -} - pub fn generate_indexed_browse_response( session: &mut S, canonical_parent: &str, @@ -594,7 +506,6 @@ pub fn generate_indexed_browse_response( path: canonical_parent.to_owned(), mime_family: (!mime_family.is_empty()).then(|| mime_family.to_owned()), }; - let context = with_mediainfo(session, &query, file_offset, file_limit, context)?; let summary = session.visit_files(&query, file_offset, file_limit, |file| { write_media_view(&mut result, object_id, &file, &context) .map_err(|_| anyhow::anyhow!("failed to construct browse XML")) @@ -620,7 +531,6 @@ pub fn generate_indexed_items_response( "#)?; let mut result = SoapResultWriter(&mut response); result.push_str(r#""#); - let context = with_mediainfo(session, &query, starting_index, requested_count, context)?; let summary = session.visit_files(&query, starting_index, requested_count, |file| { write_media_view(&mut result, object_id, &file, &context) .map_err(|_| anyhow::anyhow!("failed to construct browse XML")) diff --git a/crates/vuio-core/tests/mediainfo_integration_tests.rs b/crates/vuio-core/tests/mediainfo_integration_tests.rs index 6a2bddb..b6b3693 100644 --- a/crates/vuio-core/tests/mediainfo_integration_tests.rs +++ b/crates/vuio-core/tests/mediainfo_integration_tests.rs @@ -611,3 +611,75 @@ async fn browse_json_reports_no_media_info_when_none_was_fetched() { assert!(file["info_title"].is_null()); assert_eq!(file["info_art"], false); } + +#[tokio::test] +async fn dlna_browse_shows_filename_not_mediainfo_title_or_description() { + let temp = tempdir().unwrap(); + let database = Arc::new( + SqliteDatabase::new(temp.path().join("test.db")) + .await + .unwrap(), + ); + database.initialize().await.unwrap(); + + let media_path = temp.path().join("media"); + tokio::fs::create_dir_all(&media_path).await.unwrap(); + let file = MediaFile::new( + media_path.join("Show.Name.S02E05.1080p.mkv"), + 1024, + "video/x-matroska".to_string(), + ); + let id = database.store_media_file(&file).await.unwrap(); + + database + .bulk_store_mediainfo(&[record_for(id, 92)]) + .await + .unwrap(); + + let state = state_with(database, &temp).await; + let router = create_router(state, Surface::Primary); + + let soap_browse = r#" + + + + video/d0 + BrowseDirectChildren + * + 0 + 10 + + + +"#; + + let response = router + .oneshot( + Request::builder() + .method("POST") + .uri("/control/ContentDirectory") + .header( + "soapaction", + "\"urn:schemas-upnp-org:service:ContentDirectory:1#Browse\"", + ) + .header("content-type", "text/xml; charset=utf-8") + .extension(ConnectInfo(test_peer())) + .body(Body::from(soap_browse)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let bytes = axum::body::to_bytes(response.into_body(), 128 * 1024) + .await + .unwrap(); + let body = String::from_utf8(bytes.to_vec()).unwrap(); + + // DLNA client should see original filename, not fetched online title "Some Show" + assert!(body.contains("Show.Name.S02E05.1080p.mkv")); + assert!(!body.contains("<dc:title>Some Show</dc:title>")); + // DLNA should not contain fetched overview / description + assert!(!body.contains("A tale.")); +} + From 0e461a1665962afd65372c89df80c8a743a4acf1 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 18:13:26 +0300 Subject: [PATCH 18/23] perf(scan): read the index a page at a time when validating deletions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `validate_and_cleanup_deleted_files` loaded every fingerprint in the index as one `Vec` and then called `path.exists()` per row — a blocking syscall, once per file, on an async worker. It runs at startup and again on the periodic sweep, so a library's worth of records was materialised twice over on a schedule. Page it instead, by id, and hand each page's existence checks to the blocking pool in one hop rather than one per file. Every row is still considered — this is the pass that prunes records belonging to no configured library at all, so it cannot be scoped to a root the way a scan can. Measured at 100,000 files, release: a cold scan settles at 77 MB rather than 94, and a rescan of an unchanged library peaks at 87 MB rather than 92. Together with the streaming scan, the 100k figures are now 66 MB peak / 77 MB settled for a cold scan and 87 MB for a rescan, against 148 / 169 / 108 before this branch. The paging is what the new test covers: an index larger than one page is where a paging mistake hides, since rows past the first page would simply never be checked and deleted files would stay indexed forever. --- crates/vuio-core/src/database/mod.rs | 11 +++ .../src/database/sqlite/media_repo/basic.rs | 21 ++++ .../vuio-core/src/database/sqlite/traits.rs | 8 ++ .../vuio-core/src/lifecycle/media/scanning.rs | 99 +++++++++++++++++-- 4 files changed, 129 insertions(+), 10 deletions(-) diff --git a/crates/vuio-core/src/database/mod.rs b/crates/vuio-core/src/database/mod.rs index 8573cca..03bad89 100644 --- a/crates/vuio-core/src/database/mod.rs +++ b/crates/vuio-core/src/database/mod.rs @@ -796,6 +796,17 @@ pub trait MediaRepository: Send + Sync { canonical_prefix: &str, ) -> Result>; + /// One page of fingerprints, ordered by id, starting after `after_id`. + /// + /// For the callers that genuinely have to examine every row — the index + /// cleanup has to consider records under no configured root at all — so that + /// doing so does not mean holding every row at once. + async fn load_file_fingerprints_after( + &self, + after_id: i64, + limit: usize, + ) -> Result>; + async fn get_root_availability(&self, path: &Path) -> Result>; async fn list_root_availability(&self) -> Result>; diff --git a/crates/vuio-core/src/database/sqlite/media_repo/basic.rs b/crates/vuio-core/src/database/sqlite/media_repo/basic.rs index 496a88c..1a7f31f 100644 --- a/crates/vuio-core/src/database/sqlite/media_repo/basic.rs +++ b/crates/vuio-core/src/database/sqlite/media_repo/basic.rs @@ -105,6 +105,27 @@ impl SqliteDatabase { .await } + pub(in crate::database::sqlite) async fn load_file_fingerprints_after_impl( + &self, + after_id: i64, + limit: usize, + ) -> Result> { + self.execute_read(move |connection| { + let mut statement = connection.prepare_cached(&format!( + "SELECT {FINGERPRINT_COLUMNS} FROM media_files \ + WHERE id > ? ORDER BY id LIMIT ?" + ))?; + let fingerprints = statement + .query_map( + rusqlite::params![after_id, limit as i64], + schema::fingerprint_from_row, + )? + .collect::>>()?; + Ok(fingerprints) + }) + .await + } + pub(in crate::database::sqlite) async fn load_file_fingerprints_under_impl( &self, canonical_prefix: String, diff --git a/crates/vuio-core/src/database/sqlite/traits.rs b/crates/vuio-core/src/database/sqlite/traits.rs index 61835bb..1316b38 100644 --- a/crates/vuio-core/src/database/sqlite/traits.rs +++ b/crates/vuio-core/src/database/sqlite/traits.rs @@ -76,6 +76,14 @@ impl MediaRepository for SqliteDatabase { SqliteDatabase::load_file_fingerprints_under_impl(self, canonical_prefix.to_owned()).await } + async fn load_file_fingerprints_after( + &self, + after_id: i64, + limit: usize, + ) -> Result> { + SqliteDatabase::load_file_fingerprints_after_impl(self, after_id, limit).await + } + async fn get_root_availability(&self, path: &Path) -> Result> { SqliteDatabase::get_root_availability_impl(self, path).await } diff --git a/crates/vuio-core/src/lifecycle/media/scanning.rs b/crates/vuio-core/src/lifecycle/media/scanning.rs index 2cc8c97..5d62a29 100644 --- a/crates/vuio-core/src/lifecycle/media/scanning.rs +++ b/crates/vuio-core/src/lifecycle/media/scanning.rs @@ -92,8 +92,24 @@ pub(in crate::lifecycle) async fn validate_and_cleanup_deleted_files>(); - { - for media_file in database.load_file_fingerprints().await? { + // Walk the index a page at a time. Every row has to be considered — a record + // under no configured root at all is exactly what this prunes — but holding + // the whole table to do it made the check cost as much memory as the library. + const CLEANUP_PAGE: usize = 4096; + let mut after_id = 0_i64; + loop { + let page = database + .load_file_fingerprints_after(after_id, CLEANUP_PAGE) + .await?; + let Some(last) = page.last() else { + break; + }; + after_id = last.id; + + // Sort the page without touching the disk first, so the syscalls that do + // have to happen are the only ones that happen. + let mut to_probe: Vec = Vec::new(); + for media_file in page { total_checked += 1; let Some(root_index) = @@ -101,7 +117,7 @@ pub(in crate::lifecycle) async fn validate_and_cleanup_deleted_files 0 { info!( @@ -619,6 +643,61 @@ mod tests { assert_eq!(indexed_paths(&database).await, vec![canonical_root(&present)]); } + /// The cleanup reads the index a page at a time rather than holding all of it, so + /// an index larger than one page is the case where a paging mistake shows: rows + /// past the first page silently never get checked, and deleted files stay indexed + /// forever. Sized past `CLEANUP_PAGE` deliberately. + #[tokio::test] + async fn every_page_of_a_large_index_is_checked() { + let root = tempfile::TempDir::new().expect("root"); + + // A handful that exist, and several pages' worth that do not. + let mut survivors = Vec::new(); + for index in 0..3 { + let path = root.path().join(format!("present_{index}.mp4")); + std::fs::write(&path, b"x").expect("write"); + survivors.push(canonical_root(&path)); + } + survivors.sort(); + + let missing = 5_000; + let mut rows: Vec = Vec::with_capacity(survivors.len() + missing); + for index in 0..3 { + rows.push(MediaFile::new( + root.path().join(format!("present_{index}.mp4")), + 1024, + "video/mp4".to_string(), + )); + } + for index in 0..missing { + rows.push(MediaFile::new( + root.path().join(format!("gone_{index}.mp4")), + 1024, + "video/mp4".to_string(), + )); + } + + let temp = tempfile::TempDir::new().expect("temp dir"); + let database = Arc::new( + SqliteDatabase::new(temp.path().join("paged.db")) + .await + .expect("database"), + ); + database.initialize().await.expect("schema"); + database.bulk_store_media_files(&rows).await.expect("store"); + + let removed = + validate_and_cleanup_deleted_files(database.clone(), &[root.path().to_path_buf()], true) + .await + .expect("cleanup"); + + assert_eq!( + removed, missing, + "a row on the second page and beyond must still be checked" + ); + assert_eq!(indexed_paths(&database).await, survivors); + } + /// Nothing configured means nothing to compare against. Treating every file as an /// orphan would discard the whole index over what is almost certainly a misload. #[tokio::test] From dad71a24f357dd802cb2deec70be69162688d100 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 18:13:32 +0300 Subject: [PATCH 19/23] style(test): drop a trailing blank line rustfmt rejects --- crates/vuio-core/tests/mediainfo_integration_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/vuio-core/tests/mediainfo_integration_tests.rs b/crates/vuio-core/tests/mediainfo_integration_tests.rs index b6b3693..84f2c26 100644 --- a/crates/vuio-core/tests/mediainfo_integration_tests.rs +++ b/crates/vuio-core/tests/mediainfo_integration_tests.rs @@ -682,4 +682,3 @@ async fn dlna_browse_shows_filename_not_mediainfo_title_or_description() { // DLNA should not contain fetched overview / description assert!(!body.contains("A tale.")); } - From 3c66c1fbb33f74928938217e4e2d194bb17528fd Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 18:23:09 +0300 Subject: [PATCH 20/23] refactor(scan): size the read window to the write batch A window of 4096 fed a writer batched at 1000, so a first scan wrote 4096 rows at a time instead of the batch size the rest of the code is tuned around. Match them: one window is one write, and the flush no longer needs a threshold argument to distinguish "batch is full" from "this is the last of them". --- crates/vuio-core/src/media/scanner.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/crates/vuio-core/src/media/scanner.rs b/crates/vuio-core/src/media/scanner.rs index d137bb2..0b376ec 100644 --- a/crates/vuio-core/src/media/scanner.rs +++ b/crates/vuio-core/src/media/scanner.rs @@ -17,8 +17,9 @@ const WALK_QUEUE: usize = 4096; /// /// Only files that actually changed reach this, so on an unchanged library it /// never fills. On a first scan it is what keeps peak memory flat instead of -/// proportional to the library. -const READ_WINDOW: usize = 4096; +/// proportional to the library. Matched to [`BATCH_SIZE`] so one window is one +/// write. +const READ_WINDOW: usize = BATCH_SIZE; /// What a scan needs to know about a file it may already have indexed. /// @@ -116,17 +117,14 @@ impl MediaScanner { Ok(()) } - /// Write out whichever batch has reached `threshold`. - /// - /// Pass a threshold of 1 to flush whatever is left. + /// Write out whatever a window produced. async fn flush_batches( &self, files_to_insert: &mut Vec, files_to_update: &mut Vec, result: &mut ScanResult, - threshold: usize, ) -> Result<()> { - if files_to_insert.len() >= threshold && !files_to_insert.is_empty() { + if !files_to_insert.is_empty() { info!("Inserting batch of {} files", files_to_insert.len()); self.database_manager .bulk_store_canonical_media_files(files_to_insert) @@ -135,7 +133,7 @@ impl MediaScanner { files_to_insert.clear(); } - if files_to_update.len() >= threshold && !files_to_update.is_empty() { + if !files_to_update.is_empty() { info!("Updating batch of {} files", files_to_update.len()); self.database_manager .bulk_update_canonical_media_files(files_to_update) @@ -588,7 +586,6 @@ impl MediaScanner { &mut files_to_insert, &mut files_to_update, &mut result, - BATCH_SIZE, ) .await?; } @@ -611,7 +608,7 @@ impl MediaScanner { ) .await?; } - self.flush_batches(&mut files_to_insert, &mut files_to_update, &mut result, 1) + self.flush_batches(&mut files_to_insert, &mut files_to_update, &mut result) .await?; // The walker's own findings — which prefixes it could not read — only From 380a2a376902e58033f31debca26f4e29b8c664b Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 18:28:30 +0300 Subject: [PATCH 21/23] style: satisfy clippy on the new scan and watcher code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three lints on code added by this branch: `ScanResult` lost its `Default` when the derive was dropped, the scan's progress check spelled out a modulo that `is_multiple_of` says better, and a cache test compared a length to zero. `Default` is written out rather than derived on purpose. Every count starts at zero, but `complete` starts *true* and is cleared by anything the scan could not enumerate — a derived one would start it false and mark every scan incomplete. --- crates/vuio-core/src/media/result.rs | 9 +++++++++ crates/vuio-core/src/media/scanner.rs | 2 +- crates/vuio-core/src/watcher/file_ids.rs | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/vuio-core/src/media/result.rs b/crates/vuio-core/src/media/result.rs index 1b5fe51..8aa7600 100644 --- a/crates/vuio-core/src/media/result.rs +++ b/crates/vuio-core/src/media/result.rs @@ -42,6 +42,15 @@ pub struct ScanResult { pub complete: bool, } +impl Default for ScanResult { + /// Not derived: every count starts at zero, but `complete` starts *true* and + /// is cleared by anything that could not be enumerated. Deriving would start + /// it false and quietly mark every scan incomplete. + fn default() -> Self { + Self::new() + } +} + impl ScanResult { /// Create a new empty scan result pub fn new() -> Self { diff --git a/crates/vuio-core/src/media/scanner.rs b/crates/vuio-core/src/media/scanner.rs index 0b376ec..7a8d0a3 100644 --- a/crates/vuio-core/src/media/scanner.rs +++ b/crates/vuio-core/src/media/scanner.rs @@ -591,7 +591,7 @@ impl MediaScanner { } } - if processed % 10_000 == 0 { + if processed.is_multiple_of(10_000) { debug!("Examined {processed} files"); } } diff --git a/crates/vuio-core/src/watcher/file_ids.rs b/crates/vuio-core/src/watcher/file_ids.rs index dba7bce..c3bf75d 100644 --- a/crates/vuio-core/src/watcher/file_ids.rs +++ b/crates/vuio-core/src/watcher/file_ids.rs @@ -236,7 +236,7 @@ mod tests { for index in 0..6 { cache.add_path(&nested.join(format!("file_{index}.mp3")), RecursiveMode::NonRecursive); } - assert!(cache.len() > 0); + assert!(!cache.is_empty()); cache.remove_path(&nested); assert_eq!(cache.len(), 0); From 9f800809d5caa6a7ca9337a53cca5db0438315f5 Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 18:43:23 +0300 Subject: [PATCH 22/23] fix(config): watch the config file itself, not only its directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Narrowing the config watch from a recursive parent to a non-recursive one broke config reload on FreeBSD: three tests that had passed for years started failing, because an edit to the file was never noticed. kqueue reports per file descriptor. A watch on a directory says the directory changed and names the directory, so rewriting a file *inside* it is invisible unless that file's own descriptor is registered — and the handler identifies its config by comparing paths for equality, so a directory-named event never matches. The descriptor used to be registered by accident: notify's kqueue backend walks the tree for a recursive watch and registers every entry it finds, which is the very walk that made a config living beside a media library index the library. So watch both, neither recursively. The file, so a write into it is seen on every backend. The parent, because saving a config usually replaces it — write a temporary file, rename it over the old one — and that destroys the inode the file watch was holding, which only the directory can see. Verified by building against `notify/macos_kqueue`, which swaps the same backend in locally: the three tests fail without the file watch and pass with it. --- crates/vuio-core/src/config/mod.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/vuio-core/src/config/mod.rs b/crates/vuio-core/src/config/mod.rs index 91f51bb..2d49567 100644 --- a/crates/vuio-core/src/config/mod.rs +++ b/crates/vuio-core/src/config/mod.rs @@ -217,12 +217,30 @@ impl ConfigManager { notify::Config::default(), )?; - // Watch the config file's parent directory, one level only. The config file - // is a direct child of it, and watching recursively would follow whatever - // else happens to live there — a media tree, or a whole home directory. + // Two watches, both one level deep. + // + // The parent, because saving a config usually replaces it — write a + // temporary file, rename it over the old one — which destroys the inode a + // watch on the file was holding. Only the directory sees that. + // + // The file itself, because a directory watch is not enough to see a write + // *into* an existing file on every backend. kqueue reports per file + // descriptor: a watch on a directory reports that the directory changed and + // names the directory, so an edit that rewrites the config in place is + // invisible unless its own descriptor is registered. It used to be + // registered by accident, because the parent was watched recursively and + // the kqueue backend walks the tree registering every entry it finds — the + // same walk that made a config living beside a media library index the + // library. + // + // Neither watch is recursive. That is the point: whatever else lives in + // that directory is not our business. if let Some(parent) = config_path.parent() { debouncer.watch(parent, notify::RecursiveMode::NonRecursive)?; } + if config_path.is_file() { + debouncer.watch(&config_path, notify::RecursiveMode::NonRecursive)?; + } // Spawn task to handle debounced file events background_tasks.spawn(async move { From 7a1f88a43754a98cd89ef6f9d0a39d8e4fa32cdb Mon Sep 17 00:00:00 2001 From: vyrti Date: Fri, 14 Aug 2026 18:47:30 +0300 Subject: [PATCH 23/23] u --- crates/vuio-cast/Cargo.toml | 2 +- crates/vuio-core/Cargo.toml | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/vuio-cast/Cargo.toml b/crates/vuio-cast/Cargo.toml index 48747f8..04bfc48 100644 --- a/crates/vuio-cast/Cargo.toml +++ b/crates/vuio-cast/Cargo.toml @@ -32,7 +32,7 @@ tokio = { version = "1", features = ["net", "rt", "sync", "time", "macros", "io- tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } tokio-util = { version = "0.7", features = ["rt", "io"] } tracing = "0.1" -mdns-sd = { version = "0.20", optional = true } +mdns-sd = { version = "0.21", optional = true } axum = { version = "0.8", optional = true, default-features = false, features = ["tokio", "http1"] } [dev-dependencies] diff --git a/crates/vuio-core/Cargo.toml b/crates/vuio-core/Cargo.toml index 7f9f04e..1f6c97c 100644 --- a/crates/vuio-core/Cargo.toml +++ b/crates/vuio-core/Cargo.toml @@ -115,17 +115,17 @@ serde_json = "1.0" dirs = "6.0" hostname = "0.4" ipnet = "2.12" -http = "1.3" +http = "1.5" http-body-util = "0.1" -hyper = { version = "1.8", features = ["client", "http1"] } +hyper = { version = "1.11", features = ["client", "http1"] } hyper-util = { version = "0.1", features = ["client-legacy", "http1", "tokio"] } -bytes = "1.11" +bytes = "1.12" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } quick-xml = "0.41" percent-encoding = "2.3" sysinfo = { version = "0.39", default-features = false, features = ["system", "disk", "network"], optional = true } socket2 = { version = "0.6", features = ["all"] } -mdns-sd = { version = "0.20", default-features = false, features = ["async"] } +mdns-sd = { version = "0.21", default-features = false, features = ["async"] } vuio-cast = { path = "../vuio-cast", version = "0.0.4", default-features = false, optional = true } vuio-web = { path = "../vuio-web", version = "0.0.44", optional = true } jwalk = "0.9" @@ -133,16 +133,16 @@ tokio-stream = "0.1" hap-crypto = { version = "1.4", optional = true } hap-transport = { version = "1.3", optional = true } hap-tlv8 = { version = "1.0", optional = true } -hkdf = { version = "0.12", optional = true } -sha2 = { version = "0.10", optional = true } -num-bigint = { version = "0.4", optional = true } +hkdf = { version = "0.13", optional = true } +sha2 = { version = "0.11", optional = true } +num-bigint = { version = "0.5", optional = true } # `all` is `all-codecs` + `all-formats` + `all-meta`: every container symphonia # can demux and every tag dialect it can read, including APEv1/APEv2. Listed # rather than inherited from upstream's `default` so what ships stays auditable # and cannot drift when that default changes. SIMD is added by `casting` above. symphonia = { version = "0.6", default-features = false, features = ["all"], optional = true } -getrandom = { version = "0.3", optional = true } -plist = { version = "1.8", optional = true } +getrandom = { version = "0.4", optional = true } +plist = { version = "1.10", optional = true } hex = { version = "0.4", optional = true } rusqlite = { version = "0.40.2", features = ["bundled", "collation"] } # Only the `mediainfo` feature uses this, and only to talk to public metadata APIs