diff --git a/Cargo.toml b/Cargo.toml index 288078c..e4cada3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,25 @@ [workspace] -members = ["crates/vuio-cast", "crates/vuio-core", "crates/vuio-cli", "crates/vuio-web"] +members = [ + "crates/vuio-cast", + "crates/vuio-core", + "crates/vuio-cli", + "crates/vuio-web", + # Development tool, `publish = false`. Kept out of `vuio-cli` so the crate that + # ships the binary does not carry its dependencies. + "crates/vuio-bench", +] + +# `vuio-bench` is deliberately not a default member. Cargo unifies features across +# everything it builds in one go, so leaving it in would mean a plain +# `cargo build` — which is what the release pipeline runs — compiling `vuio-core` +# with `unstable-internals` on, for the benefit of a tool that never ships. +# Build it explicitly: `cargo run -p vuio-bench`. +default-members = [ + "crates/vuio-cast", + "crates/vuio-core", + "crates/vuio-cli", + "crates/vuio-web", +] resolver = "2" [profile.release] diff --git a/README.md b/README.md index 896dd9e..666a2e9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # VuIO Media Server A cross-platform media server written in Rust. Streams video, audio, and images to DLNA, Chromecast/Google TV, and compatible AirPlay video receivers. -Less than 18Mb of RAM needed +25Mb of RAM used for 5000 objects library Built with Tokio, Axum, and SQLite for high performance and reliability. diff --git a/config.example.toml b/config.example.toml index 7c15547..d1d4a22 100644 --- a/config.example.toml +++ b/config.example.toml @@ -57,11 +57,20 @@ recursive = true # extensions = ["mp3", "flac", "wav"] [database] +# Reclaim free space in the index at startup and shutdown; rewrites the whole file. vacuum_on_startup = false backup_enabled = false # Omit to use the platform-appropriate default location. path = "./config/media.db" -# cache_mb = 128 # Megabytes of index kept cached +# Megabytes of index cached, as a budget *per connection* — one writer plus two +# to four readers. Only a connection that runs a large query fills its share, so +# in practice resident memory grows by about this much, not a multiple of it. +# Folder browsing is served from indexes and does not care. Full-text search does: +# on a very large library it stays slow until the whole search index fits, then +# roughly halves. Measured at 500,000 files that threshold was near 192, and +# anything below it cost memory without making search faster — which is why the +# default is small. Raise it past the threshold, or not at all. +# cache_mb = 8 # The modern browser interface, served on a second port beside the built-in # dashboard. It is a second front end, not a second server: the same API, the diff --git a/crates/vuio-bench/Cargo.toml b/crates/vuio-bench/Cargo.toml new file mode 100644 index 0000000..b65eff9 --- /dev/null +++ b/crates/vuio-bench/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "vuio-bench" +version = "0.0.44" +edition = "2021" +authors = ["vyrti"] +description = "Generates large VuIO libraries for performance work. Not published." +license = "MIT OR Apache-2.0" +repository = "https://github.com/vuiodev/vuio" +# A development tool, never a release artifact. It lives in its own crate so that +# the dependencies it needs to write rows quickly — a direct SQLite handle, and +# vuio-core with its internals open — stay off `vuio-cli`, which ships the +# `vuio` binary. +publish = false + +[[bin]] +name = "vuio-bench" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0" +clap = { version = "4.6", features = ["derive"] } +rusqlite = { version = "0.40.2", features = ["bundled", "collation"] } +tokio = { version = "1.53", features = ["rt-multi-thread", "macros"] } +# `unstable-internals` is what core forbids a *dependent* crate from enabling, +# because it opens every internal module and carries no stability promise. This +# crate is `publish = false` and exists only to drive the database from the +# inside, which is the same category as core's own dev-dependency on itself. +vuio-core = { path = "../vuio-core", version = "0.0.44", features = ["unstable-internals"] } diff --git a/crates/vuio-bench/src/main.rs b/crates/vuio-bench/src/main.rs new file mode 100644 index 0000000..925e999 --- /dev/null +++ b/crates/vuio-bench/src/main.rs @@ -0,0 +1,406 @@ +//! Generates a large VuIO library, for performance work. +//! +//! The point of this tool is a database that is the right *shape*, not merely the +//! right size. An earlier version wrote its own copy of the schema and its own +//! directory tree, and the result measured very little: the full-text tables were +//! never populated, the `directories` rows did not correspond to any file's +//! `parent_path`, and the paths on disk did not match the paths in the rows. A +//! server pointed at that database browsed an empty tree and searched an empty +//! index, so the numbers described a system nobody runs. +//! +//! So the schema comes from `vuio-core` itself, and every piece of derived state — +//! the directory tree, the recursive per-family counters, both FTS indexes — is +//! built by the same `rebuild_derived_indexes` the server uses to repair itself. +//! What is left here is only the part that has to be fast: the rows. + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use anyhow::{Context, Result}; +use clap::Parser; +use vuio_core::config::generator::ConfigGenerator; +use vuio_core::config::{AppConfig, MonitoredDirectoryConfig, ValidationMode}; +use vuio_core::database::sqlite::SqliteDatabase; +use vuio_core::database::{DatabaseManager, HealthRepository}; + +/// How many rows go in one transaction. Large enough that commit overhead +/// disappears, small enough that the rollback journal stays bounded. +const CHUNK: usize = 50_000; + +const GENRES: [&str; 8] = [ + "Rock", + "Pop", + "Jazz", + "Classical", + "Electronic", + "Metal", + "Hip Hop", + "Ambient", +]; + +/// Shape of the generated tree. Real libraries are wide and shallow rather than +/// uniformly deep, and the directory counters are maintained per ancestor, so +/// these two numbers decide how much work a scan does per file. +const TRACKS_PER_ALBUM: usize = 12; +const ALBUMS_PER_ARTIST: usize = 10; + +#[derive(Parser, Debug)] +#[command(about = "Generate a large VuIO library for performance testing")] +struct Args { + /// How many media rows to index + #[arg(long, default_value_t = 100_000)] + objects: usize, + + /// Where to put the library. The database goes in `/.vuio/media.db`. + #[arg(long, default_value = "./bench-library")] + out: PathBuf, + + /// How many stub files to write to disk. Defaults to one per row. + /// + /// Every row needs a file, or the first scan deletes it: a library where + /// most rows are missing measures a deletion storm, not steady state. Lower + /// this only when the deletion path is deliberately what you want to time. + #[arg(long)] + files: Option, +} + +/// A real, silent MP3 — the same fixture `generate_test_media` uses. +/// +/// Deliberately not a handful of bytes that merely look like a header. The tag +/// reader has to actually parse these, or a cold scan measures the database +/// write path and nothing else, and the cost that dominates a real library — +/// opening and probing every file — never appears in the numbers. +const STUB_MP3_BASE64: &str = "SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU2LjM2LjEwMAAAAAAAAAAAAAAA//OEAAAAAAAAAAAAAAAAAAAAAAAASW5mbwAAAA8AAAAEAAABIADAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV6urq6urq6urq6urq6urq6urq6urq6urq6v////////////////////////////////8AAAAATGF2YzU2LjQxAAAAAAAAAAAAAAAAJAAAAAAAAAAAASDs90hvAAAAAAAAAAAAAAAAAAAA//MUZAAAAAGkAAAAAAAAA0gAAAAATEFN//MUZAMAAAGkAAAAAAAAA0gAAAAARTMu//MUZAYAAAGkAAAAAAAAA0gAAAAAOTku//MUZAkAAAGkAAAAAAAAA0gAAAAANVVV"; + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + let stub = decode_base64(STUB_MP3_BASE64).context("the embedded MP3 fixture is malformed")?; + let files_to_write = args.files.unwrap_or(args.objects).min(args.objects); + + let root = args.out.clone(); + std::fs::create_dir_all(&root) + .with_context(|| format!("could not create {}", root.display()))?; + // Canonical, because that is the form the scanner stores and compares against. + let root = root + .canonicalize() + .with_context(|| format!("could not canonicalize {}", root.display()))?; + let db_path = root.join(".vuio").join("media.db"); + + println!("VuIO benchmark library"); + println!(" objects: {}", args.objects); + println!(" files: {files_to_write} written to disk"); + println!(" library: {}", root.display()); + println!(" database: {}", db_path.display()); + println!(); + + // 1. Stub files, at the paths the rows will carry. Their modification times + // come back with them: a row whose `modified_secs` does not match its + // file reads as changed, and the first scan would re-read and rewrite the + // entire library from stubs that carry no tags. + let started = Instant::now(); + let mtimes = write_stub_files(&root, files_to_write, &stub)?; + println!( + " wrote {files_to_write} files in {:.1}s", + started.elapsed().as_secs_f64() + ); + + // 2. The real schema, from the real code. Creating the database through + // vuio-core is what guarantees this tool cannot drift from `schema.rs`. + if db_path.exists() { + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", db_path.display())); + } + } + std::fs::create_dir_all(db_path.parent().expect("the database has a parent"))?; + { + let database = SqliteDatabase::new(db_path.clone()) + .await + .context("could not create the database")?; + database + .initialize() + .await + .context("could not initialize the schema")?; + } + println!(" schema created by vuio-core"); + + // 3. The rows, as fast as SQLite will take them. + let started = Instant::now(); + insert_rows(&db_path, &root, args.objects, &mtimes, stub.len() as i64)?; + let inserted = started.elapsed(); + println!( + " inserted {} rows in {:.1}s ({:.0} rows/s)", + args.objects, + inserted.as_secs_f64(), + args.objects as f64 / inserted.as_secs_f64().max(0.001) + ); + + // 4. Every piece of derived state, built by the server's own repair path: + // the directory tree, the recursive counters, and both FTS indexes. + let started = Instant::now(); + { + let database = SqliteDatabase::new(db_path.clone()) + .await + .context("could not reopen the database")?; + let health = database + .rebuild_derived_indexes() + .await + .context("could not build the derived indexes")?; + anyhow::ensure!( + health.is_healthy, + "derived index rebuild reported problems: {:?}", + health.issues + ); + } + println!( + " built directory tree, counters and search index in {:.1}s", + started.elapsed().as_secs_f64() + ); + + // 5. A config that points the server at this library *and* this database. + // Without it the server would use its own default database path, scan the + // tree from scratch, and never open the file we just built — which is + // a silent way to measure nothing. + let config_path = write_config(&root, &db_path)?; + + report(&db_path)?; + + println!(); + println!("Run the server against it:"); + println!(" ./target/debug/vuio --config {}", config_path.display()); + Ok(()) +} + +/// Write a config naming this library and this database. +fn write_config(root: &Path, db_path: &Path) -> Result { + let mut config = AppConfig::default_for_platform(); + config.server.port = 18080; + config.media.directories = vec![MonitoredDirectoryConfig { + path: root.to_string_lossy().into_owned(), + recursive: true, + case_sensitive: None, + extensions: None, + exclude_patterns: None, + validation_mode: ValidationMode::Warn, + }]; + config.database.path = Some(db_path.to_string_lossy().into_owned()); + // The web interface would bind a second port and add noise to the profile. + config.web_ui.enabled = false; + + let rendered = ConfigGenerator::new() + .context("could not build the config generator")? + .generate_config(&config) + .context("could not render the config")?; + let config_path = root.join("vuio.toml"); + std::fs::write(&config_path, rendered) + .with_context(|| format!("could not write {}", config_path.display()))?; + Ok(config_path) +} + +/// The path a row with this index carries, relative to the library root. +/// +/// One function so the rows and the files on disk cannot disagree — which is +/// exactly how the previous generator ended up indexing paths that did not exist. +fn relative_path(index: usize) -> (String, String) { + let album = index / TRACKS_PER_ALBUM; + let artist = album / ALBUMS_PER_ARTIST; + let track = index % TRACKS_PER_ALBUM + 1; + ( + format!("Artist_{artist:05}/Album_{album:06}"), + format!("{track:02} - Track_{index}.mp3"), + ) +} + +/// Write the stubs and return each one's modification time, in row order. +fn write_stub_files(root: &Path, count: usize, stub: &[u8]) -> Result> { + let mut mtimes = Vec::with_capacity(count); + let mut last_dir = String::new(); + for index in 0..count { + let (directory, filename) = relative_path(index); + if directory != last_dir { + std::fs::create_dir_all(root.join(&directory))?; + last_dir = directory.clone(); + } + let path = root.join(&directory).join(&filename); + std::fs::write(&path, stub)?; + mtimes.push( + std::fs::metadata(&path)? + .modified()? + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs() as i64) + .unwrap_or(0), + ); + } + Ok(mtimes) +} + +fn insert_rows( + db_path: &Path, + root: &Path, + total: usize, + mtimes: &[i64], + file_size: i64, +) -> Result<()> { + let mut connection = rusqlite::Connection::open(db_path)?; + // Two of the browse indexes are declared `COLLATE natural_order`, so this + // connection has to know the collation before it can maintain them. + vuio_core::database::sqlite::register_collations(&connection)?; + // Bulk-load settings, reverted below. Safe here because the file is + // disposable: if this crashes, you regenerate it. + connection.execute_batch( + "PRAGMA synchronous = OFF; + PRAGMA journal_mode = MEMORY; + PRAGMA cache_size = -131072; + PRAGMA temp_store = MEMORY;", + )?; + + let root = root.to_string_lossy().into_owned(); + let mut done = 0usize; + while done < total { + let end = (done + CHUNK).min(total); + let transaction = connection.transaction()?; + { + let mut statement = transaction.prepare_cached(INSERT_MEDIA)?; + for index in done..end { + let (directory, filename) = relative_path(index); + let parent = format!("{root}/{directory}"); + let path = format!("{parent}/{filename}"); + let album_id = index / TRACKS_PER_ALBUM; + let artist_id = album_id / ALBUMS_PER_ARTIST; + let track = (index % TRACKS_PER_ALBUM + 1) as i64; + let year = 1980 + (index % 45) as i64; + // The file's own mtime where one exists, so the scanner sees an + // unchanged record; a fixed past value otherwise. + let stamp = mtimes.get(index).copied().unwrap_or(1_700_000_000); + statement.execute(rusqlite::params![ + (index + 1) as i64, + path, + parent, + filename, + file_size, + stamp, + "audio/mpeg", + "audio", + 215.0 + (index % 60) as f64, + format!("Track {index}"), + format!("Artist {artist_id}"), + format!("Album {album_id}"), + GENRES[index % GENRES.len()], + track, + year, + format!("Artist {artist_id}"), + 1i64, + 1i64, + TRACKS_PER_ALBUM as i64, + format!("Composer {}", artist_id % 97), + Option::::None, + 120i64, + 0i64, + Option::::None, + Option::::None, + Option::::None, + format!("{year}-01-01"), + Option::::None, + Option::::None, + Option::::None, + "mp3", + 44_100i64, + 2i64, + 16i64, + 320_000i64, + 1i64, + 0i64, + stamp, + stamp, + ])?; + } + } + transaction.commit()?; + done = end; + } + + // Back to the settings the server runs with, so what gets measured next is + // the real thing. + connection.execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + ANALYZE;", + )?; + Ok(()) +} + +/// Columns in the order `bind_media_file` uses, so the parameter list here reads +/// the same as the one in `media_repo/bulk.rs`. +const INSERT_MEDIA: &str = "\ +INSERT INTO media_files ( + id, path, parent_path, filename, size, modified_secs, mime_type, mime_family, + duration_secs, title, artist, album, genre, track_number, year, album_artist, + disc_number, disc_total, track_total, composer, comment, bpm, compilation, + sort_title, sort_artist, sort_album, release_date, + musicbrainz_track_id, musicbrainz_album_id, musicbrainz_artist_id, + codec, sample_rate, channels, bits_per_sample, bit_rate, tags_version, + subtitle_available, created_at_secs, updated_at_secs +) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, + ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29, ?30, ?31, ?32, + ?33, ?34, ?35, ?36, ?37, ?38, ?39)"; + +/// Prove the database is usable, not merely large. +/// +/// Every count here was zero or wrong in the previous generator's output, which +/// is why they are printed rather than assumed. +fn report(db_path: &Path) -> Result<()> { + let connection = rusqlite::Connection::open(db_path)?; + let count = |sql: &str| -> Result { Ok(connection.query_row(sql, [], |row| row.get(0))?) }; + + let files = count("SELECT COUNT(*) FROM media_files")?; + let directories = count("SELECT COUNT(*) FROM directories")?; + let counters = count("SELECT COUNT(*) FROM directory_mime_counts")?; + let searchable = count("SELECT COUNT(*) FROM media_fts")?; + let bytes = std::fs::metadata(db_path).map(|m| m.len()).unwrap_or(0); + + println!(); + println!(" media_files {files}"); + println!(" directories {directories}"); + println!(" directory_mime_counts {counters}"); + println!(" media_fts {searchable}"); + println!( + " database {:.1} MB", + bytes as f64 / 1_048_576.0 + ); + + anyhow::ensure!(directories > 0, "the directory tree is empty"); + anyhow::ensure!( + searchable == files, + "the search index covers {searchable} of {files} rows" + ); + Ok(()) +} + +/// Minimal base64 decoder, so the fixture can be embedded as text without a +/// dependency for one constant. +fn decode_base64(input: &str) -> Option> { + fn value(byte: u8) -> Option { + Some(match byte { + b'A'..=b'Z' => u32::from(byte - b'A'), + b'a'..=b'z' => u32::from(byte - b'a') + 26, + b'0'..=b'9' => u32::from(byte - b'0') + 52, + b'+' => 62, + b'/' => 63, + _ => return None, + }) + } + + let bytes: Vec = input.bytes().filter(|byte| *byte != b'=').collect(); + let mut out = Vec::with_capacity(bytes.len() * 3 / 4); + for chunk in bytes.chunks(4) { + let mut bits = 0u32; + for (index, byte) in chunk.iter().enumerate() { + bits |= value(*byte)? << (18 - 6 * index); + } + let produced = chunk.len() - 1; + for index in 0..produced { + out.push(((bits >> (16 - 8 * index)) & 0xff) as u8); + } + } + Some(out) +} diff --git a/crates/vuio-cast/Cargo.toml b/crates/vuio-cast/Cargo.toml index 48747f8..04bfc48 100644 --- a/crates/vuio-cast/Cargo.toml +++ b/crates/vuio-cast/Cargo.toml @@ -32,7 +32,7 @@ tokio = { version = "1", features = ["net", "rt", "sync", "time", "macros", "io- tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } tokio-util = { version = "0.7", features = ["rt", "io"] } tracing = "0.1" -mdns-sd = { version = "0.20", optional = true } +mdns-sd = { version = "0.21", optional = true } axum = { version = "0.8", optional = true, default-features = false, features = ["tokio", "http1"] } [dev-dependencies] diff --git a/crates/vuio-cli/Cargo.toml b/crates/vuio-cli/Cargo.toml index f1d12bf..7e96ebc 100644 --- a/crates/vuio-cli/Cargo.toml +++ b/crates/vuio-cli/Cargo.toml @@ -18,10 +18,6 @@ autobins = false name = "vuio" path = "src/main.rs" -[[bin]] -name = "benchmark_media" -path = "src/bin/benchmark_media.rs" - [[bin]] name = "generate_test_media" path = "src/bin/generate_test_media.rs" @@ -38,7 +34,6 @@ testdata = ["dep:audiotags"] anyhow = "1.0" audiotags = { version = "0.5", optional = true } clap = { version = "4.6", features = ["derive"] } -rusqlite = { version = "0.40.2", features = ["bundled", "collation"] } rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } reqwest = { version = "0.13", default-features = false, features = ["json", "rustls-no-provider"] } serde = { version = "1.0", features = ["derive"] } @@ -46,5 +41,5 @@ serde_json = "1.0" tokio = { version = "1.53", features = ["rt-multi-thread", "macros", "signal", "io-std", "io-util"] } tracing = "0.1" uuid = { version = "1.24", features = ["v4"] } -vuio-core = { path = "../vuio-core", version = "0.0.44", features = ["unstable-internals"] } +vuio-core = { path = "../vuio-core", version = "0.0.44" } diff --git a/crates/vuio-cli/src/bin/benchmark_media.rs b/crates/vuio-cli/src/bin/benchmark_media.rs deleted file mode 100644 index 7f0bc42..0000000 --- a/crates/vuio-cli/src/bin/benchmark_media.rs +++ /dev/null @@ -1,480 +0,0 @@ -use anyhow::{Context, Result}; -use std::cmp::Ordering; -use std::fs; -use std::path::PathBuf; -use std::time::Instant; - -fn natural_cmp(left: &str, right: &str) -> Ordering { - let left = left.to_lowercase(); - let right = right.to_lowercase(); - let mut left_chars = left.chars().peekable(); - let mut right_chars = right.chars().peekable(); - - loop { - match (left_chars.peek(), right_chars.peek()) { - (Some(a), Some(b)) if a.is_ascii_digit() && b.is_ascii_digit() => { - let left_number: String = std::iter::from_fn(|| { - left_chars.next_if(|character| character.is_ascii_digit()) - }) - .collect(); - let right_number: String = std::iter::from_fn(|| { - right_chars.next_if(|character| character.is_ascii_digit()) - }) - .collect(); - let order = left_number - .trim_start_matches('0') - .len() - .cmp(&right_number.trim_start_matches('0').len()) - .then_with(|| { - left_number - .trim_start_matches('0') - .cmp(right_number.trim_start_matches('0')) - }) - .then_with(|| left_number.len().cmp(&right_number.len())); - if order != Ordering::Equal { - return order; - } - } - (Some(_), Some(_)) => { - let order = left_chars.next().cmp(&right_chars.next()); - if order != Ordering::Equal { - return order; - } - } - _ => return left_chars.next().cmp(&right_chars.next()), - } - } -} - -const DDL_BASE: &str = r#" -CREATE TABLE IF NOT EXISTS media_files ( - id INTEGER PRIMARY KEY, - path TEXT NOT NULL, - parent_path TEXT NOT NULL, - filename TEXT NOT NULL, - size INTEGER NOT NULL, - modified_secs INTEGER NOT NULL, - mime_type TEXT NOT NULL, - mime_family TEXT NOT NULL, - duration_secs REAL, - title TEXT, - artist TEXT, - album TEXT, - genre TEXT, - track_number INTEGER, - year INTEGER, - album_artist TEXT, - disc_number INTEGER, - disc_total INTEGER, - track_total INTEGER, - composer TEXT, - comment TEXT, - bpm INTEGER, - compilation INTEGER, - sort_title TEXT, - sort_artist TEXT, - sort_album TEXT, - release_date TEXT, - musicbrainz_track_id TEXT, - musicbrainz_album_id TEXT, - musicbrainz_artist_id TEXT, - codec TEXT, - sample_rate INTEGER, - channels INTEGER, - bits_per_sample INTEGER, - bit_rate INTEGER, - tags_version INTEGER NOT NULL DEFAULT 0, - subtitle_available INTEGER NOT NULL DEFAULT 0, - created_at_secs INTEGER NOT NULL, - updated_at_secs INTEGER NOT NULL, - track_sort INTEGER GENERATED ALWAYS AS (COALESCE(track_number, 4294967296)) STORED, - disc_sort INTEGER GENERATED ALWAYS AS (COALESCE(disc_number, 1)) VIRTUAL -) STRICT; - -CREATE TABLE IF NOT EXISTS directories ( - path TEXT PRIMARY KEY, - parent_path TEXT NOT NULL, - name TEXT NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS directory_mime_counts ( - dir_path TEXT NOT NULL REFERENCES directories(path) ON DELETE CASCADE, - family TEXT NOT NULL, - count INTEGER NOT NULL, - PRIMARY KEY (dir_path, family) -) STRICT; - -CREATE TABLE IF NOT EXISTS playlists ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - description TEXT, - source_path TEXT, - created_at_secs INTEGER NOT NULL, - updated_at_secs INTEGER NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS playlist_entries ( - playlist_id INTEGER NOT NULL REFERENCES playlists(id) ON DELETE CASCADE, - media_file_id INTEGER NOT NULL REFERENCES media_files(id) ON DELETE CASCADE, - position INTEGER NOT NULL, - PRIMARY KEY (playlist_id, position) -) STRICT; - -CREATE TABLE IF NOT EXISTS root_availability ( - path TEXT PRIMARY KEY, - last_seen_secs INTEGER NOT NULL, - unavailable_since_secs INTEGER, - indexed_count INTEGER NOT NULL, - reason TEXT NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS secrets ( - key TEXT PRIMARY KEY, - value BLOB NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS media_tags ( - media_file_id INTEGER NOT NULL REFERENCES media_files(id) ON DELETE CASCADE, - key TEXT NOT NULL, - value TEXT NOT NULL, - PRIMARY KEY (media_file_id, key, value) -) STRICT; - -CREATE TABLE IF NOT EXISTS mediainfo ( - media_file_id INTEGER PRIMARY KEY REFERENCES media_files(id) ON DELETE CASCADE, - provider TEXT NOT NULL, - remote_id TEXT NOT NULL, - kind TEXT NOT NULL, - title TEXT, - original_title TEXT, - overview TEXT, - release_date TEXT, - year INTEGER, - rating REAL, - genres TEXT, - season INTEGER, - episode INTEGER, - artwork_key TEXT, - payload TEXT NOT NULL, - confidence INTEGER NOT NULL, - fetched_at INTEGER NOT NULL, - mediainfo_version INTEGER NOT NULL -) STRICT; -"#; - -const INDEXES_SQL: &str = r#" -CREATE UNIQUE INDEX IF NOT EXISTS idx_media_path ON media_files(path); -CREATE INDEX IF NOT EXISTS idx_media_dir_order - ON media_files(parent_path, disc_sort, track_sort, filename COLLATE natural_order); -CREATE INDEX IF NOT EXISTS idx_media_dir_family - ON media_files(parent_path, mime_family); -CREATE INDEX IF NOT EXISTS idx_media_album - ON media_files(album, disc_sort, track_sort, filename COLLATE natural_order); -CREATE INDEX IF NOT EXISTS idx_media_artist ON media_files(artist); -CREATE INDEX IF NOT EXISTS idx_media_genre ON media_files(genre); -CREATE INDEX IF NOT EXISTS idx_media_year ON media_files(year); -CREATE INDEX IF NOT EXISTS idx_media_album_artist ON media_files(album_artist); -CREATE INDEX IF NOT EXISTS idx_media_family ON media_files(mime_family); -CREATE INDEX IF NOT EXISTS idx_media_tags_version ON media_files(tags_version); - -CREATE INDEX IF NOT EXISTS idx_directories_parent - ON directories(parent_path, name COLLATE natural_order); -CREATE INDEX IF NOT EXISTS idx_playlists_source - ON playlists(source_path) WHERE source_path IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_playlist_entries_file - ON playlist_entries(media_file_id); -CREATE INDEX IF NOT EXISTS idx_media_tags_key - ON media_tags(key, value); -CREATE INDEX IF NOT EXISTS idx_mediainfo_confidence - ON mediainfo(confidence); -"#; - -fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - let total_objects: usize = args - .get(1) - .and_then(|s| s.parse().ok()) - .unwrap_or(10_000_000); - - let base_dir = std::env::current_dir()?; - let test_media_dir = base_dir.join("test-media"); - let config_dir = base_dir.join("target").join("release").join("config"); - let db_dir = config_dir.join("database"); - let db_path = db_dir.join("media.db"); - - println!("=== VuIO Benchmark Media Generator (Rust) ==="); - println!("Target Objects: {}", total_objects); - println!("Test Media Dir: {}", test_media_dir.display()); - println!("Database Path: {}", db_path.display()); - println!(); - - // 1. Create physical test media directory and sample files - println!( - "1. Creating physical test files in {}...", - test_media_dir.display() - ); - fs::create_dir_all(&test_media_dir)?; - - let silent_mp3 = b"ID3\x04\x00\x00\x00\x00\x00\x00\x00\xFF\xFB\x90\x44\x00\x00\x00\x00"; - let mut physical_files_created = 0; - for artist_idx in 0..10 { - let artist_dir = test_media_dir.join(format!("Artist_{:02}", artist_idx)); - for album_idx in 0..10 { - let album_dir = artist_dir.join(format!("Album_{:02}", album_idx)); - fs::create_dir_all(&album_dir)?; - for track_idx in 1..=10 { - let track_file = - album_dir.join(format!("{:02} - Track_{}.mp3", track_idx, track_idx)); - if !track_file.exists() { - fs::write(&track_file, silent_mp3)?; - physical_files_created += 1; - } - } - } - } - println!( - " Created {} physical files in test-media.\n", - physical_files_created - ); - - // 2. Prepare SQLite Database - println!( - "2. Initializing SQLite database schema at {}...", - db_path.display() - ); - fs::create_dir_all(&db_dir)?; - - // Remove existing database files - for ext in &["", "-wal", "-shm"] { - let p = PathBuf::from(format!("{}{}", db_path.display(), ext)); - if p.exists() { - let _ = fs::remove_file(p); - } - } - - let mut conn = rusqlite::Connection::open(&db_path) - .with_context(|| format!("Failed to open {}", db_path.display()))?; - - // Register natural_order collation - conn.create_collation("natural_order", |a: &str, b: &str| natural_cmp(a, b))?; - - // Pragmas for fast bulk load - conn.execute_batch( - "PRAGMA synchronous = OFF; - PRAGMA journal_mode = OFF; - PRAGMA page_size = 4096; - PRAGMA cache_size = -131072; - PRAGMA temp_store = MEMORY;", - )?; - - // Create DDL schema - conn.execute_batch(DDL_BASE)?; - conn.execute_batch("PRAGMA user_version = 4;")?; - - println!(" Schema and tables initialized.\n"); - - // 3. Pre-populate directories table - println!("3. Populating directory hierarchy in database..."); - let canonical_root = test_media_dir - .canonicalize() - .unwrap_or_else(|_| test_media_dir.clone()); - let root_str = canonical_root.to_string_lossy().to_string(); - let root_parent = canonical_root - .parent() - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_default(); - let root_name = canonical_root - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - - { - let tx = conn.transaction()?; - tx.execute( - "INSERT OR IGNORE INTO directories (path, parent_path, name) VALUES (?, ?, ?)", - rusqlite::params![&root_str, &root_parent, &root_name], - )?; - - let num_artists = 10_000.min(total_objects / 100).max(10); - { - let mut dir_stmt = tx.prepare_cached( - "INSERT OR IGNORE INTO directories (path, parent_path, name) VALUES (?, ?, ?)", - )?; - - for a in 0..num_artists { - let art_name = format!("Artist_{:04}", a); - let art_path = format!("{}/{}", root_str, art_name); - dir_stmt.execute(rusqlite::params![&art_path, &root_str, &art_name])?; - - for alb in 0..5 { - let alb_id = a * 5 + alb; - let alb_name = format!("Album_{:05}", alb_id); - let alb_path = format!("{}/{}", art_path, alb_name); - dir_stmt.execute(rusqlite::params![&alb_path, &art_path, &alb_name])?; - } - } - } - - tx.execute( - "INSERT OR REPLACE INTO directory_mime_counts (dir_path, family, count) VALUES (?, '*', ?)", - rusqlite::params![&root_str, total_objects as i64], - )?; - tx.execute( - "INSERT OR REPLACE INTO directory_mime_counts (dir_path, family, count) VALUES (?, 'audio', ?)", - rusqlite::params![&root_str, total_objects as i64], - )?; - - tx.commit()?; - } - println!(" Directory hierarchy populated.\n"); - - // 4. Bulk insert media files in transaction chunks - println!( - "4. Inserting {} media files in chunks of 50,000...", - total_objects - ); - let start_insert = Instant::now(); - let chunk_size = 50_000; - let num_artists = 10_000; - let num_albums = 100_000; - let genres = [ - "Rock", - "Pop", - "Jazz", - "Classical", - "Electronic", - "Metal", - "Hip Hop", - "Ambient", - ]; - - let insert_sql = " - INSERT INTO media_files ( - id, path, parent_path, filename, size, modified_secs, mime_type, mime_family, - duration_secs, title, artist, album, genre, track_number, year, album_artist, - disc_number, disc_total, track_total, composer, comment, bpm, compilation, - sort_title, sort_artist, sort_album, release_date, musicbrainz_track_id, - musicbrainz_album_id, musicbrainz_artist_id, codec, sample_rate, channels, - bits_per_sample, bit_rate, tags_version, subtitle_available, created_at_secs, updated_at_secs - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, - ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29, ?30, - ?31, ?32, ?33, ?34, ?35, ?36, ?37, ?38, ?39) - "; - - for chunk_start in (0..total_objects).step_by(chunk_size) { - let chunk_end = (chunk_start + chunk_size).min(total_objects); - let tx = conn.transaction()?; - { - let mut stmt = tx.prepare_cached(insert_sql)?; - for i in chunk_start..chunk_end { - let id = (i + 1) as i64; - let artist_id = i % num_artists; - let album_id = i % num_albums; - let track_num = ((i % 20) + 1) as i64; - let genre = genres[i % genres.len()]; - let parent = format!("{}/Artist_{:04}/Album_{:05}", root_str, artist_id, album_id); - let filename = format!("{:02} - Track_{}.mp3", track_num, i); - let path = format!("{}/{}", parent, filename); - let title = format!("Track {}", i); - let artist = format!("Artist {}", artist_id); - let album = format!("Album {}", album_id); - let year = (1980 + (i % 45)) as i64; - let release_date = format!("{}-01-01", year); - let size = (4_194_304 + (i % 1000) * 1024) as i64; - let duration = 215.0 + (i % 60) as f64; - let timestamp = 1700000000 + (i % 86400) as i64; - - stmt.execute(rusqlite::params![ - id, - path, - parent, - filename, - size, - timestamp, - "audio/mpeg", - "audio", - duration, - title, - artist, - album, - genre, - track_num, - year, - artist, - 1i64, - 1i64, - 20i64, - Option::::None, - Option::::None, - 120i64, - 0i64, - Option::::None, - Option::::None, - Option::::None, - release_date, - Option::::None, - Option::::None, - Option::::None, - "mp3", - 44100i64, - 2i64, - 16i64, - 320000i64, - 1i64, - 0i64, - timestamp, - timestamp, - ])?; - } - } - tx.commit()?; - - if chunk_end % 1_000_000 == 0 || chunk_end == total_objects { - let elapsed = start_insert.elapsed().as_secs_f64(); - let rate = chunk_end as f64 / elapsed; - println!( - " Inserted {:>10} / {} rows ({:>5.1}s, {:>8.0} rows/s)...", - chunk_end, total_objects, elapsed, rate - ); - } - } - - let insert_duration = start_insert.elapsed(); - println!( - " Inserted {} rows in {:.1}s.\n", - total_objects, - insert_duration.as_secs_f64() - ); - - // 5. Create B-Tree Indexes - println!("5. Creating B-Tree indexes on 10M records in SQLite..."); - let start_idx = Instant::now(); - conn.execute_batch(INDEXES_SQL)?; - println!( - " Indexes created in {:.1}s.\n", - start_idx.elapsed().as_secs_f64() - ); - - // 6. Finalize WAL and stats - println!("6. Finalizing SQLite database settings..."); - conn.execute_batch( - "PRAGMA journal_mode = WAL; - PRAGMA synchronous = NORMAL; - PRAGMA optimize;", - )?; - - drop(conn); - - let metadata = fs::metadata(&db_path)?; - let db_bytes = metadata.len(); - let db_mb = db_bytes as f64 / (1024.0 * 1024.0); - let db_gb = db_bytes as f64 / (1024.0 * 1024.0 * 1024.0); - - println!("============================================================"); - println!("BENCHMARK LIBRARY READY!"); - println!("Total Records: {}", total_objects); - println!("Database Size: {:.2} GB ({:.1} MB)", db_gb, db_mb); - println!("============================================================"); - - Ok(()) -} diff --git a/crates/vuio-core/Cargo.toml b/crates/vuio-core/Cargo.toml index 7f9f04e..1f6c97c 100644 --- a/crates/vuio-core/Cargo.toml +++ b/crates/vuio-core/Cargo.toml @@ -115,17 +115,17 @@ serde_json = "1.0" dirs = "6.0" hostname = "0.4" ipnet = "2.12" -http = "1.3" +http = "1.5" http-body-util = "0.1" -hyper = { version = "1.8", features = ["client", "http1"] } +hyper = { version = "1.11", features = ["client", "http1"] } hyper-util = { version = "0.1", features = ["client-legacy", "http1", "tokio"] } -bytes = "1.11" +bytes = "1.12" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } quick-xml = "0.41" percent-encoding = "2.3" sysinfo = { version = "0.39", default-features = false, features = ["system", "disk", "network"], optional = true } socket2 = { version = "0.6", features = ["all"] } -mdns-sd = { version = "0.20", default-features = false, features = ["async"] } +mdns-sd = { version = "0.21", default-features = false, features = ["async"] } vuio-cast = { path = "../vuio-cast", version = "0.0.4", default-features = false, optional = true } vuio-web = { path = "../vuio-web", version = "0.0.44", optional = true } jwalk = "0.9" @@ -133,16 +133,16 @@ tokio-stream = "0.1" hap-crypto = { version = "1.4", optional = true } hap-transport = { version = "1.3", optional = true } hap-tlv8 = { version = "1.0", optional = true } -hkdf = { version = "0.12", optional = true } -sha2 = { version = "0.10", optional = true } -num-bigint = { version = "0.4", optional = true } +hkdf = { version = "0.13", optional = true } +sha2 = { version = "0.11", optional = true } +num-bigint = { version = "0.5", optional = true } # `all` is `all-codecs` + `all-formats` + `all-meta`: every container symphonia # can demux and every tag dialect it can read, including APEv1/APEv2. Listed # rather than inherited from upstream's `default` so what ships stays auditable # and cannot drift when that default changes. SIMD is added by `casting` above. symphonia = { version = "0.6", default-features = false, features = ["all"], optional = true } -getrandom = { version = "0.3", optional = true } -plist = { version = "1.8", optional = true } +getrandom = { version = "0.4", optional = true } +plist = { version = "1.10", optional = true } hex = { version = "0.4", optional = true } rusqlite = { version = "0.40.2", features = ["bundled", "collation"] } # Only the `mediainfo` feature uses this, and only to talk to public metadata APIs diff --git a/crates/vuio-core/src/config/generator.rs b/crates/vuio-core/src/config/generator.rs index cde13f1..aab9240 100644 --- a/crates/vuio-core/src/config/generator.rs +++ b/crates/vuio-core/src/config/generator.rs @@ -112,6 +112,8 @@ impl ConfigGenerator { media_table["scan_playlists"] = value(config.media.scan_playlists); media_table["unavailable_root_grace_hours"] = value(config.media.unavailable_root_grace_hours as i64); + media_table["full_rescan_interval_hours"] = + value(config.media.full_rescan_interval_hours as i64); // Update supported extensions array let mut extensions_array = Array::new(); @@ -489,6 +491,7 @@ mod tests { autoplay_enabled: false, scan_playlists: false, unavailable_root_grace_hours: 168, + full_rescan_interval_hours: 24, supported_extensions: vec!["mp4".to_string(), "avi".to_string()], }, database: DatabaseConfig { @@ -601,6 +604,7 @@ mod tests { autoplay_enabled: true, scan_playlists: true, unavailable_root_grace_hours: 168, + full_rescan_interval_hours: 24, supported_extensions: vec!["mp4".to_string()], }, database: DatabaseConfig { diff --git a/crates/vuio-core/src/config/loading.rs b/crates/vuio-core/src/config/loading.rs index 59a888a..3eb7075 100644 --- a/crates/vuio-core/src/config/loading.rs +++ b/crates/vuio-core/src/config/loading.rs @@ -105,6 +105,10 @@ impl AppConfig { .ok() .and_then(|value| value.parse().ok()) .unwrap_or_else(default_unavailable_root_grace_hours), + full_rescan_interval_hours: std::env::var("VUIO_FULL_RESCAN_INTERVAL_HOURS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or_else(default_full_rescan_interval_hours), supported_extensions: vec![ "mp4".to_string(), "mkv".to_string(), @@ -370,6 +374,7 @@ impl AppConfig { autoplay_enabled: true, scan_playlists: true, unavailable_root_grace_hours: default_unavailable_root_grace_hours(), + full_rescan_interval_hours: default_full_rescan_interval_hours(), supported_extensions: platform_config.get_default_media_extensions(), }, database: DatabaseConfig { diff --git a/crates/vuio-core/src/config/mod.rs b/crates/vuio-core/src/config/mod.rs index e837c97..2d49567 100644 --- a/crates/vuio-core/src/config/mod.rs +++ b/crates/vuio-core/src/config/mod.rs @@ -13,7 +13,8 @@ mod model; pub mod validation; use model::{ - default_cache_mb, default_mediainfo_providers, default_mediainfo_timeout_seconds, + default_cache_mb, default_full_rescan_interval_hours, default_mediainfo_providers, + default_mediainfo_timeout_seconds, default_min_confidence, default_session_ttl_hours, default_unavailable_root_grace_hours, default_web_ui_port, }; @@ -78,10 +79,7 @@ pub struct ConfigManager { change_sender: broadcast::Sender, /// Held so the watcher outlives this manager; dropping it stops reloads. debouncer: Option< - notify_debouncer_full::Debouncer< - notify::RecommendedWatcher, - notify_debouncer_full::FileIdMap, - >, + notify_debouncer_full::Debouncer, >, } @@ -196,30 +194,52 @@ impl ConfigManager { cancellation: tokio_util::sync::CancellationToken, background_tasks: tokio_util::task::TaskTracker, ) -> Result< - notify_debouncer_full::Debouncer< - notify::RecommendedWatcher, - notify_debouncer_full::FileIdMap, - >, + notify_debouncer_full::Debouncer, > { - use notify_debouncer_full::{new_debouncer_opt, DebounceEventResult, Debouncer, FileIdMap}; + use notify_debouncer_full::{new_debouncer_opt, DebounceEventResult, Debouncer, NoCache}; use tokio::sync::mpsc; let (tx, mut rx) = mpsc::channel(100); - // Create debounced watcher with 500ms debounce duration - let mut debouncer: Debouncer = new_debouncer_opt( + // Create debounced watcher with 500ms debounce duration. + // + // `NoCache`, not the default file-id cache: that one stats every path under + // every watched root to keep rename ids, and the handler below cares about + // exactly one path compared by equality. Renames are never stitched here, so + // the ids would be built and never read. + let mut debouncer: Debouncer = new_debouncer_opt( Duration::from_millis(500), None, move |result: DebounceEventResult| { let _ = tx.try_send(result); }, - FileIdMap::new(), + NoCache::new(), notify::Config::default(), )?; - // Watch the config file's parent directory + // Two watches, both one level deep. + // + // The parent, because saving a config usually replaces it — write a + // temporary file, rename it over the old one — which destroys the inode a + // watch on the file was holding. Only the directory sees that. + // + // The file itself, because a directory watch is not enough to see a write + // *into* an existing file on every backend. kqueue reports per file + // descriptor: a watch on a directory reports that the directory changed and + // names the directory, so an edit that rewrites the config in place is + // invisible unless its own descriptor is registered. It used to be + // registered by accident, because the parent was watched recursively and + // the kqueue backend walks the tree registering every entry it finds — the + // same walk that made a config living beside a media library index the + // library. + // + // Neither watch is recursive. That is the point: whatever else lives in + // that directory is not our business. if let Some(parent) = config_path.parent() { - debouncer.watch(parent, notify::RecursiveMode::Recursive)?; + debouncer.watch(parent, notify::RecursiveMode::NonRecursive)?; + } + if config_path.is_file() { + debouncer.watch(&config_path, notify::RecursiveMode::NonRecursive)?; } // Spawn task to handle debounced file events diff --git a/crates/vuio-core/src/config/model.rs b/crates/vuio-core/src/config/model.rs index 537ce66..d1677fe 100644 --- a/crates/vuio-core/src/config/model.rs +++ b/crates/vuio-core/src/config/model.rs @@ -24,12 +24,39 @@ pub(super) fn default_scan_playlists() -> bool { true } +/// How often to sweep every root regardless of what the watcher reported. +/// +/// The watcher is authoritative almost all of the time; this exists for the +/// cases where it is not — a network filesystem that drops events, or a backend +/// queue that overflowed. Daily, because the cost is a full re-walk and the +/// thing it guards against is rare. +pub(super) fn default_full_rescan_interval_hours() -> u64 { + 24 +} + pub(super) fn default_unavailable_root_grace_hours() -> u64 { 168 } +/// Page cache per connection, in mebibytes. +/// +/// The budget is per connection — one writer and two to four readers — but a +/// connection only allocates pages it actually reads, and on a normal workload +/// one reader does the heavy queries. Measured on a 500,000-file library, going +/// from 8 to 128 moved resident memory by about 160 MB, not by five times the +/// difference. +/// +/// Folder browsing is index-served and flat across every setting. Full-text +/// search is the one thing that responds, and as a step rather than a curve: it +/// stays slow until the search index fits, then roughly halves. At 500,000 files +/// that step fell between 160 and 192. +/// +/// So the default is small. A larger one bought nothing measurable below the +/// step — at 500,000 files, 128 cost 170 MB more than 8 and left search exactly +/// as slow — and a server that wants the faster search has to be raised past the +/// step, not merely raised. pub(super) fn default_cache_mb() -> usize { - 128 + 8 } pub(super) fn default_true() -> bool { @@ -323,6 +350,10 @@ pub struct MediaConfig { pub scan_playlists: bool, #[serde(default = "default_unavailable_root_grace_hours")] pub unavailable_root_grace_hours: u64, + /// Hours between full sweeps of every root. `0` disables them, leaving + /// discovery entirely to the watcher. + #[serde(default = "default_full_rescan_interval_hours")] + pub full_rescan_interval_hours: u64, pub supported_extensions: Vec, } diff --git a/crates/vuio-core/src/config/template.toml b/crates/vuio-core/src/config/template.toml index 34995ff..41c4e1b 100644 --- a/crates/vuio-core/src/config/template.toml +++ b/crates/vuio-core/src/config/template.toml @@ -71,6 +71,8 @@ validation_mode = "Warn" # Platform default database location: PLACEHOLDER_DEFAULT_DATABASE_PATH [database] path = "PLACEHOLDER_DATABASE_PATH" +# Reclaim free space in the index file at startup and shutdown. Rewrites the +# whole file, so it adds time to both on a large library. vacuum_on_startup = false backup_enabled = false diff --git a/crates/vuio-core/src/config/tests.rs b/crates/vuio-core/src/config/tests.rs index 9e86966..08fb2ec 100644 --- a/crates/vuio-core/src/config/tests.rs +++ b/crates/vuio-core/src/config/tests.rs @@ -531,6 +531,65 @@ async fn overrides_hold_across_reloads_without_freezing_the_file() -> Result<()> Ok(()) } +/// A config file living inside the media tree it configures — `vuio --config +/// ./vuio.toml` run from a library folder, which is the obvious thing to do. The +/// watcher used to take its parent recursively with the debouncer's default file-id +/// cache, so starting up walked and `stat`ed the entire library and then held a map +/// entry per file for the lifetime of the process, to notice edits to one file it +/// identifies by path equality. It watches one level now, without the id cache. +#[tokio::test] +async fn config_inside_a_media_tree_still_reloads() -> Result<()> { + let temp_dir = TempDir::new()?; + let root = std::fs::canonicalize(temp_dir.path())?; + let config_path = root.join("vuio.toml"); + + // The library the config is sitting in. + let nested = root.join("Artist").join("Album"); + std::fs::create_dir_all(&nested)?; + for index in 0..64 { + std::fs::write(nested.join(format!("track_{index}.mp3")), b"x")?; + } + + let mut config = AppConfig::default(); + config.server.name = "In The Library".to_string(); + config.media.directories = vec![MonitoredDirectoryConfig { + path: root.to_string_lossy().to_string(), + recursive: true, + case_sensitive: None, + extensions: None, + exclude_patterns: None, + validation_mode: ValidationMode::Skip, + }]; + config.save_to_file(&config_path)?; + + let cancellation = tokio_util::sync::CancellationToken::new(); + let tasks = tokio_util::task::TaskTracker::new(); + let manager = ConfigManager::watching_with_overrides( + &config_path, + ConfigOverrides::default(), + cancellation.clone(), + tasks.clone(), + ) + .await?; + assert!(manager.is_watched()); + + let mut edited = AppConfig::load_from_file(&config_path)?; + edited.server.name = "Renamed In The Library".to_string(); + edited.save_to_file(&config_path)?; + tokio::time::sleep(std::time::Duration::from_millis(2000)).await; + + assert_eq!( + manager.get_config().await.server.name, + "Renamed In The Library", + "an edit to a config inside the media tree must still be picked up" + ); + + cancellation.cancel(); + tasks.close(); + tasks.wait().await; + Ok(()) +} + #[test] fn overrides_report_what_they_force() { assert!(ConfigOverrides::default().in_force().is_empty()); diff --git a/crates/vuio-core/src/database/mod.rs b/crates/vuio-core/src/database/mod.rs index 9a691fc..03bad89 100644 --- a/crates/vuio-core/src/database/mod.rs +++ b/crates/vuio-core/src/database/mod.rs @@ -618,6 +618,31 @@ pub trait DatabaseReadSession { where F: for<'a> FnMut(Self::File<'a>) -> Result<()>; + /// One page, without counting the whole result. + /// + /// [`Self::visit_files`] reports `matched`, the size of the entire result, + /// because DLNA's `TotalMatches` requires it — and computing it means + /// evaluating the query a second time. For a ranked search that is the + /// expensive half: the engine has to find and rank every hit to count them, + /// so a page of twenty costs the same as a page of one. + /// + /// Callers that page by cursor never look at `matched`. Defaulted so a + /// backend need not implement it; overriding it is what makes the saving + /// real. + fn visit_files_page( + &mut self, + query: &MediaFileQuery, + offset: usize, + limit: usize, + visitor: F, + ) -> Result + where + F: for<'a> FnMut(Self::File<'a>) -> Result<()>, + { + self.visit_files(query, offset, limit, visitor) + .map(|summary| summary.visited) + } + fn visit_direct_subdirectories( &mut self, canonical_parent: &str, @@ -760,6 +785,28 @@ pub trait MediaRepository: Send + Sync { /// Load compact scanner comparison records instead of complete media metadata. async fn load_file_fingerprints(&self) -> Result>; + /// The same, for one subtree. + /// + /// A scan compares what is on disk against what is indexed, and it only ever + /// scans one root — so loading the whole table means every other library's + /// rows are held in memory for nothing. That is most of the cost when a + /// watcher event rescans a single folder. + async fn load_file_fingerprints_under( + &self, + canonical_prefix: &str, + ) -> Result>; + + /// One page of fingerprints, ordered by id, starting after `after_id`. + /// + /// For the callers that genuinely have to examine every row — the index + /// cleanup has to consider records under no configured root at all — so that + /// doing so does not mean holding every row at once. + async fn load_file_fingerprints_after( + &self, + after_id: i64, + limit: usize, + ) -> Result>; + async fn get_root_availability(&self, path: &Path) -> Result>; async fn list_root_availability(&self) -> Result>; @@ -1129,7 +1176,7 @@ pub trait DatabaseBackend: DatabaseManager + Sized + 'static { async fn restore_backup_file(backup: &Path, destination: &Path) -> Result<()>; } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct DatabaseStats { pub total_files: usize, pub total_size: u64, diff --git a/crates/vuio-core/src/database/sqlite/directory.rs b/crates/vuio-core/src/database/sqlite/directory.rs index 38b185f..b1faf66 100644 --- a/crates/vuio-core/src/database/sqlite/directory.rs +++ b/crates/vuio-core/src/database/sqlite/directory.rs @@ -90,7 +90,15 @@ impl DirectoryDelta { } } - prune(transaction)?; + // Only when something was taken away. Neither prune statement is + // index-served — one scans the counters, the other scans `directories` + // with a probe per row — and they run once per transaction, which during + // a scan means once per thousand files. An insert-only scan cannot drive + // any count to zero, so on a large library that was millions of row + // visits looking for deletions that could not have happened. + if self.counts.values().any(|delta| *delta < 0) { + prune(transaction)?; + } Ok(()) } } diff --git a/crates/vuio-core/src/database/sqlite/media_repo/basic.rs b/crates/vuio-core/src/database/sqlite/media_repo/basic.rs index 3dedfd4..1a7f31f 100644 --- a/crates/vuio-core/src/database/sqlite/media_repo/basic.rs +++ b/crates/vuio-core/src/database/sqlite/media_repo/basic.rs @@ -105,6 +105,47 @@ impl SqliteDatabase { .await } + pub(in crate::database::sqlite) async fn load_file_fingerprints_after_impl( + &self, + after_id: i64, + limit: usize, + ) -> Result> { + self.execute_read(move |connection| { + let mut statement = connection.prepare_cached(&format!( + "SELECT {FINGERPRINT_COLUMNS} FROM media_files \ + WHERE id > ? ORDER BY id LIMIT ?" + ))?; + let fingerprints = statement + .query_map( + rusqlite::params![after_id, limit as i64], + schema::fingerprint_from_row, + )? + .collect::>>()?; + Ok(fingerprints) + }) + .await + } + + pub(in crate::database::sqlite) async fn load_file_fingerprints_under_impl( + &self, + canonical_prefix: String, + ) -> Result> { + self.execute_read(move |connection| { + // A range on `path` rather than a `LIKE`, so the primary key index + // serves it. `subtree_range` stops at a component boundary, which is + // what keeps `/media/Film` from matching `/media/Films`. + let (start, end) = SqliteDatabase::subtree_range(&canonical_prefix); + let mut statement = connection.prepare_cached(&format!( + "SELECT {FINGERPRINT_COLUMNS} FROM media_files WHERE path >= ? AND path < ?" + ))?; + let fingerprints = statement + .query_map([&start, &end], schema::fingerprint_from_row)? + .collect::>>()?; + Ok(fingerprints) + }) + .await + } + pub(in crate::database::sqlite) async fn get_files_by_paths_impl( &self, paths: &[PathBuf], diff --git a/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs b/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs index fbc7528..932b531 100644 --- a/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs +++ b/crates/vuio-core/src/database/sqlite/media_repo/bulk.rs @@ -97,23 +97,27 @@ pub(in crate::database::sqlite) fn bind_media_file(file: &MediaFile) -> Vec, media_file_id: i64, file: &MediaFile, + row_existed: bool, ) -> Result<()> { - transaction.execute( - "DELETE FROM media_tags WHERE media_file_id = ?", - [media_file_id], - )?; + if row_existed { + transaction.execute( + "DELETE FROM media_tags WHERE media_file_id = ?", + [media_file_id], + )?; + } if file.extra_tags.is_empty() { return Ok(()); } @@ -193,7 +197,7 @@ pub(in crate::database::sqlite) fn upsert_media_file( transaction .prepare_cached(UPDATE_MEDIA)? .execute(rusqlite::params_from_iter(params.iter()))?; - write_extra_tags(transaction, id, file)?; + write_extra_tags(transaction, id, file, true)?; Ok(id) } None => { @@ -206,7 +210,7 @@ pub(in crate::database::sqlite) fn upsert_media_file( .prepare_cached(INSERT_MEDIA)? .execute(rusqlite::params_from_iter(params.iter()))?; let id = transaction.last_insert_rowid(); - write_extra_tags(transaction, id, file)?; + write_extra_tags(transaction, id, file, false)?; Ok(id) } } diff --git a/crates/vuio-core/src/database/sqlite/mod.rs b/crates/vuio-core/src/database/sqlite/mod.rs index 586a5ee..1db3aea 100644 --- a/crates/vuio-core/src/database/sqlite/mod.rs +++ b/crates/vuio-core/src/database/sqlite/mod.rs @@ -38,6 +38,18 @@ mod tests; pub use session::SqliteReadSession; +/// Register the natural-order collation on a caller-supplied connection. +/// +/// Two of the browse indexes are declared `COLLATE natural_order`, so any +/// connection that writes `media_files` has to know it — including one opened +/// outside this crate. Exposed for the benchmark generator, which bulk-loads a +/// library over a direct handle; without this it would need its own copy of +/// `natural_cmp`, and a copy is a thing that drifts. +#[cfg(feature = "unstable-internals")] +pub fn register_collations(connection: &rusqlite::Connection) -> anyhow::Result<()> { + schema::register_collations(connection) +} + /// Readers held open for reuse. /// /// Reads run on Tokio's blocking pool, which is unbounded by design; without a @@ -118,6 +130,17 @@ pub struct SqliteDatabase { /// Held for the duration of a write so writers queue in async code rather /// than piling up as blocked threads inside the blocking pool. mutation_lock: tokio::sync::Mutex<()>, + /// Bumped by every write. Read-side caches compare against it to know + /// whether what they hold can still be true. + write_generation: std::sync::atomic::AtomicU64, + /// Library totals, and the generation they were computed at. + /// + /// `get_stats` aggregates the whole of `media_files` — it needs both `size` + /// and `mime_family`, which no index covers together — and it is asked for + /// by `/metrics`, `/metrics/json`, `/readyz` and the dashboard's five-second + /// poll. On a large library that is a multi-gigabyte scan several times a + /// minute to produce an answer that only changes when something is written. + stats_cache: Mutex>, } impl std::fmt::Debug for SqliteDatabase { @@ -198,6 +221,8 @@ impl SqliteDatabase { )), db_path: path, mutation_lock: tokio::sync::Mutex::new(()), + write_generation: std::sync::atomic::AtomicU64::new(0), + stats_cache: Mutex::new(None), }) } @@ -232,6 +257,11 @@ impl SqliteDatabase { F: FnOnce(&mut Connection) -> Result + Send + 'static, { let _guard = self.mutation_lock.lock().await; + // Before the write, not after: a reader that samples the generation + // mid-write must not be able to cache a result taken from the old state + // under the new number. + self.write_generation + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); let write = Arc::clone(&self.write); tokio::task::spawn_blocking(move || { let mut connection = write diff --git a/crates/vuio-core/src/database/sqlite/schema.rs b/crates/vuio-core/src/database/sqlite/schema.rs index 1477359..e9646f4 100644 --- a/crates/vuio-core/src/database/sqlite/schema.rs +++ b/crates/vuio-core/src/database/sqlite/schema.rs @@ -18,7 +18,7 @@ use crate::database::{AudioTags, FileFingerprint, FileLocation, MediaFile, Playl /// [`migrations`]; only a *newer* file — one written by a build that knows /// something this one does not — is refused, because there is no way to /// downgrade a schema without guessing at what to discard. -pub(super) const SCHEMA_VERSION: i64 = 4; +pub(super) const SCHEMA_VERSION: i64 = 5; /// Name of the collation that carries the application's natural ordering into /// SQL. Registered on every connection; see [`register_collations`]. @@ -100,7 +100,6 @@ CREATE INDEX IF NOT EXISTS idx_media_genre ON media_files(genre); CREATE INDEX IF NOT EXISTS idx_media_year ON media_files(year); CREATE INDEX IF NOT EXISTS idx_media_album_artist ON media_files(album_artist); CREATE INDEX IF NOT EXISTS idx_media_family ON media_files(mime_family); -CREATE INDEX IF NOT EXISTS idx_media_tags_version ON media_files(tags_version); -- Directories exist only by implication from the paths of files, so unlike the -- music indexes they cannot be recomputed by a query at browse time. @@ -169,7 +168,6 @@ CREATE TABLE IF NOT EXISTS media_tags ( PRIMARY KEY (media_file_id, key, value) ) STRICT; -CREATE INDEX IF NOT EXISTS idx_media_tags_key ON media_tags(key, value); -- What a public metadata service said about a file: title, synopsis, rating and -- a pointer into the artwork cache. @@ -320,6 +318,7 @@ fn migrations() -> Vec<(i64, String)> { (2, MIGRATION_V2.to_owned()), (3, MIGRATION_V3.to_owned()), (4, migration_v4()), + (5, MIGRATION_V5.to_owned()), ] } @@ -397,6 +396,20 @@ fn migration_v4() -> String { format!("{FTS_DDL}{FTS_REBUILD}") } +/// v4 → v5: drop two indexes nothing queries. +/// +/// `tags_version` never appears in a `WHERE` or `ORDER BY` — it is compared in +/// Rust, against a value the scanner already holds — and `media_tags` is only +/// ever read by `media_file_id`, which its primary key already serves. Both were +/// pure write cost: two more b-tree insertions for every row indexed. +/// +/// Dropping an index is not the destructive kind of migration the rule above +/// guards against. An index holds nothing that is not derivable from the table. +const MIGRATION_V5: &str = r#" +DROP INDEX IF EXISTS idx_media_tags_version; +DROP INDEX IF EXISTS idx_media_tags_key; +"#; + /// Columns of `media_files`, qualified so the list can be used inside joins. pub(super) const MEDIA_COLUMNS: &str = "\ media_files.id, media_files.path, media_files.filename, media_files.size, \ diff --git a/crates/vuio-core/src/database/sqlite/session.rs b/crates/vuio-core/src/database/sqlite/session.rs index 082147a..adbde53 100644 --- a/crates/vuio-core/src/database/sqlite/session.rs +++ b/crates/vuio-core/src/database/sqlite/session.rs @@ -306,6 +306,34 @@ impl DatabaseReadSession for SqliteReadSession { Ok(summary) } + fn visit_files_page( + &mut self, + query: &MediaFileQuery, + offset: usize, + limit: usize, + mut visitor: F, + ) -> Result + where + F: for<'a> FnMut(Self::File<'a>) -> Result<()>, + { + if limit == 0 { + return Ok(0); + } + let plan = query::plan(query); + let mut page_params = plan.params.clone(); + page_params.push(rusqlite::types::Value::Integer(limit as i64)); + page_params.push(rusqlite::types::Value::Integer(offset as i64)); + + let mut visited = 0; + let mut statement = self.connection.prepare_cached(&plan.page_sql())?; + let mut rows = statement.query(rusqlite::params_from_iter(page_params.iter()))?; + while let Some(row) = rows.next()? { + visitor(SqliteMediaFileView { row })?; + visited += 1; + } + Ok(visited) + } + fn visit_direct_subdirectories( &mut self, canonical_parent: &str, diff --git a/crates/vuio-core/src/database/sqlite/stats.rs b/crates/vuio-core/src/database/sqlite/stats.rs index 07a2c72..1605345 100644 --- a/crates/vuio-core/src/database/sqlite/stats.rs +++ b/crates/vuio-core/src/database/sqlite/stats.rs @@ -6,7 +6,44 @@ use super::SqliteDatabase; use crate::database::DatabaseStats; impl SqliteDatabase { + /// Library totals, recomputed only when something has been written. + /// + /// The query below is a full scan of the widest table in the schema — it + /// needs `size` and `mime_family` together, and no index covers both. It is + /// also on the path of `/readyz`, `/metrics`, `/metrics/json` and the + /// dashboard's five-second poll, so on a large library an idle server was + /// re-reading the whole table several times a minute to produce an answer + /// that had not changed. Keyed on the write generation rather than a + /// timeout, so the answer is never stale, only never recomputed for nothing. pub(super) async fn get_stats_impl(&self) -> Result { + let generation = self + .write_generation + .load(std::sync::atomic::Ordering::SeqCst); + if let Ok(cache) = self.stats_cache.lock() { + if let Some((cached_generation, stats)) = cache.as_ref() { + if *cached_generation == generation { + return Ok(stats.clone()); + } + } + } + + let stats = self.compute_stats().await?; + + if let Ok(mut cache) = self.stats_cache.lock() { + // Only if nothing was written while we were counting; otherwise the + // next caller recomputes rather than trusting a torn read. + if self + .write_generation + .load(std::sync::atomic::Ordering::SeqCst) + == generation + { + *cache = Some((generation, stats.clone())); + } + } + Ok(stats) + } + + async fn compute_stats(&self) -> Result { self.execute_read(move |connection| { // One pass over the table serves every counter, so the numbers are // consistent with each other without needing a transaction. diff --git a/crates/vuio-core/src/database/sqlite/traits.rs b/crates/vuio-core/src/database/sqlite/traits.rs index 9130c2b..1316b38 100644 --- a/crates/vuio-core/src/database/sqlite/traits.rs +++ b/crates/vuio-core/src/database/sqlite/traits.rs @@ -69,6 +69,21 @@ impl MediaRepository for SqliteDatabase { SqliteDatabase::load_file_fingerprints_impl(self).await } + async fn load_file_fingerprints_under( + &self, + canonical_prefix: &str, + ) -> Result> { + SqliteDatabase::load_file_fingerprints_under_impl(self, canonical_prefix.to_owned()).await + } + + async fn load_file_fingerprints_after( + &self, + after_id: i64, + limit: usize, + ) -> Result> { + SqliteDatabase::load_file_fingerprints_after_impl(self, after_id, limit).await + } + async fn get_root_availability(&self, path: &Path) -> Result> { SqliteDatabase::get_root_availability_impl(self, path).await } diff --git a/crates/vuio-core/src/lifecycle/media/events.rs b/crates/vuio-core/src/lifecycle/media/events.rs index 09b659d..daf5626 100644 --- a/crates/vuio-core/src/lifecycle/media/events.rs +++ b/crates/vuio-core/src/lifecycle/media/events.rs @@ -196,13 +196,13 @@ pub(in crate::lifecycle) async fn handle_file_system_event 0 { increment_content_update_id(app_state).await; } } diff --git a/crates/vuio-core/src/lifecycle/media/monitoring.rs b/crates/vuio-core/src/lifecycle/media/monitoring.rs index 58e5e61..ae9c0aa 100644 --- a/crates/vuio-core/src/lifecycle/media/monitoring.rs +++ b/crates/vuio-core/src/lifecycle/media/monitoring.rs @@ -61,6 +61,9 @@ pub(in crate::lifecycle) async fn start_file_monitoring { @@ -142,10 +145,25 @@ pub(in crate::lifecycle) async fn start_file_monitoring 0 + && last_full_sweep.elapsed() + >= std::time::Duration::from_secs(full_rescan_interval * 3600); + if full_sweep_due { + last_full_sweep = tokio::time::Instant::now(); + info!("Sweeping every library root (full rescan interval reached)"); + } + let scanner = media::MediaScanner::with_database(app_state_clone.database.clone()); for root in &configured_roots { let path = PathBuf::from(&root.path); @@ -153,6 +171,9 @@ pub(in crate::lifecycle) async fn start_file_monitoring 0 => increment_content_update_id(&app_state_clone).await, - Ok(_) => {} - Err(error) => error!("Periodic missing-file reconciliation failed: {}", error), + // Only alongside a sweep. This walks the whole index and asks + // the filesystem about every path in it, which on a large + // library is the most expensive thing the tick can do — and + // a deletion inside a watched root already arrives as an + // event and is handled per-path. Like the sweep above, this + // is the backstop for what the watcher missed, so it runs on + // the backstop's cadence rather than every five minutes. + // + // Gated on the setting, and read live so it can be changed + // without a restart. Ungated, `cleanup_deleted_files = false` + // did not prevent deletions at all — the startup scan honoured + // it and this tick then removed the same files five minutes later. + if full_sweep_due || !dirty_roots.is_empty() { + match validate_and_cleanup_deleted_files( + app_state_clone.database.clone(), + &configured_directories, + app_state_clone.current_config().media.cleanup_deleted_files, + ) + .await + { + Ok(removed) if removed > 0 => increment_content_update_id(&app_state_clone).await, + Ok(_) => {} + Err(error) => error!("Periodic missing-file reconciliation failed: {}", error), + } } } } diff --git a/crates/vuio-core/src/lifecycle/media/scanning.rs b/crates/vuio-core/src/lifecycle/media/scanning.rs index 224dc4f..5d62a29 100644 --- a/crates/vuio-core/src/lifecycle/media/scanning.rs +++ b/crates/vuio-core/src/lifecycle/media/scanning.rs @@ -12,7 +12,7 @@ fn canonical_root(path: &Path) -> PathBuf { .unwrap_or_else(|_| path.to_path_buf()) } -/// The configured library a stored path belongs to, or `None` if it belongs to none. +/// Which configured library a stored path belongs to, or `None` if it belongs to none. /// /// Matched against the root both as the config writes it and in the form the index /// stores paths in, because those routinely differ: a library reached through a symlink @@ -21,11 +21,13 @@ fn canonical_root(path: &Path) -> PathBuf { /// every file in such a library as belonging to no library at all — and for a deletion /// pass, that means discarding the entire library's index. Matching either form can only /// keep files, never remove more. -fn owning_root(path: &Path, raw: &[PathBuf], canonical: &[PathBuf]) -> Option { +/// +/// Returns the root's *index* rather than the root itself, so a caller can look up +/// whatever it has already worked out per root instead of re-deriving it per file. +fn owning_root_index(path: &Path, raw: &[PathBuf], canonical: &[PathBuf]) -> Option { raw.iter() - .chain(canonical.iter()) - .find(|root| path.starts_with(root)) - .cloned() + .position(|root| path.starts_with(root)) + .or_else(|| canonical.iter().position(|root| path.starts_with(root))) } /// Validate cached files and remove the ones that no longer belong in the index: @@ -81,15 +83,41 @@ pub(in crate::lifecycle) async fn validate_and_cleanup_deleted_files>(); - { - for media_file in database.load_file_fingerprints().await? { + // Whether each root is currently mounted, asked once. This used to be a + // `stat` of the same handful of directories once per indexed file, so a + // library of a million files made a million syscalls to answer a question + // with one answer per root. + let root_is_present = monitored_roots + .iter() + .map(|root| root.is_dir()) + .collect::>(); + + // Walk the index a page at a time. Every row has to be considered — a record + // under no configured root at all is exactly what this prunes — but holding + // the whole table to do it made the check cost as much memory as the library. + const CLEANUP_PAGE: usize = 4096; + let mut after_id = 0_i64; + loop { + let page = database + .load_file_fingerprints_after(after_id, CLEANUP_PAGE) + .await?; + let Some(last) = page.last() else { + break; + }; + after_id = last.id; + + // Sort the page without touching the disk first, so the syscalls that do + // have to happen are the only ones that happen. + let mut to_probe: Vec = Vec::new(); + for media_file in page { total_checked += 1; - let Some(configured_root) = owning_root(&media_file.path, monitored_roots, &canonical_roots) + let Some(root_index) = + owning_root_index(&media_file.path, monitored_roots, &canonical_roots) else { if prune_orphans { orphaned_count += 1; - paths_to_delete.push(media_file.path.clone()); + paths_to_delete.push(media_file.path); } continue; }; @@ -97,20 +125,28 @@ pub(in crate::lifecycle) async fn validate_and_cleanup_deleted_files 0 { info!( @@ -607,6 +643,61 @@ mod tests { assert_eq!(indexed_paths(&database).await, vec![canonical_root(&present)]); } + /// The cleanup reads the index a page at a time rather than holding all of it, so + /// an index larger than one page is the case where a paging mistake shows: rows + /// past the first page silently never get checked, and deleted files stay indexed + /// forever. Sized past `CLEANUP_PAGE` deliberately. + #[tokio::test] + async fn every_page_of_a_large_index_is_checked() { + let root = tempfile::TempDir::new().expect("root"); + + // A handful that exist, and several pages' worth that do not. + let mut survivors = Vec::new(); + for index in 0..3 { + let path = root.path().join(format!("present_{index}.mp4")); + std::fs::write(&path, b"x").expect("write"); + survivors.push(canonical_root(&path)); + } + survivors.sort(); + + let missing = 5_000; + let mut rows: Vec = Vec::with_capacity(survivors.len() + missing); + for index in 0..3 { + rows.push(MediaFile::new( + root.path().join(format!("present_{index}.mp4")), + 1024, + "video/mp4".to_string(), + )); + } + for index in 0..missing { + rows.push(MediaFile::new( + root.path().join(format!("gone_{index}.mp4")), + 1024, + "video/mp4".to_string(), + )); + } + + let temp = tempfile::TempDir::new().expect("temp dir"); + let database = Arc::new( + SqliteDatabase::new(temp.path().join("paged.db")) + .await + .expect("database"), + ); + database.initialize().await.expect("schema"); + database.bulk_store_media_files(&rows).await.expect("store"); + + let removed = + validate_and_cleanup_deleted_files(database.clone(), &[root.path().to_path_buf()], true) + .await + .expect("cleanup"); + + assert_eq!( + removed, missing, + "a row on the second page and beyond must still be checked" + ); + assert_eq!(indexed_paths(&database).await, survivors); + } + /// Nothing configured means nothing to compare against. Treating every file as an /// orphan would discard the whole index over what is almost certainly a misload. #[tokio::test] @@ -642,23 +733,23 @@ mod tests { // Stored under the canonical form: matched via the canonical root. assert_eq!( - owning_root(&canonical.join("a.mkv"), &raw, &canonicalised), - Some(canonical.clone()) + owning_root_index(&canonical.join("a.mkv"), &raw, &canonicalised), + Some(0) ); // Stored under the written form, as an older index would be: still matched. assert_eq!( - owning_root(&written.join("a.mkv"), &raw, &canonicalised), - Some(written) + owning_root_index(&written.join("a.mkv"), &raw, &canonicalised), + Some(0) ); // A file under neither is an orphan, which is the only case that deletes. assert_eq!( - owning_root(Path::new("/elsewhere/a.mkv"), &raw, &canonicalised), + owning_root_index(Path::new("/elsewhere/a.mkv"), &raw, &canonicalised), None ); // Prefix matching is by component, so a sibling with a shared prefix is not a // child: /media/films-old must not be swept up by the /media/films root. assert_eq!( - owning_root(Path::new("/private/media/films-old/a.mkv"), &raw, &canonicalised), + owning_root_index(Path::new("/private/media/films-old/a.mkv"), &raw, &canonicalised), None ); } diff --git a/crates/vuio-core/src/lifecycle/runner.rs b/crates/vuio-core/src/lifecycle/runner.rs index b191297..8e1806b 100644 --- a/crates/vuio-core/src/lifecycle/runner.rs +++ b/crates/vuio-core/src/lifecycle/runner.rs @@ -313,6 +313,21 @@ where }); } + // Announce before scanning, not after. SSDP and mDNS are how a TV finds the + // server at all, and the scan below walks the whole library — so starting + // discovery afterwards meant that on a large library nothing on the network + // could see the server until the walk finished. A client that connects + // mid-scan browses whatever is indexed so far, which is strictly better than + // being unable to find it. + let discovery_state = app_state.clone(); + let discovery_cancellation = cancellation.clone(); + services.spawn(async move { + ( + "discovery", + supervisor::run_advertisement_supervisor(discovery_state, discovery_cancellation).await, + ) + }); + // Scan only after the watcher and listeners are active. This closes the startup blind // window: a download that lands while the scan is running is either found // by the scan or delivered by the watcher (and duplicate upserts are safe). @@ -358,15 +373,6 @@ where ) }); - let discovery_state = app_state.clone(); - let discovery_cancellation = cancellation.clone(); - services.spawn(async move { - ( - "discovery", - supervisor::run_advertisement_supervisor(discovery_state, discovery_cancellation).await, - ) - }); - // Renderer discovery has nothing to do with the listener; it used to be started // beside it and would otherwise be cycled by every rebind. let tv_discovery = start_tv_discovery(app_state.clone(), cancellation.clone()); @@ -512,7 +518,10 @@ where Err(error) => warn!("Shutdown database backup failed: {}", error), } } - if let Err(e) = perform_graceful_shutdown(&database, &lifecycle_stats).await { + let compact_on_shutdown = app_state.current_config().database.vacuum_on_startup; + if let Err(e) = + perform_graceful_shutdown(&database, &lifecycle_stats, compact_on_shutdown).await + { error!("Error during graceful shutdown: {}", e); } info!("Shutdown completed in {:?}", shutdown_start.elapsed()); diff --git a/crates/vuio-core/src/lifecycle/shutdown.rs b/crates/vuio-core/src/lifecycle/shutdown.rs index b94cd26..a091c37 100644 --- a/crates/vuio-core/src/lifecycle/shutdown.rs +++ b/crates/vuio-core/src/lifecycle/shutdown.rs @@ -4,6 +4,7 @@ use super::*; pub(super) async fn perform_graceful_shutdown( database: &Arc, stats: &ApplicationStats, + compact: bool, ) -> anyhow::Result<()> { info!("Performing graceful shutdown with atomic state persistence..."); @@ -37,11 +38,17 @@ pub(super) async fn perform_graceful_shutdown( } } - // Perform database vacuum if needed (this will also ensure all data is persisted) - info!("Performing final database maintenance..."); - match database.vacuum().await { - Ok(compacted) => info!(compacted, "Final database compaction completed"), - Err(e) => warn!("Could not compact database during shutdown: {}", e), + // Only when asked. A VACUUM rewrites the entire database file, so on a large + // library this turned every stop into a multi-gigabyte copy — and it ran + // regardless of `database.vacuum_on_startup`, the setting that exists to say + // whether compaction is wanted at all. Write-ahead logging has already made + // the data durable; compaction only reclaims free pages. + if compact { + info!("Performing final database maintenance..."); + match database.vacuum().await { + Ok(compacted) => info!(compacted, "Final database compaction completed"), + Err(e) => warn!("Could not compact database during shutdown: {}", e), + } } info!("Graceful shutdown with atomic state persistence completed"); @@ -76,8 +83,9 @@ impl ShutdownCoordinator { pub async fn finalize( database: &Arc, stats: &ApplicationStats, + compact: bool, ) -> anyhow::Result<()> { - perform_graceful_shutdown(database, stats).await + perform_graceful_shutdown(database, stats, compact).await } } diff --git a/crates/vuio-core/src/media.rs b/crates/vuio-core/src/media.rs index 6708a06..ddc03b0 100644 --- a/crates/vuio-core/src/media.rs +++ b/crates/vuio-core/src/media.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use futures_util::StreamExt as _; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; diff --git a/crates/vuio-core/src/media/policy.rs b/crates/vuio-core/src/media/policy.rs index e14e7d6..30969a6 100644 --- a/crates/vuio-core/src/media/policy.rs +++ b/crates/vuio-core/src/media/policy.rs @@ -253,7 +253,6 @@ pub(super) fn swap_one_ascii_case(value: &str) -> Option { #[derive(Debug)] pub(super) struct TraversalReport { - pub(super) file_paths: Vec, pub(super) uncertain_prefixes: Vec, pub(super) errors: Vec, pub(super) root_complete: bool, diff --git a/crates/vuio-core/src/media/result.rs b/crates/vuio-core/src/media/result.rs index 4a64dbd..8aa7600 100644 --- a/crates/vuio-core/src/media/result.rs +++ b/crates/vuio-core/src/media/result.rs @@ -1,23 +1,40 @@ use super::*; /// Result of a media scanning operation +/// +/// Every outcome is a count. The records themselves live in the database by the +/// time a scan returns, and nothing has ever read them back off this struct — +/// `.len()` and `.is_empty()` were the only consumers. Retaining them meant a +/// whole [`MediaFile`] per added file, which on a large library is most of the +/// index built and dropped for a log line, and worse on real music where +/// `extra_tags` is populated. +/// +/// [`MediaFile`]: crate::database::MediaFile #[derive(Debug, Clone)] pub struct ScanResult { - /// Files that were newly added to the database - pub new_files: Vec, + /// How many files were newly added to the database. + pub new: usize, - /// Files that were updated in the database - pub updated_files: Vec, + /// How many files were updated in the database. + pub updated: usize, - /// Files that were removed from the database - pub removed_files: Vec, + /// How many files were removed from the database. + pub removed: usize, - /// Files that were unchanged - pub unchanged_files: Vec, + /// How many files were found to be unchanged. + pub unchanged: usize, /// Total number of files scanned from the file system pub total_scanned: usize, + /// How many files this scan actually opened and read. + /// + /// Distinct from `total_scanned`, which counts everything the walk saw. A + /// scan of a library that has not changed should read nothing: each file + /// costs one `stat`, and the difference between the two numbers is the work + /// avoided. + pub files_read: usize, + /// Errors encountered during scanning pub errors: Vec, @@ -25,57 +42,62 @@ pub struct ScanResult { pub complete: bool, } +impl Default for ScanResult { + /// Not derived: every count starts at zero, but `complete` starts *true* and + /// is cleared by anything that could not be enumerated. Deriving would start + /// it false and quietly mark every scan incomplete. + fn default() -> Self { + Self::new() + } +} + impl ScanResult { - /// Create a new empty scan result with pre-allocated capacity + /// Create a new empty scan result pub fn new() -> Self { Self { - new_files: Vec::with_capacity(100), - updated_files: Vec::with_capacity(50), - removed_files: Vec::with_capacity(50), - unchanged_files: Vec::with_capacity(1000), + new: 0, + updated: 0, + removed: 0, + unchanged: 0, total_scanned: 0, - errors: Vec::with_capacity(10), + files_read: 0, + errors: Vec::new(), complete: true, } } /// Merge another scan result into this one pub fn merge(&mut self, other: ScanResult) { - self.new_files.extend(other.new_files); - self.updated_files.extend(other.updated_files); - self.removed_files.extend(other.removed_files); - self.unchanged_files.extend(other.unchanged_files); + self.new += other.new; + self.updated += other.updated; + self.removed += other.removed; + self.unchanged += other.unchanged; self.total_scanned += other.total_scanned; + self.files_read += other.files_read; self.errors.extend(other.errors); self.complete &= other.complete; } /// Get the total number of changes (new + updated + removed) pub fn total_changes(&self) -> usize { - self.new_files.len() + self.updated_files.len() + self.removed_files.len() + self.new + self.updated + self.removed } - /// Get a summary string of the scan results pub fn summary(&self) -> String { format!( - "Scanned {} files: {} new, {} updated, {} removed, {} unchanged, {} errors", + "Scanned {} files ({} read): {} new, {} updated, {} removed, {} unchanged, {} errors", self.total_scanned, - self.new_files.len(), - self.updated_files.len(), - self.removed_files.len(), - self.unchanged_files.len(), + self.files_read, + self.new, + self.updated, + self.removed, + self.unchanged, self.errors.len() ) } } -impl Default for ScanResult { - fn default() -> Self { - Self::new() - } -} - /// Error that occurred during scanning #[derive(Debug, Clone)] pub struct ScanError { diff --git a/crates/vuio-core/src/media/scanner.rs b/crates/vuio-core/src/media/scanner.rs index d7dbfd6..7a8d0a3 100644 --- a/crates/vuio-core/src/media/scanner.rs +++ b/crates/vuio-core/src/media/scanner.rs @@ -7,16 +7,142 @@ pub struct MediaScanner { database_manager: Arc, } +/// How many paths the walker may run ahead of the classifier. +/// +/// Bounded so that a walk of a large library does not become a list of the +/// library: past this the walking thread blocks until the consumer catches up. +const WALK_QUEUE: usize = 4096; + +/// How many changed files to read before writing them out. +/// +/// Only files that actually changed reach this, so on an unchanged library it +/// never fills. On a first scan it is what keeps peak memory flat instead of +/// proportional to the library. Matched to [`BATCH_SIZE`] so one window is one +/// write. +const READ_WINDOW: usize = BATCH_SIZE; + +/// What a scan needs to know about a file it may already have indexed. +/// +/// Deliberately not [`FileFingerprint`]: that carries the path, and this lives +/// in a map keyed by the path, so storing it again doubled the largest +/// allocation a scan makes. +struct IndexedFile { + id: i64, + size: u64, + modified: SystemTime, + created_at: SystemTime, + tags_version: u32, + /// Set when the walk produced this path. What is left unset is what has been + /// deleted from disk — which is why the scan needs no second collection of + /// every path it saw. + seen: bool, +} + +impl IndexedFile { + /// Split a loaded record into the map's key and value, moving the path + /// rather than copying it into both halves. + fn split(fingerprint: FileFingerprint) -> (PathBuf, Self) { + let FileFingerprint { + id, + path, + size, + modified, + created_at, + tags_version, + } = fingerprint; + ( + path, + Self { + id, + size, + modified, + created_at, + tags_version, + seen: false, + }, + ) + } +} + impl MediaScanner { - fn fingerprint(file: &MediaFile) -> FileFingerprint { - FileFingerprint { - id: file.id.unwrap_or_default(), - path: file.path.clone(), - size: file.size, - modified: file.modified, - created_at: file.created_at, - tags_version: file.tags_version, + /// Read a window of changed files, several at a time, and sort them into + /// inserts and updates. + /// + /// Each read ends in `spawn_blocking`, so the work already lands on the + /// blocking pool — but awaiting them one after another meant only ever one + /// was in flight, and a scan used a single core however many the machine had. + async fn read_window( + &self, + paths: &mut Vec, + existing_files_map: &HashMap, + files_to_insert: &mut Vec, + files_to_update: &mut Vec, + result: &mut ScanResult, + concurrency: usize, + ) -> Result<()> { + let mut built = futures_util::stream::iter(paths.drain(..)) + .map(|path| async move { (path.clone(), self.create_media_file_from_path(&path).await) }) + .buffer_unordered(concurrency); + + while let Some((path, outcome)) = built.next().await { + let current_file = match outcome { + Ok(file) => file, + Err(e) => { + debug!("Failed to create MediaFile for {}: {}", path.display(), e); + result.errors.push(ScanError { + path, + error: e.to_string(), + }); + continue; + } + }; + result.files_read += 1; + + match existing_files_map.get(&path) { + Some(existing) => { + if self.fingerprint_needs_update(existing, ¤t_file) { + let mut updated = current_file; + updated.id = Some(existing.id); + updated.created_at = existing.created_at; + updated.updated_at = SystemTime::now(); + files_to_update.push(updated); + } else { + result.unchanged += 1; + } + } + None => files_to_insert.push(current_file), + } } + + Ok(()) + } + + /// Write out whatever a window produced. + async fn flush_batches( + &self, + files_to_insert: &mut Vec, + files_to_update: &mut Vec, + result: &mut ScanResult, + ) -> Result<()> { + if !files_to_insert.is_empty() { + info!("Inserting batch of {} files", files_to_insert.len()); + self.database_manager + .bulk_store_canonical_media_files(files_to_insert) + .await?; + result.new += files_to_insert.len(); + files_to_insert.clear(); + } + + if !files_to_update.is_empty() { + info!("Updating batch of {} files", files_to_update.len()); + self.database_manager + .bulk_update_canonical_media_files(files_to_update) + .await?; + result.updated += files_to_update.len(); + files_to_update.clear(); + } + + Ok(()) } /// Create a new media scanner with database manager @@ -78,9 +204,7 @@ impl MediaScanner { error: "previously populated root is unexpectedly empty; destructive reconciliation deferred" .to_owned(), }); - result - .unchanged_files - .extend(existing_files.iter().map(Self::fingerprint)); + result.unchanged += existing_files.len(); return Ok(result); } self.perform_incremental_update(&canonical_dir, existing_files, current_files) @@ -138,9 +262,7 @@ impl MediaScanner { files_to_update.push(updated_file); } else { - result - .unchanged_files - .push(Self::fingerprint(existing_file)); + result.unchanged += 1; } } None => { @@ -156,7 +278,7 @@ impl MediaScanner { if !current_paths.contains(&normalized_existing_path) { // File was removed from file system, add to bulk removal list files_to_remove.push(existing_file.path.clone()); - result.removed_files.push(Self::fingerprint(&existing_file)); + result.removed += 1; } } @@ -176,18 +298,10 @@ impl MediaScanner { file.size ); } - let insert_ids = self - .database_manager + self.database_manager .bulk_store_canonical_media_files(&files_to_insert) .await?; - - // Update result with inserted files and their IDs - for (i, mut file) in files_to_insert.into_iter().enumerate() { - if let Some(id) = insert_ids.get(i) { - file.id = Some(*id); - } - result.new_files.push(file); - } + result.new += files_to_insert.len(); } // Bulk update changed files @@ -199,7 +313,7 @@ impl MediaScanner { self.database_manager .bulk_update_canonical_media_files(&files_to_update) .await?; - result.updated_files.extend(files_to_update); + result.updated += files_to_update.len(); } // Bulk remove deleted files @@ -224,10 +338,10 @@ impl MediaScanner { // Log bulk operation summary tracing::info!( "bulk operations completed: {} inserted, {} updated, {} removed, {} unchanged", - result.new_files.len(), - result.updated_files.len(), - result.removed_files.len(), - result.unchanged_files.len() + result.new, + result.updated, + result.removed, + result.unchanged ); Ok(result) @@ -268,7 +382,33 @@ impl MediaScanner { } } - fn fingerprint_needs_update(&self, existing: &FileFingerprint, current: &MediaFile) -> bool { + /// Whether a record is stale, judged from `stat` alone. + /// + /// The same three rules as [`Self::fingerprint_needs_update`], asked before + /// the file is opened rather than after. Every rule is answerable from + /// metadata the filesystem already has, which is what makes an unchanged + /// library nearly free to re-scan. + fn stat_needs_update(existing: &IndexedFile, metadata: &std::fs::Metadata) -> bool { + if existing.size != metadata.len() { + return true; + } + // A record written by an older tag reader is stale even though its file + // is not, so it is re-read to pick up the fields the new reader knows. + if existing.tags_version < crate::platform::filesystem::TAGS_VERSION { + return true; + } + let Ok(modified) = metadata.modified() else { + return true; + }; + let difference = if existing.modified > modified { + existing.modified.duration_since(modified) + } else { + modified.duration_since(existing.modified) + }; + difference.map_or(true, |difference| difference.as_secs() > 10) + } + + fn fingerprint_needs_update(&self, existing: &IndexedFile, current: &MediaFile) -> bool { if existing.size != current.size { return true; } @@ -322,28 +462,31 @@ impl MediaScanner { canonical_root.display() ); - // Load all existing files from database once at the start (for incremental updates) + // Load the index for this subtree only. A scan compares one root against + // one root; the rest of the library would be held for nothing, and a + // watcher event that rescans a single folder used to load every row. debug!("Loading existing files from database..."); - let existing_files_map: HashMap = self + let canonical_root_str = canonical_root.to_string_lossy().into_owned(); + let mut existing_files_map: HashMap = self .database_manager - .load_file_fingerprints() + .load_file_fingerprints_under(&canonical_root_str) .await? .into_iter() - .map(|fingerprint| (fingerprint.path.clone(), fingerprint)) + .map(IndexedFile::split) .collect(); - debug!( - "Loaded {} existing files from database", - existing_files_map.len() - ); + let existing_in_root = existing_files_map.len(); + debug!("Loaded {existing_in_root} existing files from database"); - // Use jwalk for parallel directory traversal - runs in a blocking thread pool + // Walk on a blocking thread, handing paths over as they are found rather + // than collecting the library into a `Vec` first. The channel is bounded, + // so a slow consumer backs the walker up instead of buffering. let root_clone = canonical_root.clone(); let mut traversal_policy = policy.clone(); traversal_policy.root = canonical_root.clone(); + let (path_sender, path_receiver) = tokio::sync::mpsc::channel::(WALK_QUEUE); - let traversal = tokio::task::spawn_blocking(move || { + let traversal_task = tokio::task::spawn_blocking(move || { let mut report = TraversalReport { - file_paths: Vec::new(), uncertain_prefixes: Vec::new(), errors: Vec::new(), root_complete: true, @@ -352,8 +495,9 @@ impl MediaScanner { match entry { Ok(entry) if entry.file_type().is_file() => { let path = entry.path(); - if traversal_policy.allows_media(&path) { - report.file_paths.push(path); + if traversal_policy.allows_media(&path) && path_sender.blocking_send(path).is_err() { + // The consumer is gone, so the scan is over. + break; } } Ok(_) => {} @@ -374,152 +518,143 @@ impl MediaScanner { } } report - }) - .await?; - - let file_paths = traversal.file_paths; + }); - let total_files = file_paths.len(); - let existing_in_root = existing_files_map - .keys() - .filter(|path| path.starts_with(&canonical_root)) - .count(); - let suspect_empty_root = total_files == 0 && existing_in_root > 0; - info!( - "Found {} media files, processing in batches of {}", - total_files, BATCH_SIZE - ); - - // Process files in batches let mut result = ScanResult::new(); - result.errors.extend(traversal.errors); - result.complete = traversal.root_complete - && traversal.uncertain_prefixes.is_empty() - && !suspect_empty_root; - if suspect_empty_root { - result.errors.push(ScanError { - path: canonical_root.clone(), - error: "previously populated root is unexpectedly empty; destructive reconciliation deferred" - .to_owned(), - }); - } let mut files_to_insert: Vec = Vec::with_capacity(BATCH_SIZE); let mut files_to_update: Vec = Vec::with_capacity(BATCH_SIZE); - let mut current_paths: HashSet = HashSet::with_capacity(total_files); - let mut processed = 0; - - for path in file_paths { - // jwalk descendants inherit the already-canonical root. It does not - // follow file symlinks, so ordinary entries require no syscall here. - current_paths.insert(path.clone()); - - // Create MediaFile from path - let current_file = match self.create_media_file_from_path(&path).await { - Ok(f) => f, - Err(e) => { - debug!("Failed to create MediaFile for {}: {}", path.display(), e); - result.errors.push(ScanError { - path: path.clone(), - error: e.to_string(), - }); - continue; - } - }; - - // Check if file exists in database - if let Some(existing) = existing_files_map.get(&path) { - if self.fingerprint_needs_update(existing, ¤t_file) { - let mut updated = current_file; - updated.id = Some(existing.id); - updated.created_at = existing.created_at; - updated.updated_at = SystemTime::now(); - files_to_update.push(updated); + let mut processed = 0_usize; + + // Classify each path with a single `stat`, several at a time. + // + // Building a `MediaFile` canonicalizes the path, probes for a subtitle + // sidecar and, for audio, parses the entire container with symphonia. Ask + // the cheap question first, or a library that has not changed is fully + // re-read on every scan — and there is one every five minutes. + // + // Concurrent because a `stat` blocks, on a network filesystem as readily + // as a local one, and because the answers are independent. + let concurrency = std::thread::available_parallelism() + .map(|value| value.get()) + .unwrap_or(4); + let mut pending_reads: Vec = Vec::with_capacity(READ_WINDOW); + { + let mut classified = tokio_stream::wrappers::ReceiverStream::new(path_receiver) + .map(|path| async move { + let metadata = tokio::fs::metadata(&path).await.ok(); + (path, metadata) + }) + .buffer_unordered(concurrency); + + while let Some((path, metadata)) = classified.next().await { + processed += 1; + + // jwalk descendants inherit the already-canonical root. It does + // not follow file symlinks, so no syscall is needed here. + let unchanged = match (existing_files_map.get_mut(&path), metadata) { + (Some(existing), Some(metadata)) => { + // Marking the record is what replaces a second set of every + // path on disk: whatever is left unmarked is what is gone. + existing.seen = true; + !Self::stat_needs_update(existing, &metadata) + } + (Some(existing), None) => { + existing.seen = true; + false + } + // A new file, or one whose `stat` failed — + // `create_media_file_from_path` makes the same call and + // reports the error properly. + (None, _) => false, + }; + + if unchanged { + result.unchanged += 1; } else { - result.unchanged_files.push(existing.clone()); - } - } else { - files_to_insert.push(current_file); - } - - processed += 1; - - // Process batch when full - if files_to_insert.len() >= BATCH_SIZE { - info!( - "Inserting batch of {} files ({}/{})", - files_to_insert.len(), - processed, - total_files - ); - let ids = self - .database_manager - .bulk_store_canonical_media_files(&files_to_insert) - .await?; - for (i, mut file) in files_to_insert.drain(..).enumerate() { - if let Some(id) = ids.get(i) { - file.id = Some(*id); + pending_reads.push(path); + if pending_reads.len() >= READ_WINDOW { + self.read_window( + &mut pending_reads, + &existing_files_map, + &mut files_to_insert, + &mut files_to_update, + &mut result, + concurrency, + ) + .await?; + self.flush_batches( + &mut files_to_insert, + &mut files_to_update, + &mut result, + ) + .await?; } - result.new_files.push(file); } - } - - if files_to_update.len() >= BATCH_SIZE { - info!( - "Updating batch of {} files ({}/{})", - files_to_update.len(), - processed, - total_files - ); - self.database_manager - .bulk_update_canonical_media_files(&files_to_update) - .await?; - result.updated_files.append(&mut files_to_update); - } - // Progress logging every 1000 files - if processed % 1000 == 0 { - info!("Progress: {}/{} files processed", processed, total_files); + if processed.is_multiple_of(10_000) { + debug!("Examined {processed} files"); + } } } - // Process remaining files in last batch - if !files_to_insert.is_empty() { - info!("Inserting final batch of {} files", files_to_insert.len()); - let ids = self - .database_manager - .bulk_store_canonical_media_files(&files_to_insert) - .await?; - for (i, mut file) in files_to_insert.into_iter().enumerate() { - if let Some(id) = ids.get(i) { - file.id = Some(*id); - } - result.new_files.push(file); - } + if !pending_reads.is_empty() { + self.read_window( + &mut pending_reads, + &existing_files_map, + &mut files_to_insert, + &mut files_to_update, + &mut result, + concurrency, + ) + .await?; } + self.flush_batches(&mut files_to_insert, &mut files_to_update, &mut result) + .await?; - if !files_to_update.is_empty() { - info!("Updating final batch of {} files", files_to_update.len()); - self.database_manager - .bulk_update_canonical_media_files(&files_to_update) - .await?; - result.updated_files.extend(files_to_update); + // The walker's own findings — which prefixes it could not read — only + // arrive once the channel has closed, and deletion depends on them. + let traversal = traversal_task.await?; + + let total_files = processed; + let suspect_empty_root = total_files == 0 && existing_in_root > 0; + result.errors.extend(traversal.errors); + result.complete = traversal.root_complete + && traversal.uncertain_prefixes.is_empty() + && !suspect_empty_root; + if suspect_empty_root { + result.errors.push(ScanError { + path: canonical_root.clone(), + error: "previously populated root is unexpectedly empty; destructive reconciliation deferred" + .to_owned(), + }); } - // Find and remove deleted files - let files_to_remove: Vec = existing_files_map - .iter() - .filter(|(path, _)| !current_paths.contains(*path)) - .filter(|(path, _)| path.starts_with(&canonical_root)) // Only remove files under scanned directory - .filter(|(path, _)| { - traversal.root_complete - && !suspect_empty_root - && !traversal + // Whatever the walk never produced is gone from disk. + let reconcile_deletions = + traversal.root_complete && !suspect_empty_root && traversal.uncertain_prefixes.is_empty(); + let files_to_remove: Vec = if reconcile_deletions { + existing_files_map + .iter() + .filter(|(_, indexed)| !indexed.seen) + .map(|(path, _)| path.clone()) + .collect() + } else { + // A partial walk cannot tell "absent" from "unreadable". Where only + // some prefixes are in doubt, everything outside them is still + // decidable. + existing_files_map + .iter() + .filter(|(_, indexed)| !indexed.seen) + .filter(|_| traversal.root_complete && !suspect_empty_root) + .filter(|(path, _)| { + !traversal .uncertain_prefixes .iter() .any(|prefix| path.starts_with(prefix)) - }) - .map(|(_, file)| file.path.clone()) - .collect(); + }) + .map(|(path, _)| path.clone()) + .collect() + }; if !files_to_remove.is_empty() { info!( @@ -529,22 +664,18 @@ impl MediaScanner { self.database_manager .bulk_remove_media_files(&files_to_remove) .await?; - let removed_paths = files_to_remove.iter().collect::>(); - for (path, file) in existing_files_map.iter() { - if removed_paths.contains(path) { - result.removed_files.push(file.clone()); - } - } + result.removed += files_to_remove.len(); } result.total_scanned = total_files; info!( - "Scan completed: {} new, {} updated, {} removed, {} unchanged", - result.new_files.len(), - result.updated_files.len(), - result.removed_files.len(), - result.unchanged_files.len() + "Scan completed: {} new, {} updated, {} removed, {} unchanged, {} files read", + result.new, + result.updated, + result.removed, + result.unchanged, + result.files_read ); Ok(result) @@ -591,7 +722,11 @@ impl MediaScanner { tags: Default::default(), stream: Default::default(), extra_tags: Vec::new(), - tags_version: 0, + // Which reader has examined this record, not which one found + // something. A file with no readable tags — a video, or audio whose + // container will not parse — still counts as examined, or it would + // be opened again on every scan for as long as it exists. + tags_version: crate::platform::filesystem::TAGS_VERSION, subtitle_available: tokio::fs::try_exists(path.with_extension("srt")) .await .unwrap_or(false), diff --git a/crates/vuio-core/src/media/tests.rs b/crates/vuio-core/src/media/tests.rs index e674650..4597d15 100644 --- a/crates/vuio-core/src/media/tests.rs +++ b/crates/vuio-core/src/media/tests.rs @@ -72,20 +72,20 @@ async fn test_media_scanner_path_normalization() { let result = scanner.scan_directory(&temp_path).await.unwrap(); // Verify that files were found and processed - assert_eq!(result.new_files.len(), 1); - let scanned_file = &result.new_files[0]; + assert_eq!(result.new, 1); - // Verify that the path was normalized (should be canonical format) + // Verify that the path was normalized (should be canonical format). The scan + // reports counts, so the record itself is read back from the database — which + // is where the normalization has to have landed for it to matter. let expected_canonical = scanner .filesystem_manager() .get_canonical_path(&test_file_path) .unwrap(); - assert_eq!(scanned_file.path.to_string_lossy(), expected_canonical); - - // Verify the file was stored in the database with canonical path - let stored_file = db.get_file_by_path(&scanned_file.path).await.unwrap(); - assert!(stored_file.is_some()); - let stored_file = stored_file.unwrap(); + let stored_file = db + .get_file_by_path(Path::new(&expected_canonical)) + .await + .unwrap() + .expect("the scanned file must be stored under its canonical path"); assert_eq!(stored_file.path.to_string_lossy(), expected_canonical); // temp_dir dropped here, auto-cleanup @@ -95,67 +95,129 @@ async fn test_media_scanner_path_normalization() { async fn test_scan_result_operations() { let mut result1 = ScanResult::new(); result1.total_scanned = 5; - result1.new_files.push(MediaFile { - id: Some(1), - path: PathBuf::from("/test1.mp4"), - filename: "test1.mp4".to_string(), - size: 1024, - modified: SystemTime::now(), - mime_type: "video/mp4".to_string(), - duration: None, - title: None, - artist: None, - album: None, - genre: None, - track_number: None, - year: None, - album_artist: None, - tags: Default::default(), - stream: Default::default(), - extra_tags: Vec::new(), - tags_version: 0, - subtitle_available: false, - created_at: SystemTime::now(), - updated_at: SystemTime::now(), - }); + result1.new = 1; + result1.files_read = 1; let mut result2 = ScanResult::new(); result2.total_scanned = 3; - result2.updated_files.push(MediaFile { - id: Some(2), - path: PathBuf::from("/test2.mp4"), - filename: "test2.mp4".to_string(), - size: 2048, - modified: SystemTime::now(), - mime_type: "video/mp4".to_string(), - duration: None, - title: None, - artist: None, - album: None, - genre: None, - track_number: None, - year: None, - album_artist: None, - tags: Default::default(), - stream: Default::default(), - extra_tags: Vec::new(), - tags_version: 0, - subtitle_available: false, - created_at: SystemTime::now(), - updated_at: SystemTime::now(), - }); + result2.updated = 1; + result2.removed = 2; + result2.unchanged = 2; + result2.files_read = 1; + result2.complete = false; // Test merge result1.merge(result2); assert_eq!(result1.total_scanned, 8); - assert_eq!(result1.new_files.len(), 1); - assert_eq!(result1.updated_files.len(), 1); + assert_eq!(result1.new, 1); + assert_eq!(result1.updated, 1); + assert_eq!(result1.removed, 2); + assert_eq!(result1.unchanged, 2); + assert_eq!(result1.files_read, 2); + assert_eq!(result1.total_changes(), 4); + assert!( + !result1.complete, + "one incomplete half must make the whole incomplete" + ); // Test summary let summary = result1.summary(); assert!(summary.contains("8 files")); assert!(summary.contains("1 new")); assert!(summary.contains("1 updated")); + assert!(summary.contains("2 removed")); +} + +/// The scan reconciles what is on disk against what is indexed, and it does so +/// by marking the records the walk produced — so what is left unmarked is what +/// has been deleted. This is the destructive half of a scan and had no coverage +/// at all, which is uncomfortable for the one operation that removes user data. +#[tokio::test] +async fn recursive_scan_removes_what_is_gone_from_disk() { + let temp_dir = tempdir().unwrap(); + let db = Arc::new( + SqliteDatabase::new(temp_dir.path().join("deletions.db")) + .await + .unwrap(), + ); + db.initialize().await.unwrap(); + let scanner = + MediaScanner::with_filesystem_manager(Box::new(BaseFileSystemManager::new(true)), db.clone()); + + let root = temp_dir.path().join("media"); + let keep_dir = root.join("keep"); + let doomed_dir = root.join("doomed"); + tokio::fs::create_dir_all(&keep_dir).await.unwrap(); + tokio::fs::create_dir_all(&doomed_dir).await.unwrap(); + + tokio::fs::write(keep_dir.join("stays.mp4"), b"a").await.unwrap(); + let single = keep_dir.join("goes.mp4"); + tokio::fs::write(&single, b"b").await.unwrap(); + tokio::fs::write(doomed_dir.join("one.mp4"), b"c").await.unwrap(); + tokio::fs::write(doomed_dir.join("two.mp4"), b"d").await.unwrap(); + + assert_eq!(scanner.scan_directory_recursive(&root).await.unwrap().new, 4); + + // One file, and then a whole directory. + tokio::fs::remove_file(&single).await.unwrap(); + let after_file = scanner.scan_directory_recursive(&root).await.unwrap(); + assert_eq!(after_file.removed, 1, "a deleted file must leave the index"); + assert_eq!(after_file.unchanged, 3); + assert!(db.get_file_by_path(&single).await.unwrap().is_none()); + + tokio::fs::remove_dir_all(&doomed_dir).await.unwrap(); + let after_dir = scanner.scan_directory_recursive(&root).await.unwrap(); + assert_eq!( + after_dir.removed, 2, + "a directory deleted whole must take its files with it" + ); + assert_eq!(after_dir.unchanged, 1); + + let mut remaining = db.stream_all_media_files(); + let mut survivors = Vec::new(); + while let Some(file) = remaining.next().await { + survivors.push(file.unwrap()); + } + assert_eq!(survivors.len(), 1); + assert_eq!(survivors[0].filename, "stays.mp4"); +} + +/// A scan compares one root against one root. It loads only that subtree's +/// records, and a sibling library — including one whose path is a string prefix +/// of this one — must be neither examined nor deleted. +#[tokio::test] +async fn recursive_scan_leaves_other_roots_alone() { + let temp_dir = tempdir().unwrap(); + let db = Arc::new( + SqliteDatabase::new(temp_dir.path().join("roots.db")) + .await + .unwrap(), + ); + db.initialize().await.unwrap(); + let scanner = + MediaScanner::with_filesystem_manager(Box::new(BaseFileSystemManager::new(true)), db.clone()); + + // `Films` shares a prefix with `Film`, which a `LIKE 'path%'` would sweep up. + let film = temp_dir.path().join("Film"); + let films = temp_dir.path().join("Films"); + tokio::fs::create_dir_all(&film).await.unwrap(); + tokio::fs::create_dir_all(&films).await.unwrap(); + tokio::fs::write(film.join("a.mp4"), b"a").await.unwrap(); + let sibling = films.join("b.mp4"); + tokio::fs::write(&sibling, b"b").await.unwrap(); + + assert_eq!(scanner.scan_directory_recursive(&film).await.unwrap().new, 1); + assert_eq!(scanner.scan_directory_recursive(&films).await.unwrap().new, 1); + + // Rescanning `Film` must not notice, touch, or remove anything under `Films`. + let rescan = scanner.scan_directory_recursive(&film).await.unwrap(); + assert_eq!(rescan.unchanged, 1, "only its own root is compared"); + assert_eq!(rescan.removed, 0); + assert_eq!(rescan.total_scanned, 1); + assert!( + db.get_file_by_path(&sibling).await.unwrap().is_some(), + "the sibling root's file must survive a scan of Film" + ); } #[tokio::test] @@ -196,7 +258,7 @@ async fn test_recursive_scan_optimization() { // First scan to populate database let initial_result = scanner.scan_directory_recursive(&root_dir).await.unwrap(); - assert_eq!(initial_result.new_files.len(), 4); + assert_eq!(initial_result.new, 4); assert_eq!(initial_result.total_changes(), 4); // Verify all files were stored in database @@ -209,10 +271,20 @@ async fn test_recursive_scan_optimization() { // Second scan should find no changes (tests that optimization works correctly) let second_result = scanner.scan_directory_recursive(&root_dir).await.unwrap(); - assert_eq!(second_result.new_files.len(), 0); - assert_eq!(second_result.updated_files.len(), 0); - assert_eq!(second_result.unchanged_files.len(), 4); + assert_eq!(second_result.new, 0); + assert_eq!(second_result.updated, 0); + assert_eq!(second_result.unchanged, 4); assert_eq!(second_result.total_changes(), 0); + // The point of the second scan: it still visited all four files, and opened + // none of them. Reading a file means canonicalizing its path, probing for a + // subtitle sidecar and, for audio, parsing the whole container — which used + // to happen on every scan of an unchanged library, and there is one every + // five minutes. + assert_eq!(second_result.total_scanned, 4); + assert_eq!( + second_result.files_read, 0, + "a rescan of an unchanged library must not open any file" + ); // Verify the optimization is working by checking that we can handle the recursive scan // without making individual database queries for each directory @@ -238,13 +310,23 @@ async fn direct_scan_resolves_only_symlinked_media_entries() { .unwrap(), ); database.initialize().await.unwrap(); - let scanner = - MediaScanner::with_filesystem_manager(Box::new(BaseFileSystemManager::new(true)), database); + let scanner = MediaScanner::with_filesystem_manager( + Box::new(BaseFileSystemManager::new(true)), + database.clone(), + ); let result = scanner.scan_directory(&media_root).await.unwrap(); - assert_eq!(result.new_files.len(), 1); - assert_eq!(result.new_files[0].filename, "visible-name.mp4"); - assert_eq!(result.new_files[0].path, target.canonicalize().unwrap()); + assert_eq!(result.new, 1); + + // Indexed under the link's target, but named for the link the user sees. + let resolved = target.canonicalize().unwrap(); + let stored = database + .get_file_by_path(&resolved) + .await + .unwrap() + .expect("the symlinked entry must be indexed under its resolved target"); + assert_eq!(stored.filename, "visible-name.mp4"); + assert_eq!(stored.path, resolved); } #[test] fn case_policy_compares_path_components_without_changing_boundaries() { diff --git a/crates/vuio-core/src/platform/diagnostics.rs b/crates/vuio-core/src/platform/diagnostics.rs index 79e109c..ea068c5 100644 --- a/crates/vuio-core/src/platform/diagnostics.rs +++ b/crates/vuio-core/src/platform/diagnostics.rs @@ -66,19 +66,20 @@ struct DiagnosticsCollector { #[cfg(feature = "diagnostics")] impl DiagnosticsCollector { fn new() -> Self { + // `System::new()`, never `new_all()`: that walks the whole process table, + // and the only process this reports on is our own. On FreeBSD it also + // means `kinfo_getfile` — the same class of sysinfo call as the disk + // enumeration below, which faults the same way under QEMU. Every + // `AppState` builds one of these, including the ones in tests that never + // take a sample. The cost is that the first sample reports no CPU usage, + // having no earlier reading to difference against. + let mut system = sysinfo::System::new(); + // The CPU list is fixed for the life of the process and is not refreshed + // by `refresh_cpu_usage`, so take it once here. + system.refresh_cpu_list(sysinfo::CpuRefreshKind::nothing().with_cpu_usage()); + Self { - // `new_all()` walks the process table up front, which on FreeBSD - // means `kinfo_getfile` — the same class of sysinfo call as the - // disk enumeration below, and it faults the same way under QEMU. - // Every `AppState` builds one of these, including the ones in tests - // that never take a sample, so on FreeBSD the table is left to the - // `refresh_all()` in `refresh()` that would repeat it anyway. The - // cost is that the first sample reports no CPU usage, having no - // earlier reading to difference against. - #[cfg(target_os = "freebsd")] - system: sysinfo::System::new(), - #[cfg(not(target_os = "freebsd"))] - system: sysinfo::System::new_all(), + system, // sysinfo 0.39 FreeBSD disk enumeration calls getmntinfo and then // slice::from_raw_parts with a pointer that fails Rust's alignment // UB checks (abort in debug; real UB risk in release). Skip disks @@ -92,13 +93,29 @@ impl DiagnosticsCollector { } fn refresh(&mut self) -> RuntimeDiagnostics { - self.system.refresh_all(); + let pid = sysinfo::get_current_pid().ok(); + + // Refresh what is actually read below, rather than `refresh_all()`. That + // walked and stored every process on the host — a couple of megabytes + // rebuilt on every sample, and `/metrics/json` samples every five seconds + // while a dashboard is open — to answer questions about one pid. + self.system.refresh_memory(); + self.system.refresh_cpu_usage(); + if let Some(pid) = pid { + self.system.refresh_processes_specifics( + sysinfo::ProcessesToUpdate::Some(&[pid]), + true, + sysinfo::ProcessRefreshKind::nothing() + .with_memory() + .with_cpu() + .with_tasks(), + ); + } #[cfg(not(target_os = "freebsd"))] self.disks.refresh(true); self.networks.refresh(true); let load = sysinfo::System::load_average(); - let pid = sysinfo::get_current_pid().ok(); let process = pid.and_then(|pid| self.system.process(pid)); let disk_total = self @@ -221,5 +238,25 @@ mod tests { assert_eq!(second.process.pid, std::process::id()); assert!(second.system.available_memory_bytes <= second.system.total_memory_bytes); assert!(second.disks.available_bytes <= second.disks.total_bytes); + + // The sampler refreshes only the specifics these fields need, rather than + // everything sysinfo can collect, so each one is a thing that silently + // becomes zero or `None` if the wrong refresh is dropped. + assert!( + second.system.cpu_count > 0, + "the CPU list must be populated: refresh_cpu_usage does not build it" + ); + assert!( + second.system.total_memory_bytes > 0, + "system memory must be refreshed" + ); + assert!( + second.process.memory_bytes.is_some_and(|bytes| bytes > 0), + "our own process must still be in the table after a targeted refresh" + ); + assert!( + second.process.runtime_seconds.is_some(), + "our own process must still be in the table after a targeted refresh" + ); } } diff --git a/crates/vuio-core/src/watcher/file_ids.rs b/crates/vuio-core/src/watcher/file_ids.rs new file mode 100644 index 0000000..c3bf75d --- /dev/null +++ b/crates/vuio-core/src/watcher/file_ids.rs @@ -0,0 +1,255 @@ +//! A bounded file-id cache for the debounced watcher. +//! +//! `notify-debouncer-full` pairs the two halves of a rename by comparing file +//! system ids, because the `from` path no longer exists by the time the event +//! arrives. Its own [`FileIdMap`](notify_debouncer_full::FileIdMap) does that by +//! walking every watched root and keeping a `HashMap` entry for +//! every file and directory underneath — a recursive walk plus a `stat` per entry +//! at startup, and roughly half a kilobyte of resident memory per file for as long +//! as the server runs. Measured on a 500,000-file library that is a twelve-second +//! walk and about 250 MB held forever. +//! +//! Only backends without rename cookies need it at all: inotify carries a cookie +//! that pairs the halves directly, which is why upstream uses `NoCache` on Linux. +//! macOS and Windows have no cookie, so ids are the only mechanism there. +//! +//! What a paired rename buys is also smaller than it looks. A renamed *directory* +//! is handled by removing the old subtree and rescanning the new one, which is what +//! the unpaired delete-then-create pair produces anyway. A renamed *file* is the +//! only real difference: pairing keeps its row id stable and skips one tag re-read. +//! +//! So this keeps upstream's behaviour while it is cheap and stops paying for it +//! when it is not: the seed walk stops at [`DEFAULT_CAPACITY`] entries, and past +//! that point ids are remembered only for paths the watcher has actually seen an +//! event for. Libraries below the cap behave exactly as before. Above it, renaming +//! a file the watcher has not touched degrades to delete-then-create, which the +//! event handler already supports. + +use notify::RecursiveMode; +use notify_debouncer_full::file_id::{get_file_id, FileId}; +use notify_debouncer_full::FileIdCache; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use tracing::debug; + +/// How many paths to remember. Two generations are kept, so the ceiling is twice +/// this — about 25 MB at the default, against 250 MB unbounded at 500k files. +pub const DEFAULT_CAPACITY: usize = 25_000; + +/// A `FileIdCache` that never grows with the library. +/// +/// Entries live in two generations. Inserts land in `live`; when it fills, it +/// becomes `previous` and a fresh one takes over, so the oldest half is dropped +/// wholesale rather than tracked with per-entry LRU bookkeeping. Lookups check +/// both. See the module docs for what this trades away. +#[derive(Debug)] +pub struct BoundedFileIdCache { + live: HashMap, + previous: HashMap, + capacity: usize, + /// Set once the seed walk has been cut short, so it is logged only the once. + truncated: bool, +} + +impl BoundedFileIdCache { + pub fn new() -> Self { + Self::with_capacity(DEFAULT_CAPACITY) + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + live: HashMap::new(), + previous: HashMap::new(), + capacity: capacity.max(1), + truncated: false, + } + } + + /// Entries across both generations. + pub fn len(&self) -> usize { + self.live.len() + self.previous.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Insert one path, rotating the generations if `live` is full. + fn remember(&mut self, path: PathBuf, id: FileId) { + if self.live.len() >= self.capacity && !self.live.contains_key(&path) { + self.previous = std::mem::take(&mut self.live); + } + self.live.insert(path, id); + } + + /// Seed ids for a tree that already exists, stopping at the cap. + /// + /// Unlike `remember` this never rotates: rotating mid-walk would let a large + /// library evict its own entries and walk to the end for nothing. + fn seed(&mut self, root: &Path, recursive: bool) { + let mut pending = vec![root.to_path_buf()]; + while let Some(dir) = pending.pop() { + let entries = match std::fs::read_dir(&dir) { + Ok(entries) => entries, + Err(_) => continue, + }; + for entry in entries.flatten() { + if self.len() >= self.capacity { + if !self.truncated { + self.truncated = true; + debug!( + "File id cache reached its {} entry cap while seeding {}; \ + renames outside the cache will be seen as delete + create", + self.capacity, + root.display() + ); + } + return; + } + let path = entry.path(); + if let Ok(id) = get_file_id(&path) { + self.live.insert(path.clone(), id); + } + if recursive && entry.file_type().is_ok_and(|kind| kind.is_dir()) { + pending.push(path); + } + } + } + } +} + +impl Default for BoundedFileIdCache { + fn default() -> Self { + Self::new() + } +} + +impl FileIdCache for BoundedFileIdCache { + fn cached_file_id(&self, path: &Path) -> Option> { + self.live.get(path).or_else(|| self.previous.get(path)) + } + + fn add_path(&mut self, path: &Path, recursive_mode: RecursiveMode) { + if path.is_dir() { + self.seed(path, recursive_mode == RecursiveMode::Recursive); + return; + } + if let Ok(id) = get_file_id(path) { + self.remember(path.to_path_buf(), id); + } + } + + fn remove_path(&mut self, path: &Path) { + self.live.retain(|cached, _| !cached.starts_with(path)); + self.previous.retain(|cached, _| !cached.starts_with(path)); + } + + /// Deliberately does nothing. + /// + /// Upstream re-walks every root when the backend drops events — the moment the + /// system is already under load. The dropped events are handled where it + /// matters instead: the watcher marks those roots dirty and the media service + /// rescans them against the index, which needs no file ids. + fn rescan(&mut self, _roots: &[(PathBuf, RecursiveMode)]) {} +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn write_files(dir: &Path, count: usize) { + for index in 0..count { + fs::write(dir.join(format!("file_{index}.mp3")), b"x").unwrap(); + } + } + + #[test] + fn seeding_stops_at_the_cap() { + let temp = tempfile::tempdir().unwrap(); + write_files(temp.path(), 40); + + let mut cache = BoundedFileIdCache::with_capacity(10); + cache.add_path(temp.path(), RecursiveMode::Recursive); + + assert_eq!(cache.len(), 10, "the seed walk must not exceed the cap"); + } + + #[test] + fn seeding_a_small_tree_keeps_every_entry() { + let temp = tempfile::tempdir().unwrap(); + let nested = temp.path().join("Album"); + fs::create_dir(&nested).unwrap(); + write_files(temp.path(), 3); + write_files(&nested, 3); + + let mut cache = BoundedFileIdCache::with_capacity(100); + cache.add_path(temp.path(), RecursiveMode::Recursive); + + // 3 files + the directory + 3 nested files. + assert_eq!(cache.len(), 7); + assert!(cache + .cached_file_id(&nested.join("file_0.mp3")) + .is_some()); + } + + #[test] + fn non_recursive_seeding_stays_at_one_level() { + let temp = tempfile::tempdir().unwrap(); + let nested = temp.path().join("Album"); + fs::create_dir(&nested).unwrap(); + write_files(&nested, 3); + + let mut cache = BoundedFileIdCache::with_capacity(100); + cache.add_path(temp.path(), RecursiveMode::NonRecursive); + + assert_eq!(cache.len(), 1, "only the directory entry itself"); + } + + #[test] + fn individual_inserts_stay_bounded_and_keep_the_newest() { + let temp = tempfile::tempdir().unwrap(); + write_files(temp.path(), 30); + + let mut cache = BoundedFileIdCache::with_capacity(10); + for index in 0..30 { + cache.add_path(&temp.path().join(format!("file_{index}.mp3")), RecursiveMode::NonRecursive); + } + + assert!(cache.len() <= 20, "two generations of 10, at most"); + assert!( + cache + .cached_file_id(&temp.path().join("file_29.mp3")) + .is_some(), + "the most recent path must still be resolvable" + ); + } + + #[test] + fn removing_a_directory_drops_its_children_from_both_generations() { + let temp = tempfile::tempdir().unwrap(); + let nested = temp.path().join("Album"); + fs::create_dir(&nested).unwrap(); + write_files(&nested, 6); + + let mut cache = BoundedFileIdCache::with_capacity(3); + for index in 0..6 { + cache.add_path(&nested.join(format!("file_{index}.mp3")), RecursiveMode::NonRecursive); + } + assert!(!cache.is_empty()); + + cache.remove_path(&nested); + assert_eq!(cache.len(), 0); + } + + #[test] + fn rescan_does_not_rewalk() { + let temp = tempfile::tempdir().unwrap(); + write_files(temp.path(), 5); + + let mut cache = BoundedFileIdCache::with_capacity(100); + cache.rescan(&[(temp.path().to_path_buf(), RecursiveMode::Recursive)]); + + assert!(cache.is_empty(), "a dropped-event rescan must not walk"); + } +} diff --git a/crates/vuio-core/src/watcher/mod.rs b/crates/vuio-core/src/watcher/mod.rs index ad7cfd0..5280e0c 100644 --- a/crates/vuio-core/src/watcher/mod.rs +++ b/crates/vuio-core/src/watcher/mod.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use notify::{Config, RecommendedWatcher, RecursiveMode}; use notify_debouncer_full::{ - new_debouncer_opt, DebounceEventResult, DebouncedEvent, Debouncer, FileIdMap, + new_debouncer_opt, DebounceEventResult, DebouncedEvent, Debouncer, }; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -13,6 +13,9 @@ use tracing::{debug, error, info, warn}; use crate::error::AppResult as Result; use crate::media::ScanPolicy; +mod file_ids; +pub use file_ids::BoundedFileIdCache; + /// Events that can occur in the file system for media files #[derive(Debug, Clone)] pub enum FileSystemEvent { @@ -73,7 +76,7 @@ pub trait FileSystemWatcher: Send + Sync { /// Cross-platform file system watcher implementation pub struct CrossPlatformWatcher { - debouncer: Arc>>>, + debouncer: Arc>>>, event_sender: mpsc::Sender, event_receiver: Arc>>>, watched_paths: Arc>>, @@ -203,6 +206,11 @@ impl CrossPlatformWatcher { is_media_file(path) } + /// Roots that have lost watcher events since the last call, as configured. + /// + /// Returned in the form the configuration uses rather than the normalized + /// key the watch is registered under, because the caller's next move is to + /// match these against `media.directories`. pub fn take_dirty_roots(&self) -> Vec { let mut roots = self.dirty_roots.lock().unwrap_or_else(|e| e.into_inner()); roots.drain().collect() @@ -459,10 +467,15 @@ impl CrossPlatformWatcher { watched_paths.lock().unwrap_or_else(|p| p.into_inner()); let mut dirty = dirty_roots.lock().unwrap_or_else(|p| p.into_inner()); - if let Some(root) = - watched.keys().find(|root| failed_path.starts_with(root)) + // Matched on the normalized key, but recorded + // as the configured path: the caller compares + // this against the roots in the config. + if let Some(registration) = watched + .iter() + .find(|(root, _)| failed_path.starts_with(root)) + .map(|(_, registration)| registration) { - dirty.insert(root.clone()); + dirty.insert(registration.path.clone()); } } } @@ -476,11 +489,11 @@ impl CrossPlatformWatcher { dirty_roots .lock() .unwrap_or_else(|p| p.into_inner()) - .extend(watched.keys().cloned()); + .extend(watched.values().map(|registration| registration.path.clone())); } } }, - FileIdMap::new(), + BoundedFileIdCache::new(), Config::default(), )?; diff --git a/crates/vuio-core/src/web/admin.rs b/crates/vuio-core/src/web/admin.rs index af34681..5e4b642 100644 --- a/crates/vuio-core/src/web/admin.rs +++ b/crates/vuio-core/src/web/admin.rs @@ -279,6 +279,19 @@ const MEDIA_FIELDS: &[FieldSpec] = &[ Impact::Live, "How long a library that has gone offline keeps its indexed content before it is dropped.", ), + noted( + optional( + "media.full_rescan_interval_hours", + "Full rescan interval", + FieldKind::Int { min: 0, max: 8_760 }, + Impact::Live, + "How often every library is swept from scratch, in hours. 0 leaves discovery \ + entirely to the file watcher.", + ), + "Changes are normally picked up by the watcher within seconds. This sweep exists for \ + what the watcher cannot see — a network share that drops events, most often — so it \ + costs a full walk of every library each time it runs.", + ), field( "media.supported_extensions", "File extensions", @@ -296,12 +309,16 @@ const DATABASE_FIELDS: &[FieldSpec] = &[ Impact::Restart, "Where the media index lives. Leave unset for the platform default location.", ), - field( - "database.vacuum_on_startup", - "Compact at startup", - FieldKind::Bool, - Impact::NextStart, - "Reclaim space in the index file at boot. Slows startup on a large library.", + noted( + field( + "database.vacuum_on_startup", + "Compact the index", + FieldKind::Bool, + Impact::NextStart, + "Reclaim free space in the index file when the server starts and stops.", + ), + "Compaction rewrites the whole file, so on a large library it adds time to both. \ + It is not needed for durability — only to give back space left by deletions.", ), noted( field( @@ -313,12 +330,21 @@ const DATABASE_FIELDS: &[FieldSpec] = &[ ), "Applies from the next daily tick; the startup and shutdown backups need a restart.", ), - optional( - "database.cache_mb", - "Index cache", - FieldKind::Int { min: 1, max: 4_096 }, - Impact::Restart, - "Megabytes of memory the index keeps cached.", + noted( + optional( + "database.cache_mb", + "Index cache", + FieldKind::Int { min: 1, max: 4_096 }, + Impact::Restart, + "Megabytes of memory the index keeps cached, per database connection.", + ), + "A budget per connection — one writer plus two to four readers — but only a \ + connection running a large query fills its share, so resident memory grows by \ + roughly this much rather than a multiple of it. Folder browsing does not depend \ + on it. Search does, and as a step rather than a slope: on a very large library \ + it stays slow until the search index fits and then roughly halves. Raising this \ + is worth it only if it goes past that line — below it, the memory is spent and \ + nothing gets faster.", ), ]; diff --git a/crates/vuio-core/src/web/soap/content_directory.rs b/crates/vuio-core/src/web/soap/content_directory.rs index c2523cb..146cc94 100644 --- a/crates/vuio-core/src/web/soap/content_directory.rs +++ b/crates/vuio-core/src/web/soap/content_directory.rs @@ -191,9 +191,6 @@ impl ContentDirectoryHandler { autoplay_enabled: state.current_config().media.autoplay_enabled, update_id: current_update_id, bookmarks, - prefer_online_titles: state.current_config().mediainfo.prefer_online_titles, - min_confidence: state.current_config().mediainfo.min_confidence, - mediainfo: Default::default(), }; let canonical_parent = canonical_browse_path.to_string_lossy().into_owned(); let mime_family = media_type_filter.to_owned(); @@ -380,9 +377,6 @@ impl ContentDirectoryHandler { autoplay_enabled: state.current_config().media.autoplay_enabled, update_id: state.content_update_id.load(Ordering::SeqCst), bookmarks: state.bookmarks.lock().await.snapshot(), - prefer_online_titles: state.current_config().mediainfo.prefer_online_titles, - min_confidence: state.current_config().mediainfo.min_confidence, - mediainfo: Default::default(), }; let starting_index = params.starting_index as usize; let requested_count = browse_page_limit(params); diff --git a/crates/vuio-core/src/web/soap/music.rs b/crates/vuio-core/src/web/soap/music.rs index 457b12e..2d087ca 100644 --- a/crates/vuio-core/src/web/soap/music.rs +++ b/crates/vuio-core/src/web/soap/music.rs @@ -635,9 +635,6 @@ pub(super) async fn render_context( autoplay_enabled: state.current_config().media.autoplay_enabled, update_id: state.content_update_id.load(Ordering::SeqCst), bookmarks, - prefer_online_titles: state.current_config().mediainfo.prefer_online_titles, - min_confidence: state.current_config().mediainfo.min_confidence, - mediainfo: Default::default(), } } diff --git a/crates/vuio-core/src/web/ui.rs b/crates/vuio-core/src/web/ui.rs index 6ec8584..6165c18 100644 --- a/crates/vuio-core/src/web/ui.rs +++ b/crates/vuio-core/src/web/ui.rs @@ -288,8 +288,12 @@ pub async fn media_page_handler( let mut last_id = None; // Fetched titles and synopses, collected up front because the writer // cannot query the session while the session is lending it a row. + // `visit_files_page`, not `visit_files`: this listing pages by + // cursor and never reads the total, and computing that total means + // evaluating the query again — which for a ranked search is the + // expensive half. let mut ids = Vec::with_capacity(fetch_limit); - session.visit_files(&query, offset, fetch_limit, |file| { + session.visit_files_page(&query, offset, fetch_limit, |file| { if let Some(id) = file.id().filter(|id| *id > 0) { ids.push(id); } @@ -299,7 +303,7 @@ pub async fn media_page_handler( .mediainfo_overlays(&ids, min_confidence) .unwrap_or_default(); - let summary = session.visit_files(&query, offset, fetch_limit, |file| { + let visited = session.visit_files_page(&query, offset, fetch_limit, |file| { if emitted >= limit { return Ok(()); } @@ -313,7 +317,7 @@ pub async fn media_page_handler( Ok(()) })?; output.extend_from_slice(b"],\"next_cursor\":"); - let next = (summary.visited > limit).then(|| { + let next = (visited > limit).then(|| { if searching { Some((offset + emitted).to_string()) } else { diff --git a/crates/vuio-core/src/web/ui/js/browse.js b/crates/vuio-core/src/web/ui/js/browse.js index b701395..094c7a8 100644 --- a/crates/vuio-core/src/web/ui/js/browse.js +++ b/crates/vuio-core/src/web/ui/js/browse.js @@ -31,9 +31,6 @@ function render() { const matchesSearch = searchQuery === '' || file.name.toLowerCase().includes(searchQuery) || (file.title || '').toLowerCase().includes(searchQuery) - // Searching for the real title should find a file whose name is a - // release string that does not contain it. - || (file.info_title || '').toLowerCase().includes(searchQuery) || (file.artist || '').toLowerCase().includes(searchQuery) || (file.album || '').toLowerCase().includes(searchQuery); if (currentTab === 'radio') { @@ -277,9 +274,7 @@ function createFileCard(file) { `; card.querySelector('.media-icon-wrapper').innerHTML = iconSvg; const name = card.querySelector('.media-name'); - // A fetched title is the readable one, and for video it is usually the only - // title there is — nothing reads metadata out of a video file. - name.textContent = file.info_title || file.title || file.name; + name.textContent = file.title || file.name; name.title = file.name; const details = card.querySelector('.media-details'); const metadataParts = [file.artist, file.album].filter(Boolean); @@ -290,13 +285,6 @@ function createFileCard(file) { metadata.textContent = metadataParts.join(' — '); details.insertBefore(metadata, details.querySelector('.media-meta')); } - if (file.info_overview) { - const overview = document.createElement('div'); - overview.className = 'media-overview'; - overview.textContent = file.info_overview; - overview.title = file.info_overview; - details.insertBefore(overview, details.querySelector('.media-meta')); - } card.querySelector('.media-size').textContent = file.size_str; card.querySelector('.media-extension').textContent = file.ext; diff --git a/crates/vuio-core/src/web/xml/rendering.rs b/crates/vuio-core/src/web/xml/rendering.rs index fe87491..a6821ab 100644 --- a/crates/vuio-core/src/web/xml/rendering.rs +++ b/crates/vuio-core/src/web/xml/rendering.rs @@ -143,27 +143,6 @@ pub struct BrowseRenderContext { pub autoplay_enabled: bool, pub update_id: u32, pub bookmarks: HashMap, - /// Whether a fetched title outranks the one read from the file's own tags. - pub prefer_online_titles: bool, - /// Matches weaker than this are left out of `mediainfo` entirely. - pub min_confidence: u8, - /// Fetched media info for the items on this page, filled in by the response - /// generators just before rendering. Empty when nothing has been fetched. - pub mediainfo: HashMap, -} - -impl BrowseRenderContext { - /// The fetched title to show for `file_id`, if there is one and it is wanted. - fn online_title(&self, file_id: i64) -> Option<&str> { - if !self.prefer_online_titles { - return None; - } - self.mediainfo - .get(&file_id)? - .title - .as_deref() - .filter(|title| !title.is_empty()) - } } /// UPnP container classes. @@ -314,14 +293,7 @@ pub(super) fn write_media_view( let mime = file.mime_type(); let is_radio = mime == "audio/radio"; let has_srt = file.subtitle_available(); - // A fetched title is the readable one — "Arrival" rather than - // "Arrival.2016.1080p.BluRay.x264-GRP" — so it wins when there is one and the - // operator asked for it. - let title = didl_display_title( - context.online_title(file_id).or_else(|| file.title()), - file.filename(), - context.client, - ); + let title = didl_display_title(file.title(), file.filename(), context.client); write!( output, r#"{}"#, @@ -334,32 +306,6 @@ pub(super) fn write_media_view( } output.write_str("")?; - // Synopsis and genres from the fetch. Video had neither before: nothing read - // metadata out of a video file, so a TV showed a filename and nothing else. - if let Some(overlay) = context.mediainfo.get(&file_id) { - if let Some(overview) = overlay.overview.as_deref().filter(|text| !text.is_empty()) { - write!( - output, - "{}", - xml_escape(overview) - )?; - } - if !mime.starts_with("audio/") { - if let Some(genre) = overlay.genres.first() { - write!(output, "{}", xml_escape(genre))?; - } - // Audio already advertises its cover below; this is what gives a movie - // or an episode a poster for the first time. - if overlay.has_artwork { - write!( - output, - "http://{}:{}/media/{}/cover", - context.server_ip, context.server_port, file_id - )?; - } - } - } - if mime.starts_with("audio/") { if let Some(value) = file.artist() { write!(output, "{}", xml_escape(value))?; @@ -518,40 +464,6 @@ pub(super) fn write_media_view( output.write_str("") } -/// Load the fetched media info for the page about to be rendered. -/// -/// A first pass collects the ids, because the writer cannot ask the session for -/// anything while the session is lending it a row. Both passes run the same -/// indexed query on the same connection, and the first does no formatting, so the -/// cost is one extra index walk rather than a second round trip. -/// -/// A failure here is not worth failing a browse over: the page renders with local -/// metadata, exactly as it did before this feature existed. -fn with_mediainfo( - session: &mut S, - query: &MediaFileQuery, - offset: usize, - limit: usize, - mut context: BrowseRenderContext, -) -> Result { - if limit == 0 { - return Ok(context); - } - let mut ids = Vec::with_capacity(limit); - session.visit_files(query, offset, limit, |file| { - if let Some(id) = file.id().filter(|id| *id > 0) { - ids.push(id); - } - Ok(()) - })?; - - match session.mediainfo_overlays(&ids, context.min_confidence) { - Ok(overlays) => context.mediainfo = overlays, - Err(error) => tracing::debug!(%error, "Could not load media info for this page"), - } - Ok(context) -} - pub fn generate_indexed_browse_response( session: &mut S, canonical_parent: &str, @@ -594,7 +506,6 @@ pub fn generate_indexed_browse_response( path: canonical_parent.to_owned(), mime_family: (!mime_family.is_empty()).then(|| mime_family.to_owned()), }; - let context = with_mediainfo(session, &query, file_offset, file_limit, context)?; let summary = session.visit_files(&query, file_offset, file_limit, |file| { write_media_view(&mut result, object_id, &file, &context) .map_err(|_| anyhow::anyhow!("failed to construct browse XML")) @@ -620,7 +531,6 @@ pub fn generate_indexed_items_response( "#)?; let mut result = SoapResultWriter(&mut response); result.push_str(r#""#); - let context = with_mediainfo(session, &query, starting_index, requested_count, context)?; let summary = session.visit_files(&query, starting_index, requested_count, |file| { write_media_view(&mut result, object_id, &file, &context) .map_err(|_| anyhow::anyhow!("failed to construct browse XML")) diff --git a/crates/vuio-core/tests/audio_integration_tests.rs b/crates/vuio-core/tests/audio_integration_tests.rs index f93ccff..11a1997 100644 --- a/crates/vuio-core/tests/audio_integration_tests.rs +++ b/crates/vuio-core/tests/audio_integration_tests.rs @@ -91,7 +91,7 @@ async fn test_audio_implementation_and_features() { // 3. Scan the directory with MediaScanner let scanner = MediaScanner::with_database(db.clone()); let scan_result = scanner.scan_directory_recursive(&media_dir).await.unwrap(); - assert_eq!(scan_result.new_files.len(), 4); + assert_eq!(scan_result.new, 4); // 4. Verify tag metadata is correctly populated in DB // Check AC/DC @@ -295,7 +295,7 @@ async fn test_cover_art_retrieval_and_xml() { // 4. Scan the directory with MediaScanner let scanner = MediaScanner::with_database(db.clone()); let scan_result = scanner.scan_directory_recursive(&media_dir).await.unwrap(); - assert_eq!(scan_result.new_files.len(), 2); + assert_eq!(scan_result.new, 2); // 5. Get file from DB to find its assigned ID let db_file = db.get_file_by_path(&audio_path).await.unwrap().unwrap(); diff --git a/crates/vuio-core/tests/cmd.txt b/crates/vuio-core/tests/cmd.txt deleted file mode 100644 index f8bd03a..0000000 --- a/crates/vuio-core/tests/cmd.txt +++ /dev/null @@ -1,2 +0,0 @@ -cargo test test_memory_optimized_million_files --release --ignored -- --no -cargo test --release --test million_file_stress_test -- --ignored --nocapture \ No newline at end of file diff --git a/crates/vuio-core/tests/mediainfo_integration_tests.rs b/crates/vuio-core/tests/mediainfo_integration_tests.rs index 6a2bddb..84f2c26 100644 --- a/crates/vuio-core/tests/mediainfo_integration_tests.rs +++ b/crates/vuio-core/tests/mediainfo_integration_tests.rs @@ -611,3 +611,74 @@ async fn browse_json_reports_no_media_info_when_none_was_fetched() { assert!(file["info_title"].is_null()); assert_eq!(file["info_art"], false); } + +#[tokio::test] +async fn dlna_browse_shows_filename_not_mediainfo_title_or_description() { + let temp = tempdir().unwrap(); + let database = Arc::new( + SqliteDatabase::new(temp.path().join("test.db")) + .await + .unwrap(), + ); + database.initialize().await.unwrap(); + + let media_path = temp.path().join("media"); + tokio::fs::create_dir_all(&media_path).await.unwrap(); + let file = MediaFile::new( + media_path.join("Show.Name.S02E05.1080p.mkv"), + 1024, + "video/x-matroska".to_string(), + ); + let id = database.store_media_file(&file).await.unwrap(); + + database + .bulk_store_mediainfo(&[record_for(id, 92)]) + .await + .unwrap(); + + let state = state_with(database, &temp).await; + let router = create_router(state, Surface::Primary); + + let soap_browse = r#" + + + + video/d0 + BrowseDirectChildren + * + 0 + 10 + + + +"#; + + let response = router + .oneshot( + Request::builder() + .method("POST") + .uri("/control/ContentDirectory") + .header( + "soapaction", + "\"urn:schemas-upnp-org:service:ContentDirectory:1#Browse\"", + ) + .header("content-type", "text/xml; charset=utf-8") + .extension(ConnectInfo(test_peer())) + .body(Body::from(soap_browse)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let bytes = axum::body::to_bytes(response.into_body(), 128 * 1024) + .await + .unwrap(); + let body = String::from_utf8(bytes.to_vec()).unwrap(); + + // DLNA client should see original filename, not fetched online title "Some Show" + assert!(body.contains("Show.Name.S02E05.1080p.mkv")); + assert!(!body.contains("<dc:title>Some Show</dc:title>")); + // DLNA should not contain fetched overview / description + assert!(!body.contains("A tale.")); +} diff --git a/scripts/run-large-benchmarks.ps1 b/scripts/run-large-benchmarks.ps1 deleted file mode 100644 index 103485d..0000000 --- a/scripts/run-large-benchmarks.ps1 +++ /dev/null @@ -1,128 +0,0 @@ -# Large Dataset Benchmark Runner (PowerShell) -# This script runs the large dataset benchmarks with proper configuration - -param( - [switch]$Force = $false -) - -Write-Host "=== Large Dataset Benchmark Runner ===" -ForegroundColor Cyan -Write-Host "Warning: These benchmarks will create large datasets and may take hours to complete." -Write-Host "Ensure you have sufficient disk space (5-10 GB) and time available." -Write-Host "" - -# Check if user wants to continue -if (-not $Force) { - $response = Read-Host "Do you want to continue? (y/N)" - if ($response -notmatch "^[Yy]$") { - Write-Host "Benchmark cancelled." -ForegroundColor Yellow - exit 0 - } -} - -# Set environment variables for better performance -$env:RUST_LOG = "info" -$env:SQLX_OFFLINE = "true" - -# Create results directory -$timestamp = Get-Date -Format "yyyyMMdd_HHmmss" -$resultsDir = "benchmark_results_$timestamp" -New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null - -Write-Host "Results will be saved to: $resultsDir" -ForegroundColor Green -Write-Host "" - -# Function to run a benchmark and save results -function Run-Benchmark { - param( - [string]$TestName, - [string]$ResultsDir - ) - - $outputFile = Join-Path $ResultsDir "$TestName.log" - - Write-Host "Running benchmark: $TestName" -ForegroundColor Yellow - Write-Host "Output file: $outputFile" - - try { - # Run with release optimizations for realistic performance - $output = & cargo test --release --test large_dataset_benchmarks $TestName -- --ignored --nocapture 2>&1 - $output | Out-File -FilePath $outputFile -Encoding UTF8 - - if ($LASTEXITCODE -eq 0) { - Write-Host "✓ $TestName completed successfully" -ForegroundColor Green - - # Extract key metrics from output - Write-Host "Key metrics:" - $metrics = $output | Select-String -Pattern "(Total duration|Throughput|Peak memory|Files processed)" | Select-Object -First 5 - $metrics | ForEach-Object { Write-Host " $($_.Line)" } - Write-Host "" - } else { - Write-Host "✗ $TestName failed - check $outputFile for details" -ForegroundColor Red - Write-Host "" - } - } - catch { - Write-Host "✗ $TestName failed with exception: $($_.Exception.Message)" -ForegroundColor Red - $_.Exception.Message | Out-File -FilePath $outputFile -Encoding UTF8 - Write-Host "" - } -} - -# Run individual benchmarks -Write-Host "Starting large dataset benchmarks..." -ForegroundColor Cyan -Write-Host "Note: Each benchmark may take 30+ minutes to complete." -Write-Host "" - -# Benchmark 1: Million file creation -Run-Benchmark -TestName "benchmark_million_file_creation" -ResultsDir $resultsDir - -# Benchmark 2: Million file streaming -Run-Benchmark -TestName "benchmark_million_file_streaming" -ResultsDir $resultsDir - -# Benchmark 3: Database-native cleanup -Run-Benchmark -TestName "benchmark_database_native_cleanup_million_files" -ResultsDir $resultsDir - -# Benchmark 4: Directory operations -Run-Benchmark -TestName "benchmark_directory_operations_million_files" -ResultsDir $resultsDir - -# Benchmark 5: Memory bounded operations -Run-Benchmark -TestName "benchmark_memory_bounded_operations" -ResultsDir $resultsDir - -# Benchmark 6: Database maintenance -Run-Benchmark -TestName "benchmark_database_maintenance_million_files" -ResultsDir $resultsDir - -Write-Host "=== Benchmark Suite Complete ===" -ForegroundColor Cyan -Write-Host "Results saved in: $resultsDir" -ForegroundColor Green -Write-Host "" - -# Generate summary report -$summaryFile = Join-Path $resultsDir "summary.txt" -$summary = @() -$summary += "Large Dataset Benchmark Summary" -$summary += "Generated: $(Get-Date)" -$summary += "System: $env:COMPUTERNAME - $env:OS" -$summary += "" - -Get-ChildItem -Path $resultsDir -Filter "*.log" | ForEach-Object { - $benchmarkName = $_.BaseName - $summary += "=== $benchmarkName ===" - - # Extract key performance metrics - $content = Get-Content $_.FullName -ErrorAction SilentlyContinue - $metrics = $content | Select-String -Pattern "(Total duration|Throughput|Peak memory|Files processed|Database size)" - - if ($metrics) { - $metrics | ForEach-Object { $summary += $_.Line } - } else { - $summary += "No metrics found" - } - $summary += "" -} - -$summary | Out-File -FilePath $summaryFile -Encoding UTF8 - -Write-Host "Summary report generated: $summaryFile" -ForegroundColor Green -Write-Host "" -Write-Host "To view detailed results:" -Write-Host " Get-Content $summaryFile" -Write-Host " Get-ChildItem $resultsDir" \ No newline at end of file diff --git a/scripts/run-large-benchmarks.sh b/scripts/run-large-benchmarks.sh deleted file mode 100644 index 1b53f2b..0000000 --- a/scripts/run-large-benchmarks.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# Large Dataset Benchmark Runner -# This script runs the large dataset benchmarks with proper configuration - -set -e - -echo "=== Large Dataset Benchmark Runner ===" -echo "Warning: These benchmarks will create large datasets and may take hours to complete." -echo "Ensure you have sufficient disk space (5-10 GB) and time available." -echo "" - -# Check if user wants to continue -read -p "Do you want to continue? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Benchmark cancelled." - exit 0 -fi - -# Set environment variables for better performance -export RUST_LOG=info -export SQLX_OFFLINE=true - -# Create results directory -RESULTS_DIR="benchmark_results_$(date +%Y%m%d_%H%M%S)" -mkdir -p "$RESULTS_DIR" - -echo "Results will be saved to: $RESULTS_DIR" -echo "" - -# Function to run a benchmark and save results -run_benchmark() { - local test_name="$1" - local output_file="$RESULTS_DIR/${test_name}.log" - - echo "Running benchmark: $test_name" - echo "Output file: $output_file" - - # Run with release optimizations for realistic performance - if cargo test --release --test large_dataset_benchmarks "$test_name" -- --ignored --nocapture > "$output_file" 2>&1; then - echo "✓ $test_name completed successfully" - - # Extract key metrics from output - echo "Key metrics:" - grep -E "(Total duration|Throughput|Peak memory|Files processed)" "$output_file" | head -5 || true - echo "" - else - echo "✗ $test_name failed - check $output_file for details" - echo "" - fi -} - -# Run individual benchmarks -echo "Starting large dataset benchmarks..." -echo "Note: Each benchmark may take 30+ minutes to complete." -echo "" - -# Benchmark 1: Million file creation -run_benchmark "benchmark_million_file_creation" - -# Benchmark 2: Million file streaming -run_benchmark "benchmark_million_file_streaming" - -# Benchmark 3: Database-native cleanup -run_benchmark "benchmark_database_native_cleanup_million_files" - -# Benchmark 4: Directory operations -run_benchmark "benchmark_directory_operations_million_files" - -# Benchmark 5: Memory bounded operations -run_benchmark "benchmark_memory_bounded_operations" - -# Benchmark 6: Database maintenance -run_benchmark "benchmark_database_maintenance_million_files" - -echo "=== Benchmark Suite Complete ===" -echo "Results saved in: $RESULTS_DIR" -echo "" - -# Generate summary report -SUMMARY_FILE="$RESULTS_DIR/summary.txt" -echo "Large Dataset Benchmark Summary" > "$SUMMARY_FILE" -echo "Generated: $(date)" >> "$SUMMARY_FILE" -echo "System: $(uname -a)" >> "$SUMMARY_FILE" -echo "" >> "$SUMMARY_FILE" - -for log_file in "$RESULTS_DIR"/*.log; do - if [[ -f "$log_file" ]]; then - benchmark_name=$(basename "$log_file" .log) - echo "=== $benchmark_name ===" >> "$SUMMARY_FILE" - - # Extract key performance metrics - grep -E "(Total duration|Throughput|Peak memory|Files processed|Database size)" "$log_file" >> "$SUMMARY_FILE" 2>/dev/null || echo "No metrics found" >> "$SUMMARY_FILE" - echo "" >> "$SUMMARY_FILE" - fi -done - -echo "Summary report generated: $SUMMARY_FILE" -echo "" -echo "To view detailed results:" -echo " cat $RESULTS_DIR/summary.txt" -echo " ls $RESULTS_DIR/" \ No newline at end of file