Skip to content

db: Compute post-apply checksum incrementally #449

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
saleemrashid wants to merge 6 commits into
base: main
Choose a base branch
from
Open

db: Compute post-apply checksum incrementally #449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3cf3564
db: Remove duplicate WAL checksum clearing
saleemrashid Apr 8, 2026
aa5af98
db: Store only latest checksum for WAL pages
saleemrashid Apr 8, 2026
0e3547f
db: Fix switching between WAL and rollback journal
saleemrashid Apr 9, 2026
8f99591
db: Compute incremental checksum in CommitJournal/CommitWAL
saleemrashid Apr 8, 2026
129daad
store: Enable StrictVerify in tests
saleemrashid Apr 9, 2026
fa7bad4
mount: Add tests for CommitJournal incremental checksumming
saleemrashid Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 192 additions & 0 deletions cmd/litefs/mount_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,117 @@ func TestSingleNode_RecoverFromInitialRollback(t *testing.T) {
}
}

// Ensure that CommitJournal correctly computes the incremental checksum when
// pages are modified multiple times.
func TestSingleNode_JournalMultipleUpdates(t *testing.T) {
// This test only applies to the rollback journal, but it passes in WAL mode.

cmd := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
db := testingutil.OpenSQLDB(t, filepath.Join(cmd.Config.FUSE.Dir, "db"))

// Disable page cache so dirty pages are written immediately.
if _, err := db.Exec(`PRAGMA cache_size = 0`); err != nil {
t.Fatal(err)
}
Comment on lines +309 to +312
Copy link

Copilot AI Apr 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PRAGMA cache_size = 0 does not disable SQLite's page cache; it resets the cache size to the default. If the intent is to force more frequent page flush/eviction so the same pages are written multiple times within a transaction, consider using a very small cache (e.g., 1–2 pages) or update the comment to match the actual behavior.

Copilot uses AI. Check for mistakes.
Copy link
Copy Markdown
Member Author

@saleemrashid saleemrashid Apr 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does not seem to be the case? https://github.com/sqlite/sqlite/blob/a5495355c48e65411db52ac88d5b361d4d2c9f35/test/cache.test#L117


if _, err := db.Exec(`CREATE TABLE t (x)`); err != nil {
t.Fatal(err)
}
for i := 0; i < 500; i++ {
if _, err := db.Exec(`INSERT INTO t VALUES (?)`, strings.Repeat("x", 60)); err != nil {
t.Fatal(err)
}
}

tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()

// Update all pages (LiteFS will store the previous page checksum).
if _, err := tx.Exec(`UPDATE t SET x = ?`, strings.Repeat("y", 60)); err != nil {
t.Fatal(err)
}
// Update all pages again (LiteFS should not update the previous page checksum).
if _, err := tx.Exec(`UPDATE t SET x = ?`, strings.Repeat("z", 60)); err != nil {
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
}

// Ensure that CommitJournal correctly computes the checksum when the commit
// truncates dirty pages.
func TestSingleNode_JournalTruncateDirtyPages(t *testing.T) {
if testingutil.IsWALMode() {
// The page count assertion will fail in WAL mode.
t.Skip("test only applies to rollback journal, skipping")
}

cmd := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
db := testingutil.OpenSQLDB(t, filepath.Join(cmd.Config.FUSE.Dir, "db"))

// Make sure the database is truncated on commit.
if _, err := db.Exec(`PRAGMA auto_vacuum = FULL`); err != nil {
t.Fatal(err)
}

if _, err := db.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, x)`); err != nil {
t.Fatal(err)
}
for i := 0; i < 500; i++ {
if _, err := db.Exec(`INSERT INTO t (x) VALUES (?)`, strings.Repeat("x", 60)); err != nil {
t.Fatal(err)
}
}

var pageCount int
if err := db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil {
t.Fatal(err)
}
t.Logf("pre-commit pageCount = %d", pageCount)
if got, want := pageCount, 8; got <= want {
t.Fatalf("got %d pages, want more than %d", got, want)
}

tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()

// Make all pages dirty.
if _, err := tx.Exec(`UPDATE t SET x = ?`, strings.Repeat("y", 60)); err != nil {
t.Fatal(err)
}
// Truncate the database.
if _, err := tx.Exec(`DELETE FROM t WHERE id > ?`, 100); err != nil {
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}

var count int
if err := db.QueryRow(`SELECT COUNT(*) FROM t`).Scan(&count); err != nil {
t.Fatal(err)
}
if got, want := count, 100; got != want {
t.Fatalf("count=%d, want %d", got, want)
}

// Make sure the database was truncated.
if err := db.QueryRow(`PRAGMA page_count`).Scan(&pageCount); err != nil {
t.Fatal(err)
}
t.Logf("post-commit pageCount = %d", pageCount)
if got, want := pageCount, 8; got >= want {
t.Fatalf("got %d pages, want less than %d", got, want)
}
}

func TestSingleNode_DropDB(t *testing.T) {
cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
dsn := filepath.Join(cmd0.Config.FUSE.Dir, "db")
Expand Down Expand Up @@ -949,6 +1060,87 @@ func TestMultiNode_Drop(t *testing.T) {
}
}

// Ensure that CommitJournal correctly computes the checksum when
// switching from WAL mode to rollback journal mode.
func TestMultiNode_WALToJournal(t *testing.T) {
cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
waitForPrimary(t, cmd0)
cmd1 := runMountCommand(t, newMountCommand(t, t.TempDir(), cmd0))
db0 := testingutil.OpenSQLDB(t, filepath.Join(cmd0.Config.FUSE.Dir, "db"))
db1 := testingutil.OpenSQLDB(t, filepath.Join(cmd1.Config.FUSE.Dir, "db"))

// Switch to WAL mode.
if _, err := db0.Exec(`PRAGMA journal_mode = WAL`); err != nil {
t.Fatal(err)
}

// Create a simple table.
if _, err := db0.Exec(`CREATE TABLE t (x)`); err != nil {
t.Fatal(err)
}

for i := 0; i < 10; i++ {
// Write a value.
if _, err := db0.Exec(`INSERT INTO t VALUES (?)`, i); err != nil {
t.Fatal(err)
}

// Ensure it invalidates the page on the secondary.
waitForSync(t, "db", cmd0, cmd1)
var x int
if err := db1.QueryRow(`SELECT MAX(x) FROM t`).Scan(&x); err != nil {
t.Fatal(err)
} else if got, want := x, i; got != want {
t.Fatalf("count=%d, want %d", got, want)
}

// Switch to rollback journal mode after first write.
if i == 0 {
if _, err := db0.Exec(`PRAGMA journal_mode = ` + testingutil.JournalMode()); err != nil {
t.Fatal(err)
}
}
}
}

// Ensure that CommitWAL correctly computes the checksum when
// switching from rollback journal mode to WAL mode.
func TestMultiNode_JournalToWAL(t *testing.T) {
cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
waitForPrimary(t, cmd0)
cmd1 := runMountCommand(t, newMountCommand(t, t.TempDir(), cmd0))
db0 := testingutil.OpenSQLDB(t, filepath.Join(cmd0.Config.FUSE.Dir, "db"))
db1 := testingutil.OpenSQLDB(t, filepath.Join(cmd1.Config.FUSE.Dir, "db"))

// Create a simple table.
if _, err := db0.Exec(`CREATE TABLE t (x)`); err != nil {
t.Fatal(err)
}

for i := 0; i < 10; i++ {
// Write a value.
if _, err := db0.Exec(`INSERT INTO t VALUES (?)`, i); err != nil {
t.Fatal(err)
}

// Ensure it invalidates the page on the secondary.
waitForSync(t, "db", cmd0, cmd1)
var x int
if err := db1.QueryRow(`SELECT MAX(x) FROM t`).Scan(&x); err != nil {
t.Fatal(err)
} else if got, want := x, i; got != want {
t.Fatalf("count=%d, want %d", got, want)
}

// Switch to WAL mode after first write.
if i == 0 {
if _, err := db0.Exec(`PRAGMA journal_mode = WAL`); err != nil {
t.Fatal(err)
}
}
}
}

func TestMultiNode_LateJoinWithSnapshot(t *testing.T) {
cmd0 := runMountCommand(t, newMountCommand(t, t.TempDir(), nil))
waitForPrimary(t, cmd0)
Expand Down
Loading
Loading