From 0d486f90856f67f6b41102b3bf84661bd19dcb1c Mon Sep 17 00:00:00 2001 From: Jared Tobin Date: Thu, 2 Apr 2026 16:13:02 +0800 Subject: [PATCH 1/4] tapdb: fix Postgres sequence desyncs Migration 31 copied universe_leaves rows with explicit ids, leaving the BIGSERIAL sequence behind the actual data. New inserts then hit duplicate-key errors. Add migration 55 to reset the universe_leaves sequence via setval(). Also reset supply_commit_states and supply_commit_update_types (migration 40 inserted explicit ids into these enum tables); these are benign since no code path auto-increments into them, but fixing them keeps sequence state consistent. On SQLite the statements are replaced with no-ops. --- cmd/merge-sql-schemas/main.go | 30 ++++++++++++++++++- tapdb/migrations.go | 2 +- .../000055_fix_universe_leaves_seq.down.sql | 2 ++ .../000055_fix_universe_leaves_seq.up.sql | 11 +++++++ tapdb/sqlite.go | 26 +++++++++++++--- 5 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.down.sql create mode 100644 tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.up.sql diff --git a/cmd/merge-sql-schemas/main.go b/cmd/merge-sql-schemas/main.go index eac93e68aa..dd0f512742 100644 --- a/cmd/merge-sql-schemas/main.go +++ b/cmd/merge-sql-schemas/main.go @@ -7,10 +7,34 @@ import ( "path/filepath" "regexp" "sort" + "strings" _ "modernc.org/sqlite" // Register the pure-Go SQLite driver. ) +// sqliteReplacements maps Postgres-specific SQL fragments to +// SQLite-compatible no-ops. This mirrors sqliteSchemaReplacements +// in tapdb/sqlite.go. +var sqliteReplacements = map[string]string{ + "SELECT setval(pg_get_serial_sequence(" + + "'universe_leaves', 'id'), COALESCE((" + + "SELECT MAX(id) FROM universe_leaves" + + "), 1), (SELECT COUNT(*) FROM " + + "universe_leaves) > 0);": "SELECT 1;", + "SELECT setval(pg_get_serial_sequence(" + + "'supply_commit_states', 'id'), " + + "COALESCE((SELECT MAX(id) FROM " + + "supply_commit_states), 1), (SELECT " + + "COUNT(*) FROM supply_commit_states" + + ") > 0);": "SELECT 1;", + "SELECT setval(pg_get_serial_sequence(" + + "'supply_commit_update_types', 'id'), " + + "COALESCE((SELECT MAX(id) FROM " + + "supply_commit_update_types), 1), " + + "(SELECT COUNT(*) FROM " + + "supply_commit_update_types) > 0);": "SELECT 1;", +} + func main() { // Open an in-memory SQLite database. db, err := sql.Open("sqlite", ":memory:") @@ -41,7 +65,11 @@ func main() { if err != nil { log.Fatalf("failed to read file %s: %v", fname, err) } - _, err = db.Exec(string(data)) + content := string(data) + for from, to := range sqliteReplacements { + content = strings.ReplaceAll(content, from, to) + } + _, err = db.Exec(content) if err != nil { log.Fatalf("error executing migration %s: %v", fname, err) diff --git a/tapdb/migrations.go b/tapdb/migrations.go index 0f0981e7be..b316711de4 100644 --- a/tapdb/migrations.go +++ b/tapdb/migrations.go @@ -24,7 +24,7 @@ const ( // daemon. // // NOTE: This MUST be updated when a new migration is added. - LatestMigrationVersion = 54 + LatestMigrationVersion = 55 ) // DatabaseBackend is an interface that contains all methods our different diff --git a/tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.down.sql b/tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.down.sql new file mode 100644 index 0000000000..a1af64ac2f --- /dev/null +++ b/tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.down.sql @@ -0,0 +1,2 @@ +-- Sequence reset is forward-only; no rollback needed. +SELECT 1; diff --git a/tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.up.sql b/tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.up.sql new file mode 100644 index 0000000000..866732555d --- /dev/null +++ b/tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.up.sql @@ -0,0 +1,11 @@ +-- Reset BIGSERIAL sequences that were left behind by migrations +-- that inserted rows with explicit id values. +-- +-- Migration 31 copied universe_leaves rows with explicit ids. +-- Migration 40 inserted supply_commit_states (0-6) and +-- supply_commit_update_types (0-2) with explicit ids. +-- +-- On SQLite these are no-ops (replaced via sqliteSchemaReplacements). +SELECT setval(pg_get_serial_sequence('universe_leaves', 'id'), COALESCE((SELECT MAX(id) FROM universe_leaves), 1), (SELECT COUNT(*) FROM universe_leaves) > 0); +SELECT setval(pg_get_serial_sequence('supply_commit_states', 'id'), COALESCE((SELECT MAX(id) FROM supply_commit_states), 1), (SELECT COUNT(*) FROM supply_commit_states) > 0); +SELECT setval(pg_get_serial_sequence('supply_commit_update_types', 'id'), COALESCE((SELECT MAX(id) FROM supply_commit_update_types), 1), (SELECT COUNT(*) FROM supply_commit_update_types) > 0); diff --git a/tapdb/sqlite.go b/tapdb/sqlite.go index 888b64b61e..a27fd8d1d2 100644 --- a/tapdb/sqlite.go +++ b/tapdb/sqlite.go @@ -38,10 +38,28 @@ const ( ) var ( - // sqliteSchemaReplacements is a map of schema strings that need to be - // replaced for sqlite. There currently aren't any replacements, because - // the SQL files are written with SQLite compatibility in mind. - sqliteSchemaReplacements = map[string]string{} + // sqliteSchemaReplacements is a map of schema strings that need + // to be replaced for sqlite. Migration 55 uses Postgres-only + // setval() calls that must be replaced with no-ops on SQLite. + sqliteSchemaReplacements = map[string]string{ + "SELECT setval(pg_get_serial_sequence(" + + "'universe_leaves', 'id'), COALESCE((" + + "SELECT MAX(id) FROM universe_leaves" + + "), 1), (SELECT COUNT(*) FROM " + + "universe_leaves) > 0);": "SELECT 1;", + "SELECT setval(pg_get_serial_sequence(" + + "'supply_commit_states', 'id'), " + + "COALESCE((SELECT MAX(id) FROM " + + "supply_commit_states), 1), (SELECT " + + "COUNT(*) FROM supply_commit_states" + + ") > 0);": "SELECT 1;", + "SELECT setval(pg_get_serial_sequence(" + + "'supply_commit_update_types', 'id'), " + + "COALESCE((SELECT MAX(id) FROM " + + "supply_commit_update_types), 1), " + + "(SELECT COUNT(*) FROM " + + "supply_commit_update_types) > 0);": "SELECT 1;", + } ) // SqliteConfig holds all the config arguments needed to interact with our From 381edabaf4ca06c46f6dd6d5d28f8adb519ab8c5 Mon Sep 17 00:00:00 2001 From: Jared Tobin Date: Thu, 2 Apr 2026 16:13:06 +0800 Subject: [PATCH 2/4] tapdb: add generic Postgres sequence consistency test Add TestSequenceConsistency, a Postgres-only test that verifies every BIGSERIAL sequence is consistent with its table's max ID after all migrations have run. This catches migrations that recreate tables and copy rows with explicit IDs without advancing the sequence. --- tapdb/migrations_test.go | 120 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tapdb/migrations_test.go b/tapdb/migrations_test.go index 2301d3fec4..ebf25f4e6f 100644 --- a/tapdb/migrations_test.go +++ b/tapdb/migrations_test.go @@ -1279,3 +1279,123 @@ func TestMigration54Down(t *testing.T) { "delete should fail after reverting CASCADE", ) } + +// TestSequenceConsistency is a Postgres-only test that verifies every +// auto-increment sequence is consistent with its table's max ID after +// all migrations have run. This catches migrations that copy rows with +// explicit IDs (e.g. INSERT INTO new_t (id, ...) SELECT id, ... FROM +// old_t) without advancing the sequence. +func TestSequenceConsistency(t *testing.T) { + ctx := context.Background() + + db := NewTestDBWithVersion(t, 30) + + if db.Backend() != sqlc.BackendTypePostgres { + t.Skip("sequence consistency only applies to Postgres") + } + + // Seed a universe_leaves row with an explicit id = 100. + // Migration 31 will copy this row (with its explicit ID) + // into a recreated table, which won't advance the sequence. + InsertTestdata( + t, db.BaseDB, + "migrations_test_00031_dummy_data.sql", + ) + + // Reset every sequence to match its table's current max so + // that only desyncs introduced by migrations are detected. + _, err := db.ExecContext(ctx, ` + DO $$ + DECLARE + r RECORD; + BEGIN + FOR r IN + SELECT + seq.relname AS seq_name, + tbl.relname AS tbl_name, + a.attname AS col_name + FROM pg_class seq + JOIN pg_depend d + ON d.objid = seq.oid + JOIN pg_class tbl + ON d.refobjid = tbl.oid + JOIN pg_attribute a + ON a.attrelid = tbl.oid + AND a.attnum = d.refobjsubid + WHERE seq.relkind = 'S' + AND d.deptype = 'a' + LOOP + EXECUTE format( + 'SELECT setval(%L, COALESCE(' + || '(SELECT MAX(%I) FROM %I),' + || ' 1))', + r.seq_name, r.col_name, + r.tbl_name + ); + END LOOP; + END + $$; + `) + require.NoError(t, err) + + // Run all remaining migrations. + err = db.ExecuteMigrations(TargetLatest) + require.NoError(t, err) + + // Find all sequences owned by table columns. + rows, err := db.QueryContext(ctx, ` + SELECT + seq.relname AS sequence_name, + tbl.relname AS table_name, + a.attname AS column_name + FROM pg_class seq + JOIN pg_depend d ON d.objid = seq.oid + JOIN pg_class tbl ON d.refobjid = tbl.oid + JOIN pg_attribute a ON a.attrelid = tbl.oid + AND a.attnum = d.refobjsubid + WHERE seq.relkind = 'S' + AND d.deptype = 'a' + `) + require.NoError(t, err) + + defer rows.Close() + + type seqInfo struct { + seqName string + table string + column string + } + + var sequences []seqInfo + for rows.Next() { + var s seqInfo + err := rows.Scan(&s.seqName, &s.table, &s.column) + require.NoError(t, err) + sequences = append(sequences, s) + } + require.NoError(t, rows.Err()) + require.NotEmpty(t, sequences, "expected at least one sequence") + + for _, s := range sequences { + var lastValue int64 + err := db.QueryRowContext(ctx, fmt.Sprintf( + "SELECT last_value FROM %s", s.seqName, + )).Scan(&lastValue) + require.NoError(t, err) + + var maxID int64 + err = db.QueryRowContext(ctx, fmt.Sprintf( + "SELECT COALESCE(MAX(%s), 0) FROM %s", + s.column, s.table, + )).Scan(&maxID) + require.NoError(t, err) + + require.GreaterOrEqual( + t, lastValue, maxID, + "sequence %s on %s.%s is behind: "+ + "last_value=%d, max(%s)=%d", + s.seqName, s.table, s.column, + lastValue, s.column, maxID, + ) + } +} From ff64b5e428319c9427dde1461d7601e00fdc2497 Mon Sep 17 00:00:00 2001 From: Jared Tobin Date: Wed, 15 Apr 2026 11:17:40 +0800 Subject: [PATCH 3/4] tapdb: deduplicate SQLite schema replacement map Move the Postgres-to-SQLite replacement map into tapdb/sqlc/replacements.go so both tapdb/sqlite.go and cmd/merge-sql-schemas share a single source of truth. --- cmd/merge-sql-schemas/main.go | 26 ++------------------------ tapdb/sqlc/replacements.go | 25 +++++++++++++++++++++++++ tapdb/sqlite.go | 26 ++++---------------------- 3 files changed, 31 insertions(+), 46 deletions(-) create mode 100644 tapdb/sqlc/replacements.go diff --git a/cmd/merge-sql-schemas/main.go b/cmd/merge-sql-schemas/main.go index dd0f512742..6af2d06a29 100644 --- a/cmd/merge-sql-schemas/main.go +++ b/cmd/merge-sql-schemas/main.go @@ -9,32 +9,10 @@ import ( "sort" "strings" + "github.com/lightninglabs/taproot-assets/tapdb/sqlc" _ "modernc.org/sqlite" // Register the pure-Go SQLite driver. ) -// sqliteReplacements maps Postgres-specific SQL fragments to -// SQLite-compatible no-ops. This mirrors sqliteSchemaReplacements -// in tapdb/sqlite.go. -var sqliteReplacements = map[string]string{ - "SELECT setval(pg_get_serial_sequence(" + - "'universe_leaves', 'id'), COALESCE((" + - "SELECT MAX(id) FROM universe_leaves" + - "), 1), (SELECT COUNT(*) FROM " + - "universe_leaves) > 0);": "SELECT 1;", - "SELECT setval(pg_get_serial_sequence(" + - "'supply_commit_states', 'id'), " + - "COALESCE((SELECT MAX(id) FROM " + - "supply_commit_states), 1), (SELECT " + - "COUNT(*) FROM supply_commit_states" + - ") > 0);": "SELECT 1;", - "SELECT setval(pg_get_serial_sequence(" + - "'supply_commit_update_types', 'id'), " + - "COALESCE((SELECT MAX(id) FROM " + - "supply_commit_update_types), 1), " + - "(SELECT COUNT(*) FROM " + - "supply_commit_update_types) > 0);": "SELECT 1;", -} - func main() { // Open an in-memory SQLite database. db, err := sql.Open("sqlite", ":memory:") @@ -66,7 +44,7 @@ func main() { log.Fatalf("failed to read file %s: %v", fname, err) } content := string(data) - for from, to := range sqliteReplacements { + for from, to := range sqlc.SQLiteSchemaReplacements { content = strings.ReplaceAll(content, from, to) } _, err = db.Exec(content) diff --git a/tapdb/sqlc/replacements.go b/tapdb/sqlc/replacements.go new file mode 100644 index 0000000000..236645ba6e --- /dev/null +++ b/tapdb/sqlc/replacements.go @@ -0,0 +1,25 @@ +package sqlc + +// SQLiteSchemaReplacements maps Postgres-specific SQL fragments to +// SQLite-compatible no-ops. Used by both the runtime migration +// layer (tapdb/sqlite.go) and the schema merge tool +// (cmd/merge-sql-schemas). +var SQLiteSchemaReplacements = map[string]string{ + "SELECT setval(pg_get_serial_sequence(" + + "'universe_leaves', 'id'), COALESCE((" + + "SELECT MAX(id) FROM universe_leaves" + + "), 1), (SELECT COUNT(*) FROM " + + "universe_leaves) > 0);": "SELECT 1;", + "SELECT setval(pg_get_serial_sequence(" + + "'supply_commit_states', 'id'), " + + "COALESCE((SELECT MAX(id) FROM " + + "supply_commit_states), 1), (SELECT " + + "COUNT(*) FROM supply_commit_states" + + ") > 0);": "SELECT 1;", + "SELECT setval(pg_get_serial_sequence(" + + "'supply_commit_update_types', 'id'), " + + "COALESCE((SELECT MAX(id) FROM " + + "supply_commit_update_types), 1), " + + "(SELECT COUNT(*) FROM " + + "supply_commit_update_types) > 0);": "SELECT 1;", +} diff --git a/tapdb/sqlite.go b/tapdb/sqlite.go index a27fd8d1d2..b6a6a50e9b 100644 --- a/tapdb/sqlite.go +++ b/tapdb/sqlite.go @@ -38,28 +38,10 @@ const ( ) var ( - // sqliteSchemaReplacements is a map of schema strings that need - // to be replaced for sqlite. Migration 55 uses Postgres-only - // setval() calls that must be replaced with no-ops on SQLite. - sqliteSchemaReplacements = map[string]string{ - "SELECT setval(pg_get_serial_sequence(" + - "'universe_leaves', 'id'), COALESCE((" + - "SELECT MAX(id) FROM universe_leaves" + - "), 1), (SELECT COUNT(*) FROM " + - "universe_leaves) > 0);": "SELECT 1;", - "SELECT setval(pg_get_serial_sequence(" + - "'supply_commit_states', 'id'), " + - "COALESCE((SELECT MAX(id) FROM " + - "supply_commit_states), 1), (SELECT " + - "COUNT(*) FROM supply_commit_states" + - ") > 0);": "SELECT 1;", - "SELECT setval(pg_get_serial_sequence(" + - "'supply_commit_update_types', 'id'), " + - "COALESCE((SELECT MAX(id) FROM " + - "supply_commit_update_types), 1), " + - "(SELECT COUNT(*) FROM " + - "supply_commit_update_types) > 0);": "SELECT 1;", - } + // sqliteSchemaReplacements maps Postgres-specific SQL + // fragments to SQLite-compatible no-ops. The canonical map + // lives in tapdb/sqlc/replacements.go. + sqliteSchemaReplacements = sqlc.SQLiteSchemaReplacements ) // SqliteConfig holds all the config arguments needed to interact with our From 68f650c27fec4a15553d13ae5a5b527a2d8bbe1d Mon Sep 17 00:00:00 2001 From: Jared Tobin Date: Thu, 2 Apr 2026 16:51:33 +0800 Subject: [PATCH 4/4] docs: add release note --- docs/release-notes/release-notes-0.8.0.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release-notes/release-notes-0.8.0.md b/docs/release-notes/release-notes-0.8.0.md index c2d0495fb7..5ea7c9708a 100644 --- a/docs/release-notes/release-notes-0.8.0.md +++ b/docs/release-notes/release-notes-0.8.0.md @@ -89,6 +89,11 @@ fixes `DecodeAssetPayReq` so `GenesisInfo` is populated consistently, including group-key invoice decodes. +* [PR#2047](https://github.com/lightninglabs/taproot-assets/pull/2047) + fixes a bug that affected legacy Postgres-backed universe servers that + were upgraded to v0.6.0, by which universe leaf inserts could result in + duplicate-key errors. + # New Features ## Functional Enhancements