tapdb: fix postgres sequence desyncs

cmd/merge-sql-schemas/main.go

@@ -7,7 +7,9 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 
+	"github.com/lightninglabs/taproot-assets/tapdb/sqlc"
 	_ "modernc.org/sqlite" // Register the pure-Go SQLite driver.
 )
 
@@ -41,7 +43,11 @@ func main() {
 		if err != nil {
 			log.Fatalf("failed to read file %s: %v", fname, err)
 		}
-		_, err = db.Exec(string(data))
+		content := string(data)
+		for from, to := range sqlc.SQLiteSchemaReplacements {
+			content = strings.ReplaceAll(content, from, to)
+		}
+		_, err = db.Exec(content)
 		if err != nil {
 			log.Fatalf("error executing migration %s: %v", fname,
 				err)

docs/release-notes/release-notes-0.8.0.md

@@ -89,6 +89,11 @@
   fixes `DecodeAssetPayReq` so `GenesisInfo` is populated consistently,
   including group-key invoice decodes.
 
+* [PR#2047](https://github.com/lightninglabs/taproot-assets/pull/2047)
+  fixes a bug that affected legacy Postgres-backed universe servers that
+  were upgraded to v0.6.0, by which universe leaf inserts could result in
+  duplicate-key errors.
+
 # New Features
 
 ## Functional Enhancements

tapdb/migrations.go

@@ -24,7 +24,7 @@ const (
 	// daemon.
 	//
 	// NOTE: This MUST be updated when a new migration is added.
-	LatestMigrationVersion = 54
+	LatestMigrationVersion = 55
 )
 
 // DatabaseBackend is an interface that contains all methods our different

tapdb/migrations_test.go

@@ -1279,3 +1279,123 @@ func TestMigration54Down(t *testing.T) {
 		"delete should fail after reverting CASCADE",
 	)
 }
+
+// TestSequenceConsistency is a Postgres-only test that verifies every
+// auto-increment sequence is consistent with its table's max ID after
+// all migrations have run. This catches migrations that copy rows with
+// explicit IDs (e.g. INSERT INTO new_t (id, ...) SELECT id, ... FROM
+// old_t) without advancing the sequence.
+func TestSequenceConsistency(t *testing.T) {
+	ctx := context.Background()
+
+	db := NewTestDBWithVersion(t, 30)
+
+	if db.Backend() != sqlc.BackendTypePostgres {
+		t.Skip("sequence consistency only applies to Postgres")
+	}
+
+	// Seed a universe_leaves row with an explicit id = 100.
+	// Migration 31 will copy this row (with its explicit ID)
+	// into a recreated table, which won't advance the sequence.
+	InsertTestdata(
+		t, db.BaseDB,
+		"migrations_test_00031_dummy_data.sql",
+	)
+
+	// Reset every sequence to match its table's current max so
+	// that only desyncs introduced by migrations are detected.
+	_, err := db.ExecContext(ctx, `
+		DO $$
+		DECLARE
+			r RECORD;
+		BEGIN
+			FOR r IN
+				SELECT
+					seq.relname  AS seq_name,
+					tbl.relname  AS tbl_name,
+					a.attname    AS col_name
+				FROM pg_class seq
+				JOIN pg_depend d
+					ON d.objid = seq.oid
+				JOIN pg_class tbl
+					ON d.refobjid = tbl.oid
+				JOIN pg_attribute a
+					ON a.attrelid = tbl.oid
+					AND a.attnum = d.refobjsubid
+				WHERE seq.relkind = 'S'
+				  AND d.deptype  = 'a'
+			LOOP
+				EXECUTE format(
+					'SELECT setval(%L, COALESCE('
+					  || '(SELECT MAX(%I) FROM %I),'
+					  || ' 1))',
+					r.seq_name, r.col_name,
+					r.tbl_name
+				);
+			END LOOP;
+		END
+		$$;
+	`)
+	require.NoError(t, err)
+
+	// Run all remaining migrations.
+	err = db.ExecuteMigrations(TargetLatest)
+	require.NoError(t, err)
+
+	// Find all sequences owned by table columns.
+	rows, err := db.QueryContext(ctx, `
+		SELECT
+			seq.relname  AS sequence_name,
+			tbl.relname  AS table_name,
+			a.attname    AS column_name
+		FROM pg_class seq
+		JOIN pg_depend d    ON d.objid = seq.oid
+		JOIN pg_class tbl   ON d.refobjid = tbl.oid
+		JOIN pg_attribute a ON a.attrelid = tbl.oid
+		                   AND a.attnum = d.refobjsubid
+		WHERE seq.relkind = 'S'
+		  AND d.deptype = 'a'
+	`)
+	require.NoError(t, err)
+
+	defer rows.Close()
+
+	type seqInfo struct {
+		seqName string
+		table   string
+		column  string
+	}
+
+	var sequences []seqInfo
+	for rows.Next() {
+		var s seqInfo
+		err := rows.Scan(&s.seqName, &s.table, &s.column)
+		require.NoError(t, err)
+		sequences = append(sequences, s)
+	}
+	require.NoError(t, rows.Err())
+	require.NotEmpty(t, sequences, "expected at least one sequence")
+
+	for _, s := range sequences {
+		var lastValue int64
+		err := db.QueryRowContext(ctx, fmt.Sprintf(
+			"SELECT last_value FROM %s", s.seqName,
+		)).Scan(&lastValue)
+		require.NoError(t, err)
+
+		var maxID int64
+		err = db.QueryRowContext(ctx, fmt.Sprintf(
+			"SELECT COALESCE(MAX(%s), 0) FROM %s",
+			s.column, s.table,
+		)).Scan(&maxID)
+		require.NoError(t, err)
+
+		require.GreaterOrEqual(
+			t, lastValue, maxID,
+			"sequence %s on %s.%s is behind: "+
+				"last_value=%d, max(%s)=%d",
+			s.seqName, s.table, s.column,
+			lastValue, s.column, maxID,
+		)
+	}
+}

tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.down.sql

@@ -0,0 +1,2 @@
+-- Sequence reset is forward-only; no rollback needed.
+SELECT 1;

tapdb/sqlc/migrations/000055_fix_universe_leaves_seq.up.sql

@@ -0,0 +1,11 @@
+-- Reset BIGSERIAL sequences that were left behind by migrations
+-- that inserted rows with explicit id values.
+--
+-- Migration 31 copied universe_leaves rows with explicit ids.
+-- Migration 40 inserted supply_commit_states (0-6) and
+-- supply_commit_update_types (0-2) with explicit ids.
+--
+-- On SQLite these are no-ops (replaced via sqliteSchemaReplacements).
+SELECT setval(pg_get_serial_sequence('universe_leaves', 'id'), COALESCE((SELECT MAX(id) FROM universe_leaves), 1), (SELECT COUNT(*) FROM universe_leaves) > 0);
+SELECT setval(pg_get_serial_sequence('supply_commit_states', 'id'), COALESCE((SELECT MAX(id) FROM supply_commit_states), 1), (SELECT COUNT(*) FROM supply_commit_states) > 0);
+SELECT setval(pg_get_serial_sequence('supply_commit_update_types', 'id'), COALESCE((SELECT MAX(id) FROM supply_commit_update_types), 1), (SELECT COUNT(*) FROM supply_commit_update_types) > 0);

tapdb/sqlc/replacements.go

@@ -0,0 +1,25 @@
+package sqlc
+
+// SQLiteSchemaReplacements maps Postgres-specific SQL fragments to
+// SQLite-compatible no-ops. Used by both the runtime migration
+// layer (tapdb/sqlite.go) and the schema merge tool
+// (cmd/merge-sql-schemas).
+var SQLiteSchemaReplacements = map[string]string{
+	"SELECT setval(pg_get_serial_sequence(" +
+		"'universe_leaves', 'id'), COALESCE((" +
+		"SELECT MAX(id) FROM universe_leaves" +
+		"), 1), (SELECT COUNT(*) FROM " +
+		"universe_leaves) > 0);": "SELECT 1;",
+	"SELECT setval(pg_get_serial_sequence(" +
+		"'supply_commit_states', 'id'), " +
+		"COALESCE((SELECT MAX(id) FROM " +
+		"supply_commit_states), 1), (SELECT " +
+		"COUNT(*) FROM supply_commit_states" +
+		") > 0);": "SELECT 1;",
+	"SELECT setval(pg_get_serial_sequence(" +
+		"'supply_commit_update_types', 'id'), " +
+		"COALESCE((SELECT MAX(id) FROM " +
+		"supply_commit_update_types), 1), " +
+		"(SELECT COUNT(*) FROM " +
+		"supply_commit_update_types) > 0);": "SELECT 1;",
+}

tapdb/sqlite.go

@@ -38,10 +38,10 @@ const (
 )
 
 var (
-	// sqliteSchemaReplacements is a map of schema strings that need to be
-	// replaced for sqlite. There currently aren't any replacements, because
-	// the SQL files are written with SQLite compatibility in mind.
-	sqliteSchemaReplacements = map[string]string{}
+	// sqliteSchemaReplacements maps Postgres-specific SQL
+	// fragments to SQLite-compatible no-ops. The canonical map
+	// lives in tapdb/sqlc/replacements.go.
+	sqliteSchemaReplacements = sqlc.SQLiteSchemaReplacements
 )
 
 // SqliteConfig holds all the config arguments needed to interact with our