fix(spruce): Prevent OOM during ROM cache population

cache/populate.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	DefaultRomPageSize           = 1000
+	DefaultRomPageSize           = 250
 	MaxConcurrentPlatformFetches = 10
 )
 
@@ -174,9 +174,9 @@ func (cm *Manager) fetchPlatformGames(platform romm.Platform, opts *fetchOpts) (
 		client = romm.NewClientFromHost(cm.host, cm.config.GetApiTimeout())
 	}
 
-	var allGames []romm.Rom
 	offset := 0
 	expectedTotal := 0
+	totalSaved := 0
 
 	for {
 		q := romm.GetRomsQuery{
@@ -198,42 +198,50 @@ func (cm *Manager) fetchPlatformGames(platform romm.Platform, opts *fetchOpts) (
 
 		if offset == 0 {
 			expectedTotal = res.Total
-			// Pre-allocate the exact slice capacity to prevent memory spikes
-			allGames = make([]romm.Rom, 0, expectedTotal)
 		}
 
-		allGames = append(allGames, res.Items...)
+		// Persist each page before fetching the next one. Retaining every ROM for a
+		// platform here can exhaust 128MB handhelds, especially when WithFiles adds
+		// large nested payloads. SavePlatformGames already replaces rows per game,
+		// so page-sized transactions preserve both full and incremental semantics.
+		if len(res.Items) > 0 {
+			if err := cm.SavePlatformGames(platform.ID, res.Items); err != nil {
+				return 0, err
+			}
+			totalSaved += len(res.Items)
+		}
 
 		if opts.onProgress != nil && len(res.Items) > 0 {
 			opts.onProgress(len(res.Items))
 		}
 		if opts.onPctProgress != nil && expectedTotal > 0 {
-			pct := float64(len(allGames)) / float64(expectedTotal)
+			pct := float64(totalSaved) / float64(expectedTotal)
 			if pct > 1.0 {
 				pct = 1.0
 			}
 			opts.onPctProgress.Store(pct)
 		}
 
-		if len(allGames) >= expectedTotal || len(res.Items) == 0 || len(res.Items) < DefaultRomPageSize {
+		if (expectedTotal > 0 && totalSaved >= expectedTotal) || len(res.Items) == 0 || len(res.Items) < DefaultRomPageSize {
 			break
 		}
 
 		offset += len(res.Items)
+		runtime.GC()
 	}
 
 	if opts.updatedAfter != "" {
 		logger.Debug("Fetched updated platform games",
 			"platform", platform.Name,
-			"count", len(allGames),
+			"count", totalSaved,
 			"updated_after", opts.updatedAfter)
 	} else {
 		logger.Debug("Cached platform games",
 			"platform", platform.Name,
-			"count", len(allGames))
+			"count", totalSaved)
 	}
 
-	return len(allGames), cm.SavePlatformGames(platform.ID, allGames)
+	return totalSaved, nil
 }
 
 func (cm *Manager) fetchAndCacheCollectionsWithProgress(progress *atomic.Float64, progressStart, progressEnd float64) int {

cache/populate_test.go

@@ -0,0 +1,86 @@
+package cache
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	"grout/romm"
+)
+
+func TestFetchPlatformGamesPersistsEachPageBeforeFetchingNext(t *testing.T) {
+	cm := newTestManager(t)
+	const platformID = 42
+	const total = 501
+
+	var offsets []int
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/roms" {
+			t.Errorf("path = %q, want /api/roms", r.URL.Path)
+			http.NotFound(w, r)
+			return
+		}
+
+		limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
+		offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
+		offsets = append(offsets, offset)
+		if limit != DefaultRomPageSize {
+			t.Errorf("limit = %d, want %d", limit, DefaultRomPageSize)
+		}
+
+		// The previous page must already be committed before the next request.
+		// Retaining all pages until the end is what exhausts 128MB handhelds.
+		if offset > 0 {
+			var cached int
+			if err := cm.db.QueryRow("SELECT COUNT(*) FROM games").Scan(&cached); err != nil {
+				t.Errorf("count cached games: %v", err)
+			} else if cached != offset {
+				t.Errorf("before offset %d request, cached games = %d", offset, cached)
+			}
+		}
+
+		end := offset + limit
+		if end > total {
+			end = total
+		}
+		items := make([]romm.Rom, 0, end-offset)
+		for id := offset + 1; id <= end; id++ {
+			items = append(items, romm.Rom{
+				ID:             id,
+				PlatformID:     platformID,
+				PlatformFSSlug: "test",
+				Name:           "Game " + strconv.Itoa(id),
+			})
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(romm.PaginatedRoms{
+			Items: items, Total: total, Limit: limit, Offset: offset,
+		}); err != nil {
+			t.Errorf("encode response: %v", err)
+		}
+	}))
+	defer server.Close()
+
+	count, err := cm.fetchPlatformGames(romm.Platform{ID: platformID, Name: "Test"}, &fetchOpts{
+		client: romm.NewClient(server.URL),
+	})
+	if err != nil {
+		t.Fatalf("fetchPlatformGames: %v", err)
+	}
+	if count != total {
+		t.Fatalf("count = %d, want %d", count, total)
+	}
+
+	wantOffsets := []int{0, 250, 500}
+	if len(offsets) != len(wantOffsets) {
+		t.Fatalf("offsets = %v, want %v", offsets, wantOffsets)
+	}
+	for i := range wantOffsets {
+		if offsets[i] != wantOffsets[i] {
+			t.Fatalf("offsets = %v, want %v", offsets, wantOffsets)
+		}
+	}
+}