diff --git a/cache/populate.go b/cache/populate.go index 3519fecd..d8d00b1d 100644 --- a/cache/populate.go +++ b/cache/populate.go @@ -11,7 +11,7 @@ import ( ) const ( - DefaultRomPageSize = 1000 + DefaultRomPageSize = 250 MaxConcurrentPlatformFetches = 10 ) @@ -174,9 +174,9 @@ func (cm *Manager) fetchPlatformGames(platform romm.Platform, opts *fetchOpts) ( client = romm.NewClientFromHost(cm.host, cm.config.GetApiTimeout()) } - var allGames []romm.Rom offset := 0 expectedTotal := 0 + totalSaved := 0 for { q := romm.GetRomsQuery{ @@ -198,42 +198,50 @@ func (cm *Manager) fetchPlatformGames(platform romm.Platform, opts *fetchOpts) ( if offset == 0 { expectedTotal = res.Total - // Pre-allocate the exact slice capacity to prevent memory spikes - allGames = make([]romm.Rom, 0, expectedTotal) } - allGames = append(allGames, res.Items...) + // Persist each page before fetching the next one. Retaining every ROM for a + // platform here can exhaust 128MB handhelds, especially when WithFiles adds + // large nested payloads. SavePlatformGames already replaces rows per game, + // so page-sized transactions preserve both full and incremental semantics. + if len(res.Items) > 0 { + if err := cm.SavePlatformGames(platform.ID, res.Items); err != nil { + return 0, err + } + totalSaved += len(res.Items) + } if opts.onProgress != nil && len(res.Items) > 0 { opts.onProgress(len(res.Items)) } if opts.onPctProgress != nil && expectedTotal > 0 { - pct := float64(len(allGames)) / float64(expectedTotal) + pct := float64(totalSaved) / float64(expectedTotal) if pct > 1.0 { pct = 1.0 } opts.onPctProgress.Store(pct) } - if len(allGames) >= expectedTotal || len(res.Items) == 0 || len(res.Items) < DefaultRomPageSize { + if (expectedTotal > 0 && totalSaved >= expectedTotal) || len(res.Items) == 0 || len(res.Items) < DefaultRomPageSize { break } offset += len(res.Items) + runtime.GC() } if opts.updatedAfter != "" { logger.Debug("Fetched updated platform games", "platform", platform.Name, - "count", len(allGames), + "count", totalSaved, "updated_after", opts.updatedAfter) } else { logger.Debug("Cached platform games", "platform", platform.Name, - "count", len(allGames)) + "count", totalSaved) } - return len(allGames), cm.SavePlatformGames(platform.ID, allGames) + return totalSaved, nil } func (cm *Manager) fetchAndCacheCollectionsWithProgress(progress *atomic.Float64, progressStart, progressEnd float64) int { diff --git a/cache/populate_test.go b/cache/populate_test.go new file mode 100644 index 00000000..a2f1b94f --- /dev/null +++ b/cache/populate_test.go @@ -0,0 +1,86 @@ +package cache + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "grout/romm" +) + +func TestFetchPlatformGamesPersistsEachPageBeforeFetchingNext(t *testing.T) { + cm := newTestManager(t) + const platformID = 42 + const total = 501 + + var offsets []int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/roms" { + t.Errorf("path = %q, want /api/roms", r.URL.Path) + http.NotFound(w, r) + return + } + + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + offset, _ := strconv.Atoi(r.URL.Query().Get("offset")) + offsets = append(offsets, offset) + if limit != DefaultRomPageSize { + t.Errorf("limit = %d, want %d", limit, DefaultRomPageSize) + } + + // The previous page must already be committed before the next request. + // Retaining all pages until the end is what exhausts 128MB handhelds. + if offset > 0 { + var cached int + if err := cm.db.QueryRow("SELECT COUNT(*) FROM games").Scan(&cached); err != nil { + t.Errorf("count cached games: %v", err) + } else if cached != offset { + t.Errorf("before offset %d request, cached games = %d", offset, cached) + } + } + + end := offset + limit + if end > total { + end = total + } + items := make([]romm.Rom, 0, end-offset) + for id := offset + 1; id <= end; id++ { + items = append(items, romm.Rom{ + ID: id, + PlatformID: platformID, + PlatformFSSlug: "test", + Name: "Game " + strconv.Itoa(id), + }) + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(romm.PaginatedRoms{ + Items: items, Total: total, Limit: limit, Offset: offset, + }); err != nil { + t.Errorf("encode response: %v", err) + } + })) + defer server.Close() + + count, err := cm.fetchPlatformGames(romm.Platform{ID: platformID, Name: "Test"}, &fetchOpts{ + client: romm.NewClient(server.URL), + }) + if err != nil { + t.Fatalf("fetchPlatformGames: %v", err) + } + if count != total { + t.Fatalf("count = %d, want %d", count, total) + } + + wantOffsets := []int{0, 250, 500} + if len(offsets) != len(wantOffsets) { + t.Fatalf("offsets = %v, want %v", offsets, wantOffsets) + } + for i := range wantOffsets { + if offsets[i] != wantOffsets[i] { + t.Fatalf("offsets = %v, want %v", offsets, wantOffsets) + } + } +}