From 2185cd7765617009bdd12449ba0434117d2c7e10 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 12 Aug 2026 22:42:53 +0400 Subject: [PATCH 01/15] Add `hardcache local status` command --- .github/workflows/go.yml | 4 +- CHANGELOG.md | 7 ++ README.md | 19 ++- internal/caches/local/local.go | 19 +++ internal/caches/local/local_test.go | 25 ++++ internal/go/cache/cache_extra.go | 31 +++++ main.go | 50 +++++--- main_status.go | 176 ++++++++++++++++++++++++++++ main_status_test.go | 81 +++++++++++++ 9 files changed, 389 insertions(+), 23 deletions(-) create mode 100644 main_status.go create mode 100644 main_status_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 55dd1b3..513ba69 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -28,10 +28,10 @@ jobs: run: git config --global core.autocrlf input - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@v6 - name: Setup Go - uses: actions/setup-go@v7 + uses: actions/setup-go@v6 with: go-version: ${{ matrix.go }} diff --git a/CHANGELOG.md b/CHANGELOG.md index b49aa89..4686942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## Unreleased + +## What's Changed + +* Add `hardcache local status` command with human-readable and `--json` output. +* Move trim-only flags (`--unused-for`, `--max-size`) under `trim` and `trimd` commands. + ## [v0.2.0](https://github.com/AlekSi/hardcache/releases/tag/v0.2.0) (2025-12-07) ## What's Changed diff --git a/README.md b/README.md index 66c1267..01ed9a8 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@ Hardcache is a tool for managing the Go build cache. -The initial public version supports only a more flexible trimming policy of a standard local cache. -More functionality will be published soon, including support for `GOCACHEPROG`. +It currently supports local cache status reporting and a more flexible trimming policy +of a standard local cache. More functionality will be published soon, including support +for `GOCACHEPROG`. ## Installation @@ -69,6 +70,20 @@ hardcache local trimd --unused-for=2w --max-size=10GB --interval=1h Using `trimd` subcommand instead of `trim` triggers trimming every specified interval. +#### Status + +Display current cache and disk usage stats: + +``` +hardcache local status +``` + +Use compact JSON output for scripting: + +``` +hardcache local status --json +``` + ## Credits This tool is written by me, Alexey Palazhchenko, diff --git a/internal/caches/local/local.go b/internal/caches/local/local.go index 300cfad..efb0dd4 100644 --- a/internal/caches/local/local.go +++ b/internal/caches/local/local.go @@ -20,6 +20,14 @@ type Cache struct { l *slog.Logger } +// Stats describes local cache state. +type Stats struct { + Entries int + Bytes int64 + Oldest *time.Time + Newest *time.Time +} + // New creates a new [Cache]. func New(dir string, cutoff *time.Time, maxSize *int64, l *slog.Logger) (*Cache, error) { dc, err := cache.Open(dir) @@ -68,6 +76,17 @@ func (c *Cache) TrimForce() (before, freed int64) { return c.dc.TrimForce(c.cutoff, c.maxSize, c.l) } +// Status returns current local cache statistics. +func (c *Cache) Status() Stats { + s := c.dc.Stats(c.l) + return Stats{ + Entries: s.Entries, + Bytes: s.Bytes, + Oldest: s.Oldest, + Newest: s.Newest, + } +} + // check interfaces var ( _ cache.Cache = (*Cache)(nil) diff --git a/internal/caches/local/local_test.go b/internal/caches/local/local_test.go index 29c5067..257dbb3 100644 --- a/internal/caches/local/local_test.go +++ b/internal/caches/local/local_test.go @@ -182,3 +182,28 @@ func TestTrimSizePart(t *testing.T) { shoulda.BeEqual(t, freed, int64(60_023_595)) shoulda.BeLess(t, before-freed, maxSize) } + +func TestStatusFixture(t *testing.T) { + t.Parallel() + + c := musta.NotFail(New(setup(t), nil, nil, logger(t)))(t) + + stats := c.Status() + shoulda.BeEqual(t, stats.Entries, 1219) + shoulda.BeEqual(t, stats.Bytes, int64(109_518_524)) + musta.NotBeZero(t, stats.Oldest) + musta.NotBeZero(t, stats.Newest) + shoulda.CompareLess(t, *stats.Oldest, *stats.Newest, time.Time.Compare) +} + +func TestStatusEmpty(t *testing.T) { + t.Parallel() + + c := musta.NotFail(New(t.TempDir(), nil, nil, logger(t)))(t) + + stats := c.Status() + shoulda.BeEqual(t, stats.Entries, 0) + shoulda.BeEqual(t, stats.Bytes, int64(0)) + shoulda.BeZero(t, stats.Oldest) + shoulda.BeZero(t, stats.Newest) +} diff --git a/internal/go/cache/cache_extra.go b/internal/go/cache/cache_extra.go index fd43c91..137c862 100644 --- a/internal/go/cache/cache_extra.go +++ b/internal/go/cache/cache_extra.go @@ -20,6 +20,14 @@ import ( // EntryNotFoundError is exported for use in other packages. type EntryNotFoundError = entryNotFoundError +// Stats describes cache state derived from a full directory scan. +type Stats struct { + Entries int + Bytes int64 + Oldest *time.Time + Newest *time.Time +} + // fileInfo represents information about a file or directory with executable in the cache. // The order of fields is weird to make struct smaller. type fileInfo struct { @@ -128,6 +136,29 @@ func (c *DiskCache) TrimForce(cutoff *time.Time, maxSize *int64, l *slog.Logger) return } +// Stats scans the cache directory and returns aggregate statistics. +func (c *DiskCache) Stats(l *slog.Logger) Stats { + files, bytes := c.read(l) + stats := Stats{ + Entries: len(files), + Bytes: bytes, + } + + for _, fi := range files { + modTime := fi.ModTime + if stats.Oldest == nil || modTime.Before(*stats.Oldest) { + v := modTime + stats.Oldest = &v + } + if stats.Newest == nil || modTime.After(*stats.Newest) { + v := modTime + stats.Newest = &v + } + } + + return stats +} + // read reads the entire cache directory. func (c *DiskCache) read(l *slog.Logger) (files []fileInfo, before int64) { files = make([]fileInfo, 0, 256) diff --git a/main.go b/main.go index 030c259..9ee6908 100644 --- a/main.go +++ b/main.go @@ -23,13 +23,21 @@ import ( //nolint:vet // for readability var cli struct { Local struct { - Dir string `default:"${local_dir_default}" type:"path" help:"Directory to use."` - UnusedFor unit.Duration `default:"5d" help:"Always remove entries unused for this duration. Pass 0 to disable."` - MaxSize string `default:"0GB" help:"${local_max_size_help}"` + Dir string `default:"${local_dir_default}" type:"path" help:"Directory to use."` + + Status struct { + JSON bool `help:"Output as compact JSON."` + } `cmd:"" help:"Show local cache status."` + + Trim struct { + UnusedFor unit.Duration `default:"5d" help:"Always remove entries unused for this duration. Pass 0 to disable."` + MaxSize string `default:"0GB" help:"${local_max_size_help}"` + } `cmd:"" help:"Trim local cache."` - Trim struct{} `cmd:"" help:"Trim local cache."` Trimd struct { - Interval unit.Duration `short:"i" default:"1h" help:"Interval between trimmings."` + UnusedFor unit.Duration `default:"5d" help:"Always remove entries unused for this duration. Pass 0 to disable."` + MaxSize string `default:"0GB" help:"${local_max_size_help}"` + Interval unit.Duration `short:"i" default:"1h" help:"Interval between trimmings."` } `cmd:"" help:"Trim local cache continuously."` } `cmd:""` @@ -52,26 +60,26 @@ var GOCACHE = sync.OnceValue(func() string { return strings.TrimSpace(string(b)) }) -// localTrim force-trims local cache according to CLI flags. -func localTrim(l *slog.Logger) error { - if cli.Local.UnusedFor < 0 { - return fmt.Errorf("--unused-for cannot be negative: %d", cli.Local.UnusedFor) +// localTrim force-trims local cache according to parameters. +func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog.Logger) error { + if unusedFor < 0 { + return fmt.Errorf("--unused-for cannot be negative: %d", unusedFor) } var cutoff *time.Time - if cli.Local.UnusedFor > 0 { - c := time.Now().Add(-time.Duration(cli.Local.UnusedFor)) + if unusedFor > 0 { + c := time.Now().Add(-time.Duration(unusedFor)) cutoff = &c } var b unit.Bytes - if strings.HasSuffix(cli.Local.MaxSize, "%") { + if strings.HasSuffix(maxSizeValue, "%") { var p unit.Percentage - if err := p.UnmarshalText([]byte(cli.Local.MaxSize)); err != nil { + if err := p.UnmarshalText([]byte(maxSizeValue)); err != nil { return err } - total, _, err := local.DiskInfo(cli.Local.Dir) + total, _, err := local.DiskInfo(dir) if err != nil { return err } @@ -86,7 +94,7 @@ func localTrim(l *slog.Logger) error { slog.String("max_size", b.String()), ) } else { - if err := b.UnmarshalText([]byte(cli.Local.MaxSize)); err != nil { + if err := b.UnmarshalText([]byte(maxSizeValue)); err != nil { return err } @@ -102,7 +110,7 @@ func localTrim(l *slog.Logger) error { maxSize = (*int64)(&b) } - c, err := local.New(cli.Local.Dir, cutoff, maxSize, l) + c, err := local.New(dir, cutoff, maxSize, l) if err != nil { return err } @@ -151,17 +159,21 @@ func main() { defer cancel() switch kongCtx.Command() { + case "local status": + err := localStatus(cli.Local.Dir, cli.Local.Status.JSON, l) + kongCtx.FatalIfErrorf(err) + case "local trim": - if time.Duration(cli.Local.UnusedFor) > 5*24*time.Hour { + if time.Duration(cli.Local.Trim.UnusedFor) > 5*24*time.Hour { l.Info("Note: this command should be invoked more often than once per day to keep the cache.") } - err := localTrim(l) + err := localTrim(cli.Local.Dir, cli.Local.Trim.UnusedFor, cli.Local.Trim.MaxSize, l) kongCtx.FatalIfErrorf(err) case "local trimd": for { - err := localTrim(l) + err := localTrim(cli.Local.Dir, cli.Local.Trimd.UnusedFor, cli.Local.Trimd.MaxSize, l) kongCtx.FatalIfErrorf(err) select { diff --git a/main_status.go b/main_status.go new file mode 100644 index 0000000..fa70561 --- /dev/null +++ b/main_status.go @@ -0,0 +1,176 @@ +package main + +import ( + "encoding/json" + "fmt" + "log/slog" + "math" + "os" + "strings" + "time" + + "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/unit" +) + +// localStatusReport contains all calculated values used by status output modes. +type localStatusReport struct { + Directory string + CacheEntries int + CacheBytes int64 + CacheOldest *time.Time + CacheNewest *time.Time + DiskTotalBytes int64 + DiskUsedBytes int64 + DiskFreeBytes int64 + DiskUsedPercent float64 + DiskFreePercent float64 + CacheOfTotalPercent float64 +} + +type jsonLocalStatus struct { + Directory string `json:"directory"` + Cache struct { + Entries int `json:"entries"` + Bytes int64 `json:"bytes"` + Human string `json:"human"` + Oldest *string `json:"oldest"` + Newest *string `json:"newest"` + } `json:"cache"` + Disk struct { + TotalBytes int64 `json:"total_bytes"` + TotalHuman string `json:"total_human"` + UsedBytes int64 `json:"used_bytes"` + UsedHuman string `json:"used_human"` + UsedPercent float64 `json:"used_percent"` + FreeBytes int64 `json:"free_bytes"` + FreeHuman string `json:"free_human"` + FreePercent float64 `json:"free_percent"` + } `json:"disk"` + CacheOfTotalPercent float64 `json:"cache_of_total_percent"` +} + +func localStatus(dir string, asJSON bool, l *slog.Logger) error { + c, err := local.New(dir, nil, nil, l) + if err != nil { + return err + } + + cacheStats := c.Status() + total, free, err := local.DiskInfo(dir) + if err != nil { + return err + } + + report := newLocalStatusReport(dir, cacheStats, total, free) + + var out string + if asJSON { + out, err = renderLocalStatusJSON(report) + if err != nil { + return err + } + } else { + out = renderLocalStatusText(report) + } + + _, err = os.Stdout.WriteString(out) + return err +} + +func newLocalStatusReport(dir string, stats local.Stats, total, free int64) localStatusReport { + used := total - free + if used < 0 { + used = 0 + } + + return localStatusReport{ + Directory: dir, + CacheEntries: stats.Entries, + CacheBytes: stats.Bytes, + CacheOldest: stats.Oldest, + CacheNewest: stats.Newest, + DiskTotalBytes: total, + DiskUsedBytes: used, + DiskFreeBytes: free, + DiskUsedPercent: round2(percentage(used, total)), + DiskFreePercent: round2(percentage(free, total)), + CacheOfTotalPercent: round2(percentage(stats.Bytes, total)), + } +} + +func renderLocalStatusText(report localStatusReport) string { + var b strings.Builder + + fmt.Fprintf(&b, "Directory: %s\n", report.Directory) + fmt.Fprintf(&b, "Cache entries: %d\n", report.CacheEntries) + fmt.Fprintf(&b, "Cache size: %s\n", formatSizeWithRaw(report.CacheBytes)) + fmt.Fprintf(&b, "Oldest entry: %s\n", formatLocalTime(report.CacheOldest)) + fmt.Fprintf(&b, "Newest entry: %s\n", formatLocalTime(report.CacheNewest)) + fmt.Fprintf(&b, "Disk total: %s\n", formatSizeWithRaw(report.DiskTotalBytes)) + fmt.Fprintf(&b, "Disk used: %s (%.2f%%)\n", formatSizeWithRaw(report.DiskUsedBytes), report.DiskUsedPercent) + fmt.Fprintf(&b, "Disk free: %s (%.2f%%)\n", formatSizeWithRaw(report.DiskFreeBytes), report.DiskFreePercent) + fmt.Fprintf(&b, "Cache of total disk: %.2f%%\n", report.CacheOfTotalPercent) + + return b.String() +} + +func renderLocalStatusJSON(report localStatusReport) (string, error) { + payload := jsonLocalStatus{ + Directory: report.Directory, + CacheOfTotalPercent: report.CacheOfTotalPercent, + } + payload.Cache.Entries = report.CacheEntries + payload.Cache.Bytes = report.CacheBytes + payload.Cache.Human = unit.Bytes(report.CacheBytes).String() + payload.Cache.Oldest = formatLocalTimePtr(report.CacheOldest) + payload.Cache.Newest = formatLocalTimePtr(report.CacheNewest) + payload.Disk.TotalBytes = report.DiskTotalBytes + payload.Disk.TotalHuman = unit.Bytes(report.DiskTotalBytes).String() + payload.Disk.UsedBytes = report.DiskUsedBytes + payload.Disk.UsedHuman = unit.Bytes(report.DiskUsedBytes).String() + payload.Disk.UsedPercent = report.DiskUsedPercent + payload.Disk.FreeBytes = report.DiskFreeBytes + payload.Disk.FreeHuman = unit.Bytes(report.DiskFreeBytes).String() + payload.Disk.FreePercent = report.DiskFreePercent + + res, err := json.Marshal(payload) + if err != nil { + return "", err + } + + return string(res) + "\n", nil +} + +func formatSizeWithRaw(size int64) string { + return fmt.Sprintf("%s (%d bytes)", unit.Bytes(size).String(), size) +} + +func formatLocalTime(ts *time.Time) string { + if ts == nil { + return "n/a" + } + + return ts.Local().Format(time.RFC3339) +} + +func formatLocalTimePtr(ts *time.Time) *string { + if ts == nil { + return nil + } + + res := ts.Local().Format(time.RFC3339) + return &res +} + +func percentage(value, total int64) float64 { + if total <= 0 { + return 0 + } + + return float64(value) / float64(total) * 100 +} + +func round2(v float64) float64 { + return math.Round(v*100) / 100 +} diff --git a/main_status_test.go b/main_status_test.go new file mode 100644 index 0000000..c6ca317 --- /dev/null +++ b/main_status_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/shoulda" + "github.com/AlekSi/shoulda/musta" +) + +func TestStatusTextFormatting(t *testing.T) { + oldest := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + newest := time.Date(2026, time.January, 3, 4, 5, 6, 0, time.UTC) + report := newLocalStatusReport("/tmp/cache", local.Stats{ + Entries: 3, + Bytes: 1024, + Oldest: &oldest, + Newest: &newest, + }, 10*1024, 4*1024) + + actual := renderLocalStatusText(report) + + shoulda.SatisfyWith(t, actual, "Directory: /tmp/cache", strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache entries: 3", strings.Contains) + shoulda.SatisfyWith(t, actual, fmt.Sprintf("Cache size: %s", formatSizeWithRaw(1024)), strings.Contains) + shoulda.SatisfyWith(t, actual, fmt.Sprintf("Oldest entry: %s", oldest.Local().Format(time.RFC3339)), strings.Contains) + shoulda.SatisfyWith(t, actual, fmt.Sprintf("Newest entry: %s", newest.Local().Format(time.RFC3339)), strings.Contains) + shoulda.SatisfyWith(t, actual, fmt.Sprintf("Disk total: %s", formatSizeWithRaw(10*1024)), strings.Contains) + shoulda.SatisfyWith(t, actual, fmt.Sprintf("Disk used: %s (60.00%%)", formatSizeWithRaw(6*1024)), strings.Contains) + shoulda.SatisfyWith(t, actual, fmt.Sprintf("Disk free: %s (40.00%%)", formatSizeWithRaw(4*1024)), strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache of total disk: 10.00%", strings.Contains) +} + +func TestStatusTextFormattingEmpty(t *testing.T) { + report := newLocalStatusReport("/tmp/cache", local.Stats{}, 100, 25) + + actual := renderLocalStatusText(report) + + shoulda.SatisfyWith(t, actual, "Cache entries: 0", strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache size: 0B (0 bytes)", strings.Contains) + shoulda.SatisfyWith(t, actual, "Oldest entry: n/a", strings.Contains) + shoulda.SatisfyWith(t, actual, "Newest entry: n/a", strings.Contains) +} + +func TestStatusJSONCompact(t *testing.T) { + oldest := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + newest := time.Date(2026, time.January, 3, 4, 5, 6, 0, time.UTC) + report := newLocalStatusReport("/tmp/cache", local.Stats{ + Entries: 3, + Bytes: 1024, + Oldest: &oldest, + Newest: &newest, + }, 10*1024, 4*1024) + + actual, err := renderLocalStatusJSON(report) + musta.NoError(t, err) + shoulda.SatisfyWith(t, actual, "\n", strings.HasSuffix) + + var got jsonLocalStatus + err = json.Unmarshal([]byte(actual), &got) + musta.NoError(t, err) + + shoulda.BeEqual(t, got.Directory, "/tmp/cache") + shoulda.BeEqual(t, got.Cache.Entries, 3) + shoulda.BeEqual(t, got.Cache.Bytes, int64(1024)) + shoulda.NotBeZero(t, got.Cache.Human) + musta.NotBeZero(t, got.Cache.Oldest) + musta.NotBeZero(t, got.Cache.Newest) + shoulda.BeEqual(t, *got.Cache.Oldest, oldest.Local().Format(time.RFC3339)) + shoulda.BeEqual(t, *got.Cache.Newest, newest.Local().Format(time.RFC3339)) + shoulda.BeEqual(t, got.Disk.TotalBytes, int64(10*1024)) + shoulda.BeEqual(t, got.Disk.UsedBytes, int64(6*1024)) + shoulda.BeEqual(t, got.Disk.FreeBytes, int64(4*1024)) + shoulda.BeEqual(t, got.Disk.UsedPercent, 60) + shoulda.BeEqual(t, got.Disk.FreePercent, 40) + shoulda.BeEqual(t, got.CacheOfTotalPercent, 10) +} From bb5b5528a6718895d98ba07067de3371124b3d9f Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 12 Aug 2026 22:43:52 +0400 Subject: [PATCH 02/15] Revert --- .github/workflows/go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 513ba69..55dd1b3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -28,10 +28,10 @@ jobs: run: git config --global core.autocrlf input - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 with: go-version: ${{ matrix.go }} From 9e16e45fcb22420fc8f4dbd0250bc03abc5d8fba Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 12 Aug 2026 22:46:02 +0400 Subject: [PATCH 03/15] WIP --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 01ed9a8..7a8b142 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,20 @@ It can be overridden with `--dir` flag: hardcache local --dir=/tmp/cache ... ``` +#### Status + +Display current cache and disk usage stats: + +``` +hardcache local status +``` + +Use compact JSON output for scripting: + +``` +hardcache local status --json +``` + #### Manual trimming Go standard build cache does not support [disabling trimming](https://github.com/golang/go/issues/69565), @@ -70,20 +84,6 @@ hardcache local trimd --unused-for=2w --max-size=10GB --interval=1h Using `trimd` subcommand instead of `trim` triggers trimming every specified interval. -#### Status - -Display current cache and disk usage stats: - -``` -hardcache local status -``` - -Use compact JSON output for scripting: - -``` -hardcache local status --json -``` - ## Credits This tool is written by me, Alexey Palazhchenko, From 9d7d764dfdf3523a1d277d845fb9afc0418631cf Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 12 Aug 2026 22:57:10 +0400 Subject: [PATCH 04/15] WIP --- main_status.go | 172 ++++++++++++++++---------------------------- main_status_test.go | 111 ++++++++++++++-------------- 2 files changed, 115 insertions(+), 168 deletions(-) diff --git a/main_status.go b/main_status.go index fa70561..df09076 100644 --- a/main_status.go +++ b/main_status.go @@ -6,29 +6,13 @@ import ( "log/slog" "math" "os" - "strings" "time" "github.com/AlekSi/hardcache/internal/caches/local" "github.com/AlekSi/hardcache/internal/unit" ) -// localStatusReport contains all calculated values used by status output modes. -type localStatusReport struct { - Directory string - CacheEntries int - CacheBytes int64 - CacheOldest *time.Time - CacheNewest *time.Time - DiskTotalBytes int64 - DiskUsedBytes int64 - DiskFreeBytes int64 - DiskUsedPercent float64 - DiskFreePercent float64 - CacheOfTotalPercent float64 -} - -type jsonLocalStatus struct { +type localStatusOutput struct { Directory string `json:"directory"` Cache struct { Entries int `json:"entries"` @@ -56,121 +40,85 @@ func localStatus(dir string, asJSON bool, l *slog.Logger) error { return err } - cacheStats := c.Status() + stats := c.Status() total, free, err := local.DiskInfo(dir) if err != nil { return err } - report := newLocalStatusReport(dir, cacheStats, total, free) - - var out string + output := newLocalStatusOutput(dir, stats, total, free) if asJSON { - out, err = renderLocalStatusJSON(report) - if err != nil { - return err - } - } else { - out = renderLocalStatusText(report) + return json.NewEncoder(os.Stdout).Encode(output) } - _, err = os.Stdout.WriteString(out) + _, err = fmt.Fprint(os.Stdout, output) return err } -func newLocalStatusReport(dir string, stats local.Stats, total, free int64) localStatusReport { - used := total - free - if used < 0 { - used = 0 - } +func newLocalStatusOutput(dir string, stats local.Stats, total, free int64) localStatusOutput { + used := max(total-free, 0) + percent := func(value int64) float64 { + if total <= 0 { + return 0 + } - return localStatusReport{ - Directory: dir, - CacheEntries: stats.Entries, - CacheBytes: stats.Bytes, - CacheOldest: stats.Oldest, - CacheNewest: stats.Newest, - DiskTotalBytes: total, - DiskUsedBytes: used, - DiskFreeBytes: free, - DiskUsedPercent: round2(percentage(used, total)), - DiskFreePercent: round2(percentage(free, total)), - CacheOfTotalPercent: round2(percentage(stats.Bytes, total)), + return math.Round(float64(value)/float64(total)*10_000) / 100 } -} - -func renderLocalStatusText(report localStatusReport) string { - var b strings.Builder - - fmt.Fprintf(&b, "Directory: %s\n", report.Directory) - fmt.Fprintf(&b, "Cache entries: %d\n", report.CacheEntries) - fmt.Fprintf(&b, "Cache size: %s\n", formatSizeWithRaw(report.CacheBytes)) - fmt.Fprintf(&b, "Oldest entry: %s\n", formatLocalTime(report.CacheOldest)) - fmt.Fprintf(&b, "Newest entry: %s\n", formatLocalTime(report.CacheNewest)) - fmt.Fprintf(&b, "Disk total: %s\n", formatSizeWithRaw(report.DiskTotalBytes)) - fmt.Fprintf(&b, "Disk used: %s (%.2f%%)\n", formatSizeWithRaw(report.DiskUsedBytes), report.DiskUsedPercent) - fmt.Fprintf(&b, "Disk free: %s (%.2f%%)\n", formatSizeWithRaw(report.DiskFreeBytes), report.DiskFreePercent) - fmt.Fprintf(&b, "Cache of total disk: %.2f%%\n", report.CacheOfTotalPercent) - return b.String() -} - -func renderLocalStatusJSON(report localStatusReport) (string, error) { - payload := jsonLocalStatus{ - Directory: report.Directory, - CacheOfTotalPercent: report.CacheOfTotalPercent, + res := localStatusOutput{ + Directory: dir, + CacheOfTotalPercent: percent(stats.Bytes), } - payload.Cache.Entries = report.CacheEntries - payload.Cache.Bytes = report.CacheBytes - payload.Cache.Human = unit.Bytes(report.CacheBytes).String() - payload.Cache.Oldest = formatLocalTimePtr(report.CacheOldest) - payload.Cache.Newest = formatLocalTimePtr(report.CacheNewest) - payload.Disk.TotalBytes = report.DiskTotalBytes - payload.Disk.TotalHuman = unit.Bytes(report.DiskTotalBytes).String() - payload.Disk.UsedBytes = report.DiskUsedBytes - payload.Disk.UsedHuman = unit.Bytes(report.DiskUsedBytes).String() - payload.Disk.UsedPercent = report.DiskUsedPercent - payload.Disk.FreeBytes = report.DiskFreeBytes - payload.Disk.FreeHuman = unit.Bytes(report.DiskFreeBytes).String() - payload.Disk.FreePercent = report.DiskFreePercent - - res, err := json.Marshal(payload) - if err != nil { - return "", err + res.Cache.Entries = stats.Entries + res.Cache.Bytes = stats.Bytes + res.Cache.Human = unit.Bytes(stats.Bytes).String() + if stats.Oldest != nil { + oldest := stats.Oldest.Local().Format(time.RFC3339) + res.Cache.Oldest = &oldest } - - return string(res) + "\n", nil -} - -func formatSizeWithRaw(size int64) string { - return fmt.Sprintf("%s (%d bytes)", unit.Bytes(size).String(), size) -} - -func formatLocalTime(ts *time.Time) string { - if ts == nil { - return "n/a" + if stats.Newest != nil { + newest := stats.Newest.Local().Format(time.RFC3339) + res.Cache.Newest = &newest } - - return ts.Local().Format(time.RFC3339) + res.Disk.TotalBytes = total + res.Disk.TotalHuman = unit.Bytes(total).String() + res.Disk.UsedBytes = used + res.Disk.UsedHuman = unit.Bytes(used).String() + res.Disk.UsedPercent = percent(used) + res.Disk.FreeBytes = free + res.Disk.FreeHuman = unit.Bytes(free).String() + res.Disk.FreePercent = percent(free) + + return res } -func formatLocalTimePtr(ts *time.Time) *string { - if ts == nil { - return nil +func (s localStatusOutput) String() string { + oldest, newest := "n/a", "n/a" + if s.Cache.Oldest != nil { + oldest = *s.Cache.Oldest } - - res := ts.Local().Format(time.RFC3339) - return &res -} - -func percentage(value, total int64) float64 { - if total <= 0 { - return 0 + if s.Cache.Newest != nil { + newest = *s.Cache.Newest } - return float64(value) / float64(total) * 100 -} - -func round2(v float64) float64 { - return math.Round(v*100) / 100 + return fmt.Sprintf(`Directory: %s +Cache entries: %d +Cache size: %s (%d bytes) +Oldest entry: %s +Newest entry: %s +Disk total: %s (%d bytes) +Disk used: %s (%d bytes) (%.2f%%) +Disk free: %s (%d bytes) (%.2f%%) +Cache of total disk: %.2f%% +`, + s.Directory, + s.Cache.Entries, + s.Cache.Human, s.Cache.Bytes, + oldest, + newest, + s.Disk.TotalHuman, s.Disk.TotalBytes, + s.Disk.UsedHuman, s.Disk.UsedBytes, s.Disk.UsedPercent, + s.Disk.FreeHuman, s.Disk.FreeBytes, s.Disk.FreePercent, + s.CacheOfTotalPercent, + ) } diff --git a/main_status_test.go b/main_status_test.go index c6ca317..241b58c 100644 --- a/main_status_test.go +++ b/main_status_test.go @@ -3,79 +3,78 @@ package main import ( "encoding/json" "fmt" + "log/slog" + "os/exec" + "path/filepath" "strings" "testing" "time" "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/unit" "github.com/AlekSi/shoulda" "github.com/AlekSi/shoulda/musta" ) -func TestStatusTextFormatting(t *testing.T) { - oldest := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) - newest := time.Date(2026, time.January, 3, 4, 5, 6, 0, time.UTC) - report := newLocalStatusReport("/tmp/cache", local.Stats{ - Entries: 3, - Bytes: 1024, - Oldest: &oldest, - Newest: &newest, - }, 10*1024, 4*1024) +func TestLocalStatusOutput(t *testing.T) { + t.Parallel() - actual := renderLocalStatusText(report) + src := filepath.Join("internal", "testdata", "local") + dst := t.TempDir() + b, err := exec.Command("cp", "-a", src, dst).CombinedOutput() + musta.NoErrorf(t, err, "%s", b) + dir := filepath.Join(dst, "local") - shoulda.SatisfyWith(t, actual, "Directory: /tmp/cache", strings.Contains) - shoulda.SatisfyWith(t, actual, "Cache entries: 3", strings.Contains) - shoulda.SatisfyWith(t, actual, fmt.Sprintf("Cache size: %s", formatSizeWithRaw(1024)), strings.Contains) - shoulda.SatisfyWith(t, actual, fmt.Sprintf("Oldest entry: %s", oldest.Local().Format(time.RFC3339)), strings.Contains) - shoulda.SatisfyWith(t, actual, fmt.Sprintf("Newest entry: %s", newest.Local().Format(time.RFC3339)), strings.Contains) - shoulda.SatisfyWith(t, actual, fmt.Sprintf("Disk total: %s", formatSizeWithRaw(10*1024)), strings.Contains) - shoulda.SatisfyWith(t, actual, fmt.Sprintf("Disk used: %s (60.00%%)", formatSizeWithRaw(6*1024)), strings.Contains) - shoulda.SatisfyWith(t, actual, fmt.Sprintf("Disk free: %s (40.00%%)", formatSizeWithRaw(4*1024)), strings.Contains) - shoulda.SatisfyWith(t, actual, "Cache of total disk: 10.00%", strings.Contains) + c := musta.NotFail(local.New(dir, nil, nil, slog.Default()))(t) + stats := c.Status() + total, free := musta.NotFail2(local.DiskInfo(dir))(t) + output := newLocalStatusOutput(dir, stats, total, free) + + oldest := time.Date(2025, time.November, 17, 17, 12, 57, 524467000, time.UTC).Local().Format(time.RFC3339) + newest := time.Date(2025, time.November, 17, 17, 13, 7, 284400000, time.UTC).Local().Format(time.RFC3339) + + shoulda.BeEqual(t, output.Cache.Entries, 1219) + shoulda.BeEqual(t, output.Cache.Bytes, int64(109_518_524)) + shoulda.BeEqual(t, output.Cache.Human, "109MB") + shoulda.BeEqual(t, *output.Cache.Oldest, oldest) + shoulda.BeEqual(t, *output.Cache.Newest, newest) + + t.Run("text", func(t *testing.T) { + actual := output.String() + shoulda.SatisfyWith(t, actual, "Directory: "+dir, strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache entries: 1219", strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache size: 109MB (109518524 bytes)", strings.Contains) + shoulda.SatisfyWith(t, actual, "Oldest entry: "+oldest, strings.Contains) + shoulda.SatisfyWith(t, actual, "Newest entry: "+newest, strings.Contains) + shoulda.SatisfyWith(t, actual, + fmt.Sprintf("Disk total: %s (%d bytes)", unit.Bytes(total), total), strings.Contains) + shoulda.SatisfyWith(t, actual, + fmt.Sprintf("Disk free: %s (%d bytes)", unit.Bytes(free), free), strings.Contains) + }) + + t.Run("JSON", func(t *testing.T) { + actual, err := json.Marshal(output) + musta.NoError(t, err) + shoulda.BeZero(t, strings.Contains(string(actual), "\n")) + + var got localStatusOutput + musta.NoError(t, json.Unmarshal(actual, &got)) + shoulda.BeDeepEqual(t, got, output) + }) } -func TestStatusTextFormattingEmpty(t *testing.T) { - report := newLocalStatusReport("/tmp/cache", local.Stats{}, 100, 25) +func TestLocalStatusOutputEmpty(t *testing.T) { + t.Parallel() - actual := renderLocalStatusText(report) + dir := t.TempDir() + c := musta.NotFail(local.New(dir, nil, nil, slog.Default()))(t) + stats := c.Status() + total, free := musta.NotFail2(local.DiskInfo(dir))(t) + output := newLocalStatusOutput(dir, stats, total, free) + actual := output.String() shoulda.SatisfyWith(t, actual, "Cache entries: 0", strings.Contains) shoulda.SatisfyWith(t, actual, "Cache size: 0B (0 bytes)", strings.Contains) shoulda.SatisfyWith(t, actual, "Oldest entry: n/a", strings.Contains) shoulda.SatisfyWith(t, actual, "Newest entry: n/a", strings.Contains) } - -func TestStatusJSONCompact(t *testing.T) { - oldest := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) - newest := time.Date(2026, time.January, 3, 4, 5, 6, 0, time.UTC) - report := newLocalStatusReport("/tmp/cache", local.Stats{ - Entries: 3, - Bytes: 1024, - Oldest: &oldest, - Newest: &newest, - }, 10*1024, 4*1024) - - actual, err := renderLocalStatusJSON(report) - musta.NoError(t, err) - shoulda.SatisfyWith(t, actual, "\n", strings.HasSuffix) - - var got jsonLocalStatus - err = json.Unmarshal([]byte(actual), &got) - musta.NoError(t, err) - - shoulda.BeEqual(t, got.Directory, "/tmp/cache") - shoulda.BeEqual(t, got.Cache.Entries, 3) - shoulda.BeEqual(t, got.Cache.Bytes, int64(1024)) - shoulda.NotBeZero(t, got.Cache.Human) - musta.NotBeZero(t, got.Cache.Oldest) - musta.NotBeZero(t, got.Cache.Newest) - shoulda.BeEqual(t, *got.Cache.Oldest, oldest.Local().Format(time.RFC3339)) - shoulda.BeEqual(t, *got.Cache.Newest, newest.Local().Format(time.RFC3339)) - shoulda.BeEqual(t, got.Disk.TotalBytes, int64(10*1024)) - shoulda.BeEqual(t, got.Disk.UsedBytes, int64(6*1024)) - shoulda.BeEqual(t, got.Disk.FreeBytes, int64(4*1024)) - shoulda.BeEqual(t, got.Disk.UsedPercent, 60) - shoulda.BeEqual(t, got.Disk.FreePercent, 40) - shoulda.BeEqual(t, got.CacheOfTotalPercent, 10) -} From 79114f46fc714763155177f962eee3c0d9f7e8dc Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Wed, 12 Aug 2026 23:29:26 +0400 Subject: [PATCH 05/15] WIP --- .../commands/local_status.go | 11 +- internal/commands/local_status_test.go | 83 +++++++++++++ internal/commands/local_trim.go | 110 ++++++++++++++++++ main.go | 95 ++------------- main_status_test.go | 80 ------------- 5 files changed, 206 insertions(+), 173 deletions(-) rename main_status.go => internal/commands/local_status.go (91%) create mode 100644 internal/commands/local_status_test.go create mode 100644 internal/commands/local_trim.go delete mode 100644 main_status_test.go diff --git a/main_status.go b/internal/commands/local_status.go similarity index 91% rename from main_status.go rename to internal/commands/local_status.go index df09076..6ae628b 100644 --- a/main_status.go +++ b/internal/commands/local_status.go @@ -1,11 +1,11 @@ -package main +package commands import ( "encoding/json" "fmt" + "io" "log/slog" "math" - "os" "time" "github.com/AlekSi/hardcache/internal/caches/local" @@ -34,7 +34,8 @@ type localStatusOutput struct { CacheOfTotalPercent float64 `json:"cache_of_total_percent"` } -func localStatus(dir string, asJSON bool, l *slog.Logger) error { +// LocalStatus writes local cache and disk usage statistics to out. +func LocalStatus(dir string, asJSON bool, out io.Writer, l *slog.Logger) error { c, err := local.New(dir, nil, nil, l) if err != nil { return err @@ -48,10 +49,10 @@ func localStatus(dir string, asJSON bool, l *slog.Logger) error { output := newLocalStatusOutput(dir, stats, total, free) if asJSON { - return json.NewEncoder(os.Stdout).Encode(output) + return json.NewEncoder(out).Encode(output) } - _, err = fmt.Fprint(os.Stdout, output) + _, err = fmt.Fprint(out, output) return err } diff --git a/internal/commands/local_status_test.go b/internal/commands/local_status_test.go new file mode 100644 index 0000000..56220f9 --- /dev/null +++ b/internal/commands/local_status_test.go @@ -0,0 +1,83 @@ +package commands + +import ( + "encoding/json" + "log/slog" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/AlekSi/shoulda" + "github.com/AlekSi/shoulda/musta" +) + +// setup copies the testdata cache to a test-specific temporary directory. +func setup(t testing.TB) string { + t.Helper() + + src := filepath.Join("..", "testdata", "local") + dst := t.TempDir() + b, err := exec.Command("cp", "-a", src, dst).CombinedOutput() + musta.NoErrorf(t, err, "%s", b) + + return filepath.Join(dst, "local") +} + +func TestLocalStatus(t *testing.T) { + t.Parallel() + + dir := setup(t) + oldest := time.Date(2025, time.November, 17, 17, 12, 57, 524467000, time.UTC).Local().Format(time.RFC3339) + newest := time.Date(2025, time.November, 17, 17, 13, 7, 284400000, time.UTC).Local().Format(time.RFC3339) + + t.Run("text", func(t *testing.T) { + var output strings.Builder + musta.NoError(t, LocalStatus(dir, false, &output, slog.Default())) + + actual := output.String() + shoulda.SatisfyWith(t, actual, "Directory: "+dir, strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache entries: 1219", strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache size: 109MB (109518524 bytes)", strings.Contains) + shoulda.SatisfyWith(t, actual, "Oldest entry: "+oldest, strings.Contains) + shoulda.SatisfyWith(t, actual, "Newest entry: "+newest, strings.Contains) + shoulda.SatisfyWith(t, actual, "Disk total: ", strings.Contains) + shoulda.SatisfyWith(t, actual, "Disk free: ", strings.Contains) + }) + + t.Run("JSON", func(t *testing.T) { + var output strings.Builder + musta.NoError(t, LocalStatus(dir, true, &output, slog.Default())) + + actual := output.String() + shoulda.SatisfyWith(t, actual, "\n", strings.HasSuffix) + shoulda.BeEqual(t, strings.Count(actual, "\n"), 1) + + var got localStatusOutput + musta.NoError(t, json.Unmarshal([]byte(actual), &got)) + shoulda.BeEqual(t, got.Directory, dir) + shoulda.BeEqual(t, got.Cache.Entries, 1219) + shoulda.BeEqual(t, got.Cache.Bytes, int64(109_518_524)) + shoulda.BeEqual(t, got.Cache.Human, "109MB") + musta.NotBeZero(t, got.Cache.Oldest) + musta.NotBeZero(t, got.Cache.Newest) + shoulda.BeEqual(t, *got.Cache.Oldest, oldest) + shoulda.BeEqual(t, *got.Cache.Newest, newest) + shoulda.BeGreater(t, got.Disk.TotalBytes, int64(0)) + shoulda.BeEqual(t, got.Disk.UsedBytes+got.Disk.FreeBytes, got.Disk.TotalBytes) + }) +} + +func TestLocalStatusEmpty(t *testing.T) { + t.Parallel() + + var output strings.Builder + musta.NoError(t, LocalStatus(t.TempDir(), false, &output, slog.Default())) + + actual := output.String() + shoulda.SatisfyWith(t, actual, "Cache entries: 0", strings.Contains) + shoulda.SatisfyWith(t, actual, "Cache size: 0B (0 bytes)", strings.Contains) + shoulda.SatisfyWith(t, actual, "Oldest entry: n/a", strings.Contains) + shoulda.SatisfyWith(t, actual, "Newest entry: n/a", strings.Contains) +} diff --git a/internal/commands/local_trim.go b/internal/commands/local_trim.go new file mode 100644 index 0000000..3788d18 --- /dev/null +++ b/internal/commands/local_trim.go @@ -0,0 +1,110 @@ +package commands + +import ( + "context" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/unit" +) + +// LocalTrim force-trims a local cache according to the given parameters. +func LocalTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog.Logger) error { + if time.Duration(unusedFor) > 5*24*time.Hour { + l.Info("Note: this command should be invoked more often than once per day to keep the cache.") + } + + return localTrim(dir, unusedFor, maxSizeValue, l) +} + +func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog.Logger) error { + if unusedFor < 0 { + return fmt.Errorf("--unused-for cannot be negative: %d", unusedFor) + } + + var cutoff *time.Time + if unusedFor > 0 { + c := time.Now().Add(-time.Duration(unusedFor)) + cutoff = &c + } + + var b unit.Bytes + if strings.HasSuffix(maxSizeValue, "%") { + var p unit.Percentage + if err := p.UnmarshalText([]byte(maxSizeValue)); err != nil { + return err + } + + total, _, err := local.DiskInfo(dir) + if err != nil { + return err + } + + b = unit.Bytes(total / 100 * int64(p)) + + l.Debug( + "Calculated max size from percentage of total disk size", + slog.Int64("disk_size_bytes", total), + slog.String("disk_size", unit.Bytes(total).String()), + slog.Int64("max_size_bytes", int64(b)), + slog.String("max_size", b.String()), + ) + } else { + if err := b.UnmarshalText([]byte(maxSizeValue)); err != nil { + return err + } + + l.Debug("Max size", slog.Int64("max_size_bytes", int64(b)), slog.String("max_size", b.String())) + } + + if b < 0 { + return fmt.Errorf("--max-size cannot be negative: %d", b) + } + + var maxSize *int64 + if b > 0 { + maxSize = (*int64)(&b) + } + + c, err := local.New(dir, cutoff, maxSize, l) + if err != nil { + return err + } + + before, freed := c.TrimForce() + l.Debug( + "Local cache trimmed", + slog.Int64("before_bytes", before), slog.Int64("freed_bytes", freed), + ) + l.Info( + "Local cache trimmed", + slog.String("before", unit.Bytes(before).String()), slog.String("freed", unit.Bytes(freed).String()), + ) + + return nil +} + +// LocalTrimd continuously trims a local cache until ctx is canceled. +func LocalTrimd( + ctx context.Context, + dir string, + unusedFor unit.Duration, + maxSizeValue string, + interval unit.Duration, + l *slog.Logger, +) error { + for { + if err := localTrim(dir, unusedFor, maxSizeValue, l); err != nil { + return err + } + + select { + case <-ctx.Done(): + return nil + case <-time.After(time.Duration(interval)): + } + } +} diff --git a/main.go b/main.go index 9ee6908..cf02393 100644 --- a/main.go +++ b/main.go @@ -2,18 +2,16 @@ package main import ( "context" - "fmt" "log" "log/slog" "os" "os/exec" "strings" "sync" - "time" "github.com/alecthomas/kong" - "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/commands" "github.com/AlekSi/hardcache/internal/sigterm" "github.com/AlekSi/hardcache/internal/unit" ) @@ -60,74 +58,6 @@ var GOCACHE = sync.OnceValue(func() string { return strings.TrimSpace(string(b)) }) -// localTrim force-trims local cache according to parameters. -func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog.Logger) error { - if unusedFor < 0 { - return fmt.Errorf("--unused-for cannot be negative: %d", unusedFor) - } - - var cutoff *time.Time - if unusedFor > 0 { - c := time.Now().Add(-time.Duration(unusedFor)) - cutoff = &c - } - - var b unit.Bytes - if strings.HasSuffix(maxSizeValue, "%") { - var p unit.Percentage - if err := p.UnmarshalText([]byte(maxSizeValue)); err != nil { - return err - } - - total, _, err := local.DiskInfo(dir) - if err != nil { - return err - } - - b = unit.Bytes(total / 100 * int64(p)) - - l.Debug( - "Calculated max size from percentage of total disk size", - slog.Int64("disk_size_bytes", total), - slog.String("disk_size", unit.Bytes(total).String()), - slog.Int64("max_size_bytes", int64(b)), - slog.String("max_size", b.String()), - ) - } else { - if err := b.UnmarshalText([]byte(maxSizeValue)); err != nil { - return err - } - - l.Debug("Max size", slog.Int64("max_size_bytes", int64(b)), slog.String("max_size", b.String())) - } - - if b < 0 { - return fmt.Errorf("--max-size cannot be negative: %d", b) - } - - var maxSize *int64 - if b > 0 { - maxSize = (*int64)(&b) - } - - c, err := local.New(dir, cutoff, maxSize, l) - if err != nil { - return err - } - - before, freed := c.TrimForce() - l.Debug( - "Local cache trimmed", - slog.Int64("before_bytes", before), slog.Int64("freed_bytes", freed), - ) - l.Info( - "Local cache trimmed", - slog.String("before", unit.Bytes(before).String()), slog.String("freed", unit.Bytes(freed).String()), - ) - - return nil -} - func main() { opts := []kong.Option{ kong.Name("hardcache"), @@ -160,29 +90,18 @@ func main() { switch kongCtx.Command() { case "local status": - err := localStatus(cli.Local.Dir, cli.Local.Status.JSON, l) + err := commands.LocalStatus(cli.Local.Dir, cli.Local.Status.JSON, os.Stdout, l) kongCtx.FatalIfErrorf(err) case "local trim": - if time.Duration(cli.Local.Trim.UnusedFor) > 5*24*time.Hour { - l.Info("Note: this command should be invoked more often than once per day to keep the cache.") - } - - err := localTrim(cli.Local.Dir, cli.Local.Trim.UnusedFor, cli.Local.Trim.MaxSize, l) + err := commands.LocalTrim(cli.Local.Dir, cli.Local.Trim.UnusedFor, cli.Local.Trim.MaxSize, l) kongCtx.FatalIfErrorf(err) case "local trimd": - for { - err := localTrim(cli.Local.Dir, cli.Local.Trimd.UnusedFor, cli.Local.Trimd.MaxSize, l) - kongCtx.FatalIfErrorf(err) - - select { - case <-ctx.Done(): - return - case <-time.After(time.Duration(cli.Local.Trimd.Interval)): - // nothing - } - } + err := commands.LocalTrimd( + ctx, cli.Local.Dir, cli.Local.Trimd.UnusedFor, cli.Local.Trimd.MaxSize, cli.Local.Trimd.Interval, l, + ) + kongCtx.FatalIfErrorf(err) default: kongCtx.Fatalf("unknown command: %q", kongCtx.Command()) diff --git a/main_status_test.go b/main_status_test.go deleted file mode 100644 index 241b58c..0000000 --- a/main_status_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "log/slog" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/AlekSi/hardcache/internal/caches/local" - "github.com/AlekSi/hardcache/internal/unit" - "github.com/AlekSi/shoulda" - "github.com/AlekSi/shoulda/musta" -) - -func TestLocalStatusOutput(t *testing.T) { - t.Parallel() - - src := filepath.Join("internal", "testdata", "local") - dst := t.TempDir() - b, err := exec.Command("cp", "-a", src, dst).CombinedOutput() - musta.NoErrorf(t, err, "%s", b) - dir := filepath.Join(dst, "local") - - c := musta.NotFail(local.New(dir, nil, nil, slog.Default()))(t) - stats := c.Status() - total, free := musta.NotFail2(local.DiskInfo(dir))(t) - output := newLocalStatusOutput(dir, stats, total, free) - - oldest := time.Date(2025, time.November, 17, 17, 12, 57, 524467000, time.UTC).Local().Format(time.RFC3339) - newest := time.Date(2025, time.November, 17, 17, 13, 7, 284400000, time.UTC).Local().Format(time.RFC3339) - - shoulda.BeEqual(t, output.Cache.Entries, 1219) - shoulda.BeEqual(t, output.Cache.Bytes, int64(109_518_524)) - shoulda.BeEqual(t, output.Cache.Human, "109MB") - shoulda.BeEqual(t, *output.Cache.Oldest, oldest) - shoulda.BeEqual(t, *output.Cache.Newest, newest) - - t.Run("text", func(t *testing.T) { - actual := output.String() - shoulda.SatisfyWith(t, actual, "Directory: "+dir, strings.Contains) - shoulda.SatisfyWith(t, actual, "Cache entries: 1219", strings.Contains) - shoulda.SatisfyWith(t, actual, "Cache size: 109MB (109518524 bytes)", strings.Contains) - shoulda.SatisfyWith(t, actual, "Oldest entry: "+oldest, strings.Contains) - shoulda.SatisfyWith(t, actual, "Newest entry: "+newest, strings.Contains) - shoulda.SatisfyWith(t, actual, - fmt.Sprintf("Disk total: %s (%d bytes)", unit.Bytes(total), total), strings.Contains) - shoulda.SatisfyWith(t, actual, - fmt.Sprintf("Disk free: %s (%d bytes)", unit.Bytes(free), free), strings.Contains) - }) - - t.Run("JSON", func(t *testing.T) { - actual, err := json.Marshal(output) - musta.NoError(t, err) - shoulda.BeZero(t, strings.Contains(string(actual), "\n")) - - var got localStatusOutput - musta.NoError(t, json.Unmarshal(actual, &got)) - shoulda.BeDeepEqual(t, got, output) - }) -} - -func TestLocalStatusOutputEmpty(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - c := musta.NotFail(local.New(dir, nil, nil, slog.Default()))(t) - stats := c.Status() - total, free := musta.NotFail2(local.DiskInfo(dir))(t) - output := newLocalStatusOutput(dir, stats, total, free) - - actual := output.String() - shoulda.SatisfyWith(t, actual, "Cache entries: 0", strings.Contains) - shoulda.SatisfyWith(t, actual, "Cache size: 0B (0 bytes)", strings.Contains) - shoulda.SatisfyWith(t, actual, "Oldest entry: n/a", strings.Contains) - shoulda.SatisfyWith(t, actual, "Newest entry: n/a", strings.Contains) -} From d3f9e42ff836d13e61c2d2758e1256ae767ab10d Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 00:12:19 +0400 Subject: [PATCH 06/15] WIP --- Makefile | 1 + internal/commands/local_trim.go | 41 ++++++++++++++++- internal/commands/local_trim_test.go | 67 ++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 internal/commands/local_trim_test.go diff --git a/Makefile b/Makefile index efd99a1..a79787c 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,5 @@ test: run: go build -race -o bin/ bin/hardcache --help + bin/hardcache local status mkdir -p tmp/cache diff --git a/internal/commands/local_trim.go b/internal/commands/local_trim.go index 3788d18..c6fffff 100644 --- a/internal/commands/local_trim.go +++ b/internal/commands/local_trim.go @@ -75,13 +75,50 @@ func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog } before, freed := c.TrimForce() + stats := c.Status() + total, free, err := local.DiskInfo(dir) + if err != nil { + return err + } + + status := newLocalStatusOutput(dir, stats, total, free) + if before < 0 { + before = stats.Bytes + freed + } + + oldest, newest := "n/a", "n/a" + if status.Cache.Oldest != nil { + oldest = *status.Cache.Oldest + } + if status.Cache.Newest != nil { + newest = *status.Cache.Newest + } + l.Debug( "Local cache trimmed", - slog.Int64("before_bytes", before), slog.Int64("freed_bytes", freed), + slog.Int64("before_bytes", before), + slog.Int64("after_bytes", stats.Bytes), + slog.Int64("freed_bytes", freed), ) l.Info( "Local cache trimmed", - slog.String("before", unit.Bytes(before).String()), slog.String("freed", unit.Bytes(freed).String()), + slog.String("directory", status.Directory), + slog.String("before", fmt.Sprintf("%s (%d bytes)", unit.Bytes(before), before)), + slog.String("freed", fmt.Sprintf("%s (%d bytes)", unit.Bytes(freed), freed)), + slog.Group("cache", + slog.Int("entries", status.Cache.Entries), + slog.String("size", fmt.Sprintf("%s (%d bytes)", status.Cache.Human, status.Cache.Bytes)), + slog.String("oldest", oldest), + slog.String("newest", newest), + ), + slog.Group("disk", + slog.String("total", fmt.Sprintf("%s (%d bytes)", status.Disk.TotalHuman, status.Disk.TotalBytes)), + slog.String("used", fmt.Sprintf("%s (%d bytes)", status.Disk.UsedHuman, status.Disk.UsedBytes)), + slog.String("used_percent", fmt.Sprintf("%.2f%%", status.Disk.UsedPercent)), + slog.String("free", fmt.Sprintf("%s (%d bytes)", status.Disk.FreeHuman, status.Disk.FreeBytes)), + slog.String("free_percent", fmt.Sprintf("%.2f%%", status.Disk.FreePercent)), + ), + slog.String("cache_of_total_disk", fmt.Sprintf("%.2f%%", status.CacheOfTotalPercent)), ) return nil diff --git a/internal/commands/local_trim_test.go b/internal/commands/local_trim_test.go new file mode 100644 index 0000000..6ea49dc --- /dev/null +++ b/internal/commands/local_trim_test.go @@ -0,0 +1,67 @@ +package commands + +import ( + "context" + "log/slog" + "strings" + "testing" + "time" + + "github.com/AlekSi/shoulda" + "github.com/AlekSi/shoulda/musta" + + "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/unit" +) + +func TestLocalTrimStatistics(t *testing.T) { + t.Parallel() + + dir := setup(t) + var output strings.Builder + l := slog.New(slog.NewTextHandler(&output, nil)) + + musta.NoError(t, LocalTrim(dir, 0, "50MB", l)) + + stats := musta.NotFail(local.New(dir, nil, nil, l))(t).Status() + shoulda.BeEqual(t, stats.Bytes, int64(49_494_929)) + + actual := output.String() + for _, expected := range []string{ + `msg="Local cache trimmed"`, + `directory=`, + `before="109MB (109518524 bytes)"`, + `freed="60MB (60023595 bytes)"`, + `cache.entries=`, + `cache.size="49MB (49494929 bytes)"`, + `cache.oldest=`, + `cache.newest=`, + `disk.total=`, + `disk.used=`, + `disk.used_percent=`, + `disk.free=`, + `disk.free_percent=`, + `cache_of_total_disk=`, + } { + shoulda.SatisfyWith(t, actual, expected, strings.Contains) + } +} + +func TestLocalTrimdStatistics(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + var output strings.Builder + l := slog.New(slog.NewTextHandler(&output, nil)) + musta.NoError(t, LocalTrimd(ctx, t.TempDir(), 0, "0GB", unit.Duration(time.Hour), l)) + + actual := output.String() + shoulda.SatisfyWith(t, actual, `msg="Local cache trimmed"`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.entries=0`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.size="0B (0 bytes)"`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.oldest=n/a`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.newest=n/a`, strings.Contains) + shoulda.SatisfyWith(t, actual, `disk.total=`, strings.Contains) +} From b72403d07ef37d97bc55da7f4d560a458f8bdd9b Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 08:19:16 +0400 Subject: [PATCH 07/15] WIP --- internal/commands/local_status.go | 16 ++++--- internal/commands/local_status_test.go | 6 +-- internal/commands/local_trim.go | 60 ++++++++++---------------- internal/commands/local_trim_test.go | 24 +---------- internal/commands/local_trimd.go | 38 ++++++++++++++++ internal/commands/local_trimd_test.go | 36 ++++++++++++++++ main.go | 20 ++++++--- 7 files changed, 126 insertions(+), 74 deletions(-) create mode 100644 internal/commands/local_trimd.go create mode 100644 internal/commands/local_trimd_test.go diff --git a/internal/commands/local_status.go b/internal/commands/local_status.go index 6ae628b..fc6361e 100644 --- a/internal/commands/local_status.go +++ b/internal/commands/local_status.go @@ -34,21 +34,27 @@ type localStatusOutput struct { CacheOfTotalPercent float64 `json:"cache_of_total_percent"` } +// LocalStatusOpts contains flag values for [LocalStatus]. +type LocalStatusOpts struct { + Dir string + JSON bool +} + // LocalStatus writes local cache and disk usage statistics to out. -func LocalStatus(dir string, asJSON bool, out io.Writer, l *slog.Logger) error { - c, err := local.New(dir, nil, nil, l) +func LocalStatus(opts *LocalStatusOpts, out io.Writer, l *slog.Logger) error { + c, err := local.New(opts.Dir, nil, nil, l) if err != nil { return err } stats := c.Status() - total, free, err := local.DiskInfo(dir) + total, free, err := local.DiskInfo(opts.Dir) if err != nil { return err } - output := newLocalStatusOutput(dir, stats, total, free) - if asJSON { + output := newLocalStatusOutput(opts.Dir, stats, total, free) + if opts.JSON { return json.NewEncoder(out).Encode(output) } diff --git a/internal/commands/local_status_test.go b/internal/commands/local_status_test.go index 56220f9..4ce606a 100644 --- a/internal/commands/local_status_test.go +++ b/internal/commands/local_status_test.go @@ -34,7 +34,7 @@ func TestLocalStatus(t *testing.T) { t.Run("text", func(t *testing.T) { var output strings.Builder - musta.NoError(t, LocalStatus(dir, false, &output, slog.Default())) + musta.NoError(t, LocalStatus(&LocalStatusOpts{Dir: dir}, &output, slog.Default())) actual := output.String() shoulda.SatisfyWith(t, actual, "Directory: "+dir, strings.Contains) @@ -48,7 +48,7 @@ func TestLocalStatus(t *testing.T) { t.Run("JSON", func(t *testing.T) { var output strings.Builder - musta.NoError(t, LocalStatus(dir, true, &output, slog.Default())) + musta.NoError(t, LocalStatus(&LocalStatusOpts{Dir: dir, JSON: true}, &output, slog.Default())) actual := output.String() shoulda.SatisfyWith(t, actual, "\n", strings.HasSuffix) @@ -73,7 +73,7 @@ func TestLocalStatusEmpty(t *testing.T) { t.Parallel() var output strings.Builder - musta.NoError(t, LocalStatus(t.TempDir(), false, &output, slog.Default())) + musta.NoError(t, LocalStatus(&LocalStatusOpts{Dir: t.TempDir()}, &output, slog.Default())) actual := output.String() shoulda.SatisfyWith(t, actual, "Cache entries: 0", strings.Contains) diff --git a/internal/commands/local_trim.go b/internal/commands/local_trim.go index c6fffff..6b9b689 100644 --- a/internal/commands/local_trim.go +++ b/internal/commands/local_trim.go @@ -1,7 +1,6 @@ package commands import ( - "context" "fmt" "log/slog" "strings" @@ -11,34 +10,41 @@ import ( "github.com/AlekSi/hardcache/internal/unit" ) +// LocalTrimOpts contains flag values for [LocalTrim]. +type LocalTrimOpts struct { + Dir string + UnusedFor unit.Duration + MaxSize string +} + // LocalTrim force-trims a local cache according to the given parameters. -func LocalTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog.Logger) error { - if time.Duration(unusedFor) > 5*24*time.Hour { +func LocalTrim(opts *LocalTrimOpts, l *slog.Logger) error { + if time.Duration(opts.UnusedFor) > 5*24*time.Hour { l.Info("Note: this command should be invoked more often than once per day to keep the cache.") } - return localTrim(dir, unusedFor, maxSizeValue, l) + return localTrim(opts, l) } -func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog.Logger) error { - if unusedFor < 0 { - return fmt.Errorf("--unused-for cannot be negative: %d", unusedFor) +func localTrim(opts *LocalTrimOpts, l *slog.Logger) error { + if opts.UnusedFor < 0 { + return fmt.Errorf("--unused-for cannot be negative: %d", opts.UnusedFor) } var cutoff *time.Time - if unusedFor > 0 { - c := time.Now().Add(-time.Duration(unusedFor)) + if opts.UnusedFor > 0 { + c := time.Now().Add(-time.Duration(opts.UnusedFor)) cutoff = &c } var b unit.Bytes - if strings.HasSuffix(maxSizeValue, "%") { + if strings.HasSuffix(opts.MaxSize, "%") { var p unit.Percentage - if err := p.UnmarshalText([]byte(maxSizeValue)); err != nil { + if err := p.UnmarshalText([]byte(opts.MaxSize)); err != nil { return err } - total, _, err := local.DiskInfo(dir) + total, _, err := local.DiskInfo(opts.Dir) if err != nil { return err } @@ -53,7 +59,7 @@ func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog slog.String("max_size", b.String()), ) } else { - if err := b.UnmarshalText([]byte(maxSizeValue)); err != nil { + if err := b.UnmarshalText([]byte(opts.MaxSize)); err != nil { return err } @@ -69,19 +75,19 @@ func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog maxSize = (*int64)(&b) } - c, err := local.New(dir, cutoff, maxSize, l) + c, err := local.New(opts.Dir, cutoff, maxSize, l) if err != nil { return err } before, freed := c.TrimForce() stats := c.Status() - total, free, err := local.DiskInfo(dir) + total, free, err := local.DiskInfo(opts.Dir) if err != nil { return err } - status := newLocalStatusOutput(dir, stats, total, free) + status := newLocalStatusOutput(opts.Dir, stats, total, free) if before < 0 { before = stats.Bytes + freed } @@ -123,25 +129,3 @@ func localTrim(dir string, unusedFor unit.Duration, maxSizeValue string, l *slog return nil } - -// LocalTrimd continuously trims a local cache until ctx is canceled. -func LocalTrimd( - ctx context.Context, - dir string, - unusedFor unit.Duration, - maxSizeValue string, - interval unit.Duration, - l *slog.Logger, -) error { - for { - if err := localTrim(dir, unusedFor, maxSizeValue, l); err != nil { - return err - } - - select { - case <-ctx.Done(): - return nil - case <-time.After(time.Duration(interval)): - } - } -} diff --git a/internal/commands/local_trim_test.go b/internal/commands/local_trim_test.go index 6ea49dc..16b4670 100644 --- a/internal/commands/local_trim_test.go +++ b/internal/commands/local_trim_test.go @@ -1,17 +1,14 @@ package commands import ( - "context" "log/slog" "strings" "testing" - "time" "github.com/AlekSi/shoulda" "github.com/AlekSi/shoulda/musta" "github.com/AlekSi/hardcache/internal/caches/local" - "github.com/AlekSi/hardcache/internal/unit" ) func TestLocalTrimStatistics(t *testing.T) { @@ -21,7 +18,7 @@ func TestLocalTrimStatistics(t *testing.T) { var output strings.Builder l := slog.New(slog.NewTextHandler(&output, nil)) - musta.NoError(t, LocalTrim(dir, 0, "50MB", l)) + musta.NoError(t, LocalTrim(&LocalTrimOpts{Dir: dir, MaxSize: "50MB"}, l)) stats := musta.NotFail(local.New(dir, nil, nil, l))(t).Status() shoulda.BeEqual(t, stats.Bytes, int64(49_494_929)) @@ -46,22 +43,3 @@ func TestLocalTrimStatistics(t *testing.T) { shoulda.SatisfyWith(t, actual, expected, strings.Contains) } } - -func TestLocalTrimdStatistics(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - var output strings.Builder - l := slog.New(slog.NewTextHandler(&output, nil)) - musta.NoError(t, LocalTrimd(ctx, t.TempDir(), 0, "0GB", unit.Duration(time.Hour), l)) - - actual := output.String() - shoulda.SatisfyWith(t, actual, `msg="Local cache trimmed"`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.entries=0`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.size="0B (0 bytes)"`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.oldest=n/a`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.newest=n/a`, strings.Contains) - shoulda.SatisfyWith(t, actual, `disk.total=`, strings.Contains) -} diff --git a/internal/commands/local_trimd.go b/internal/commands/local_trimd.go new file mode 100644 index 0000000..a9bbfaf --- /dev/null +++ b/internal/commands/local_trimd.go @@ -0,0 +1,38 @@ +package commands + +import ( + "context" + "log/slog" + "time" + + "github.com/AlekSi/hardcache/internal/unit" +) + +// LocalTrimdOpts contains flag values for [LocalTrimd]. +type LocalTrimdOpts struct { + Dir string + UnusedFor unit.Duration + MaxSize string + Interval unit.Duration +} + +// LocalTrimd continuously trims a local cache until ctx is canceled. +func LocalTrimd(ctx context.Context, opts *LocalTrimdOpts, l *slog.Logger) error { + trimOpts := &LocalTrimOpts{ + Dir: opts.Dir, + UnusedFor: opts.UnusedFor, + MaxSize: opts.MaxSize, + } + + for { + if err := localTrim(trimOpts, l); err != nil { + return err + } + + select { + case <-ctx.Done(): + return nil + case <-time.After(time.Duration(opts.Interval)): + } + } +} diff --git a/internal/commands/local_trimd_test.go b/internal/commands/local_trimd_test.go new file mode 100644 index 0000000..245169c --- /dev/null +++ b/internal/commands/local_trimd_test.go @@ -0,0 +1,36 @@ +package commands + +import ( + "context" + "log/slog" + "strings" + "testing" + "time" + + "github.com/AlekSi/hardcache/internal/unit" + "github.com/AlekSi/shoulda" + "github.com/AlekSi/shoulda/musta" +) + +func TestLocalTrimdStatistics(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + var output strings.Builder + l := slog.New(slog.NewTextHandler(&output, nil)) + musta.NoError(t, LocalTrimd(ctx, &LocalTrimdOpts{ + Dir: t.TempDir(), + MaxSize: "0GB", + Interval: unit.Duration(time.Hour), + }, l)) + + actual := output.String() + shoulda.SatisfyWith(t, actual, `msg="Local cache trimmed"`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.entries=0`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.size="0B (0 bytes)"`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.oldest=n/a`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.newest=n/a`, strings.Contains) + shoulda.SatisfyWith(t, actual, `disk.total=`, strings.Contains) +} diff --git a/main.go b/main.go index cf02393..8432846 100644 --- a/main.go +++ b/main.go @@ -90,17 +90,27 @@ func main() { switch kongCtx.Command() { case "local status": - err := commands.LocalStatus(cli.Local.Dir, cli.Local.Status.JSON, os.Stdout, l) + err := commands.LocalStatus(&commands.LocalStatusOpts{ + Dir: cli.Local.Dir, + JSON: cli.Local.Status.JSON, + }, os.Stdout, l) kongCtx.FatalIfErrorf(err) case "local trim": - err := commands.LocalTrim(cli.Local.Dir, cli.Local.Trim.UnusedFor, cli.Local.Trim.MaxSize, l) + err := commands.LocalTrim(&commands.LocalTrimOpts{ + Dir: cli.Local.Dir, + UnusedFor: cli.Local.Trim.UnusedFor, + MaxSize: cli.Local.Trim.MaxSize, + }, l) kongCtx.FatalIfErrorf(err) case "local trimd": - err := commands.LocalTrimd( - ctx, cli.Local.Dir, cli.Local.Trimd.UnusedFor, cli.Local.Trimd.MaxSize, cli.Local.Trimd.Interval, l, - ) + err := commands.LocalTrimd(ctx, &commands.LocalTrimdOpts{ + Dir: cli.Local.Dir, + UnusedFor: cli.Local.Trimd.UnusedFor, + MaxSize: cli.Local.Trimd.MaxSize, + Interval: cli.Local.Trimd.Interval, + }, l) kongCtx.FatalIfErrorf(err) default: From e39da8b117082305bc22fbbd04ac1639387d2b9a Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 08:45:38 +0400 Subject: [PATCH 08/15] WIP --- internal/caches/local/local_test.go | 40 ++++++--------- internal/caches/local/localtest/localtest.go | 25 ++++++++++ internal/commands/local_status_test.go | 28 +++++------ internal/commands/local_trim.go | 6 +-- internal/commands/local_trim_test.go | 3 +- internal/commands/local_trimd.go | 6 ++- internal/commands/local_trimd_test.go | 25 +++++++--- main.go | 51 ++++++++++---------- 8 files changed, 105 insertions(+), 79 deletions(-) create mode 100644 internal/caches/local/localtest/localtest.go diff --git a/internal/caches/local/local_test.go b/internal/caches/local/local_test.go index 257dbb3..d508aac 100644 --- a/internal/caches/local/local_test.go +++ b/internal/caches/local/local_test.go @@ -4,30 +4,16 @@ import ( "encoding/hex" "log/slog" "math" - "os/exec" - "path/filepath" "testing" "time" "github.com/AlekSi/shoulda" "github.com/AlekSi/shoulda/musta" + "github.com/AlekSi/hardcache/internal/caches/local/localtest" "github.com/AlekSi/hardcache/internal/go/cache" ) -// setup copies testdata cache to a test-specific temporary directory. -func setup(t testing.TB) string { - t.Helper() - - src := filepath.Join("..", "..", "testdata", "local") - dst := t.TempDir() - - b, err := exec.Command("cp", "-a", src, dst).CombinedOutput() - musta.NoErrorf(t, err, "%s", b) - - return filepath.Join(dst, "local") -} - // logger returns a [slog.Logger] for the given test. func logger(t testing.TB) *slog.Logger { t.Helper() @@ -66,7 +52,7 @@ func outputID(t testing.TB, s string) cache.OutputID { func TestCache(t *testing.T) { t.Parallel() - dir := setup(t) + dir := localtest.Setup(t) c := musta.NotFail(cache.Open(dir))(t) @@ -108,7 +94,7 @@ func TestCache(t *testing.T) { func TestTrimNoop(t *testing.T) { t.Parallel() - c, err := New(setup(t), nil, nil, logger(t)) + c, err := New(localtest.Setup(t), nil, nil, logger(t)) musta.NoError(t, err) before, freed := c.TrimForce() @@ -120,7 +106,7 @@ func TestTrimCutoffNone(t *testing.T) { t.Parallel() cutoff := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) - c := musta.NotFail(New(setup(t), &cutoff, nil, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), &cutoff, nil, logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -131,7 +117,7 @@ func TestTrimCutoffAll(t *testing.T) { t.Parallel() cutoff := time.Date(2999, time.January, 1, 0, 0, 0, 0, time.UTC) - c := musta.NotFail(New(setup(t), &cutoff, nil, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), &cutoff, nil, logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -142,7 +128,7 @@ func TestTrimCutoffPart(t *testing.T) { t.Parallel() cutoff := time.Date(2025, time.November, 17, 17, 13, 0, 0, time.UTC) - c := musta.NotFail(New(setup(t), &cutoff, nil, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), &cutoff, nil, logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -153,7 +139,7 @@ func TestTrimSizeNone(t *testing.T) { t.Parallel() maxSize := int64(math.MaxInt64) - c := musta.NotFail(New(setup(t), nil, &maxSize, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -164,7 +150,7 @@ func TestTrimSizeAll(t *testing.T) { t.Parallel() maxSize := int64(0) - c := musta.NotFail(New(setup(t), nil, &maxSize, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -175,7 +161,7 @@ func TestTrimSizePart(t *testing.T) { t.Parallel() maxSize := int64(50_000_000) - c := musta.NotFail(New(setup(t), nil, &maxSize, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -186,7 +172,7 @@ func TestTrimSizePart(t *testing.T) { func TestStatusFixture(t *testing.T) { t.Parallel() - c := musta.NotFail(New(setup(t), nil, nil, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, nil, logger(t)))(t) stats := c.Status() shoulda.BeEqual(t, stats.Entries, 1219) @@ -199,7 +185,11 @@ func TestStatusFixture(t *testing.T) { func TestStatusEmpty(t *testing.T) { t.Parallel() - c := musta.NotFail(New(t.TempDir(), nil, nil, logger(t)))(t) + maxSize := int64(0) + c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) + before, freed := c.TrimForce() + shoulda.BeEqual(t, before, int64(109_518_524)) + shoulda.BeEqual(t, freed, before) stats := c.Status() shoulda.BeEqual(t, stats.Entries, 0) diff --git a/internal/caches/local/localtest/localtest.go b/internal/caches/local/localtest/localtest.go new file mode 100644 index 0000000..bbd6901 --- /dev/null +++ b/internal/caches/local/localtest/localtest.go @@ -0,0 +1,25 @@ +package localtest + +import ( + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/AlekSi/shoulda/musta" +) + +// Setup copies testdata cache to a test-specific temporary directory. +func Setup(t testing.TB) string { + t.Helper() + + _, filename, _, ok := runtime.Caller(0) + musta.BeTrue(t, ok) + src := filepath.Join(filepath.Dir(filename), "..", "..", "..", "testdata", "local") + dst := t.ArtifactDir() + + b, err := exec.Command("cp", "-a", src, dst).CombinedOutput() + musta.NoErrorf(t, err, "%s", b) + + return filepath.Join(dst, "local") +} diff --git a/internal/commands/local_status_test.go b/internal/commands/local_status_test.go index 4ce606a..865f70a 100644 --- a/internal/commands/local_status_test.go +++ b/internal/commands/local_status_test.go @@ -3,32 +3,21 @@ package commands import ( "encoding/json" "log/slog" - "os/exec" - "path/filepath" "strings" "testing" "time" "github.com/AlekSi/shoulda" "github.com/AlekSi/shoulda/musta" -) - -// setup copies the testdata cache to a test-specific temporary directory. -func setup(t testing.TB) string { - t.Helper() - - src := filepath.Join("..", "testdata", "local") - dst := t.TempDir() - b, err := exec.Command("cp", "-a", src, dst).CombinedOutput() - musta.NoErrorf(t, err, "%s", b) - return filepath.Join(dst, "local") -} + "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/caches/local/localtest" +) func TestLocalStatus(t *testing.T) { t.Parallel() - dir := setup(t) + dir := localtest.Setup(t) oldest := time.Date(2025, time.November, 17, 17, 12, 57, 524467000, time.UTC).Local().Format(time.RFC3339) newest := time.Date(2025, time.November, 17, 17, 13, 7, 284400000, time.UTC).Local().Format(time.RFC3339) @@ -72,8 +61,15 @@ func TestLocalStatus(t *testing.T) { func TestLocalStatusEmpty(t *testing.T) { t.Parallel() + dir := localtest.Setup(t) + maxSize := int64(0) + c := musta.NotFail(local.New(dir, nil, &maxSize, slog.Default()))(t) + before, freed := c.TrimForce() + shoulda.BeEqual(t, before, int64(109_518_524)) + shoulda.BeEqual(t, freed, before) + var output strings.Builder - musta.NoError(t, LocalStatus(&LocalStatusOpts{Dir: t.TempDir()}, &output, slog.Default())) + musta.NoError(t, LocalStatus(&LocalStatusOpts{Dir: dir}, &output, slog.Default())) actual := output.String() shoulda.SatisfyWith(t, actual, "Cache entries: 0", strings.Contains) diff --git a/internal/commands/local_trim.go b/internal/commands/local_trim.go index 6b9b689..44308be 100644 --- a/internal/commands/local_trim.go +++ b/internal/commands/local_trim.go @@ -23,17 +23,17 @@ func LocalTrim(opts *LocalTrimOpts, l *slog.Logger) error { l.Info("Note: this command should be invoked more often than once per day to keep the cache.") } - return localTrim(opts, l) + return localTrim(opts, time.Now, l) } -func localTrim(opts *LocalTrimOpts, l *slog.Logger) error { +func localTrim(opts *LocalTrimOpts, now func() time.Time, l *slog.Logger) error { if opts.UnusedFor < 0 { return fmt.Errorf("--unused-for cannot be negative: %d", opts.UnusedFor) } var cutoff *time.Time if opts.UnusedFor > 0 { - c := time.Now().Add(-time.Duration(opts.UnusedFor)) + c := now().Add(-time.Duration(opts.UnusedFor)) cutoff = &c } diff --git a/internal/commands/local_trim_test.go b/internal/commands/local_trim_test.go index 16b4670..e4c4f19 100644 --- a/internal/commands/local_trim_test.go +++ b/internal/commands/local_trim_test.go @@ -9,12 +9,13 @@ import ( "github.com/AlekSi/shoulda/musta" "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/caches/local/localtest" ) func TestLocalTrimStatistics(t *testing.T) { t.Parallel() - dir := setup(t) + dir := localtest.Setup(t) var output strings.Builder l := slog.New(slog.NewTextHandler(&output, nil)) diff --git a/internal/commands/local_trimd.go b/internal/commands/local_trimd.go index a9bbfaf..6f66bc5 100644 --- a/internal/commands/local_trimd.go +++ b/internal/commands/local_trimd.go @@ -24,15 +24,17 @@ func LocalTrimd(ctx context.Context, opts *LocalTrimdOpts, l *slog.Logger) error MaxSize: opts.MaxSize, } + t := time.Tick(time.Duration(opts.Interval)) + for { - if err := localTrim(trimOpts, l); err != nil { + if err := localTrim(trimOpts, time.Now, l); err != nil { return err } select { case <-ctx.Done(): return nil - case <-time.After(time.Duration(opts.Interval)): + case <-t: } } } diff --git a/internal/commands/local_trimd_test.go b/internal/commands/local_trimd_test.go index 245169c..42b0d3a 100644 --- a/internal/commands/local_trimd_test.go +++ b/internal/commands/local_trimd_test.go @@ -3,13 +3,17 @@ package commands import ( "context" "log/slog" + "strconv" "strings" "testing" "time" - "github.com/AlekSi/hardcache/internal/unit" "github.com/AlekSi/shoulda" "github.com/AlekSi/shoulda/musta" + + "github.com/AlekSi/hardcache/internal/caches/local" + "github.com/AlekSi/hardcache/internal/caches/local/localtest" + "github.com/AlekSi/hardcache/internal/unit" ) func TestLocalTrimdStatistics(t *testing.T) { @@ -18,19 +22,26 @@ func TestLocalTrimdStatistics(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() + dir := localtest.Setup(t) var output strings.Builder l := slog.New(slog.NewTextHandler(&output, nil)) musta.NoError(t, LocalTrimd(ctx, &LocalTrimdOpts{ - Dir: t.TempDir(), - MaxSize: "0GB", + Dir: dir, + MaxSize: "50MB", Interval: unit.Duration(time.Hour), }, l)) + stats := musta.NotFail(local.New(dir, nil, nil, l))(t).Status() + shoulda.BeEqual(t, stats.Bytes, int64(49_494_929)) + shoulda.BeGreater(t, stats.Entries, 0) + actual := output.String() shoulda.SatisfyWith(t, actual, `msg="Local cache trimmed"`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.entries=0`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.size="0B (0 bytes)"`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.oldest=n/a`, strings.Contains) - shoulda.SatisfyWith(t, actual, `cache.newest=n/a`, strings.Contains) + shoulda.SatisfyWith(t, actual, `before="109MB (109518524 bytes)"`, strings.Contains) + shoulda.SatisfyWith(t, actual, `freed="60MB (60023595 bytes)"`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.entries=`+strconv.Itoa(stats.Entries), strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.size="49MB (49494929 bytes)"`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.oldest=`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.newest=`, strings.Contains) shoulda.SatisfyWith(t, actual, `disk.total=`, strings.Contains) } diff --git a/main.go b/main.go index 8432846..10682b6 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,22 @@ import ( "github.com/AlekSi/hardcache/internal/unit" ) +// GOCACHE returns the Go build cache directory. +var GOCACHE = sync.OnceValue(func() string { + // in theory, someone might not have go in the PATH + if v := os.Getenv("GOCACHE"); v != "" { + return v + } + + // that handles `go env -w` and the default value + b, err := exec.Command("go", "env", "GOCACHE").Output() + if err != nil { + panic(err) + } + + return strings.TrimSpace(string(b)) +}) + // cli represents CLI arguments and flags. // //nolint:vet // for readability @@ -28,12 +44,12 @@ var cli struct { } `cmd:"" help:"Show local cache status."` Trim struct { - UnusedFor unit.Duration `default:"5d" help:"Always remove entries unused for this duration. Pass 0 to disable."` + UnusedFor unit.Duration `default:"5d" help:"${local_unused_for_help}"` MaxSize string `default:"0GB" help:"${local_max_size_help}"` } `cmd:"" help:"Trim local cache."` Trimd struct { - UnusedFor unit.Duration `default:"5d" help:"Always remove entries unused for this duration. Pass 0 to disable."` + UnusedFor unit.Duration `default:"5d" help:"${local_unused_for_help}"` MaxSize string `default:"0GB" help:"${local_max_size_help}"` Interval unit.Duration `short:"i" default:"1h" help:"Interval between trimmings."` } `cmd:"" help:"Trim local cache continuously."` @@ -42,28 +58,13 @@ var cli struct { Debug bool `help:"Enable debug logging."` } -// GOCACHE returns the Go build cache directory. -var GOCACHE = sync.OnceValue(func() string { - // in theory, someone might not have go in the PATH - if v := os.Getenv("GOCACHE"); v != "" { - return v - } - - // that handles `go env -w` and the default value - b, err := exec.Command("go", "env", "GOCACHE").Output() - if err != nil { - panic(err) - } - - return strings.TrimSpace(string(b)) -}) - func main() { opts := []kong.Option{ kong.Name("hardcache"), kong.Description("Tool for managing the Go build cache."), kong.Vars{ - "local_dir_default": GOCACHE(), + "local_dir_default": GOCACHE(), + "local_unused_for_help": "Always remove entries unused for this duration. Pass 0 to disable.", "local_max_size_help": "Remove entries, starting from least recently used, " + "if cache size is larger than this value. " + "Supports MiB, GB, etc. suffixes, or percentage of the total disk space (e.g., 5%). " + @@ -88,32 +89,32 @@ func main() { ctx, cancel := sigterm.Ctx(context.Background()) defer cancel() + var err error switch kongCtx.Command() { case "local status": - err := commands.LocalStatus(&commands.LocalStatusOpts{ + err = commands.LocalStatus(&commands.LocalStatusOpts{ Dir: cli.Local.Dir, JSON: cli.Local.Status.JSON, }, os.Stdout, l) - kongCtx.FatalIfErrorf(err) case "local trim": - err := commands.LocalTrim(&commands.LocalTrimOpts{ + err = commands.LocalTrim(&commands.LocalTrimOpts{ Dir: cli.Local.Dir, UnusedFor: cli.Local.Trim.UnusedFor, MaxSize: cli.Local.Trim.MaxSize, }, l) - kongCtx.FatalIfErrorf(err) case "local trimd": - err := commands.LocalTrimd(ctx, &commands.LocalTrimdOpts{ + err = commands.LocalTrimd(ctx, &commands.LocalTrimdOpts{ Dir: cli.Local.Dir, UnusedFor: cli.Local.Trimd.UnusedFor, MaxSize: cli.Local.Trimd.MaxSize, Interval: cli.Local.Trimd.Interval, }, l) - kongCtx.FatalIfErrorf(err) default: kongCtx.Fatalf("unknown command: %q", kongCtx.Command()) } + + kongCtx.FatalIfErrorf(err) } From e6197273ddb420797fef1ac26b85faed8c78a023 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 08:52:56 +0400 Subject: [PATCH 09/15] WIP --- internal/caches/local/local_test.go | 26 ++++++++++++------------ internal/commands/local_status.go | 6 ++---- internal/commands/local_status_test.go | 3 +-- internal/commands/local_trim.go | 5 ++--- internal/go/cache/cache.go | 4 ++-- internal/go/cache/cache_extra.go | 6 ++---- internal/go/cache/hash.go | 4 +--- internal/go/lockedfile/transform_test.go | 6 +++--- 8 files changed, 26 insertions(+), 34 deletions(-) diff --git a/internal/caches/local/local_test.go b/internal/caches/local/local_test.go index d508aac..9fc51b7 100644 --- a/internal/caches/local/local_test.go +++ b/internal/caches/local/local_test.go @@ -105,8 +105,9 @@ func TestTrimNoop(t *testing.T) { func TestTrimCutoffNone(t *testing.T) { t.Parallel() - cutoff := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) - c := musta.NotFail(New(localtest.Setup(t), &cutoff, nil, logger(t)))(t) + c := musta.NotFail(New( + localtest.Setup(t), new(time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)), nil, logger(t), + ))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -116,8 +117,9 @@ func TestTrimCutoffNone(t *testing.T) { func TestTrimCutoffAll(t *testing.T) { t.Parallel() - cutoff := time.Date(2999, time.January, 1, 0, 0, 0, 0, time.UTC) - c := musta.NotFail(New(localtest.Setup(t), &cutoff, nil, logger(t)))(t) + c := musta.NotFail(New( + localtest.Setup(t), new(time.Date(2999, time.January, 1, 0, 0, 0, 0, time.UTC)), nil, logger(t), + ))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -127,8 +129,9 @@ func TestTrimCutoffAll(t *testing.T) { func TestTrimCutoffPart(t *testing.T) { t.Parallel() - cutoff := time.Date(2025, time.November, 17, 17, 13, 0, 0, time.UTC) - c := musta.NotFail(New(localtest.Setup(t), &cutoff, nil, logger(t)))(t) + c := musta.NotFail(New( + localtest.Setup(t), new(time.Date(2025, time.November, 17, 17, 13, 0, 0, time.UTC)), nil, logger(t), + ))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -138,8 +141,7 @@ func TestTrimCutoffPart(t *testing.T) { func TestTrimSizeNone(t *testing.T) { t.Parallel() - maxSize := int64(math.MaxInt64) - c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, new(int64(math.MaxInt64)), logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -149,8 +151,7 @@ func TestTrimSizeNone(t *testing.T) { func TestTrimSizeAll(t *testing.T) { t.Parallel() - maxSize := int64(0) - c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, new(int64(0)), logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -161,7 +162,7 @@ func TestTrimSizePart(t *testing.T) { t.Parallel() maxSize := int64(50_000_000) - c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, new(maxSize), logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) @@ -185,8 +186,7 @@ func TestStatusFixture(t *testing.T) { func TestStatusEmpty(t *testing.T) { t.Parallel() - maxSize := int64(0) - c := musta.NotFail(New(localtest.Setup(t), nil, &maxSize, logger(t)))(t) + c := musta.NotFail(New(localtest.Setup(t), nil, new(int64(0)), logger(t)))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) shoulda.BeEqual(t, freed, before) diff --git a/internal/commands/local_status.go b/internal/commands/local_status.go index fc6361e..4e21e96 100644 --- a/internal/commands/local_status.go +++ b/internal/commands/local_status.go @@ -80,12 +80,10 @@ func newLocalStatusOutput(dir string, stats local.Stats, total, free int64) loca res.Cache.Bytes = stats.Bytes res.Cache.Human = unit.Bytes(stats.Bytes).String() if stats.Oldest != nil { - oldest := stats.Oldest.Local().Format(time.RFC3339) - res.Cache.Oldest = &oldest + res.Cache.Oldest = new(stats.Oldest.Local().Format(time.RFC3339)) } if stats.Newest != nil { - newest := stats.Newest.Local().Format(time.RFC3339) - res.Cache.Newest = &newest + res.Cache.Newest = new(stats.Newest.Local().Format(time.RFC3339)) } res.Disk.TotalBytes = total res.Disk.TotalHuman = unit.Bytes(total).String() diff --git a/internal/commands/local_status_test.go b/internal/commands/local_status_test.go index 865f70a..4dff9c5 100644 --- a/internal/commands/local_status_test.go +++ b/internal/commands/local_status_test.go @@ -62,8 +62,7 @@ func TestLocalStatusEmpty(t *testing.T) { t.Parallel() dir := localtest.Setup(t) - maxSize := int64(0) - c := musta.NotFail(local.New(dir, nil, &maxSize, slog.Default()))(t) + c := musta.NotFail(local.New(dir, nil, new(int64(0)), slog.Default()))(t) before, freed := c.TrimForce() shoulda.BeEqual(t, before, int64(109_518_524)) shoulda.BeEqual(t, freed, before) diff --git a/internal/commands/local_trim.go b/internal/commands/local_trim.go index 44308be..0437a07 100644 --- a/internal/commands/local_trim.go +++ b/internal/commands/local_trim.go @@ -33,8 +33,7 @@ func localTrim(opts *LocalTrimOpts, now func() time.Time, l *slog.Logger) error var cutoff *time.Time if opts.UnusedFor > 0 { - c := now().Add(-time.Duration(opts.UnusedFor)) - cutoff = &c + cutoff = new(now().Add(-time.Duration(opts.UnusedFor))) } var b unit.Bytes @@ -72,7 +71,7 @@ func localTrim(opts *LocalTrimOpts, now func() time.Time, l *slog.Logger) error var maxSize *int64 if b > 0 { - maxSize = (*int64)(&b) + maxSize = new(int64(b)) } c, err := local.New(opts.Dir, cutoff, maxSize, l) diff --git a/internal/go/cache/cache.go b/internal/go/cache/cache.go index 3d6c791..62e850c 100644 --- a/internal/go/cache/cache.go +++ b/internal/go/cache/cache.go @@ -99,7 +99,7 @@ func Open(dir string) (*DiskCache, error) { if !info.IsDir() { return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} } - for i := 0; i < 256; i++ { + for i := range 256 { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) if err := os.MkdirAll(name, 0o777); err != nil { return nil, err @@ -369,7 +369,7 @@ func (c *DiskCache) Trim() error { // We subtract an additional mtimeInterval // to account for the imprecision of our "last used" mtimes. cutoff := now.Add(-trimLimit - mtimeInterval) - for i := 0; i < 256; i++ { + for i := range 256 { subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i)) c.trimSubdir(subdir, cutoff) } diff --git a/internal/go/cache/cache_extra.go b/internal/go/cache/cache_extra.go index 137c862..8a031bd 100644 --- a/internal/go/cache/cache_extra.go +++ b/internal/go/cache/cache_extra.go @@ -147,12 +147,10 @@ func (c *DiskCache) Stats(l *slog.Logger) Stats { for _, fi := range files { modTime := fi.ModTime if stats.Oldest == nil || modTime.Before(*stats.Oldest) { - v := modTime - stats.Oldest = &v + stats.Oldest = new(modTime) } if stats.Newest == nil || modTime.After(*stats.Newest) { - v := modTime - stats.Newest = &v + stats.Newest = new(modTime) } } diff --git a/internal/go/cache/hash.go b/internal/go/cache/hash.go index 4f79c31..0a2dbe3 100644 --- a/internal/go/cache/hash.go +++ b/internal/go/cache/hash.go @@ -48,9 +48,7 @@ var hashSalt = []byte(stripExperiment(runtime.Version())) // stripExperiment strips any GOEXPERIMENT configuration from the Go // version string. func stripExperiment(version string) string { - if i := strings.Index(version, " X:"); i >= 0 { - return version[:i] - } + version, _, _ = strings.Cut(version, " X:") return version } diff --git a/internal/go/lockedfile/transform_test.go b/internal/go/lockedfile/transform_test.go index 096d258..864cd71 100644 --- a/internal/go/lockedfile/transform_test.go +++ b/internal/go/lockedfile/transform_test.go @@ -40,7 +40,7 @@ func TestTransform(t *testing.T) { const maxChunkWords = 8 << 10 buf := make([]byte, 2*maxChunkWords*8) - for i := uint64(0); i < 2*maxChunkWords; i++ { + for i := range uint64(2 * maxChunkWords) { binary.LittleEndian.PutUint64(buf[i*8:], i) } if err := lockedfile.Write(path, bytes.NewReader(buf[:8]), 0o666); err != nil { @@ -55,7 +55,7 @@ func TestTransform(t *testing.T) { sem := make(chan bool, parallel) - for n := attempts; n > 0; n-- { + for range attempts { sem <- true go func() { defer func() { <-sem }() @@ -96,7 +96,7 @@ func TestTransform(t *testing.T) { }() } - for n := parallel; n > 0; n-- { + for range parallel { sem <- true } } From c50dd59e02bb074d2e54902bd61e8f468fd007cc Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 09:34:27 +0400 Subject: [PATCH 10/15] WIP --- internal/caches/local/local.go | 28 ++++--- internal/caches/local/local_test.go | 13 +++- internal/commands/local_status.go | 51 ++++++++----- internal/commands/local_status_test.go | 8 ++ internal/commands/local_trim.go | 20 ++--- internal/commands/local_trim_test.go | 4 +- internal/commands/local_trimd_test.go | 4 +- internal/go/cache/cache.go | 48 ++++++++---- internal/go/cache/cache_extra.go | 75 ++++++++++++++++--- internal/go/cache/hash.go | 7 +- internal/go/cache/hash_test.go | 2 +- internal/go/cacheprog/cacheprog.go | 2 + .../lockedfile/internal/filelock/filelock.go | 8 -- .../internal/filelock/filelock_windows.go | 12 +-- internal/go/lockedfile/lockedfile.go | 5 ++ internal/go/lockedfile/transform_test.go | 6 +- internal/go/mmap/mmap_windows.go | 8 +- 17 files changed, 208 insertions(+), 93 deletions(-) diff --git a/internal/caches/local/local.go b/internal/caches/local/local.go index efb0dd4..d59ac22 100644 --- a/internal/caches/local/local.go +++ b/internal/caches/local/local.go @@ -21,11 +21,15 @@ type Cache struct { } // Stats describes local cache state. +// Oldest and Newest are action-entry add times. LeastRecentlyUsed and +// MostRecentlyUsed are approximate last-use times. type Stats struct { - Entries int - Bytes int64 - Oldest *time.Time - Newest *time.Time + Entries int + Bytes int64 + Oldest *time.Time + Newest *time.Time + LeastRecentlyUsed *time.Time + MostRecentlyUsed *time.Time } // New creates a new [Cache]. @@ -76,14 +80,16 @@ func (c *Cache) TrimForce() (before, freed int64) { return c.dc.TrimForce(c.cutoff, c.maxSize, c.l) } -// Status returns current local cache statistics. -func (c *Cache) Status() Stats { +// Stats returns current local cache statistics. +func (c *Cache) Stats() *Stats { s := c.dc.Stats(c.l) - return Stats{ - Entries: s.Entries, - Bytes: s.Bytes, - Oldest: s.Oldest, - Newest: s.Newest, + return &Stats{ + Entries: s.Entries, + Bytes: s.Bytes, + Oldest: s.Oldest, + Newest: s.Newest, + LeastRecentlyUsed: s.LeastRecentlyUsed, + MostRecentlyUsed: s.MostRecentlyUsed, } } diff --git a/internal/caches/local/local_test.go b/internal/caches/local/local_test.go index 9fc51b7..8e8c85e 100644 --- a/internal/caches/local/local_test.go +++ b/internal/caches/local/local_test.go @@ -175,12 +175,19 @@ func TestStatusFixture(t *testing.T) { c := musta.NotFail(New(localtest.Setup(t), nil, nil, logger(t)))(t) - stats := c.Status() + stats := c.Stats() shoulda.BeEqual(t, stats.Entries, 1219) shoulda.BeEqual(t, stats.Bytes, int64(109_518_524)) musta.NotBeZero(t, stats.Oldest) musta.NotBeZero(t, stats.Newest) + musta.NotBeZero(t, stats.LeastRecentlyUsed) + musta.NotBeZero(t, stats.MostRecentlyUsed) + shoulda.BeEqual(t, stats.Oldest.UnixNano(), int64(1_763_399_577_524_486_000)) + shoulda.BeEqual(t, stats.Newest.UnixNano(), int64(1_763_399_587_284_280_000)) + shoulda.BeEqual(t, stats.LeastRecentlyUsed.UnixNano(), int64(1_763_399_577_524_467_000)) + shoulda.BeEqual(t, stats.MostRecentlyUsed.UnixNano(), int64(1_763_399_587_284_400_000)) shoulda.CompareLess(t, *stats.Oldest, *stats.Newest, time.Time.Compare) + shoulda.CompareLess(t, *stats.LeastRecentlyUsed, *stats.MostRecentlyUsed, time.Time.Compare) } func TestStatusEmpty(t *testing.T) { @@ -191,9 +198,11 @@ func TestStatusEmpty(t *testing.T) { shoulda.BeEqual(t, before, int64(109_518_524)) shoulda.BeEqual(t, freed, before) - stats := c.Status() + stats := c.Stats() shoulda.BeEqual(t, stats.Entries, 0) shoulda.BeEqual(t, stats.Bytes, int64(0)) shoulda.BeZero(t, stats.Oldest) shoulda.BeZero(t, stats.Newest) + shoulda.BeZero(t, stats.LeastRecentlyUsed) + shoulda.BeZero(t, stats.MostRecentlyUsed) } diff --git a/internal/commands/local_status.go b/internal/commands/local_status.go index 4e21e96..e74c673 100644 --- a/internal/commands/local_status.go +++ b/internal/commands/local_status.go @@ -15,11 +15,13 @@ import ( type localStatusOutput struct { Directory string `json:"directory"` Cache struct { - Entries int `json:"entries"` - Bytes int64 `json:"bytes"` - Human string `json:"human"` - Oldest *string `json:"oldest"` - Newest *string `json:"newest"` + Entries int `json:"entries"` + Bytes int64 `json:"bytes"` + Human string `json:"human"` + Oldest *string `json:"oldest"` + Newest *string `json:"newest"` + LeastRecentlyUsed *string `json:"least_recently_used"` + MostRecentlyUsed *string `json:"most_recently_used"` } `json:"cache"` Disk struct { TotalBytes int64 `json:"total_bytes"` @@ -47,7 +49,7 @@ func LocalStatus(opts *LocalStatusOpts, out io.Writer, l *slog.Logger) error { return err } - stats := c.Status() + stats := c.Stats() total, free, err := local.DiskInfo(opts.Dir) if err != nil { return err @@ -62,7 +64,7 @@ func LocalStatus(opts *LocalStatusOpts, out io.Writer, l *slog.Logger) error { return err } -func newLocalStatusOutput(dir string, stats local.Stats, total, free int64) localStatusOutput { +func newLocalStatusOutput(dir string, stats *local.Stats, total, free int64) localStatusOutput { used := max(total-free, 0) percent := func(value int64) float64 { if total <= 0 { @@ -79,12 +81,17 @@ func newLocalStatusOutput(dir string, stats local.Stats, total, free int64) loca res.Cache.Entries = stats.Entries res.Cache.Bytes = stats.Bytes res.Cache.Human = unit.Bytes(stats.Bytes).String() - if stats.Oldest != nil { - res.Cache.Oldest = new(stats.Oldest.Local().Format(time.RFC3339)) - } - if stats.Newest != nil { - res.Cache.Newest = new(stats.Newest.Local().Format(time.RFC3339)) + formatTime := func(t *time.Time) *string { + if t == nil { + return nil + } + + return new(t.Local().Format(time.RFC3339)) } + res.Cache.Oldest = formatTime(stats.Oldest) + res.Cache.Newest = formatTime(stats.Newest) + res.Cache.LeastRecentlyUsed = formatTime(stats.LeastRecentlyUsed) + res.Cache.MostRecentlyUsed = formatTime(stats.MostRecentlyUsed) res.Disk.TotalBytes = total res.Disk.TotalHuman = unit.Bytes(total).String() res.Disk.UsedBytes = used @@ -98,12 +105,12 @@ func newLocalStatusOutput(dir string, stats local.Stats, total, free int64) loca } func (s localStatusOutput) String() string { - oldest, newest := "n/a", "n/a" - if s.Cache.Oldest != nil { - oldest = *s.Cache.Oldest - } - if s.Cache.Newest != nil { - newest = *s.Cache.Newest + formatTime := func(t *string) string { + if t == nil { + return "n/a" + } + + return *t } return fmt.Sprintf(`Directory: %s @@ -111,6 +118,8 @@ Cache entries: %d Cache size: %s (%d bytes) Oldest entry: %s Newest entry: %s +Least recently used: %s +Most recently used: %s Disk total: %s (%d bytes) Disk used: %s (%d bytes) (%.2f%%) Disk free: %s (%d bytes) (%.2f%%) @@ -119,8 +128,10 @@ Cache of total disk: %.2f%% s.Directory, s.Cache.Entries, s.Cache.Human, s.Cache.Bytes, - oldest, - newest, + formatTime(s.Cache.Oldest), + formatTime(s.Cache.Newest), + formatTime(s.Cache.LeastRecentlyUsed), + formatTime(s.Cache.MostRecentlyUsed), s.Disk.TotalHuman, s.Disk.TotalBytes, s.Disk.UsedHuman, s.Disk.UsedBytes, s.Disk.UsedPercent, s.Disk.FreeHuman, s.Disk.FreeBytes, s.Disk.FreePercent, diff --git a/internal/commands/local_status_test.go b/internal/commands/local_status_test.go index 4dff9c5..2a27fe8 100644 --- a/internal/commands/local_status_test.go +++ b/internal/commands/local_status_test.go @@ -31,6 +31,8 @@ func TestLocalStatus(t *testing.T) { shoulda.SatisfyWith(t, actual, "Cache size: 109MB (109518524 bytes)", strings.Contains) shoulda.SatisfyWith(t, actual, "Oldest entry: "+oldest, strings.Contains) shoulda.SatisfyWith(t, actual, "Newest entry: "+newest, strings.Contains) + shoulda.SatisfyWith(t, actual, "Least recently used: "+oldest, strings.Contains) + shoulda.SatisfyWith(t, actual, "Most recently used: "+newest, strings.Contains) shoulda.SatisfyWith(t, actual, "Disk total: ", strings.Contains) shoulda.SatisfyWith(t, actual, "Disk free: ", strings.Contains) }) @@ -51,8 +53,12 @@ func TestLocalStatus(t *testing.T) { shoulda.BeEqual(t, got.Cache.Human, "109MB") musta.NotBeZero(t, got.Cache.Oldest) musta.NotBeZero(t, got.Cache.Newest) + musta.NotBeZero(t, got.Cache.LeastRecentlyUsed) + musta.NotBeZero(t, got.Cache.MostRecentlyUsed) shoulda.BeEqual(t, *got.Cache.Oldest, oldest) shoulda.BeEqual(t, *got.Cache.Newest, newest) + shoulda.BeEqual(t, *got.Cache.LeastRecentlyUsed, oldest) + shoulda.BeEqual(t, *got.Cache.MostRecentlyUsed, newest) shoulda.BeGreater(t, got.Disk.TotalBytes, int64(0)) shoulda.BeEqual(t, got.Disk.UsedBytes+got.Disk.FreeBytes, got.Disk.TotalBytes) }) @@ -75,4 +81,6 @@ func TestLocalStatusEmpty(t *testing.T) { shoulda.SatisfyWith(t, actual, "Cache size: 0B (0 bytes)", strings.Contains) shoulda.SatisfyWith(t, actual, "Oldest entry: n/a", strings.Contains) shoulda.SatisfyWith(t, actual, "Newest entry: n/a", strings.Contains) + shoulda.SatisfyWith(t, actual, "Least recently used: n/a", strings.Contains) + shoulda.SatisfyWith(t, actual, "Most recently used: n/a", strings.Contains) } diff --git a/internal/commands/local_trim.go b/internal/commands/local_trim.go index 0437a07..5eaa7bd 100644 --- a/internal/commands/local_trim.go +++ b/internal/commands/local_trim.go @@ -80,7 +80,7 @@ func localTrim(opts *LocalTrimOpts, now func() time.Time, l *slog.Logger) error } before, freed := c.TrimForce() - stats := c.Status() + stats := c.Stats() total, free, err := local.DiskInfo(opts.Dir) if err != nil { return err @@ -91,12 +91,12 @@ func localTrim(opts *LocalTrimOpts, now func() time.Time, l *slog.Logger) error before = stats.Bytes + freed } - oldest, newest := "n/a", "n/a" - if status.Cache.Oldest != nil { - oldest = *status.Cache.Oldest - } - if status.Cache.Newest != nil { - newest = *status.Cache.Newest + formatTime := func(t *string) string { + if t == nil { + return "n/a" + } + + return *t } l.Debug( @@ -113,8 +113,10 @@ func localTrim(opts *LocalTrimOpts, now func() time.Time, l *slog.Logger) error slog.Group("cache", slog.Int("entries", status.Cache.Entries), slog.String("size", fmt.Sprintf("%s (%d bytes)", status.Cache.Human, status.Cache.Bytes)), - slog.String("oldest", oldest), - slog.String("newest", newest), + slog.String("oldest", formatTime(status.Cache.Oldest)), + slog.String("newest", formatTime(status.Cache.Newest)), + slog.String("least_recently_used", formatTime(status.Cache.LeastRecentlyUsed)), + slog.String("most_recently_used", formatTime(status.Cache.MostRecentlyUsed)), ), slog.Group("disk", slog.String("total", fmt.Sprintf("%s (%d bytes)", status.Disk.TotalHuman, status.Disk.TotalBytes)), diff --git a/internal/commands/local_trim_test.go b/internal/commands/local_trim_test.go index e4c4f19..739b1ba 100644 --- a/internal/commands/local_trim_test.go +++ b/internal/commands/local_trim_test.go @@ -21,7 +21,7 @@ func TestLocalTrimStatistics(t *testing.T) { musta.NoError(t, LocalTrim(&LocalTrimOpts{Dir: dir, MaxSize: "50MB"}, l)) - stats := musta.NotFail(local.New(dir, nil, nil, l))(t).Status() + stats := musta.NotFail(local.New(dir, nil, nil, l))(t).Stats() shoulda.BeEqual(t, stats.Bytes, int64(49_494_929)) actual := output.String() @@ -34,6 +34,8 @@ func TestLocalTrimStatistics(t *testing.T) { `cache.size="49MB (49494929 bytes)"`, `cache.oldest=`, `cache.newest=`, + `cache.least_recently_used=`, + `cache.most_recently_used=`, `disk.total=`, `disk.used=`, `disk.used_percent=`, diff --git a/internal/commands/local_trimd_test.go b/internal/commands/local_trimd_test.go index 42b0d3a..cb82914 100644 --- a/internal/commands/local_trimd_test.go +++ b/internal/commands/local_trimd_test.go @@ -31,7 +31,7 @@ func TestLocalTrimdStatistics(t *testing.T) { Interval: unit.Duration(time.Hour), }, l)) - stats := musta.NotFail(local.New(dir, nil, nil, l))(t).Status() + stats := musta.NotFail(local.New(dir, nil, nil, l))(t).Stats() shoulda.BeEqual(t, stats.Bytes, int64(49_494_929)) shoulda.BeGreater(t, stats.Entries, 0) @@ -43,5 +43,7 @@ func TestLocalTrimdStatistics(t *testing.T) { shoulda.SatisfyWith(t, actual, `cache.size="49MB (49494929 bytes)"`, strings.Contains) shoulda.SatisfyWith(t, actual, `cache.oldest=`, strings.Contains) shoulda.SatisfyWith(t, actual, `cache.newest=`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.least_recently_used=`, strings.Contains) + shoulda.SatisfyWith(t, actual, `cache.most_recently_used=`, strings.Contains) shoulda.SatisfyWith(t, actual, `disk.total=`, strings.Contains) } diff --git a/internal/go/cache/cache.go b/internal/go/cache/cache.go index 62e850c..c9576d4 100644 --- a/internal/go/cache/cache.go +++ b/internal/go/cache/cache.go @@ -99,7 +99,7 @@ func Open(dir string) (*DiskCache, error) { if !info.IsDir() { return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} } - for i := range 256 { + for i := 0; i < 256; i++ { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) if err := os.MkdirAll(name, 0o777); err != nil { return nil, err @@ -151,9 +151,6 @@ const ( // GODEBUG=gocacheverify=1. const verify = false -// DebugTest is set when GODEBUG=gocachetest=1 is in the environment. -const DebugTest = false - // Get looks up the action ID in the cache, // returning the corresponding output ID and file size, if any. // Note that finding an output ID does not guarantee that the @@ -356,32 +353,53 @@ func (c *DiskCache) Trim() error { // trim time is too far in the future, attempt the trim anyway. It's possible that // the cache was full when the corruption happened. Attempting a trim on // an empty cache is cheap, so there wouldn't be a big performance hit in that case. - if data, err := lockedfile.Read(filepath.Join(c.dir, "trim.txt")); err == nil { + skipTrim := func(data []byte) bool { if t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil { lastTrim := time.Unix(t, 0) if d := now.Sub(lastTrim); d < trimInterval && d > -mtimeInterval { - return nil + return true } } + return false + } + // Check to see if we need a trim. Do this check separately from the lockedfile.Transform + // so that we can skip getting an exclusive lock in the common case. + if data, err := lockedfile.Read(filepath.Join(c.dir, "trim.txt")); err == nil { + if skipTrim(data) { + return nil + } + } + + errFileChanged := errors.New("file changed") + + // Write the new timestamp before we start trimming to reduce the chance that multiple invocations + // try to trim at the same time, causing contention in CI (#76314). + err := lockedfile.Transform(filepath.Join(c.dir, "trim.txt"), func(data []byte) ([]byte, error) { + if skipTrim(data) { + // The timestamp in the file no longer meets the criteria for us to + // do a trim. It must have been updated by another go command invocation + // since we last read it. Skip the trim. + return nil, errFileChanged + } + return fmt.Appendf(nil, "%d", now.Unix()), nil + }) + if errors.Is(err, errors.ErrUnsupported) { + return err + } + if errors.Is(err, errFileChanged) { + // Skip the trim because we don't need it anymore. + return nil } // Trim each of the 256 subdirectories. // We subtract an additional mtimeInterval // to account for the imprecision of our "last used" mtimes. cutoff := now.Add(-trimLimit - mtimeInterval) - for i := range 256 { + for i := 0; i < 256; i++ { subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i)) c.trimSubdir(subdir, cutoff) } - // Ignore errors from here: if we don't write the complete timestamp, the - // cache will appear older than it is, and we'll trim it again next time. - var b bytes.Buffer - fmt.Fprintf(&b, "%d", now.Unix()) - if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0o666); err != nil { - return err - } - return nil } diff --git a/internal/go/cache/cache_extra.go b/internal/go/cache/cache_extra.go index 8a031bd..d05fe5f 100644 --- a/internal/go/cache/cache_extra.go +++ b/internal/go/cache/cache_extra.go @@ -5,6 +5,7 @@ package cache import ( "bytes" + "errors" "fmt" "log/slog" "os" @@ -21,17 +22,21 @@ import ( type EntryNotFoundError = entryNotFoundError // Stats describes cache state derived from a full directory scan. +// Oldest and Newest are add times from action entries. Data entries do not store add times. +// LeastRecentlyUsed and MostRecentlyUsed are approximate last-use times from filesystem mtimes. type Stats struct { - Entries int - Bytes int64 - Oldest *time.Time - Newest *time.Time + Entries int + Bytes int64 + Oldest *time.Time + Newest *time.Time + LeastRecentlyUsed *time.Time + MostRecentlyUsed *time.Time } // fileInfo represents information about a file or directory with executable in the cache. // The order of fields is weird to make struct smaller. type fileInfo struct { - ModTime time.Time // file modification time, or directory modification time + ModTime time.Time // file (or directory for executable) last use time Name string // file name, or directory name for executable Size int64 // file size, or executable size } @@ -137,20 +142,47 @@ func (c *DiskCache) TrimForce(cutoff *time.Time, maxSize *int64, l *slog.Logger) } // Stats scans the cache directory and returns aggregate statistics. -func (c *DiskCache) Stats(l *slog.Logger) Stats { +func (c *DiskCache) Stats(l *slog.Logger) *Stats { files, bytes := c.read(l) - stats := Stats{ + stats := &Stats{ Entries: len(files), Bytes: bytes, } for _, fi := range files { modTime := fi.ModTime - if stats.Oldest == nil || modTime.Before(*stats.Oldest) { - stats.Oldest = new(modTime) + if stats.LeastRecentlyUsed == nil || modTime.Before(*stats.LeastRecentlyUsed) { + stats.LeastRecentlyUsed = new(modTime) } - if stats.Newest == nil || modTime.After(*stats.Newest) { - stats.Newest = new(modTime) + if stats.MostRecentlyUsed == nil || modTime.After(*stats.MostRecentlyUsed) { + stats.MostRecentlyUsed = new(modTime) + } + + if !strings.HasSuffix(fi.Name, "-a") { + continue + } + + path := filepath.Join(c.dir, fi.Name[:2], fi.Name) + entry, err := os.ReadFile(path) + if err != nil { + l.Debug("Failed to read action entry", slog.String("name", path), slog.String("error", err.Error())) + continue + } + if !validEntry(entry) { + l.Debug("Invalid action entry", slog.String("name", path)) + continue + } + + added, err := parseEntryTime(entry[entryTimeOffset : entryTimeOffset+entryTimeSize]) + if err != nil { + l.Debug("Failed to parse action entry time", slog.String("name", path), slog.String("error", err.Error())) + continue + } + if stats.Oldest == nil || added.Before(*stats.Oldest) { + stats.Oldest = new(added) + } + if stats.Newest == nil || added.After(*stats.Newest) { + stats.Newest = new(added) } } @@ -219,3 +251,24 @@ func (c *DiskCache) read(l *slog.Logger) (files []fileInfo, before int64) { return } + +func parseEntryTime(b []byte) (time.Time, error) { + ns, err := strconv.ParseInt(strings.TrimLeft(string(b), " "), 10, 64) + if err != nil { + return time.Time{}, err + } + if ns < 0 { + return time.Time{}, errors.New("negative timestamp") + } + + return time.Unix(0, ns), nil +} + +func validEntry(entry []byte) bool { + return len(entry) == entrySize && + entry[0] == 'v' && entry[1] == '1' && entry[2] == ' ' && + entry[3+hexSize] == ' ' && + entry[3+hexSize+1+hexSize] == ' ' && + entry[3+hexSize+1+hexSize+1+20] == ' ' && + entry[entrySize-1] == '\n' +} diff --git a/internal/go/cache/hash.go b/internal/go/cache/hash.go index 0a2dbe3..27d2756 100644 --- a/internal/go/cache/hash.go +++ b/internal/go/cache/hash.go @@ -48,7 +48,12 @@ var hashSalt = []byte(stripExperiment(runtime.Version())) // stripExperiment strips any GOEXPERIMENT configuration from the Go // version string. func stripExperiment(version string) string { - version, _, _ = strings.Cut(version, " X:") + if i := strings.Index(version, " X:"); i >= 0 { + return version[:i] + } + if i := strings.Index(version, "-X:"); i >= 0 { + return version[:i] + } return version } diff --git a/internal/go/cache/hash_test.go b/internal/go/cache/hash_test.go index 391669f..a035677 100644 --- a/internal/go/cache/hash_test.go +++ b/internal/go/cache/hash_test.go @@ -34,7 +34,7 @@ func TestHashFile(t *testing.T) { name := f.Name() fmt.Fprintf(f, "hello world") defer os.Remove(name) - if err = f.Close(); err != nil { + if err := f.Close(); err != nil { t.Fatal(err) } diff --git a/internal/go/cacheprog/cacheprog.go b/internal/go/cacheprog/cacheprog.go index 9379636..168df83 100644 --- a/internal/go/cacheprog/cacheprog.go +++ b/internal/go/cacheprog/cacheprog.go @@ -122,5 +122,7 @@ type Response struct { // DiskPath is the absolute path on disk of the body corresponding to a // "get" (on cache hit) or "put" request's ActionID. + // By convention, cached files are stored without any filename extensions. + // Some tools may filter out files with extensions. DiskPath string `json:",omitempty"` } diff --git a/internal/go/lockedfile/internal/filelock/filelock.go b/internal/go/lockedfile/internal/filelock/filelock.go index d373318..f0452f0 100644 --- a/internal/go/lockedfile/internal/filelock/filelock.go +++ b/internal/go/lockedfile/internal/filelock/filelock.go @@ -8,7 +8,6 @@ package filelock import ( - "errors" "io/fs" ) @@ -74,10 +73,3 @@ func (lt lockType) String() string { return "Unlock" } } - -// IsNotSupported returns a boolean indicating whether the error is known to -// report that a function is not supported (possibly for a specific input). -// It is satisfied by errors.ErrUnsupported as well as some syscall errors. -func IsNotSupported(err error) bool { - return errors.Is(err, errors.ErrUnsupported) -} diff --git a/internal/go/lockedfile/internal/filelock/filelock_windows.go b/internal/go/lockedfile/internal/filelock/filelock_windows.go index 7ce5580..647ee99 100644 --- a/internal/go/lockedfile/internal/filelock/filelock_windows.go +++ b/internal/go/lockedfile/internal/filelock/filelock_windows.go @@ -7,9 +7,9 @@ package filelock import ( + "internal/syscall/windows" "io/fs" - - "golang.org/x/sys/windows" + "syscall" ) type lockType uint32 @@ -30,9 +30,9 @@ func lock(f File, lt lockType) error { // However, LockFileEx still requires an OVERLAPPED structure, // which contains the file offset of the beginning of the lock range. // We want to lock the entire file, so we leave the offset as zero. - ol := new(windows.Overlapped) + ol := new(syscall.Overlapped) - err := windows.LockFileEx(windows.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) + err := windows.LockFileEx(syscall.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) if err != nil { return &fs.PathError{ Op: lt.String(), @@ -44,8 +44,8 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - ol := new(windows.Overlapped) - err := windows.UnlockFileEx(windows.Handle(f.Fd()), reserved, allBytes, allBytes, ol) + ol := new(syscall.Overlapped) + err := windows.UnlockFileEx(syscall.Handle(f.Fd()), reserved, allBytes, allBytes, ol) if err != nil { return &fs.PathError{ Op: "Unlock", diff --git a/internal/go/lockedfile/lockedfile.go b/internal/go/lockedfile/lockedfile.go index 05f5ba5..a26dbec 100644 --- a/internal/go/lockedfile/lockedfile.go +++ b/internal/go/lockedfile/lockedfile.go @@ -94,6 +94,11 @@ func (f *File) Close() error { err := closeFile(f.osFile.File) f.cleanup.Stop() + // f may be dead at the moment after we access f.cleanup, + // so the cleanup can fire before Stop completes. Keep f + // alive while we call Stop. See the documentation for + // runtime.Cleanup.Stop. + runtime.KeepAlive(f) return err } diff --git a/internal/go/lockedfile/transform_test.go b/internal/go/lockedfile/transform_test.go index 864cd71..096d258 100644 --- a/internal/go/lockedfile/transform_test.go +++ b/internal/go/lockedfile/transform_test.go @@ -40,7 +40,7 @@ func TestTransform(t *testing.T) { const maxChunkWords = 8 << 10 buf := make([]byte, 2*maxChunkWords*8) - for i := range uint64(2 * maxChunkWords) { + for i := uint64(0); i < 2*maxChunkWords; i++ { binary.LittleEndian.PutUint64(buf[i*8:], i) } if err := lockedfile.Write(path, bytes.NewReader(buf[:8]), 0o666); err != nil { @@ -55,7 +55,7 @@ func TestTransform(t *testing.T) { sem := make(chan bool, parallel) - for range attempts { + for n := attempts; n > 0; n-- { sem <- true go func() { defer func() { <-sem }() @@ -96,7 +96,7 @@ func TestTransform(t *testing.T) { }() } - for range parallel { + for n := parallel; n > 0; n-- { sem <- true } } diff --git a/internal/go/mmap/mmap_windows.go b/internal/go/mmap/mmap_windows.go index cd50d86..a188512 100644 --- a/internal/go/mmap/mmap_windows.go +++ b/internal/go/mmap/mmap_windows.go @@ -6,10 +6,10 @@ package mmap import ( "fmt" + "internal/syscall/windows" "os" + "syscall" "unsafe" - - "golang.org/x/sys/windows" ) func mmapFile(f *os.File) (Data, error) { @@ -21,12 +21,12 @@ func mmapFile(f *os.File) (Data, error) { if size == 0 { return Data{f, nil}, nil } - h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil, windows.PAGE_READONLY, 0, 0, nil) + h, err := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, 0, 0, nil) if err != nil { return Data{}, fmt.Errorf("CreateFileMapping %s: %w", f.Name(), err) } - addr, err := windows.MapViewOfFile(h, windows.FILE_MAP_READ, 0, 0, 0) + addr, err := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, 0) if err != nil { return Data{}, fmt.Errorf("MapViewOfFile %s: %w", f.Name(), err) } From 949d71be519e4a53ada481fa9c182526d39ed894 Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 09:37:10 +0400 Subject: [PATCH 11/15] WIP --- internal/go/lockedfile/internal/filelock/filelock_windows.go | 3 ++- internal/go/mmap/mmap_windows.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/go/lockedfile/internal/filelock/filelock_windows.go b/internal/go/lockedfile/internal/filelock/filelock_windows.go index 647ee99..f9d7878 100644 --- a/internal/go/lockedfile/internal/filelock/filelock_windows.go +++ b/internal/go/lockedfile/internal/filelock/filelock_windows.go @@ -7,9 +7,10 @@ package filelock import ( - "internal/syscall/windows" "io/fs" "syscall" + + "golang.org/x/sys/windows" ) type lockType uint32 diff --git a/internal/go/mmap/mmap_windows.go b/internal/go/mmap/mmap_windows.go index a188512..256fab4 100644 --- a/internal/go/mmap/mmap_windows.go +++ b/internal/go/mmap/mmap_windows.go @@ -6,10 +6,11 @@ package mmap import ( "fmt" - "internal/syscall/windows" "os" "syscall" "unsafe" + + "golang.org/x/sys/windows" ) func mmapFile(f *os.File) (Data, error) { From 55b69833dfa1ce66810c9baeab0c8480a43ff50e Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 09:38:29 +0400 Subject: [PATCH 12/15] WIP --- .../go/lockedfile/internal/filelock/filelock_windows.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/go/lockedfile/internal/filelock/filelock_windows.go b/internal/go/lockedfile/internal/filelock/filelock_windows.go index f9d7878..7ce5580 100644 --- a/internal/go/lockedfile/internal/filelock/filelock_windows.go +++ b/internal/go/lockedfile/internal/filelock/filelock_windows.go @@ -8,7 +8,6 @@ package filelock import ( "io/fs" - "syscall" "golang.org/x/sys/windows" ) @@ -31,9 +30,9 @@ func lock(f File, lt lockType) error { // However, LockFileEx still requires an OVERLAPPED structure, // which contains the file offset of the beginning of the lock range. // We want to lock the entire file, so we leave the offset as zero. - ol := new(syscall.Overlapped) + ol := new(windows.Overlapped) - err := windows.LockFileEx(syscall.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) + err := windows.LockFileEx(windows.Handle(f.Fd()), uint32(lt), reserved, allBytes, allBytes, ol) if err != nil { return &fs.PathError{ Op: lt.String(), @@ -45,8 +44,8 @@ func lock(f File, lt lockType) error { } func unlock(f File) error { - ol := new(syscall.Overlapped) - err := windows.UnlockFileEx(syscall.Handle(f.Fd()), reserved, allBytes, allBytes, ol) + ol := new(windows.Overlapped) + err := windows.UnlockFileEx(windows.Handle(f.Fd()), reserved, allBytes, allBytes, ol) if err != nil { return &fs.PathError{ Op: "Unlock", From 96a1bdff7ee1d7a3c0d07a02cbfb5174c1ba4fef Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 13 Aug 2026 09:41:16 +0400 Subject: [PATCH 13/15] WIP --- internal/go/cache/cache_extra.go | 37 +++++++++++--------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/internal/go/cache/cache_extra.go b/internal/go/cache/cache_extra.go index d05fe5f..bb5bfcd 100644 --- a/internal/go/cache/cache_extra.go +++ b/internal/go/cache/cache_extra.go @@ -5,7 +5,6 @@ package cache import ( "bytes" - "errors" "fmt" "log/slog" "os" @@ -21,6 +20,11 @@ import ( // EntryNotFoundError is exported for use in other packages. type EntryNotFoundError = entryNotFoundError +const ( + actionEntryTimeSize = 20 + actionEntryTimeOffset = entrySize - actionEntryTimeSize - 1 +) + // Stats describes cache state derived from a full directory scan. // Oldest and Newest are add times from action entries. Data entries do not store add times. // LeastRecentlyUsed and MostRecentlyUsed are approximate last-use times from filesystem mtimes. @@ -168,16 +172,22 @@ func (c *DiskCache) Stats(l *slog.Logger) *Stats { l.Debug("Failed to read action entry", slog.String("name", path), slog.String("error", err.Error())) continue } - if !validEntry(entry) { + if len(entry) != entrySize { l.Debug("Invalid action entry", slog.String("name", path)) continue } - added, err := parseEntryTime(entry[entryTimeOffset : entryTimeOffset+entryTimeSize]) + b := entry[actionEntryTimeOffset : actionEntryTimeOffset+actionEntryTimeSize] + ns, err := strconv.ParseInt(strings.TrimLeft(string(b), " "), 10, 64) if err != nil { l.Debug("Failed to parse action entry time", slog.String("name", path), slog.String("error", err.Error())) continue } + if ns < 0 { + l.Debug("Invalid action entry time", slog.String("name", path), slog.Int64("timestamp", ns)) + continue + } + added := time.Unix(0, ns) if stats.Oldest == nil || added.Before(*stats.Oldest) { stats.Oldest = new(added) } @@ -251,24 +261,3 @@ func (c *DiskCache) read(l *slog.Logger) (files []fileInfo, before int64) { return } - -func parseEntryTime(b []byte) (time.Time, error) { - ns, err := strconv.ParseInt(strings.TrimLeft(string(b), " "), 10, 64) - if err != nil { - return time.Time{}, err - } - if ns < 0 { - return time.Time{}, errors.New("negative timestamp") - } - - return time.Unix(0, ns), nil -} - -func validEntry(entry []byte) bool { - return len(entry) == entrySize && - entry[0] == 'v' && entry[1] == '1' && entry[2] == ' ' && - entry[3+hexSize] == ' ' && - entry[3+hexSize+1+hexSize] == ' ' && - entry[3+hexSize+1+hexSize+1+20] == ' ' && - entry[entrySize-1] == '\n' -} From 3c95294ea29cad94297cea87eededf1cc8b5690c Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Sun, 16 Aug 2026 09:20:56 +0400 Subject: [PATCH 14/15] Bump Go version --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d312157..f72e625 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/AlekSi/hardcache go 1.26 -toolchain go1.26.5 +toolchain go1.26.6 // https://go.dev/doc/godebug#go-120 // https://pkg.go.dev/archive/tar#Reader.Next From c694396a79ebdee4be21f5893e1cb66c3bbe04fb Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Sun, 16 Aug 2026 09:44:21 +0400 Subject: [PATCH 15/15] WIP --- internal/go/mmap/mmap_windows.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/go/mmap/mmap_windows.go b/internal/go/mmap/mmap_windows.go index 256fab4..cd50d86 100644 --- a/internal/go/mmap/mmap_windows.go +++ b/internal/go/mmap/mmap_windows.go @@ -7,7 +7,6 @@ package mmap import ( "fmt" "os" - "syscall" "unsafe" "golang.org/x/sys/windows" @@ -22,12 +21,12 @@ func mmapFile(f *os.File) (Data, error) { if size == 0 { return Data{f, nil}, nil } - h, err := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, 0, 0, nil) + h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil, windows.PAGE_READONLY, 0, 0, nil) if err != nil { return Data{}, fmt.Errorf("CreateFileMapping %s: %w", f.Name(), err) } - addr, err := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, 0) + addr, err := windows.MapViewOfFile(h, windows.FILE_MAP_READ, 0, 0, 0) if err != nil { return Data{}, fmt.Errorf("MapViewOfFile %s: %w", f.Name(), err) }