From 24f9603b751b1d2ba26775e59c50b602721e0236 Mon Sep 17 00:00:00 2001 From: jvoisin Date: Sun, 5 Jul 2026 22:34:36 +0200 Subject: [PATCH] perf(sanitizer): avoid Token() allocations in stripIter Tokenizer.Token() allocates a Token struct and parses the full attribute slice for every start tag, even though stripIter only consumes text tokens. Switch to checking the TokenType from Next() and calling Text() only for text tokens. Shared by StripTags and TruncateHTML, both called per entry: StripTags 247KB -> 71KB/op (-71%), 12623 -> 4423 allocs (-65%) TruncateHTML 19KB -> 7.9KB/op (-59%), 819 -> 297 allocs (-64%) Both roughly 2x faster on tag-heavy input. --- internal/reader/sanitizer/strip_tags.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/reader/sanitizer/strip_tags.go b/internal/reader/sanitizer/strip_tags.go index 8c75e557692..d99c4152831 100644 --- a/internal/reader/sanitizer/strip_tags.go +++ b/internal/reader/sanitizer/strip_tags.go @@ -33,13 +33,18 @@ func StripTags(input string) string { func stripIter(src io.Reader, yield func(string) bool) error { tokenizer := html.NewTokenizer(src) - for tokenizer.Next() != html.ErrorToken { - token := tokenizer.Token() - if token.Type != html.TextToken { + for { + tokenType := tokenizer.Next() + if tokenType == html.ErrorToken { + break + } + if tokenType != html.TextToken { continue } - if !yield(token.Data) { + // Use Text() instead of Token() to avoid allocating a Token + // struct (and parsing attribute slices) for every non-text tag. + if !yield(string(tokenizer.Text())) { break } }