diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 27f2c545..f5afe7f0 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,3 +35,6 @@ jobs: - name: Run Tests run: go test -trimpath -race ./... + + - name: Run Fuzz Tests + run: go test -fuzz=FuzzAppendSanitizedMetricName -fuzztime=5s ./datadog diff --git a/.gitignore b/.gitignore index 73c9f056..bfe5fa03 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,6 @@ _testmain.go *~ # Commands -/dogstatsd \ No newline at end of file +/dogstatsd + +/datadog/testdata/fuzz diff --git a/datadog/serializer.go b/datadog/serializer.go index db8f479c..00c90f8c 100644 --- a/datadog/serializer.go +++ b/datadog/serializer.go @@ -84,7 +84,14 @@ func (s *serializer) AppendMeasures(b []byte, _ time.Time, measures ...stats.Mea return b } -var accentMap [256]byte +// latin1SupplementMap maps Unicode codepoints U+00C0-U+00FF (Latin-1 Supplement) +// to their unaccented ASCII equivalents. This is used to handle common accented +// characters in metric names. +// +// Note: This array is indexed by codepoint values (e.g., U+00E9 for é), which +// numerically match the byte values in the Latin-1 encoding. The mapping handles +// 2-byte UTF-8 sequences that decode to these codepoints. +var latin1SupplementMap [256]byte // valid[byte] = 1 if the ASCII char is allowed, 0 otherwise. var valid = [256]bool{ @@ -93,106 +100,106 @@ var valid = [256]bool{ func init() { // Initialize all to identity mapping - for i := range accentMap { - accentMap[i] = byte(i) + for i := range latin1SupplementMap { + latin1SupplementMap[i] = byte(i) } // Latin-1 Supplement mappings (0xC0-0xFF) // Uppercase A variants - accentMap[0xC0] = 'A' // À - accentMap[0xC1] = 'A' // Á - accentMap[0xC2] = 'A' // Â - accentMap[0xC3] = 'A' // Ã - accentMap[0xC4] = 'A' // Ä - accentMap[0xC5] = 'A' // Å - accentMap[0xC6] = 'A' // Æ -> A (could be "AE" but single char is simpler) + latin1SupplementMap[0xC0] = 'A' // À + latin1SupplementMap[0xC1] = 'A' // Á + latin1SupplementMap[0xC2] = 'A' // Â + latin1SupplementMap[0xC3] = 'A' // Ã + latin1SupplementMap[0xC4] = 'A' // Ä + latin1SupplementMap[0xC5] = 'A' // Å + latin1SupplementMap[0xC6] = 'A' // Æ -> A (could be "AE" but single char is simpler) // Uppercase C - accentMap[0xC7] = 'C' // Ç + latin1SupplementMap[0xC7] = 'C' // Ç // Uppercase E variants - accentMap[0xC8] = 'E' // È - accentMap[0xC9] = 'E' // É - accentMap[0xCA] = 'E' // Ê - accentMap[0xCB] = 'E' // Ë + latin1SupplementMap[0xC8] = 'E' // È + latin1SupplementMap[0xC9] = 'E' // É + latin1SupplementMap[0xCA] = 'E' // Ê + latin1SupplementMap[0xCB] = 'E' // Ë // Uppercase I variants - accentMap[0xCC] = 'I' // Ì - accentMap[0xCD] = 'I' // Í - accentMap[0xCE] = 'I' // Î - accentMap[0xCF] = 'I' // Ï + latin1SupplementMap[0xCC] = 'I' // Ì + latin1SupplementMap[0xCD] = 'I' // Í + latin1SupplementMap[0xCE] = 'I' // Î + latin1SupplementMap[0xCF] = 'I' // Ï // Uppercase D, N - accentMap[0xD0] = 'D' // Ð - accentMap[0xD1] = 'N' // Ñ + latin1SupplementMap[0xD0] = 'D' // Ð + latin1SupplementMap[0xD1] = 'N' // Ñ // Uppercase O variants - accentMap[0xD2] = 'O' // Ò - accentMap[0xD3] = 'O' // Ó - accentMap[0xD4] = 'O' // Ô - accentMap[0xD5] = 'O' // Õ - accentMap[0xD6] = 'O' // Ö - accentMap[0xD8] = 'O' // Ø + latin1SupplementMap[0xD2] = 'O' // Ò + latin1SupplementMap[0xD3] = 'O' // Ó + latin1SupplementMap[0xD4] = 'O' // Ô + latin1SupplementMap[0xD5] = 'O' // Õ + latin1SupplementMap[0xD6] = 'O' // Ö + latin1SupplementMap[0xD8] = 'O' // Ø // Uppercase U variants - accentMap[0xD9] = 'U' // Ù - accentMap[0xDA] = 'U' // Ú - accentMap[0xDB] = 'U' // Û - accentMap[0xDC] = 'U' // Ü + latin1SupplementMap[0xD9] = 'U' // Ù + latin1SupplementMap[0xDA] = 'U' // Ú + latin1SupplementMap[0xDB] = 'U' // Û + latin1SupplementMap[0xDC] = 'U' // Ü // Uppercase Y - accentMap[0xDD] = 'Y' // Ý - accentMap[0xDE] = 'T' // Þ (Thorn) + latin1SupplementMap[0xDD] = 'Y' // Ý + latin1SupplementMap[0xDE] = 'T' // Þ (Thorn) // Lowercase sharp s - accentMap[0xDF] = 's' // ß + latin1SupplementMap[0xDF] = 's' // ß // Lowercase a variants - accentMap[0xE0] = 'a' // à - accentMap[0xE1] = 'a' // á - accentMap[0xE2] = 'a' // â - accentMap[0xE3] = 'a' // ã - accentMap[0xE4] = 'a' // ä - accentMap[0xE5] = 'a' // å - accentMap[0xE6] = 'a' // æ -> a (could be "ae" but single char is simpler) + latin1SupplementMap[0xE0] = 'a' // à + latin1SupplementMap[0xE1] = 'a' // á + latin1SupplementMap[0xE2] = 'a' // â + latin1SupplementMap[0xE3] = 'a' // ã + latin1SupplementMap[0xE4] = 'a' // ä + latin1SupplementMap[0xE5] = 'a' // å + latin1SupplementMap[0xE6] = 'a' // æ -> a (could be "ae" but single char is simpler) // Lowercase c - accentMap[0xE7] = 'c' // ç + latin1SupplementMap[0xE7] = 'c' // ç // Lowercase e variants - accentMap[0xE8] = 'e' // è - accentMap[0xE9] = 'e' // é - accentMap[0xEA] = 'e' // ê - accentMap[0xEB] = 'e' // ë + latin1SupplementMap[0xE8] = 'e' // è + latin1SupplementMap[0xE9] = 'e' // é + latin1SupplementMap[0xEA] = 'e' // ê + latin1SupplementMap[0xEB] = 'e' // ë // Lowercase i variants - accentMap[0xEC] = 'i' // ì - accentMap[0xED] = 'i' // í - accentMap[0xEE] = 'i' // î - accentMap[0xEF] = 'i' // ï + latin1SupplementMap[0xEC] = 'i' // ì + latin1SupplementMap[0xED] = 'i' // í + latin1SupplementMap[0xEE] = 'i' // î + latin1SupplementMap[0xEF] = 'i' // ï // Lowercase d, n - accentMap[0xF0] = 'd' // ð - accentMap[0xF1] = 'n' // ñ + latin1SupplementMap[0xF0] = 'd' // ð + latin1SupplementMap[0xF1] = 'n' // ñ // Lowercase o variants - accentMap[0xF2] = 'o' // ò - accentMap[0xF3] = 'o' // ó - accentMap[0xF4] = 'o' // ô - accentMap[0xF5] = 'o' // õ - accentMap[0xF6] = 'o' // ö - accentMap[0xF8] = 'o' // ø + latin1SupplementMap[0xF2] = 'o' // ò + latin1SupplementMap[0xF3] = 'o' // ó + latin1SupplementMap[0xF4] = 'o' // ô + latin1SupplementMap[0xF5] = 'o' // õ + latin1SupplementMap[0xF6] = 'o' // ö + latin1SupplementMap[0xF8] = 'o' // ø // Lowercase u variants - accentMap[0xF9] = 'u' // ù - accentMap[0xFA] = 'u' // ú - accentMap[0xFB] = 'u' // û - accentMap[0xFC] = 'u' // ü + latin1SupplementMap[0xF9] = 'u' // ù + latin1SupplementMap[0xFA] = 'u' // ú + latin1SupplementMap[0xFB] = 'u' // û + latin1SupplementMap[0xFC] = 'u' // ü // Lowercase y - accentMap[0xFD] = 'y' // ý - accentMap[0xFE] = 't' // þ (thorn) - accentMap[0xFF] = 'y' // ÿ + latin1SupplementMap[0xFD] = 'y' // ý + latin1SupplementMap[0xFE] = 't' // þ (thorn) + latin1SupplementMap[0xFF] = 'y' // ÿ for c := '0'; c <= '9'; c++ { valid[c] = true @@ -246,7 +253,7 @@ func appendSanitizedMetricName(dst []byte, raw string) []byte { // Map common accented characters (U+00C0-U+00FF range) if codepoint >= 0xC0 && codepoint <= 0xFF { - mapped := accentMap[codepoint] + mapped := latin1SupplementMap[codepoint] if valid[mapped] { dst = append(dst, mapped) nameLen++ @@ -262,11 +269,17 @@ func appendSanitizedMetricName(dst []byte, raw string) []byte { nameLen++ lastWasRepl = true } - } else if !lastWasRepl { + } else { // Everything else (3-byte, 4-byte sequences, invalid chars) - dst = append(dst, replacement) - nameLen++ - lastWasRepl = true + // Skip continuation bytes (0x80-0xBF) to avoid creating invalid UTF-8 + for i+1 < len(raw) && (raw[i+1]&0xC0) == 0x80 { + i++ + } + if !lastWasRepl { + dst = append(dst, replacement) + nameLen++ + lastWasRepl = true + } } if nameLen >= maxLen { diff --git a/datadog/serializer_test.go b/datadog/serializer_test.go index a01da24b..1975d4b2 100644 --- a/datadog/serializer_test.go +++ b/datadog/serializer_test.go @@ -253,6 +253,7 @@ func TestAppendSanitizedMetricName(t *testing.T) { // consecutive illegal chars collapse {"", "foo!!!@@@###bar", "foo_bar"}, {"app.", "foo!!!@@@###bar", "app.foo_bar"}, + {"", "0_ 0", "0__0"}, // legitimate underscore followed by space -> two underscores (one legit, one replacement) // Unicode / accent folding - existing cases {"", "🐳docker.stats", "docker.stats"}, @@ -304,6 +305,26 @@ func TestAppendSanitizedMetricName(t *testing.T) { {"", "price¤amount", "price_amount"}, // ¤ (currency sign, U+00A4) {"prefix_", "data÷by×time", "prefix_data_by_time"}, // multiple unmapped chars with prefix + // Edge cases for UTF-8 continuation byte handling + {"", "test\xC3\xA9\x80suffix", "teste_suffix"}, // é followed by orphaned continuation byte + {"", "metric\xE2\x82", "metric"}, // incomplete 3-byte sequence (skips continuation byte, no replacement at end) + {"", "test\xF0\x9F", "test"}, // incomplete 4-byte emoji (no continuation bytes, no replacement at end) + {"", "test\xC0\x80", "test"}, // overlong encoding of NULL (invalid, skips continuation byte) + {"", "\x80\x81\x82\xBF", "_truncated_"}, // all invalid continuation bytes + {"", "valid\xC3\xA9\xC3\xA8text", "valideetext"}, // adjacent 2-byte sequences (é è) + {"", "test\xED\xA0\x80invalid", "test_invalid"}, // invalid surrogate half (UTF-16 artifact) + {"", "hello\xF4\x90\x80\x80", "hello"}, // codepoint beyond U+10FFFF (invalid, skips all continuation bytes) + + // Latin-1 Supplement boundary characters + {"", "À", "A"}, // U+00C0 (first mapped character) + {"", "ÿ", "y"}, // U+00FF (last mapped character) + {"", "\u00C0test\u00FF", "Atesty"}, // boundaries with text + + // Mixed valid and invalid sequences + {"", "café\xE2\x82résumé", "cafe_resume"}, // valid + incomplete + valid + {"", "test\xF0\x9F\x98\x80end", "test_end"}, // valid 4-byte emoji (should become single _) + {"", "\xF0\x9F\x98\x80\xF0\x9F\x98\x81", "_truncated_"}, // two valid 4-byte emojis (all invalid, only punctuation) + // empty or only illegal {"", "", "_unnamed_"}, {"", "!!!", "_truncated_"}, @@ -355,3 +376,168 @@ func TestAppendSanitizedMetricName(t *testing.T) { } }) } + +// FuzzAppendSanitizedMetricName performs fuzz testing to discover edge cases +// in metric name sanitization, particularly around UTF-8 handling. +func FuzzAppendSanitizedMetricName(f *testing.F) { + // Seed corpus with interesting test cases + f.Add("") + f.Add("simple") + f.Add("café") + f.Add("🤡test") + f.Add("\xff\xfe") + f.Add("test\xC3\xA9\x80suffix") + f.Add("\xF0\x9F\x98\x80") // valid emoji + f.Add("\xED\xA0\x80") // invalid surrogate + f.Add("naïve.users") + f.Add(strings.Repeat("x", 300)) // over maxLen + + f.Fuzz(func(t *testing.T, input string) { + buf := appendSanitizedMetricName(nil, input) + result := string(buf) + + // Invariant 1: Output must be valid UTF-8 + // (Note: The serializer.Write() method also validates, but sanitization should produce valid UTF-8) + if !utf8.ValidString(result) { + t.Errorf("Output is not valid UTF-8 for input %q: got %q", input, result) + } + + // Invariant 2: Output length must not exceed maxLen (except special fallback cases) + if len(result) > maxLen && result != "_unnamed_" && result != "_truncated_" { + t.Errorf("Output exceeds maxLen: %d > %d for input %q", len(result), maxLen, input) + } + + // Invariant 3: Output must not start/end with trim chars (except special cases) + if len(result) > 0 && result != "_unnamed_" && result != "_truncated_" { + if isTrim(result[0]) { + t.Errorf("Output starts with trim char for input %q: %q", input, result) + } + if isTrim(result[len(result)-1]) { + t.Errorf("Output ends with trim char for input %q: %q", input, result) + } + } + + // Invariant 4: Every ASCII byte in output must be valid or replacement + for i := 0; i < len(result); i++ { + b := result[i] + // Only check ASCII range (< 128), as valid UTF-8 multi-byte sequences are allowed + if b < 128 && !valid[b] && b != replacement { + t.Errorf("Invalid ASCII byte in output at position %d: 0x%02X for input %q", i, b, input) + } + } + + // Invariant 5: Empty input should produce "_unnamed_" + if input == "" && result != "_unnamed_" { + t.Errorf("Empty input should produce '_unnamed_', got %q", result) + } + }) +} + +// TestSanitizationPreservesUTF8Validity ensures that the full pipeline +// (sanitize → Write → output) maintains UTF-8 validity, even with problematic inputs. +func TestSanitizationPreservesUTF8Validity(t *testing.T) { + testCases := []struct { + name string + input string + }{ + {"normal metric", "normal.metric"}, + {"Latin-1 accents", "café.résumé"}, + {"orphaned continuation byte", "test\xC3\xA9\x80suffix"}, + {"incomplete 3-byte", "metric\xE2\x82"}, + {"valid emoji", string([]byte{0xF0, 0x9F, 0x98, 0x80})}, + {"incomplete emoji", string([]byte{0xF0, 0x9F})}, + {"overlong encoding", "test\xC0\x80"}, + {"invalid surrogate", "test\xED\xA0\x80invalid"}, + {"mixed valid/invalid", "café\xE2\x82résumé"}, + {"all continuation bytes", "\x80\x81\x82\xBF"}, + {"multiple emojis", "test🤡and🎯more"}, + {"Chinese characters", "测试.metrics"}, + {"Cyrillic text", "тест.data"}, + {"mixed scripts", "café测试résumé🤡тест"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockBuffer := &MockWriteCloser{Buffer: &bytes.Buffer{}} + s := &serializer{ + conn: mockBuffer, + bufferSize: MaxBufferSize, + } + + measure := stats.Measure{ + Name: tc.input, + Fields: []stats.Field{ + stats.MakeField("count", 1, stats.Counter), + }, + Tags: []stats.Tag{ + stats.T("tagname", tc.input), + }, + } + + buf := s.AppendMeasure(nil, measure) + _, err := s.Write(buf) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + + output := mockBuffer.String() + if !utf8.ValidString(output) { + t.Errorf("Final output is not valid UTF-8: %q", output) + } + + // Additional check: ensure output doesn't contain raw invalid bytes + for i, r := range output { + if r == utf8.RuneError { + _, size := utf8.DecodeRuneInString(output[i:]) + if size == 1 { + // This is an actual decoding error, not the valid U+FFFD character + t.Errorf("Output contains invalid UTF-8 at position %d", i) + } + } + } + }) + } +} + +// BenchmarkAppendSanitizedMetricName measures performance of metric name sanitization +// across different input types to ensure the implementation is efficient. +func BenchmarkAppendSanitizedMetricName(b *testing.B) { + benchmarks := []struct { + name string + input string + }{ + {"simple ASCII", "simple.metric.name"}, + {"with dashes and underscores", "my_app-server.request_count"}, + {"Latin-1 accents", "café.résumé.münchen"}, + {"mixed accents", "naïve.señor.Zürich"}, + {"emoji in middle", "performance🚀metrics🎯data"}, + {"Chinese characters", "测试.metrics.数据"}, + {"Cyrillic text", "тест.данные.метрики"}, + {"mixed scripts", "app.测试.тест.café"}, + {"invalid UTF-8", "test\xC3\xA9\x80\xE2\x82invalid"}, + {"over maxLen", strings.Repeat("x", 300)}, + {"special chars", "foo!!!@@@###bar|||:::"}, + {"mostly trim chars", "...___---...___---"}, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + buf := make([]byte, 0, 512) + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf = appendSanitizedMetricName(buf[:0], bm.input) + } + }) + } + + // Benchmark with prefix (common use case) + b.Run("with prefix", func(b *testing.B) { + buf := make([]byte, 0, 512) + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf = buf[:0] + buf = append(buf, "myapp."...) + buf = appendSanitizedMetricName(buf, "café.metrics") + } + }) +}