Skip to content

Commit 7437698

Browse files
Split a glob class too large to list into its ranges [patch] (#196)
* Split a glob class too large to list into its ranges [patch] ToGlobCharacterList gave up on a class that would list more than 4096 characters, and ResolveGlob read that as a half-typed token to ignore, so a complete class such as "[가-힣a-z]*" matched every item, and as an exclusion hid nothing. A class that is not negated is now split instead: each letter or digit range is matched natively, the rest is listed, and the token matches if any piece does. A negated class, or one whose split would give the token more than 16 alternatives, is listed in full however long it is, which is slower to match but no longer matches everything. A half-typed token is still ignored. Fixes #184 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017vfJW4tA1FYwsQrBAjGqs7 * Hold only the visible overloads to the README's documented defaults Ports the fix from #186 (for #183). The README API test failed on main because the binary-compatibility overloads deliberately carry no defaults, so this PR's CI was red for a reason outside its change. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017vfJW4tA1FYwsQrBAjGqs7 --------- Co-authored-by: Claude <noreply@anthropic.com>
1 parent 5f420d9 commit 7437698

3 files changed

Lines changed: 156 additions & 36 deletions

File tree

‎TextFilter.Test/ReadmeTests.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ namespace TextFilter.Test;
44

55
using System;
66
using System.Collections.Generic;
7+
using System.ComponentModel;
78
using System.IO;
89
using System.Linq;
910
using System.Reflection;
@@ -89,7 +90,10 @@ public void EveryMethodInTheReadmeApiReferenceExistsWithItsDocumentedDefaults()
8990
foreach (Match row in rows)
9091
{
9192
string methodName = row.Groups["method"].Value;
92-
MethodInfo[] overloads = [.. methods.Where(m => m.Name == methodName)];
93+
// The binary-compatibility overloads are hidden from IntelliSense and deliberately carry no
94+
// defaults, since defaults on both sets would make calls ambiguous. The README documents the
95+
// visible overloads, so only those are held to its defaults.
96+
MethodInfo[] overloads = [.. methods.Where(m => m.Name == methodName && !IsHiddenFromCallers(m))];
9397
if (overloads.Length == 0)
9498
{
9599
problems.Add($"{methodName} does not exist");
@@ -115,6 +119,9 @@ public void EveryMethodInTheReadmeApiReferenceExistsWithItsDocumentedDefaults()
115119
Assert.IsEmpty(problems, string.Join("; ", problems));
116120
}
117121

122+
private static bool IsHiddenFromCallers(MethodInfo method) =>
123+
method.GetCustomAttribute<EditorBrowsableAttribute>()?.State == EditorBrowsableState.Never;
124+
118125
[TestMethod]
119126
public void ReadmeMatchOptionsSampleGivesTheDocumentedResults()
120127
{

‎TextFilter.Test/TextFilterTests.cs‎

Lines changed: 61 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1031,14 +1031,72 @@ public void GlobClassWithATrailingDashMatchesTheDashLiterally()
10311031
[DataRow("x[a^]", "x[a^]")]
10321032
[DataRow("x[^a", "x[^a")]
10331033
public void ExpandGlobClassesRewritesOnlyClassesDotNetGlobMisreads(string token, string expected) =>
1034-
Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token));
1034+
CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray());
10351035

10361036
[TestMethod]
10371037
[DataRow("[#-!]")]
1038-
[DataRow("[\u0001-\uffff_]")]
10391038
public void ExpandGlobClassesRejectsClassesThatCannotBeListed(string token) =>
10401039
Assert.IsNull(TextFilter.ExpandGlobClasses(token));
10411040

1041+
[TestMethod]
1042+
[DataRow("[가-힣a-z]*", new[] { "[가-힣]*", "[a-z]*" })]
1043+
[DataRow("x[一-龥0-9_]", new[] { "x[一-龥]", "x[0-9]", "x[_]" })]
1044+
[DataRow("[가-힣a-z][一-龥0-9]", new[] { "[가-힣][一-龥]", "[가-힣][0-9]", "[a-z][一-龥]", "[a-z][0-9]" })]
1045+
[DataRow("[가-힣z-a]", new[] { "[가-힣]" })]
1046+
public void ExpandGlobClassesSplitsAClassTooLargeToListIntoItsRanges(string token, string[] expected) =>
1047+
CollectionAssert.AreEqual(expected, TextFilter.ExpandGlobClasses(token)?.ToArray());
1048+
1049+
[TestMethod]
1050+
[DataRow("[\u0001-\uffff_]")]
1051+
[DataRow("[!가-힣a-z]")]
1052+
[DataRow("[A-힣]")]
1053+
public void ExpandGlobClassesListsAClassTooLargeToListThatCannotBeSplit(string token)
1054+
{
1055+
IReadOnlyList<string>? expanded = TextFilter.ExpandGlobClasses(token);
1056+
Assert.IsNotNull(expanded);
1057+
Assert.HasCount(1, expanded);
1058+
Assert.IsGreaterThan(4096, expanded[0].Length);
1059+
}
1060+
1061+
[TestMethod]
1062+
[DataRow("README.md", "[가-힣a-z]*", false)]
1063+
[DataRow("README.md", "[一-龥a-z]*", false)]
1064+
[DataRow("README.md", "[一-龥0-9]*", false)]
1065+
// Matching is case-sensitive, so "README.md" is not among the members and is not excluded.
1066+
[DataRow("README.md", "-[一-龥a-z]*", true)]
1067+
[DataRow("README.md", "-[一-龥A-Z]*", false)]
1068+
[DataRow("@", "[A-힣]", false)]
1069+
[DataRow("readme.md", "[가-힣a-z]*", true)]
1070+
[DataRow("한국.md", "[가-힣a-z]*", true)]
1071+
[DataRow("中文", "[一-龥a-z]*", true)]
1072+
[DataRow("7z", "[一-龥0-9]*", true)]
1073+
[DataRow("readme.md", "-[一-龥a-z]*", false)]
1074+
[DataRow("中文", "-[一-龥a-z]*", false)]
1075+
[DataRow("README.md", "+[가-힣a-z]*", false)]
1076+
[DataRow("한국", "+[가-힣a-z]*", true)]
1077+
[DataRow("A", "[A-힣]", true)]
1078+
[DataRow("힣", "[A-힣]", true)]
1079+
[DataRow("a\\b", "a[A-힣]b", true)]
1080+
[DataRow("README.md", "[!가-힣a-z]*", true)]
1081+
[DataRow("readme.md", "[!가-힣a-z]*", false)]
1082+
[DataRow("한국", "[!가-힣a-z]*", false)]
1083+
[DataRow("x_", "x[가-힣a-z_]", true)]
1084+
[DataRow("x-", "x[가-힣a-z_]", false)]
1085+
public void GlobClassTooLargeToListMatchesOnlyItsMembers(string text, string filter, bool expected) =>
1086+
Assert.AreEqual(expected, TextFilter.IsMatch(text, filter, TextFilterType.Glob, TextFilterMatchOptions.ByWholeString));
1087+
1088+
[TestMethod]
1089+
public void GlobClassTooLargeToListStillLeavesAHalfTypedTokenIgnored()
1090+
{
1091+
string[] items = ["README.md", "한국.md"];
1092+
string[] korean = ["한국.md"];
1093+
string[] readme = ["README.md"];
1094+
CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]*", TextFilterType.Glob).ToArray());
1095+
CollectionAssert.AreEqual(readme, TextFilter.Filter(items, "-[가-힣a-z]*", TextFilterType.Glob).ToArray());
1096+
CollectionAssert.AreEqual(items, TextFilter.Filter(items, "file[0-", TextFilterType.Glob).ToArray());
1097+
CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]* file[0-", TextFilterType.Glob).ToArray());
1098+
}
1099+
10421100
[TestMethod]
10431101
[DataRow("[0-9a-f]", "0 5 9 a c f", "g G - x _")]
10441102
[DataRow("[!0-9a-f]", "g - x _", "0 5 a c f")]
@@ -1144,7 +1202,7 @@ public void GlobClassRangeSpanningAPathSeparatorMatchesIt(string pattern, string
11441202
[DataRow("[]]]", "[]][]]")]
11451203
[DataRow("*[[][0-9]]", "*[[][0-9][]]")]
11461204
public void ExpandGlobClassesWritesALiteralBracketAfterAClassAsItsOwnClass(string token, string expected) =>
1147-
Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token));
1205+
CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray());
11481206

11491207
[TestMethod]
11501208
[DataRow("*[[][0-9]]", "items[3]", "items[3")]

‎TextFilter/TextFilter.cs‎

Lines changed: 87 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -505,18 +505,26 @@ internal static IReadOnlyList<string> ExpandGlobStarSegments(string maskedToken)
505505
// letter or digit range is left for DotNet.Glob to match natively, since it reads that form
506506
// correctly and it may span far more characters than are worth listing. A class with no closing
507507
// "]" is left alone and stays unparseable.
508-
// Returns null for a class that cannot be written as a list: one whose ranges span more than
509-
// MaxGlobClassMembers characters, or that has no members at all because every range is reversed.
508+
// A class that would list more than MaxGlobClassMembers characters ("[가-힣a-z]") is costly to match
509+
// as a list, so a class that is not negated is split instead: each letter or digit range in it is
510+
// matched natively, the rest is listed, and the token becomes one alternative per piece, matching if
511+
// any of them does. A negated class cannot be split that way, since it must miss every piece at once,
512+
// so it is listed in full however long it is, as is a class whose split would give the token more than
513+
// MaxGlobClassAlternatives alternatives.
514+
// Returns the token's alternatives, or null for a class that has no members at all because every
515+
// range is reversed.
510516
private const int MaxGlobClassMembers = 4096;
511517

512-
internal static string? ExpandGlobClasses(string token)
518+
private const int MaxGlobClassAlternatives = 16;
519+
520+
internal static IReadOnlyList<string>? ExpandGlobClasses(string token)
513521
{
514522
if (!token.Contains(']'))
515523
{
516-
return token;
524+
return [token];
517525
}
518526

519-
StringBuilder result = new(token.Length);
527+
List<StringBuilder> results = [new(token.Length)];
520528
int index = 0;
521529
while (index < token.Length)
522530
{
@@ -533,29 +541,66 @@ internal static IReadOnlyList<string> ExpandGlobStarSegments(string maskedToken)
533541
}
534542

535543
string content = token[contentStart..close];
536-
string opening = negated ? "[!" : "[";
537-
string? rewritten = !content.Contains('-') || IsNativeGlobRange(content)
538-
? opening + content + "]"
539-
: ToGlobCharacterList(content, negated);
544+
List<string>? rewritten = RewriteGlobClass(content, negated, MaxGlobClassAlternatives / results.Count);
540545
if (rewritten is null)
541546
{
542547
return null;
543548
}
544549

545-
result.Append(token, index, open - index).Append(rewritten);
550+
string literal = token[index..open];
551+
results = [.. results.SelectMany(result => rewritten.Select(piece => new StringBuilder(result.ToString()).Append(literal).Append(piece)))];
546552
index = close + 1;
547553

548554
// DotNet.Glob reads a "]" straight after a class as one more member of it, so "[a]]" would
549555
// match "a" and not "a]". Each such "]" is written as the one-member class "[]]", which it
550556
// reads as the literal the pattern means.
551557
while (index < token.Length && token[index] == ']')
552558
{
553-
result.Append("[]]");
559+
results.ForEach(result => result.Append("[]]"));
554560
index++;
555561
}
556562
}
557563

558-
return result.Append(token, index, token.Length - index).ToString();
564+
string tail = token[index..];
565+
return [.. results.Select(result => result.Append(tail).ToString())];
566+
}
567+
568+
// Returns the pieces a class is matched as, any of which may match, at most maxPieces of them; or
569+
// null if the class has no members.
570+
private static List<string>? RewriteGlobClass(string content, bool negated, int maxPieces)
571+
{
572+
if (!content.Contains('-') || IsNativeGlobRange(content))
573+
{
574+
return [(negated ? "[!" : "[") + content + "]"];
575+
}
576+
577+
List<char> singles = [];
578+
List<(char First, char Last)> ranges = [];
579+
ReadGlobClassMembers(content, singles, ranges);
580+
581+
long memberCount = singles.Count + ranges.Sum(range => Math.Max(0L, range.Last - range.First + 1L));
582+
if (!negated && memberCount > MaxGlobClassMembers)
583+
{
584+
// A reversed range has no members, so it adds no piece.
585+
List<(char First, char Last)> listedRanges = [.. ranges.Where(range => range.First <= range.Last && !IsNativeGlobRange(range.First, range.Last))];
586+
List<string> pieces = [.. ranges
587+
.Where(range => range.First <= range.Last && IsNativeGlobRange(range.First, range.Last))
588+
.Select(range => $"[{range.First}-{range.Last}]")
589+
.Distinct(StringComparer.Ordinal)];
590+
591+
string? rest = ToGlobCharacterList(CollectGlobClassMembers(singles, listedRanges), negated: false);
592+
if (rest is not null)
593+
{
594+
pieces.Add(rest);
595+
}
596+
597+
if (pieces.Count <= maxPieces)
598+
{
599+
return pieces;
600+
}
601+
}
602+
603+
return ToGlobCharacterList(CollectGlobClassMembers(singles, ranges), negated) is string list ? [list] : null;
559604
}
560605

561606
// Returns the index of the "]" closing the class opened at "open", or -1 if it is never closed.
@@ -577,47 +622,55 @@ private static int FindGlobClassEnd(string token, int open, out bool negated, ou
577622
// A range that spans "/" or "\" is listed instead, so the separator in it is masked like any other
578623
// literal one; left native, it would never contain the masked separator in the text ("[A-z]").
579624
private static bool IsNativeGlobRange(string content) =>
580-
content.Length == 3 && content[1] == '-' && content[0] != ']'
581-
&& ((char.IsLetter(content[0]) && char.IsLetter(content[2])) || (char.IsAsciiDigit(content[0]) && char.IsAsciiDigit(content[2])))
582-
&& !SpansPathSeparator(content[0], content[2]);
625+
content.Length == 3 && content[1] == '-' && content[0] != ']' && IsNativeGlobRange(content[0], content[2]);
626+
627+
private static bool IsNativeGlobRange(char first, char last) =>
628+
((char.IsLetter(first) && char.IsLetter(last)) || (char.IsAsciiDigit(first) && char.IsAsciiDigit(last)))
629+
&& !SpansPathSeparator(first, last);
583630

584631
private static bool SpansPathSeparator(char first, char last) =>
585632
(first <= '/' && '/' <= last) || (first <= '\\' && '\\' <= last);
586633

587-
// Reads the class members with the usual glob rules (a leading "]" and a leading or trailing "-" are
588-
// literal, "x-y" is an inclusive range) and writes them in an order DotNet.Glob reads as a plain list:
589-
// "]" first, so it does not close the class, then "-", so no member pair looks like a range, and "!"
590-
// never first, so it is not read as negation.
591-
private static string? ToGlobCharacterList(string content, bool negated)
634+
// Reads the class members with the usual glob rules: a leading "]" and a leading or trailing "-" are
635+
// literal, and "x-y" is an inclusive range.
636+
private static void ReadGlobClassMembers(string content, List<char> singles, List<(char First, char Last)> ranges)
592637
{
593-
SortedSet<char> members = [];
594638
int i = 0;
595639
while (i < content.Length)
596640
{
597-
char first = content[i];
598641
// A leading "]" can start a range too: "[]-a]" is "]".."a", as in POSIX.
599642
bool isRange = i + 2 < content.Length && content[i + 1] == '-';
600643
if (!isRange)
601644
{
602-
members.Add(first);
645+
singles.Add(content[i]);
603646
i++;
604647
continue;
605648
}
606649

607-
char last = content[i + 2];
608-
if (last - first + members.Count >= MaxGlobClassMembers)
609-
{
610-
return null;
611-
}
650+
ranges.Add((content[i], content[i + 2]));
651+
i += 3;
652+
}
653+
}
612654

655+
private static SortedSet<char> CollectGlobClassMembers(List<char> singles, List<(char First, char Last)> ranges)
656+
{
657+
SortedSet<char> members = [.. singles];
658+
foreach ((char first, char last) in ranges)
659+
{
613660
for (int member = first; member <= last; member++)
614661
{
615662
members.Add((char)member);
616663
}
617-
618-
i += 3;
619664
}
620665

666+
return members;
667+
}
668+
669+
// Writes the members in an order DotNet.Glob reads as a plain list: "]" first, so it does not close
670+
// the class, then "-", so no member pair looks like a range, and "!" never first, so it is not read
671+
// as negation. Returns null if there are no members.
672+
private static string? ToGlobCharacterList(SortedSet<char> members, bool negated)
673+
{
621674
if (members.Count == 0)
622675
{
623676
return null;
@@ -663,12 +716,14 @@ private static bool SpansPathSeparator(char first, char last) =>
663716
{
664717
try
665718
{
666-
string? expandedToken = ExpandGlobClasses(filterToken);
719+
IReadOnlyList<string>? expandedTokens = ExpandGlobClasses(filterToken);
667720
// A token made only of "**/" segments expands to the empty string as well as to "*/".
668721
// Glob.Parse throws on an empty pattern, which would mark the whole token unparseable and
669722
// discard the valid expansions with it. An empty pattern could only match empty text, which
670723
// never reaches the matcher, so drop it; a token with nothing left is still unparseable.
671-
Glob[]? parsed = expandedToken is null ? null : [.. ExpandGlobStarSegments(MaskPathSeparators(expandedToken))
724+
Glob[]? parsed = expandedTokens is null ? null : [.. expandedTokens
725+
.SelectMany(expandedToken => ExpandGlobStarSegments(MaskPathSeparators(expandedToken)))
726+
.Distinct(StringComparer.Ordinal)
672727
.Where(pattern => pattern.Length != 0)
673728
.Select(pattern => caseSensitivity is TextFilterCaseSensitivity.CaseInsensitive
674729
? Glob.Parse(pattern, CaseInsensitiveGlobOptions)

0 commit comments

Comments
 (0)