Split a glob class too large to list into its ranges [patch]

TextFilter.Test/TextFilterTests.cs

@@ -1031,14 +1031,72 @@ public void GlobClassWithATrailingDashMatchesTheDashLiterally()
 	[DataRow("x[a^]", "x[a^]")]
 	[DataRow("x[^a", "x[^a")]
 	public void ExpandGlobClassesRewritesOnlyClassesDotNetGlobMisreads(string token, string expected) =>
-		Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token));
+		CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray());
 
 	[TestMethod]
 	[DataRow("[#-!]")]
-	[DataRow("[\u0001-\uffff_]")]
 	public void ExpandGlobClassesRejectsClassesThatCannotBeListed(string token) =>
 		Assert.IsNull(TextFilter.ExpandGlobClasses(token));
 
+	[TestMethod]
+	[DataRow("[가-힣a-z]*", new[] { "[가-힣]*", "[a-z]*" })]
+	[DataRow("x[一-龥0-9_]", new[] { "x[一-龥]", "x[0-9]", "x[_]" })]
+	[DataRow("[가-힣a-z][一-龥0-9]", new[] { "[가-힣][一-龥]", "[가-힣][0-9]", "[a-z][一-龥]", "[a-z][0-9]" })]
+	[DataRow("[가-힣z-a]", new[] { "[가-힣]" })]
+	public void ExpandGlobClassesSplitsAClassTooLargeToListIntoItsRanges(string token, string[] expected) =>
+		CollectionAssert.AreEqual(expected, TextFilter.ExpandGlobClasses(token)?.ToArray());
+
+	[TestMethod]
+	[DataRow("[\u0001-\uffff_]")]
+	[DataRow("[!가-힣a-z]")]
+	[DataRow("[A-힣]")]
+	public void ExpandGlobClassesListsAClassTooLargeToListThatCannotBeSplit(string token)
+	{
+		IReadOnlyList<string>? expanded = TextFilter.ExpandGlobClasses(token);
+		Assert.IsNotNull(expanded);
+		Assert.HasCount(1, expanded);
+		Assert.IsGreaterThan(4096, expanded[0].Length);
+	}
+
+	[TestMethod]
+	[DataRow("README.md", "[가-힣a-z]*", false)]
+	[DataRow("README.md", "[一-龥a-z]*", false)]
+	[DataRow("README.md", "[一-龥0-9]*", false)]
+	// Matching is case-sensitive, so "README.md" is not among the members and is not excluded.
+	[DataRow("README.md", "-[一-龥a-z]*", true)]
+	[DataRow("README.md", "-[一-龥A-Z]*", false)]
+	[DataRow("@", "[A-힣]", false)]
+	[DataRow("readme.md", "[가-힣a-z]*", true)]
+	[DataRow("한국.md", "[가-힣a-z]*", true)]
+	[DataRow("中文", "[一-龥a-z]*", true)]
+	[DataRow("7z", "[一-龥0-9]*", true)]
+	[DataRow("readme.md", "-[一-龥a-z]*", false)]
+	[DataRow("中文", "-[一-龥a-z]*", false)]
+	[DataRow("README.md", "+[가-힣a-z]*", false)]
+	[DataRow("한국", "+[가-힣a-z]*", true)]
+	[DataRow("A", "[A-힣]", true)]
+	[DataRow("힣", "[A-힣]", true)]
+	[DataRow("a\\b", "a[A-힣]b", true)]
+	[DataRow("README.md", "[!가-힣a-z]*", true)]
+	[DataRow("readme.md", "[!가-힣a-z]*", false)]
+	[DataRow("한국", "[!가-힣a-z]*", false)]
+	[DataRow("x_", "x[가-힣a-z_]", true)]
+	[DataRow("x-", "x[가-힣a-z_]", false)]
+	public void GlobClassTooLargeToListMatchesOnlyItsMembers(string text, string filter, bool expected) =>
+		Assert.AreEqual(expected, TextFilter.IsMatch(text, filter, TextFilterType.Glob, TextFilterMatchOptions.ByWholeString));
+
+	[TestMethod]
+	public void GlobClassTooLargeToListStillLeavesAHalfTypedTokenIgnored()
+	{
+		string[] items = ["README.md", "한국.md"];
+		string[] korean = ["한국.md"];
+		string[] readme = ["README.md"];
+		CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]*", TextFilterType.Glob).ToArray());
+		CollectionAssert.AreEqual(readme, TextFilter.Filter(items, "-[가-힣a-z]*", TextFilterType.Glob).ToArray());
+		CollectionAssert.AreEqual(items, TextFilter.Filter(items, "file[0-", TextFilterType.Glob).ToArray());
+		CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]* file[0-", TextFilterType.Glob).ToArray());
+	}
+
 	[TestMethod]
 	[DataRow("[0-9a-f]", "0 5 9 a c f", "g G - x _")]
 	[DataRow("[!0-9a-f]", "g - x _", "0 5 a c f")]
@@ -1144,7 +1202,7 @@ public void GlobClassRangeSpanningAPathSeparatorMatchesIt(string pattern, string
 	[DataRow("[]]]", "[]][]]")]
 	[DataRow("*[[][0-9]]", "*[[][0-9][]]")]
 	public void ExpandGlobClassesWritesALiteralBracketAfterAClassAsItsOwnClass(string token, string expected) =>
-		Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token));
+		CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray());
 
 	[TestMethod]
 	[DataRow("*[[][0-9]]", "items[3]", "items[3")]

TextFilter/TextFilter.cs

@@ -505,18 +505,26 @@ internal static IReadOnlyList<string> ExpandGlobStarSegments(string maskedToken)
 	// letter or digit range is left for DotNet.Glob to match natively, since it reads that form
 	// correctly and it may span far more characters than are worth listing. A class with no closing
 	// "]" is left alone and stays unparseable.
-	// Returns null for a class that cannot be written as a list: one whose ranges span more than
-	// MaxGlobClassMembers characters, or that has no members at all because every range is reversed.
+	// A class that would list more than MaxGlobClassMembers characters ("[가-힣a-z]") is costly to match
+	// as a list, so a class that is not negated is split instead: each letter or digit range in it is
+	// matched natively, the rest is listed, and the token becomes one alternative per piece, matching if
+	// any of them does. A negated class cannot be split that way, since it must miss every piece at once,
+	// so it is listed in full however long it is, as is a class whose split would give the token more than
+	// MaxGlobClassAlternatives alternatives.
+	// Returns the token's alternatives, or null for a class that has no members at all because every
+	// range is reversed.
 	private const int MaxGlobClassMembers = 4096;
 
-	internal static string? ExpandGlobClasses(string token)
+	private const int MaxGlobClassAlternatives = 16;
+
+	internal static IReadOnlyList<string>? ExpandGlobClasses(string token)
 	{
 		if (!token.Contains(']'))
 		{
-			return token;
+			return [token];
 		}
 
-		StringBuilder result = new(token.Length);
+		List<StringBuilder> results = [new(token.Length)];
 		int index = 0;
 		while (index < token.Length)
 		{
@@ -533,29 +541,66 @@ internal static IReadOnlyList<string> ExpandGlobStarSegments(string maskedToken)
 			}
 
 			string content = token[contentStart..close];
-			string opening = negated ? "[!" : "[";
-			string? rewritten = !content.Contains('-') || IsNativeGlobRange(content)
-				? opening + content + "]"
-				: ToGlobCharacterList(content, negated);
+			List<string>? rewritten = RewriteGlobClass(content, negated, MaxGlobClassAlternatives / results.Count);
 			if (rewritten is null)
 			{
 				return null;
 			}
 
-			result.Append(token, index, open - index).Append(rewritten);
+			string literal = token[index..open];
+			results = [.. results.SelectMany(result => rewritten.Select(piece => new StringBuilder(result.ToString()).Append(literal).Append(piece)))];
 			index = close + 1;
 
 			// DotNet.Glob reads a "]" straight after a class as one more member of it, so "[a]]" would
 			// match "a" and not "a]". Each such "]" is written as the one-member class "[]]", which it
 			// reads as the literal the pattern means.
 			while (index < token.Length && token[index] == ']')
 			{
-				result.Append("[]]");
+				results.ForEach(result => result.Append("[]]"));
 				index++;
 			}
 		}
 
-		return result.Append(token, index, token.Length - index).ToString();
+		string tail = token[index..];
+		return [.. results.Select(result => result.Append(tail).ToString())];
+	}
+
+	// Returns the pieces a class is matched as, any of which may match, at most maxPieces of them; or
+	// null if the class has no members.
+	private static List<string>? RewriteGlobClass(string content, bool negated, int maxPieces)
+	{
+		if (!content.Contains('-') || IsNativeGlobRange(content))
+		{
+			return [(negated ? "[!" : "[") + content + "]"];
+		}
+
+		List<char> singles = [];
+		List<(char First, char Last)> ranges = [];
+		ReadGlobClassMembers(content, singles, ranges);
+
+		long memberCount = singles.Count + ranges.Sum(range => Math.Max(0L, range.Last - range.First + 1L));
+		if (!negated && memberCount > MaxGlobClassMembers)
+		{
+			// A reversed range has no members, so it adds no piece.
+			List<(char First, char Last)> listedRanges = [.. ranges.Where(range => range.First <= range.Last && !IsNativeGlobRange(range.First, range.Last))];
+			List<string> pieces = [.. ranges
+				.Where(range => range.First <= range.Last && IsNativeGlobRange(range.First, range.Last))
+				.Select(range => $"[{range.First}-{range.Last}]")
+				.Distinct(StringComparer.Ordinal)];
+
+			string? rest = ToGlobCharacterList(CollectGlobClassMembers(singles, listedRanges), negated: false);
+			if (rest is not null)
+			{
+				pieces.Add(rest);
+			}
+
+			if (pieces.Count <= maxPieces)
+			{
+				return pieces;
+			}
+		}
+
+		return ToGlobCharacterList(CollectGlobClassMembers(singles, ranges), negated) is string list ? [list] : null;
 	}
 
 	// Returns the index of the "]" closing the class opened at "open", or -1 if it is never closed.
@@ -577,47 +622,55 @@ private static int FindGlobClassEnd(string token, int open, out bool negated, ou
 	// A range that spans "/" or "\" is listed instead, so the separator in it is masked like any other
 	// literal one; left native, it would never contain the masked separator in the text ("[A-z]").
 	private static bool IsNativeGlobRange(string content) =>
-		content.Length == 3 && content[1] == '-' && content[0] != ']'
-		&& ((char.IsLetter(content[0]) && char.IsLetter(content[2])) || (char.IsAsciiDigit(content[0]) && char.IsAsciiDigit(content[2])))
-		&& !SpansPathSeparator(content[0], content[2]);
+		content.Length == 3 && content[1] == '-' && content[0] != ']' && IsNativeGlobRange(content[0], content[2]);
+
+	private static bool IsNativeGlobRange(char first, char last) =>
+		((char.IsLetter(first) && char.IsLetter(last)) || (char.IsAsciiDigit(first) && char.IsAsciiDigit(last)))
+		&& !SpansPathSeparator(first, last);
 
 	private static bool SpansPathSeparator(char first, char last) =>
 		(first <= '/' && '/' <= last) || (first <= '\\' && '\\' <= last);
 
-	// Reads the class members with the usual glob rules (a leading "]" and a leading or trailing "-" are
-	// literal, "x-y" is an inclusive range) and writes them in an order DotNet.Glob reads as a plain list:
-	// "]" first, so it does not close the class, then "-", so no member pair looks like a range, and "!"
-	// never first, so it is not read as negation.
-	private static string? ToGlobCharacterList(string content, bool negated)
+	// Reads the class members with the usual glob rules: a leading "]" and a leading or trailing "-" are
+	// literal, and "x-y" is an inclusive range.
+	private static void ReadGlobClassMembers(string content, List<char> singles, List<(char First, char Last)> ranges)
 	{
-		SortedSet<char> members = [];
 		int i = 0;
 		while (i < content.Length)
 		{
-			char first = content[i];
 			// A leading "]" can start a range too: "[]-a]" is "]".."a", as in POSIX.
 			bool isRange = i + 2 < content.Length && content[i + 1] == '-';
 			if (!isRange)
 			{
-				members.Add(first);
+				singles.Add(content[i]);
 				i++;
 				continue;
 			}
 
-			char last = content[i + 2];
-			if (last - first + members.Count >= MaxGlobClassMembers)
-			{
-				return null;
-			}
+			ranges.Add((content[i], content[i + 2]));
+			i += 3;
+		}
+	}
 
+	private static SortedSet<char> CollectGlobClassMembers(List<char> singles, List<(char First, char Last)> ranges)
+	{
+		SortedSet<char> members = [.. singles];
+		foreach ((char first, char last) in ranges)
+		{
 			for (int member = first; member <= last; member++)
 			{
 				members.Add((char)member);
 			}
-
-			i += 3;
 		}
 
+		return members;
+	}
+
+	// Writes the members in an order DotNet.Glob reads as a plain list: "]" first, so it does not close
+	// the class, then "-", so no member pair looks like a range, and "!" never first, so it is not read
+	// as negation. Returns null if there are no members.
+	private static string? ToGlobCharacterList(SortedSet<char> members, bool negated)
+	{
 		if (members.Count == 0)
 		{
 			return null;
@@ -663,12 +716,14 @@ private static bool SpansPathSeparator(char first, char last) =>
 		{
 			try
 			{
-				string? expandedToken = ExpandGlobClasses(filterToken);
+				IReadOnlyList<string>? expandedTokens = ExpandGlobClasses(filterToken);
 				// A token made only of "**/" segments expands to the empty string as well as to "*/".
 				// Glob.Parse throws on an empty pattern, which would mark the whole token unparseable and
 				// discard the valid expansions with it. An empty pattern could only match empty text, which
 				// never reaches the matcher, so drop it; a token with nothing left is still unparseable.
-				Glob[]? parsed = expandedToken is null ? null : [.. ExpandGlobStarSegments(MaskPathSeparators(expandedToken))
+				Glob[]? parsed = expandedTokens is null ? null : [.. expandedTokens
+					.SelectMany(expandedToken => ExpandGlobStarSegments(MaskPathSeparators(expandedToken)))
+					.Distinct(StringComparer.Ordinal)
 					.Where(pattern => pattern.Length != 0)
 					.Select(pattern => caseSensitivity is TextFilterCaseSensitivity.CaseInsensitive
 						? Glob.Parse(pattern, CaseInsensitiveGlobOptions)