Skip to content

test: add ToolErrorClassifierTests (closes #28) #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chiliec wants to merge 1 commit into
base: main
Choose a base branch
from
Open

test: add ToolErrorClassifierTests (closes #28) #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Agent.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@
AB112233445566778899050A /* PackageAutoDiscoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = AB112233445566778899050B /* PackageAutoDiscoveryTests.swift */; };
AB112233445566778899060A /* PackageGenerationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = AB112233445566778899060B /* PackageGenerationTests.swift */; };
AB112233445566778899070A /* ScriptServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = AB112233445566778899070B /* ScriptServiceTests.swift */; };
AB1122334455667788990C70 /* ToolErrorClassifierTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = AB1122334455667788990C71 /* ToolErrorClassifierTests.swift */; };
AB112233445566778899080A /* ScriptingBridgesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = AB112233445566778899080B /* ScriptingBridgesTests.swift */; };
ABC7EAA6F77F494989A1A61F /* AgentTools+AppBridge.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC627819110EDB4BE4A30406 /* AgentTools+AppBridge.swift */; };
AD0E84FCAA3A3D6AF61ECD16 /* SavedScripts.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4709078FC05AA3587261B14 /* SavedScripts.swift */; };
Expand Down Expand Up @@ -422,6 +423,7 @@
AB112233445566778899050B /* PackageAutoDiscoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PackageAutoDiscoveryTests.swift; sourceTree = "<group>"; };
AB112233445566778899060B /* PackageGenerationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PackageGenerationTests.swift; sourceTree = "<group>"; };
AB112233445566778899070B /* ScriptServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScriptServiceTests.swift; sourceTree = "<group>"; };
AB1122334455667788990C71 /* ToolErrorClassifierTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ToolErrorClassifierTests.swift; sourceTree = "<group>"; };
AB112233445566778899080B /* ScriptingBridgesTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScriptingBridgesTests.swift; sourceTree = "<group>"; };
ABBDF459529494A3928D101A /* AgenticCoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AgenticCoreTests.swift; sourceTree = "<group>"; };
AC627819110EDB4BE4A30406 /* AgentTools+AppBridge.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AgentTools+AppBridge.swift"; sourceTree = "<group>"; };
Expand Down Expand Up @@ -908,6 +910,7 @@
AB112233445566778899050B /* PackageAutoDiscoveryTests.swift */,
AB112233445566778899060B /* PackageGenerationTests.swift */,
AB112233445566778899070B /* ScriptServiceTests.swift */,
AB1122334455667788990C71 /* ToolErrorClassifierTests.swift */,
AB112233445566778899080B /* ScriptingBridgesTests.swift */,
);
path = AgentTests;
Expand Down Expand Up @@ -1466,6 +1469,7 @@
AB112233445566778899050A /* PackageAutoDiscoveryTests.swift in Sources */,
AB112233445566778899060A /* PackageGenerationTests.swift in Sources */,
AB112233445566778899070A /* ScriptServiceTests.swift in Sources */,
AB1122334455667788990C70 /* ToolErrorClassifierTests.swift in Sources */,
AB112233445566778899080A /* ScriptingBridgesTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
124 changes: 124 additions & 0 deletions AgentTests/ToolErrorClassifierTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import Testing
import Foundation
@testable import Agent_

@Suite("ToolErrorClassifier")
struct ToolErrorClassifierTests {

// MARK: - Each rule matches (code + a non-empty hint)

@Test("edit_file exact-match failure -> old_string_not_found")
func oldStringNotFound() {
let err = ToolErrorClassifier.classify(tool: "edit_file", output: "Error: old_string not found in file")
#expect(err?.code == "old_string_not_found")
#expect(!(err?.hint.isEmpty ?? true))
}

@Test("old_string with 'no match' also classifies as old_string_not_found")
func oldStringNoMatch() {
let err = ToolErrorClassifier.classify(tool: "edit_file", output: "old_string: no match")
#expect(err?.code == "old_string_not_found")
}

@Test("multiple matches -> ambiguous_match")
func ambiguousMatch() {
#expect(ToolErrorClassifier.classify(tool: "edit_file", output: "Found multiple matches")?.code == "ambiguous_match")
#expect(ToolErrorClassifier.classify(tool: "edit_file", output: "more than one match found")?.code == "ambiguous_match")
}

@Test("no such file -> file_not_found")
func noSuchFile() {
#expect(ToolErrorClassifier.classify(tool: "read_file", output: "No such file or directory")?.code == "file_not_found")
#expect(ToolErrorClassifier.classify(tool: "shell", output: "path does not exist")?.code == "file_not_found")
}

@Test("generic 'not found' only classifies as file_not_found for file tools")
func notFoundNarrowedByTool() {
// A file tool with a bare "not found" -> file_not_found.
#expect(ToolErrorClassifier.classify(tool: "read_file", output: "resource not found")?.code == "file_not_found")
// A non-file tool with a bare "not found" should NOT be file_not_found
// (no rule matches it) — this guards the tool-narrowing condition.
#expect(ToolErrorClassifier.classify(tool: "web_search", output: "not found") == nil)
}

@Test("permission denied -> permission_denied")
func permissionDenied() {
#expect(ToolErrorClassifier.classify(tool: "shell", output: "Permission denied")?.code == "permission_denied")
#expect(ToolErrorClassifier.classify(tool: "shell", output: "operation not permitted")?.code == "permission_denied")
}

@Test("build failed -> build_failed")
func buildFailed() {
#expect(ToolErrorClassifier.classify(tool: "xcode_build", output: "** BUILD FAILED **")?.code == "build_failed")
#expect(ToolErrorClassifier.classify(tool: "build", output: "main.swift:10: error: expected ')'")?.code == "build_failed")
}

@Test("timeout -> timeout")
func timeout() {
#expect(ToolErrorClassifier.classify(tool: "shell", output: "Operation timed out")?.code == "timeout")
#expect(ToolErrorClassifier.classify(tool: "shell", output: "request timeout")?.code == "timeout")
}

@Test("connection errors -> network_unreachable")
func networkUnreachable() {
#expect(ToolErrorClassifier.classify(tool: "http", output: "Connection refused")?.code == "network_unreachable")
#expect(ToolErrorClassifier.classify(tool: "http", output: "could not connect to host")?.code == "network_unreachable")
#expect(ToolErrorClassifier.classify(tool: "http", output: "You are offline")?.code == "network_unreachable")
}

@Test("script syntax error -> script_syntax_error (only for script tools)")
func scriptSyntaxError() {
#expect(ToolErrorClassifier.classify(tool: "applescript_tool", output: "Syntax Error: expected end of line")?.code == "script_syntax_error")
// A syntax error from a non-script tool is not classified by this rule.
#expect(ToolErrorClassifier.classify(tool: "read_file", output: "syntax error") == nil)
}

@Test("rate limit -> rate_limited")
func rateLimited() {
#expect(ToolErrorClassifier.classify(tool: "llm", output: "Rate limit exceeded")?.code == "rate_limited")
#expect(ToolErrorClassifier.classify(tool: "llm", output: "HTTP 429 Too Many Requests")?.code == "rate_limited")
}

// MARK: - No match / fallback

@Test("unrecognized output returns nil")
func noMatchReturnsNil() {
#expect(ToolErrorClassifier.classify(tool: "shell", output: "everything worked fine") == nil)
#expect(ToolErrorClassifier.classify(tool: "shell", output: "") == nil)
}

// MARK: - Matching is case-insensitive

@Test("classification is case-insensitive")
func caseInsensitive() {
#expect(ToolErrorClassifier.classify(tool: "shell", output: "PERMISSION DENIED")?.code == "permission_denied")
#expect(ToolErrorClassifier.classify(tool: "shell", output: "Timed Out")?.code == "timeout")
}

// MARK: - First-match-wins ordering

@Test("first matching rule wins when output hits several")
func firstMatchWins() {
// Contains both "old_string ... not found" (rule 1) and "permission denied"
// (a later rule). Rule 1 is ordered first, so it must win.
let err = ToolErrorClassifier.classify(
tool: "edit_file",
output: "old_string not found; also permission denied"
)
#expect(err?.code == "old_string_not_found")
}

// MARK: - annotation()

@Test("annotation wraps code + hint for a classified error")
func annotationForClassified() {
let note = ToolErrorClassifier.annotation(tool: "shell", output: "Permission denied")
#expect(note != nil)
#expect(note?.contains("[error_code: permission_denied]") == true)
}

@Test("annotation returns nil for unclassified output")
func annotationForUnclassified() {
#expect(ToolErrorClassifier.annotation(tool: "shell", output: "all good") == nil)
}
}
Loading