Skip to content
Open
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions src/diff.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,17 @@ fn tokenize_mixed(s: &str) -> Vec<&str> {
// Standalone '#'
tokens.push(&s[start..i]);
}
} else if c == '$' && !s[..i].ends_with('\\') {
// Inline equation: keep `$...$` as one atomic token.
// A `$` preceded by a backslash is an escaped
// literal dollar sign in markup, not a math delimiter.
let start = i;
i += 1;
match s[i..].find('$') {
Some(off) => i += off + 1,
None => i = s.len(),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
}
tokens.push(&s[start..i]);
Comment thread
Copilot marked this conversation as resolved.
Comment thread
Copilot marked this conversation as resolved.
Comment thread
Copilot marked this conversation as resolved.
Outdated
} else if c == '@' {
// Typst reference: @label (alphanumeric, hyphens, underscores, colons, periods)
let start = i;
Expand Down Expand Up @@ -543,6 +554,19 @@ mod tests {
assert_eq!(tokens, vec!["<sample-widget_anchor>"]);
}

#[test]
fn test_tokenize_inline_equation_is_atomic() {
let tokens = tokenize_mixed("is $t = 12.4 \"hour\"$ for");
assert_eq!(tokens, vec!["is", " ", "$t = 12.4 \"hour\"$", " ", "for"]);
}

#[test]
fn test_tokenize_escaped_dollar_is_not_equation() {
let tokens = tokenize_mixed("\\$5 and $x$");
assert!(tokens.contains(&"$x$"));
assert!(!tokens.iter().any(|t| t.starts_with("$5")));
}
Comment thread
Copilot marked this conversation as resolved.

#[test]
fn test_tokenize_cjk_char_level() {
// CJK characters should be tokenized individually (no whitespace boundaries).
Expand Down