Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions packages/lang-core/src/parser/__tests__/parser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -235,3 +235,55 @@ describe("orphaned statements", () => {
expect(result.meta.orphaned).toHaveLength(0);
});
});

describe("markdown fences and multiline comments in strings", () => {
it("preserves js markdown fences inside strings", () => {
const code = 'root = Title("```js\\nconsole.log(\\"Hello World\\");\\n```")';
const result = parse(code, schema);
expect(result.meta.errors).toHaveLength(0);
expect(result.root).not.toBeNull();
expect(result.root?.props.text).toBe('```js\nconsole.log("Hello World");\n```');
});

it("preserves python fenced code blocks inside strings", () => {
const code = "root = Title(\"```python\\nprint('hi')\\n```\")";
const result = parse(code, schema);
expect(result.meta.errors).toHaveLength(0);
expect(result.root).not.toBeNull();
expect(result.root?.props.text).toBe("```python\nprint('hi')\n```");
});

it("preserves // inside multiline strings", () => {
const code = `root = Title("
Hello // World
https://example.com/
")`;
const result = parse(code, schema);
expect(result.meta.errors).toHaveLength(0);
expect(result.root).not.toBeNull();
expect(result.root?.props.text).toBe("\nHello // World\nhttps://example.com/\n");
});

it("preserves # inside multiline strings", () => {
const code = `root = Title("
Hello # World
https://example.com/
")`;
const result = parse(code, schema);
expect(result.meta.errors).toHaveLength(0);
expect(result.root).not.toBeNull();
expect(result.root?.props.text).toBe("\nHello # World\nhttps://example.com/\n");
});

it("does not treat apostrophes as string delimiters when stripping fences", () => {
const code = `Here's the code:

\`\`\`js
root = Title("hello")
\`\`\``;
const result = parse(code, schema);
expect(result.meta.errors).toHaveLength(0);
expect(result.root).not.toBeNull();
expect(result.root?.props.text).toBe("hello");
});
});
62 changes: 44 additions & 18 deletions packages/lang-core/src/parser/parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -246,15 +246,52 @@ function buildResult(
// Public API
// ─────────────────────────────────────────────────────────────────────────────

function skipString(input: string, start: number): number {
if (input[start] !== '"') return start;
let i = start + 1;
while (i < input.length) {
const c = input[i];
if (c === "\\") {
i += 2; // skip escape character and the escaped character
} else if (c === '"') {
return i + 1; // return index after the closing quote
} else {
i++;
}
}
return i; // return length if string was unclosed
}

/** Extract code from markdown fences, or return as-is if no fences found.
* String-context-aware: skips ``` inside double-quoted strings. */
export function stripFences(input: string): string {
const blocks: string[] = [];
let i = 0;

while (i < input.length) {
// Look for opening ```
const fenceStart = input.indexOf("```", i);
// Scan for opening ``` while tracking string context
let fenceStart = -1;
while (i < input.length) {
const nextI = skipString(input, i);
if (nextI > i) {
i = nextI;
continue;
}

const c = input[i];
if (
c === "`" &&
i + 1 < input.length &&
input[i + 1] === "`" &&
i + 2 < input.length &&
input[i + 2] === "`"
) {
fenceStart = i;
break;
}
i++;
}

if (fenceStart === -1) break;

// Skip language tag until newline
Expand All @@ -269,26 +306,15 @@ export function stripFences(input: string): string {
j++; // skip the newline

// Scan for closing ``` while tracking string context
let inStr = false;
let closePos = -1;
let k = j;
while (k < input.length) {
const c = input[k];
if (inStr) {
if (c === "\\" && k + 1 < input.length) {
k += 2; // skip escaped character
continue;
}
if (c === '"') inStr = false;
k++;
continue;
}
// Not in string
if (c === '"') {
inStr = true;
k++;
const nextK = skipString(input, k);
if (nextK > k) {
k = nextK;
continue;
}
const c = input[k];
if (
c === "`" &&
k + 1 < input.length &&
Expand Down Expand Up @@ -333,10 +359,10 @@ export function stripFences(input: string): string {

/** Strip // and # line comments outside of strings (handles both " and ' delimiters). */
function stripComments(input: string): string {
let inStr: false | '"' | "'" = false;
return input
.split("\n")
.map((line) => {
let inStr: false | '"' | "'" = false;
for (let i = 0; i < line.length; i++) {
const c = line[i];
if (inStr) {
Expand Down
Loading