diff --git a/src/ai/experience-compactor.ts b/src/ai/experience-compactor.ts index 81f0f56..966aa67 100644 --- a/src/ai/experience-compactor.ts +++ b/src/ai/experience-compactor.ts @@ -1,12 +1,12 @@ import { unlinkSync } from 'node:fs'; import { basename } from 'node:path'; import dedent from 'dedent'; -import { type Tokens, marked } from 'marked'; import { z } from 'zod'; import { type ExperienceFile, type ExperienceTracker, RECENT_WINDOW_DAYS } from '../experience-tracker.js'; import { Observability } from '../observability.js'; import { createDebug, log, tag } from '../utils/logger.js'; import { mdq } from '../utils/markdown-query.js'; +import { isNonReusableCode } from '../utils/step-analyzer.js'; import { generalizeUrl, hasDynamicUrlSegment } from '../utils/url-matcher.js'; import type { Agent } from './agent.js'; import type { Provider } from './provider.js'; @@ -438,28 +438,9 @@ export class ExperienceCompactor implements Agent { } function listSections(content: string): { title: string; raw: string }[] { - const tokens = marked.lexer(content); - const sections: { title: string; raw: string }[] = []; - - let currentHeading: string | null = null; - let currentRaw = ''; - - const flush = () => { - if (currentHeading !== null) sections.push({ title: currentHeading, raw: currentRaw }); - }; - - for (const token of tokens) { - const raw = (token as any).raw || ''; - if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) { - flush(); - currentHeading = (token as Tokens.Heading).text.trim(); - currentRaw = raw; - continue; - } - if (currentHeading !== null) currentRaw += raw; - } - flush(); - return sections; + const sections = mdq(content).query('section2').each(); + const metas = mdq(content).query('section2').meta(); + return sections.map((q, i) => ({ title: metas[i].text.trim(), raw: q.text() })); } function dropEmptySections(content: string): string { @@ -483,7 +464,7 @@ function dropNonReusableSections(content: string): string { for (const section of sections) { const raw = section.text(); - if (!/\bI\.clickXY\s*\(/.test(raw)) continue; + if (!isNonReusableCode(raw)) continue; result = result.replace(raw, ''); } diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 0b453cb..823fa2f 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -280,29 +280,36 @@ export class Planner extends PlannerBase implements Agent { } private cleanExperienceFlows(text: string): string | null { - const seenTitles = new Set(); let result = text; + const seenTitles = new Set(); + for (const selector of ['section2', 'section3']) { + result = mdq(result) + .query(selector) + .replaceEach((section) => { + const heading = section.query('heading').text().trim(); + const withoutHeadings = mdq(section.text()).query('heading').replace(''); + const body = mdq(withoutHeadings).query('hr').replace('').trim(); + if (body && !seenTitles.has(heading)) { + seenTitles.add(heading); + return section.text(); + } + return ''; + }); + } - for (const section of [...mdq(text).query('section2').each(), ...mdq(text).query('section3').each()]) { - const heading = section.query('heading').text().trim(); - const body = mdq(section.text()) - .query('heading') - .replace('') - .replace(/^---\s*$/gm, '') - .trim(); - - if (!body || seenTitles.has(heading)) { - result = result.replace(section.text(), ''); - continue; - } - seenTitles.add(heading); - - const blockquotes = section.query('blockquote').each(); - if (blockquotes.length <= 10) continue; - for (const bq of blockquotes.slice(10)) { - result = result.replace(bq.text(), ''); - } - result = result.replace(section.text().trim(), `${section.text().trim()}\n> ... and ${blockquotes.length - 10} more discoveries`); + const trimmedTitles = new Set(); + for (const selector of ['section2', 'section3']) { + result = mdq(result) + .query(selector) + .replaceEach((section) => { + const heading = section.query('heading').text().trim(); + if (trimmedTitles.has(heading)) return section.text(); + const count = section.query('blockquote').count(); + if (count <= 10) return section.text(); + const kept = mdq(section.text()).query('blockquote[10:]').replace(''); + trimmedTitles.add(heading); + return `${kept.trimEnd()}\n> ... and ${count - 10} more discoveries\n`; + }); } return result.trim() || null; @@ -382,16 +389,17 @@ export class Planner extends PlannerBase implements Agent { deep: true, }); let plannerResearch = mdq(research).query('code').replace(''); - for (const table of mdq(plannerResearch).query('table').each()) { - const rawTable = table.text(); - const rows = table.toJson(); - if (rows.length === 0 || !rows[0].Element) continue; - const elementWithType = rows.map((r) => ({ - Element: r.Element, - Type: r.Type || '', - })); - plannerResearch = plannerResearch.replace(rawTable, jsonToTable(elementWithType, ['Element', 'Type'])); - } + plannerResearch = mdq(plannerResearch) + .query('table') + .replaceEach((table) => { + const rows = table.toJson(); + if (rows.length === 0 || !rows[0].Element) return table.text(); + const elementWithType = rows.map((r) => ({ + Element: r.Element, + Type: r.Type || '', + })); + return jsonToTable(elementWithType, ['Element', 'Type']); + }); const hasFocusedOverlay = hasFocusedSection(plannerResearch); const focusNote = hasFocusedOverlay ? "IMPORTANT: One section is marked as **Focused** — this is the user's current focus area. Concentrate testing on the Focused section FIRST — test all interactions inside it before planning tests for the rest of the page." : ''; diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index a6c77f0..9236241 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -314,11 +314,9 @@ export class Researcher extends ResearcherBase implements Agent { researchFile = saveResearch(stateHash, result.text, combinedHtml); } - const summaryMatch = result.text.match(/## Summary\s*\n+([\s\S]*?)(?=\n##|$)/i); - if (summaryMatch) { - const summaryLine = summaryMatch[1].trim().split('\n')[0].trim().slice(0, 200); - if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine); - } + const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim(); + const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200); + if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine); tag('multiline').log(formatResearchSummary(result.text, { visionUsed: this.hasScreenshotToAnalyze })); tag('success').log('Research complete'); diff --git a/src/ai/researcher/parser.ts b/src/ai/researcher/parser.ts index f6cab4d..7dc0f24 100644 --- a/src/ai/researcher/parser.ts +++ b/src/ai/researcher/parser.ts @@ -94,7 +94,7 @@ export function extractContainerFromBlockquote(sectionMarkdown: string): string } export function parseResearchSections(markdown: string): ResearchSection[] { - const hasExtendedResearch = markdown.includes('\n# Extended Research') || markdown.startsWith('# Extended Research'); + const hasExtendedResearch = mdq(markdown).query('section1(~"Extended Research")').count() > 0; return parseSections(markdown) .filter((s) => !SKIP_SECTIONS.has(s.name.toLowerCase()) && !s.name.toLowerCase().includes('data:')) diff --git a/src/experience-tracker.ts b/src/experience-tracker.ts index 54a45e0..2d56ba4 100644 --- a/src/experience-tracker.ts +++ b/src/experience-tracker.ts @@ -1,7 +1,6 @@ import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import { basename, join } from 'node:path'; import matter from 'gray-matter'; -import { type Tokens, marked } from 'marked'; import type { ActionResult } from './action-result.js'; import { ConfigParser } from './config.js'; import { KnowledgeTracker } from './knowledge-tracker.js'; @@ -159,7 +158,12 @@ export class ExperienceTracker { this.ensureExperienceFile(state); const stateHash = state.getStateHash(); const { content, data } = this.readExperienceFile(stateHash); - if (content.includes(action.code)) { + if ( + mdq(content) + .query('code') + .meta() + .some((m) => m.text.trim() === action.code.trim()) + ) { debugLog('Skipping duplicate action', action.code); return; } @@ -187,7 +191,12 @@ export class ExperienceTracker { const stateHash = state.getStateHash(); const { content, data } = this.readExperienceFile(stateHash); - if (content.includes(body)) { + if ( + mdq(content) + .query('section2') + .each() + .some((s) => s.text().trim() === body.trim()) + ) { debugLog('Skipping duplicate flow body'); return; } @@ -244,9 +253,14 @@ export class ExperienceTracker { }); }) .map((experience) => { - const lines = experience.content.split('\n'); - if (lines.length <= maxLines) return experience; - return { ...experience, content: lines.slice(0, maxLines).join('\n') }; + if (experience.content.split('\n').length <= maxLines) return experience; + let content = experience.content; + while (content.split('\n').length > maxLines) { + const sections = mdq(content).query('section2').each(); + if (sections.length <= 1) break; + content = sections[sections.length - 1].replace(''); + } + return { ...experience, content }; }); } @@ -393,49 +407,22 @@ export class ExperienceTracker { } function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] { - const tokens = marked.lexer(content); - const result: { index: number; level: 2 | 3; title: string }[] = []; - let index = 0; - for (const token of tokens) { - if (token.type !== 'heading') continue; - const heading = token as Tokens.Heading; - if (heading.depth !== 2 && heading.depth !== 3) continue; - index++; - result.push({ index, level: heading.depth as 2 | 3, title: heading.text }); - } - return result; + return mdq(content) + .query('heading') + .meta() + .filter((m) => m.depth === 2 || m.depth === 3) + .map((m, i) => ({ index: i + 1, level: m.depth as 2 | 3, title: m.text })); } function extractHeadingSection(content: string, sectionIndex: number): { title: string; body: string } | null { - const tokens = marked.lexer(content); - const matching: { tokenIdx: number; depth: number; text: string }[] = []; - - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i]; - if (token.type !== 'heading') continue; - const heading = token as Tokens.Heading; - if (heading.depth !== 2 && heading.depth !== 3) continue; - matching.push({ tokenIdx: i, depth: heading.depth, text: heading.text }); - } + const sections = mdq(content).query('section').each(); + const metas = mdq(content).query('section').meta(); + const matching = metas.map((meta, i) => ({ meta, section: sections[i] })).filter((entry) => entry.meta.depth === 2 || entry.meta.depth === 3); if (sectionIndex < 1 || sectionIndex > matching.length) return null; const target = matching[sectionIndex - 1]; - let endTokenIdx = tokens.length; - for (let j = target.tokenIdx + 1; j < tokens.length; j++) { - const token = tokens[j]; - if (token.type !== 'heading') continue; - if ((token as Tokens.Heading).depth <= target.depth) { - endTokenIdx = j; - break; - } - } - - const body = tokens - .slice(target.tokenIdx, endTokenIdx) - .map((t) => (t as any).raw || '') - .join(''); - return { title: target.text, body }; + return { title: target.meta.text, body: target.section.text() }; } function indexToLetters(index: number): string { @@ -504,23 +491,14 @@ function generateActionContent(title: string, code: string, explanation?: string } function renderAsHowTo(content: string): string { - const tokens = marked.lexer(content); - let result = ''; - for (const token of tokens) { - if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) { - const text = (token as Tokens.Heading).text.trim(); - if (text.startsWith('FLOW:')) { - result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`; - continue; - } - if (text.startsWith('ACTION:')) { - result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`; - continue; - } - } - result += (token as any).raw || ''; - } - return result; + return mdq(content) + .query('h2') + .replaceEach((heading) => { + const text = heading.meta()[0]?.text.trim() || ''; + if (text.startsWith('FLOW:')) return `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`; + if (text.startsWith('ACTION:')) return `## HOW to ${text.slice(7).trim()} (single-step)\n\n`; + return heading.text(); + }); } export interface ExperienceFile { diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index af5ae63..13eeca4 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -7,6 +7,7 @@ import { ConfigParser } from './config.js'; import { getCliName } from './utils/cli-name.ts'; import { createDebug, pluralize, tag } from './utils/logger.js'; import { loadMarkdownFiles } from './utils/markdown-files.js'; +import { mdq } from './utils/markdown-query.js'; import { isSecretName, registerSecret } from './utils/secrets.js'; import { slugify } from './utils/strings.js'; @@ -186,7 +187,7 @@ export class KnowledgeTracker { return this.knowledgeFiles.map((knowledge) => { const content = knowledge.content.trim(); - const firstLine = content.split('\n')[0]?.trim() || ''; + const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || ''; return { url: knowledge.url, firstLine, diff --git a/src/utils/markdown-query.ts b/src/utils/markdown-query.ts index 4ebee4a..f421195 100644 --- a/src/utils/markdown-query.ts +++ b/src/utils/markdown-query.ts @@ -408,6 +408,10 @@ export class MarkdownQuery { } replace(content: string): string { + return this.replaceEach(() => content); + } + + replaceEach(replacer: (match: MarkdownQuery, index: number) => string): string { const sorted = [...this.matches].sort((a, b) => a.start - b.start); const kept: MatchedRange[] = []; @@ -418,10 +422,11 @@ export class MarkdownQuery { lastEnd = range.start + range.length; } + const replacements = kept.map((range, index) => replacer(new MarkdownQuery(this.source, [range]), index)); let result = this.source; for (let i = kept.length - 1; i >= 0; i--) { const range = kept[i]; - result = result.slice(0, range.start) + content + result.slice(range.start + range.length); + result = result.slice(0, range.start) + replacements[i] + result.slice(range.start + range.length); } return result; @@ -459,6 +464,15 @@ export class MarkdownQuery { each(): MarkdownQuery[] { return this.matches.map((m) => new MarkdownQuery(this.source, [m])); } + + meta(): Array<{ type: string; depth: number | null; text: string }> { + return this.matches.map((range) => { + const token = range.token as any; + let depth: number | null = null; + if (token.type === 'heading') depth = token.depth; + return { type: token.type, depth, text: getTokenText(range.token) }; + }); + } } export function mdq(source: string): MarkdownQuery { diff --git a/tests/unit/experience-tracker.test.ts b/tests/unit/experience-tracker.test.ts index b940d64..37d0b30 100644 --- a/tests/unit/experience-tracker.test.ts +++ b/tests/unit/experience-tracker.test.ts @@ -368,6 +368,32 @@ describe('ExperienceTracker', () => { }); }); + describe('getSuccessfulExperience rendering', () => { + it('rewrites FLOW/ACTION headings to HOW to phrasing', () => { + const state = new ActionResult({ html: '', url: 'https://example.com/howto', title: 'HowTo' }); + const body = ['## FLOW: create a test', '', '* Click create', '', '```js', 'I.click("Create")', '```', '', '---', '', '## ACTION: save the form', '', '```js', 'I.click("Save")', '```', ''].join('\n'); + experienceTracker.writeExperienceFile(state.getStateHash(), body, { url: '/howto', title: 'HowTo' }); + + const combined = experienceTracker.getSuccessfulExperience(state).join('\n'); + + expect(combined).toContain('## HOW to create a test (multi-step)'); + expect(combined).toContain('## HOW to save the form (single-step)'); + expect(combined).not.toContain('## FLOW:'); + expect(combined).not.toContain('## ACTION:'); + }); + + it('rewrites and terminates when a title contains the marker', () => { + const state = new ActionResult({ html: '', url: 'https://example.com/edge', title: 'Edge' }); + const body = ['## FLOW: undo FLOW: steps', '', '* step', '', '---', ''].join('\n'); + experienceTracker.writeExperienceFile(state.getStateHash(), body, { url: '/edge', title: 'Edge' }); + + const combined = experienceTracker.getSuccessfulExperience(state).join('\n'); + + expect(combined).toContain('## HOW to undo FLOW: steps (multi-step)'); + expect(combined).not.toContain('## FLOW:'); + }); + }); + describe('table-of-contents lettering', () => { it('assigns contiguous fileTags when a zero-section file sorts first', () => { const url = '/lettering-page'; diff --git a/tests/unit/markdown-query.test.ts b/tests/unit/markdown-query.test.ts index 46cafe4..7ed4bca 100644 --- a/tests/unit/markdown-query.test.ts +++ b/tests/unit/markdown-query.test.ts @@ -592,6 +592,51 @@ Not a section. }); }); + describe('meta', () => { + it('returns heading type, depth, and unwrapped text', () => { + const metas = mdq(sampleMarkdown).query('h2').meta(); + expect(metas).toHaveLength(3); + expect(metas[0]).toEqual({ type: 'heading', depth: 2, text: 'API' }); + expect(metas[1].text).toBe('Settings'); + expect(metas[2].text).toBe('FAQ'); + expect(metas.every((m) => !m.text.includes('#'))).toBe(true); + }); + + it('returns section heading depth and title without markup', () => { + const metas = mdq(sampleMarkdown).query('section("API")').meta(); + expect(metas).toHaveLength(1); + expect(metas[0]).toEqual({ type: 'heading', depth: 2, text: 'API' }); + }); + + it('returns h3 section depth', () => { + const metas = mdq(sampleMarkdown).query('section3(~"Auth")').meta(); + expect(metas).toHaveLength(1); + expect(metas[0].depth).toBe(3); + expect(metas[0].text).toBe('Authentication'); + }); + + it('returns code body without fences and null depth', () => { + const metas = mdq(sampleMarkdown).query('code').meta(); + expect(metas).toHaveLength(2); + expect(metas[0].type).toBe('code'); + expect(metas[0].depth).toBeNull(); + expect(metas[0].text).toContain('const token = getToken()'); + expect(metas[0].text).not.toContain('```'); + }); + + it('returns paragraph text with null depth', () => { + const metas = mdq(sampleMarkdown).query('paragraph(~"intro")').meta(); + expect(metas).toHaveLength(1); + expect(metas[0].type).toBe('paragraph'); + expect(metas[0].depth).toBeNull(); + expect(metas[0].text).toContain('intro paragraph'); + }); + + it('returns empty array when no matches', () => { + expect(mdq(sampleMarkdown).query('h6').meta()).toEqual([]); + }); + }); + describe('replace', () => { it('should replace matched content', () => { const result = mdq(sampleMarkdown).query('heading("FAQ")').replace('## Questions\n'); @@ -622,6 +667,15 @@ Not a section. const result = mdq(md).query('section').replace('REPLACED\n'); expect(result).toBe('REPLACED\n'); }); + + it('should replace each match without stale offsets', () => { + const md = '## Short\n\nText\n\n## Much Longer Heading\n\nMore\n'; + const result = mdq(md) + .query('h2') + .replaceEach((heading, index) => `## ${index + 1}: ${heading.meta()[0].text}\n\n`); + + expect(result).toBe('## 1: Short\n\nText\n\n## 2: Much Longer Heading\n\nMore\n'); + }); }); describe('composable chaining', () => {