refactor: route all markdown parsing through mdq() (plan 022)

src/ai/experience-compactor.ts

@@ -1,11 +1,11 @@
 import { unlinkSync } from 'node:fs';
 import dedent from 'dedent';
-import { type Tokens, marked } from 'marked';
 import { z } from 'zod';
 import { type ExperienceFile, type ExperienceTracker, RECENT_WINDOW_DAYS } from '../experience-tracker.js';
 import { Observability } from '../observability.js';
 import { createDebug, log, tag } from '../utils/logger.js';
 import { mdq } from '../utils/markdown-query.js';
+import { isNonReusableCode } from '../utils/step-analyzer.js';
 import { generalizeUrl, hasDynamicUrlSegment } from '../utils/url-matcher.js';
 import type { Agent } from './agent.js';
 import type { Provider } from './provider.js';
@@ -438,28 +438,9 @@ export class ExperienceCompactor implements Agent {
 }
 
 function listSections(content: string): { title: string; raw: string }[] {
-  const tokens = marked.lexer(content);
-  const sections: { title: string; raw: string }[] = [];
-
-  let currentHeading: string | null = null;
-  let currentRaw = '';
-
-  const flush = () => {
-    if (currentHeading !== null) sections.push({ title: currentHeading, raw: currentRaw });
-  };
-
-  for (const token of tokens) {
-    const raw = (token as any).raw || '';
-    if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
-      flush();
-      currentHeading = (token as Tokens.Heading).text.trim();
-      currentRaw = raw;
-      continue;
-    }
-    if (currentHeading !== null) currentRaw += raw;
-  }
-  flush();
-  return sections;
+  const sections = mdq(content).query('section2').each();
+  const metas = mdq(content).query('section2').meta();
+  return sections.map((q, i) => ({ title: metas[i].text.trim(), raw: q.text() }));
 }
 
 function dropEmptySections(content: string): string {
@@ -483,7 +464,7 @@ function dropNonReusableSections(content: string): string {
 
   for (const section of sections) {
     const raw = section.text();
-    if (!/\bI\.clickXY\s*\(/.test(raw)) continue;
+    if (!isNonReusableCode(raw)) continue;
     result = result.replace(raw, '');
   }
 

src/ai/planner.ts

@@ -280,29 +280,43 @@ export class Planner extends PlannerBase implements Agent {
   }
 
   private cleanExperienceFlows(text: string): string | null {
-    const seenTitles = new Set<string>();
     let result = text;
 
-    for (const section of [...mdq(text).query('section2').each(), ...mdq(text).query('section3').each()]) {
-      const heading = section.query('heading').text().trim();
-      const body = mdq(section.text())
-        .query('heading')
-        .replace('')
-        .replace(/^---\s*$/gm, '')
-        .trim();
-
-      if (!body || seenTitles.has(heading)) {
-        result = result.replace(section.text(), '');
-        continue;
+    while (true) {
+      const sections = [...mdq(result).query('section2').each(), ...mdq(result).query('section3').each()];
+      const seenTitles = new Set<string>();
+      let removed = false;
+      for (const section of sections) {
+        const heading = section.query('heading').text().trim();
+        const withoutHeadings = mdq(section.text()).query('heading').replace('');
+        const body = mdq(withoutHeadings).query('hr').replace('').trim();
+        if (body && !seenTitles.has(heading)) {
+          seenTitles.add(heading);
+          continue;
+        }
+        result = section.replace('');
+        removed = true;
+        break;
       }
-      seenTitles.add(heading);
+      if (!removed) break;
+    }
 
-      const blockquotes = section.query('blockquote').each();
-      if (blockquotes.length <= 10) continue;
-      for (const bq of blockquotes.slice(10)) {
-        result = result.replace(bq.text(), '');
+    const trimmedTitles = new Set<string>();
+    while (true) {
+      const sections = [...mdq(result).query('section2').each(), ...mdq(result).query('section3').each()];
+      let trimmed = false;
+      for (const section of sections) {
+        const heading = section.query('heading').text().trim();
+        if (trimmedTitles.has(heading)) continue;
+        const count = section.query('blockquote').count();
+        if (count <= 10) continue;
+        const kept = mdq(section.text()).query('blockquote[10:]').replace('');
+        result = section.replace(`${kept.trimEnd()}\n> ... and ${count - 10} more discoveries\n`);
+        trimmedTitles.add(heading);
+        trimmed = true;
+        break;
       }
-      result = result.replace(section.text().trim(), `${section.text().trim()}\n> ... and ${blockquotes.length - 10} more discoveries`);
+      if (!trimmed) break;
     }
 
     return result.trim() || null;
@@ -381,15 +395,16 @@ export class Planner extends PlannerBase implements Agent {
       deep: true,
     });
     let plannerResearch = mdq(research).query('code').replace('');
-    for (const table of mdq(plannerResearch).query('table').each()) {
-      const rawTable = table.text();
+    const tableCount = mdq(plannerResearch).query('table').count();
+    for (let i = 0; i < tableCount; i++) {
+      const table = mdq(plannerResearch).query('table').each()[i];
       const rows = table.toJson();
       if (rows.length === 0 || !rows[0].Element) continue;
       const elementWithType = rows.map((r) => ({
         Element: r.Element,
         Type: r.Type || '',
       }));
-      plannerResearch = plannerResearch.replace(rawTable, jsonToTable(elementWithType, ['Element', 'Type']));
+      plannerResearch = table.replace(jsonToTable(elementWithType, ['Element', 'Type']));
     }
 
     const hasFocusedOverlay = hasFocusedSection(plannerResearch);

src/ai/researcher.ts

@@ -315,11 +315,9 @@ export class Researcher extends ResearcherBase implements Agent {
         researchFile = saveResearch(stateHash, result.text, combinedHtml);
       }
 
-      const summaryMatch = result.text.match(/## Summary\s*\n+([\s\S]*?)(?=\n##|$)/i);
-      if (summaryMatch) {
-        const summaryLine = summaryMatch[1].trim().split('\n')[0].trim().slice(0, 200);
-        if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine);
-      }
+      const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim();
+      const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200);
+      if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine);
 
       tag('multiline').log(formatResearchSummary(result.text, { visionUsed: this.hasScreenshotToAnalyze }));
       tag('success').log('Research complete');

src/ai/researcher/parser.ts

@@ -94,7 +94,7 @@ export function extractContainerFromBlockquote(sectionMarkdown: string): string
 }
 
 export function parseResearchSections(markdown: string): ResearchSection[] {
-  const hasExtendedResearch = markdown.includes('\n# Extended Research') || markdown.startsWith('# Extended Research');
+  const hasExtendedResearch = mdq(markdown).query('section1(~"Extended Research")').count() > 0;
 
   return parseSections(markdown)
     .filter((s) => !SKIP_SECTIONS.has(s.name.toLowerCase()) && !s.name.toLowerCase().includes('data:'))

src/experience-tracker.ts

@@ -1,7 +1,6 @@
 import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
 import { basename, dirname, join } from 'node:path';
 import matter from 'gray-matter';
-import { type Tokens, marked } from 'marked';
 import type { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { KnowledgeTracker } from './knowledge-tracker.js';
@@ -178,7 +177,12 @@ export class ExperienceTracker {
     this.ensureExperienceFile(state);
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
-    if (content.includes(action.code)) {
+    if (
+      mdq(content)
+        .query('code')
+        .meta()
+        .some((m) => m.text.trim() === action.code.trim())
+    ) {
       debugLog('Skipping duplicate action', action.code);
       return;
     }
@@ -206,7 +210,12 @@ export class ExperienceTracker {
     const stateHash = state.getStateHash();
     const { content, data } = this.readExperienceFile(stateHash);
 
-    if (content.includes(body)) {
+    if (
+      mdq(content)
+        .query('section2')
+        .each()
+        .some((s) => s.text().trim() === body.trim())
+    ) {
       debugLog('Skipping duplicate flow body');
       return;
     }
@@ -277,9 +286,14 @@ export class ExperienceTracker {
         });
       })
       .map((experience) => {
-        const lines = experience.content.split('\n');
-        if (lines.length <= maxLines) return experience;
-        return { ...experience, content: lines.slice(0, maxLines).join('\n') };
+        if (experience.content.split('\n').length <= maxLines) return experience;
+        let content = experience.content;
+        while (content.split('\n').length > maxLines) {
+          const sections = mdq(content).query('section2').each();
+          if (sections.length <= 1) break;
+          content = sections[sections.length - 1].replace('');
+        }
+        return { ...experience, content };
       });
   }
 
@@ -424,49 +438,22 @@ export class ExperienceTracker {
 }
 
 function listTocHeadings(content: string): { index: number; level: 2 | 3; title: string }[] {
-  const tokens = marked.lexer(content);
-  const result: { index: number; level: 2 | 3; title: string }[] = [];
-  let index = 0;
-  for (const token of tokens) {
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    index++;
-    result.push({ index, level: heading.depth as 2 | 3, title: heading.text });
-  }
-  return result;
+  return mdq(content)
+    .query('heading')
+    .meta()
+    .filter((m) => m.depth === 2 || m.depth === 3)
+    .map((m, i) => ({ index: i + 1, level: m.depth as 2 | 3, title: m.text }));
 }
 
 function extractHeadingSection(content: string, sectionIndex: number): { title: string; body: string } | null {
-  const tokens = marked.lexer(content);
-  const matching: { tokenIdx: number; depth: number; text: string }[] = [];
-
-  for (let i = 0; i < tokens.length; i++) {
-    const token = tokens[i];
-    if (token.type !== 'heading') continue;
-    const heading = token as Tokens.Heading;
-    if (heading.depth !== 2 && heading.depth !== 3) continue;
-    matching.push({ tokenIdx: i, depth: heading.depth, text: heading.text });
-  }
+  const sections = mdq(content).query('section').each();
+  const metas = mdq(content).query('section').meta();
+  const matching = metas.map((meta, i) => ({ meta, section: sections[i] })).filter((entry) => entry.meta.depth === 2 || entry.meta.depth === 3);
 
   if (sectionIndex < 1 || sectionIndex > matching.length) return null;
 
   const target = matching[sectionIndex - 1];
-  let endTokenIdx = tokens.length;
-  for (let j = target.tokenIdx + 1; j < tokens.length; j++) {
-    const token = tokens[j];
-    if (token.type !== 'heading') continue;
-    if ((token as Tokens.Heading).depth <= target.depth) {
-      endTokenIdx = j;
-      break;
-    }
-  }
-
-  const body = tokens
-    .slice(target.tokenIdx, endTokenIdx)
-    .map((t) => (t as any).raw || '')
-    .join('');
-  return { title: target.text, body };
+  return { title: target.meta.text, body: target.section.text() };
 }
 
 function indexToLetters(index: number): string {
@@ -535,21 +522,18 @@ function generateActionContent(title: string, code: string, explanation?: string
 }
 
 function renderAsHowTo(content: string): string {
-  const tokens = marked.lexer(content);
-  let result = '';
-  for (const token of tokens) {
-    if (token.type === 'heading' && (token as Tokens.Heading).depth === 2) {
-      const text = (token as Tokens.Heading).text.trim();
-      if (text.startsWith('FLOW:')) {
-        result += `## HOW to ${text.slice(5).trim()} (multi-step)\n\n`;
-        continue;
-      }
-      if (text.startsWith('ACTION:')) {
-        result += `## HOW to ${text.slice(7).trim()} (single-step)\n\n`;
-        continue;
-      }
+  let result = content;
+  for (const { marker, suffix } of [
+    { marker: 'FLOW:', suffix: 'multi-step' },
+    { marker: 'ACTION:', suffix: 'single-step' },
+  ]) {
+    while (true) {
+      const headings = mdq(result).query('h2').meta();
+      const idx = headings.findIndex((h) => h.text.trim().startsWith(marker));
+      if (idx === -1) break;
+      const title = headings[idx].text.trim().slice(marker.length).trim();
+      result = mdq(result).query(`h2[${idx}]`).replace(`## HOW to ${title} (${suffix})\n\n`);
     }
-    result += (token as any).raw || '';
   }
   return result;
 }

src/knowledge-tracker.ts

@@ -5,6 +5,7 @@ import { ActionResult } from './action-result.js';
 import { ConfigParser } from './config.js';
 import { getCliName } from './utils/cli-name.ts';
 import { createDebug } from './utils/logger.js';
+import { mdq } from './utils/markdown-query.js';
 
 const debugLog = createDebug('explorbot:knowledge-tracker');
 
@@ -190,7 +191,7 @@ export class KnowledgeTracker {
 
     return this.knowledgeFiles.map((knowledge) => {
       const content = knowledge.content.trim();
-      const firstLine = content.split('\n')[0]?.trim() || '';
+      const firstLine = mdq(content).meta()[0]?.text.split('\n')[0]?.trim() || '';
       return {
         url: knowledge.url,
         firstLine,

src/utils/markdown-query.ts

@@ -459,6 +459,15 @@ export class MarkdownQuery {
   each(): MarkdownQuery[] {
     return this.matches.map((m) => new MarkdownQuery(this.source, [m]));
   }
+
+  meta(): Array<{ type: string; depth: number | null; text: string }> {
+    return this.matches.map((range) => {
+      const token = range.token as any;
+      let depth: number | null = null;
+      if (token.type === 'heading') depth = token.depth;
+      return { type: token.type, depth, text: getTokenText(range.token) };
+    });
+  }
 }
 
 export function mdq(source: string): MarkdownQuery {

tests/unit/experience-tracker.test.ts

@@ -349,6 +349,32 @@ describe('ExperienceTracker', () => {
     });
   });
 
+  describe('getSuccessfulExperience rendering', () => {
+    it('rewrites FLOW/ACTION headings to HOW to phrasing', () => {
+      const state = new ActionResult({ html: '<html></html>', url: 'https://example.com/howto', title: 'HowTo' });
+      const body = ['## FLOW: create a test', '', '* Click create', '', '```js', 'I.click("Create")', '```', '', '---', '', '## ACTION: save the form', '', '```js', 'I.click("Save")', '```', ''].join('\n');
+      experienceTracker.writeExperienceFile(state.getStateHash(), body, { url: '/howto', title: 'HowTo' });
+
+      const combined = experienceTracker.getSuccessfulExperience(state).join('\n');
+
+      expect(combined).toContain('## HOW to create a test (multi-step)');
+      expect(combined).toContain('## HOW to save the form (single-step)');
+      expect(combined).not.toContain('## FLOW:');
+      expect(combined).not.toContain('## ACTION:');
+    });
+
+    it('rewrites and terminates when a title contains the marker', () => {
+      const state = new ActionResult({ html: '<html></html>', url: 'https://example.com/edge', title: 'Edge' });
+      const body = ['## FLOW: undo FLOW: steps', '', '* step', '', '---', ''].join('\n');
+      experienceTracker.writeExperienceFile(state.getStateHash(), body, { url: '/edge', title: 'Edge' });
+
+      const combined = experienceTracker.getSuccessfulExperience(state).join('\n');
+
+      expect(combined).toContain('## HOW to undo FLOW: steps (multi-step)');
+      expect(combined).not.toContain('## FLOW:');
+    });
+  });
+
   describe('table-of-contents lettering', () => {
     it('assigns contiguous fileTags when a zero-section file sorts first', () => {
       const url = '/lettering-page';