Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions LLM-Prompt-Builder/.claude/skills/verify/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,19 @@ Analytics' own load of the object) - the header of the file has the details.
What no stand-in shows is the Options pane itself: a change to the options
group still needs a look in a real report.

### A slow suite in parts

`VERIFY_SHARD=i/n` runs part `i` (0-based) of `n` of `verify-new-item`,
`verify-outcomes` or `verify-reads`, so the parts can run side by side, each
against a mock of its own (`MOCK_PORT`, `VERIFY_BASE`). `run-all.cjs` does not
split; a runner that does starts one mock per part. `outsideShard` in
`page-waits.js` decides: `verify-outcomes` and `verify-reads` take every
`n`-th road; `verify-new-item` takes whole blocks (templates, RAG setups,
optimizations), because its roads read what earlier roads of their block left,
and every part also runs the three references. The parts hold every check of
the whole suite. `verify-round-trip` is not split: its last check adds up
what all of its roads wrote.

## By hand

1. Copy `mock-server.js`, the suites, `source-census.js`, `page-waits.js`,
Expand Down
19 changes: 18 additions & 1 deletion LLM-Prompt-Builder/.claude/skills/verify/page-waits.js
Original file line number Diff line number Diff line change
Expand Up @@ -178,4 +178,21 @@ async function chooseFolder(page, folder, prefix = '#app-obj-RGB-source', timeou
await page.waitForSelector(dialog, { state: 'detached', timeout });
}

module.exports = { trackRequests, trackPagesOf, quiet, dialogShown, clickInDialog, dialogsGone, gate, chooseFolder };
/**
* VERIFY_SHARD=i/n runs every n-th road of a suite, starting at road i (0-based), so the parts of a slow suite can
* run side by side, each with a mock of its own (MOCK_PORT, VERIFY_BASE). Round-robin rather than in blocks: the
* slow roads of a suite sit together, and every part gets some of them. Unset, every road runs. A road that later
* roads read from (`always`, such as the reference of verify-new-item) runs in every part and is not counted. A
* suite whose roads read what earlier roads of the same block left passes `block`: whole blocks go to the parts.
*/
const SHARD = /^(\d+)\/(\d+)$/.exec(process.env.VERIFY_SHARD || '');
let roadsSeen = 0;
function outsideShard(name, { always = false, block } = {}) {
if (always) return false;
const index = block === undefined ? roadsSeen++ : block;
if (!SHARD || index % Number(SHARD[2]) === Number(SHARD[1])) return false;
console.log(`---- ${name}: in another part (VERIFY_SHARD ${SHARD[0]})`);
return true;
}

module.exports = { trackRequests, trackPagesOf, quiet, dialogShown, clickInDialog, dialogsGone, gate, chooseFolder, outsideShard };
9 changes: 8 additions & 1 deletion LLM-Prompt-Builder/.claude/skills/verify/verify-new-item.js
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
*/
const { chromium } = require('playwright');
const fetch = require('./mock-fetch.cjs');
const { trackPagesOf, quiet, dialogShown, dialogsGone, gate, chooseFolder } = require('./page-waits.cjs');
const { trackPagesOf, quiet, dialogShown, dialogsGone, gate, chooseFolder, outsideShard } = require('./page-waits.cjs');

const BASE = (process.env.VERIFY_BASE || 'http://localhost:4173').replace(/\/+$/, '');
const results = [];
Expand Down Expand Up @@ -345,7 +345,12 @@ function same(got, wanted, label, fieldsOnly = false) {
return page;
};
// Each road starts on a page of its own, as a person who opens the report.
// VERIFY_SHARD splits this suite by block, not by road: the roads of a block read what earlier roads of the same
// block left (the fields left alone, the existing items), and the reference of a kind is what the later roads
// of that kind compare with, so every part takes the references.
let block = 0;
const road = async (name, run) => {
if (outsideShard(name, { always: name.includes(': the reference - '), block })) return;
console.log(`---- ${name}`);
const page = await newPage();
try {
Expand Down Expand Up @@ -1015,6 +1020,7 @@ function same(got, wanted, label, fieldsOnly = false) {
// ============================================================================
// RAG setups (the RAG Builder, RGB)
// ============================================================================
block = 1;
const rgb = (suffix) => `#app-obj-RGB-${suffix}`;
const ragState = async () => (await fetch(BASE + '/__ragstate')).json();
const openRagBuilder = async (page) => {
Expand Down Expand Up @@ -1550,6 +1556,7 @@ function same(got, wanted, label, fieldsOnly = false) {
// optimization tracker then stands, and every later job is found finished
// on its first poll.
// ============================================================================
block = 2;
// What the Optimize step says about a job.
const optimizeRun = async (page) => ({
log: await page.isVisible(lpb('optimize-log')),
Expand Down
3 changes: 2 additions & 1 deletion LLM-Prompt-Builder/.claude/skills/verify/verify-outcomes.js
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
*/
const { chromium } = require('playwright');
const fetch = require('./mock-fetch.cjs');
const { trackPagesOf, quiet, dialogShown, clickInDialog, dialogsGone, gate } = require('./page-waits.cjs');
const { trackPagesOf, quiet, dialogShown, clickInDialog, dialogsGone, gate, outsideShard } = require('./page-waits.cjs');
const { parse, walk, callsInSource, sourceFiles } = require('./source-census.cjs');

const BASE = (process.env.VERIFY_BASE || 'http://localhost:4173').replace(/\/+$/, '');
Expand Down Expand Up @@ -404,6 +404,7 @@ function writesCensus() {
return page;
};
const road = async (name, run, options) => {
if (outsideShard(name)) return;
console.log(`---- ${name}`);
pageErrors = [];
const page = await newPage(options);
Expand Down
21 changes: 15 additions & 6 deletions LLM-Prompt-Builder/.claude/skills/verify/verify-reads.js
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
*/
const { chromium } = require('playwright');
const fetch = require('./mock-fetch.cjs');
const { trackPagesOf, quiet } = require('./page-waits.cjs');
const { trackPagesOf, quiet, outsideShard } = require('./page-waits.cjs');
const fs = require('fs');
const path = require('path');
const { PROJECT, parse, walk, callsInSource, sourceFiles, loadStoredJson } = require('./source-census.cjs');
Expand Down Expand Up @@ -880,6 +880,7 @@ async function shapeChecks({ STORED_SHAPES, TRACKER_META_KEYS, NUMBER_FIELD_OPTI
return page;
};
const road = async (name, run) => {
if (outsideShard(name)) return;
console.log(`---- ${name}`);
pageErrors = [];
const page = await newPage();
Expand Down Expand Up @@ -1141,14 +1142,19 @@ async function shapeChecks({ STORED_SHAPES, TRACKER_META_KEYS, NUMBER_FIELD_OPTI
const rows = [{ ...row, runId: 1, model: '', systemPrompt: 'Every field', userPrompt: 'of a row' }, { ...row, runId: 1, model: 'demo_llm', best_prompt: 1 }];
await page.route('**/files/files/trk-1/content', (route) => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(rows) }));
await choose(page, 'LPB', '#LPB-prompt-dropdown', 'model-used');
// The tracker is on Build & Test: from another step its Load button is
// hidden, and a click on it waited out Playwright's 30 seconds and was
// swallowed - the run was drawn but never loaded.
await goStep(page, 'LPB', 'build');
await waitUntil(async () => (await savedRuns(page)) > 0, 'the run is drawn', 15000).catch(() => {});
await page.click(`${lpb('pet-0')} .pet-run-load`).catch(() => {});
const clicked = await page.click(`${lpb('pet-0')} .pet-run-load`, { timeout: 5000 }).then(() => true, () => false);
await quiet(page);
const loaded = clicked && (await page.inputValue(lpb('system-prompt'))) === 'Every field';
const line = await textOf(page, lpb('pet-save-only-result'));
const text = await textOf(page, lpb('pet'));
step(
(await savedRuns(page)) === 1 && line === '' && !text.includes('[object Object]') && pageErrors.length === 0,
`a tracker whose rows hold ${how} is drawn and loaded (runs: ${await savedRuns(page)}; line: "${clip(line)}"; [object Object] drawn: ${text.includes('[object Object]')}; page errors: ${pageErrors.join(' | ') || 'none'})`
(await savedRuns(page)) === 1 && loaded && line === '' && !text.includes('[object Object]') && pageErrors.length === 0,
`a tracker whose rows hold ${how} is drawn and loaded (runs: ${await savedRuns(page)}; loaded: ${loaded}; line: "${clip(line)}"; [object Object] drawn: ${text.includes('[object Object]')}; page errors: ${pageErrors.join(' | ') || 'none'})`
);
});
await road(`an optimization history whose entry holds ${how}`, async (page) => {
Expand All @@ -1163,11 +1169,14 @@ async function shapeChecks({ STORED_SHAPES, TRACKER_META_KEYS, NUMBER_FIELD_OPTI
const drawn = (await page.$$(entries)).length;
const text = await textOf(page, lpb('pane-optimize'));
const unread = (await page.$$(`${lpb('pane-optimize')} .pb-optimize-history-unread`)).length;
await page.click(`${lpb('pane-optimize')} .pb-optimize-history-load`).catch(() => {});
// An entry with every optional field empty may offer nothing to load:
// its button then never becomes clickable, and the default 30 seconds
// were waited out for nothing. Whether it loaded is said, not required.
const loaded = await page.click(`${lpb('pane-optimize')} .pb-optimize-history-load`, { timeout: 5000 }).then(() => true, () => false);
await quiet(page);
step(
drawn === 2 && unread === 0 && !text.includes('[object Object]') && pageErrors.length === 0,
`an optimization history whose entry holds ${how} - as a run and as a comparison - is drawn and loaded (entries: ${drawn}; line that it cannot be shown: ${unread}; [object Object] drawn: ${text.includes('[object Object]')}; page errors: ${pageErrors.join(' | ') || 'none'})`
`an optimization history whose entry holds ${how} - as a run and as a comparison - is drawn and loaded (entries: ${drawn}; loaded: ${loaded}; line that it cannot be shown: ${unread}; [object Object] drawn: ${text.includes('[object Object]')}; page errors: ${pageErrors.join(' | ') || 'none'})`
);
});
}
Expand Down
Loading