Repository navigation
Expand file tree
/
Copy pathtest_sbi.py
More file actions
865 lines (796 loc) · 41.7 KB
/
Copy pathtest_sbi.py
File metadata and controls
865 lines (796 loc) · 41.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
import asyncio
import argparse
import base64
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.parse
import urllib.request
# Force stdout to UTF-8 on Windows
if sys.platform == "win32":
try:
sys.stdout.reconfigure(encoding="utf-8")
sys.stderr.reconfigure(encoding="utf-8")
except Exception:
pass
DEFAULT_BROWSER_CANDIDATES = [
r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
r"C:\Program Files\chrome-win\chrome.exe",
]
DEFAULT_PORT = 9880
DEFAULT_DEBUG_PORT = 9337
TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_img")
PACK_ALIAS = {
"pvpmen": "Pvpmen",
}
def normalize_pack_name(raw):
# Strip " (2)" style counter suffixes and brackets; keep letters/digits/_
s = re.sub(r"\s*\(\d+\)\s*$", "", raw).strip()
s = s.replace("[", "").replace("]", "")
s = re.sub(r"\s+", "_", s)
s = re.sub(r"[^A-Za-z0-9_\-]", "", s)
return s
def parse_test_images():
tests = []
for name in sorted(os.listdir(TEST_DIR)):
m = re.match(r"^(Large|Small)\s*-\s*(.+)\.(png|jpe?g)$", name, re.I)
if not m:
continue
scale = m.group(1).lower()
expected = normalize_pack_name(m.group(2))
low = expected.lower()
if low in PACK_ALIAS:
expected = PACK_ALIAS[low]
tests.append((name, scale, expected))
return tests
def parse_synthetic_manifest(tests_dir, tiers=None):
"""Read test_img/synthetic/manifest.json and emit (relpath, scale, expected_pack_name) triples.
relpath is relative to test_img/synthetic/ so that the caller constructs
the HTTP URL as base/test_img/synthetic/<relpath>.
"""
manifest_path = os.path.join(tests_dir, 'synthetic', 'manifest.json')
if not os.path.exists(manifest_path):
return []
with open(manifest_path, 'r', encoding='utf-8') as fh:
manifest = json.load(fh)
images = manifest.get('images', [])
tests = []
for img in images:
if tiers and img.get('tier') not in tiers:
continue
rel = img.get('file')
if not rel:
continue
expected = img.get('packId', '')
if not expected:
continue
normalized = normalize_pack_name(expected)
tests.append((rel.replace('\\', '/'), 'synthetic', normalized))
return tests
def parse_args():
parser = argparse.ArgumentParser(description="Run SBI image matching regression tests.")
group = parser.add_mutually_exclusive_group()
group.add_argument("--image", help="Run one test image by exact or case-insensitive substring match.")
group.add_argument("--filter", help="Run test images whose filename matches this regex.")
group.add_argument("--synthetic", action="store_true",
help="Evaluate the synthetic corpus (test_img/synthetic/manifest.json) instead of the 9-shot regression set.")
parser.add_argument("--fail-fast", action="store_true", help="Stop after the first failed test.")
parser.add_argument("--quiet", action="store_true", help="Only print compact per-image results and summary.")
parser.add_argument("--verbose", action="store_true", help="Print slot features and top-10 rows.")
parser.add_argument("--diagnostics-json", help="Write full candidate diagnostics for offline scoring analysis.")
parser.add_argument("--no-timings", action="store_true", help="Hide per-image timing output.")
parser.add_argument("--base-url", help="Test a deployed site instead of starting the local HTTP server.")
parser.add_argument("--benchmark", choices=("warm", "cold"), help="Run repeated warm- or cold-cache measurements.")
parser.add_argument("--runs", type=int, default=5, help="Runs per image in benchmark mode (default: 5).")
parser.add_argument("--force-fallback", action="store_true", help="Force the global coarse fallback path.")
parser.add_argument("--benchmark-groups", type=int, help="Inflate the browser corpus to this many groups for performance testing.")
parser.add_argument("--enforce-budgets", action="store_true", help="Fail when benchmark p95 exceeds SBI budgets.")
parser.add_argument("--port", type=int, default=DEFAULT_PORT, help="HTTP server port.")
parser.add_argument("--debug-port", type=int, default=DEFAULT_DEBUG_PORT, help="Browser CDP port.")
parser.add_argument("--browser", help="Browser executable path. Defaults to Edge if available, then Chrome.")
parser.add_argument("--synthetic-tiers", default="light",
help="Comma-separated synthetic tiers to evaluate (default: light).")
parser.add_argument("--limit", type=int, default=0,
help="Cap the number of evaluated images (0 = no cap). Useful for baseline sampling.")
parser.add_argument("--headless", default="--headless", help="Browser headless flag, e.g. --headless or --headless=new.")
parser.add_argument('--matcher', choices=('legacy', 'aligned'), default='legacy')
parser.add_argument('--transfer-report', help='Write CDP transfer evidence; requires --benchmark cold or warm.')
args = parser.parse_args()
if args.transfer_report and not args.benchmark:
parser.error('--transfer-report requires --benchmark cold or warm')
return args
def filter_tests(tests, args):
if getattr(args, 'synthetic', False):
return tests
if args.image:
needle = args.image.lower()
exact = [test for test in tests if test[0].lower() == needle]
return exact or [test for test in tests if needle in test[0].lower()]
if args.filter:
pattern = re.compile(args.filter, re.I)
return [test for test in tests if pattern.search(test[0])]
return tests
def fmt_seconds(value):
return f"{value:.2f}s"
def percentile(values, quantile):
ordered = sorted(value for value in values if isinstance(value, (int, float)))
if not ordered:
return None
index = max(0, min(len(ordered) - 1, int((len(ordered) * quantile + 0.999999)) - 1))
return ordered[index]
def resolve_browser_path(path):
if path:
if not os.path.isfile(path):
raise FileNotFoundError(f"Browser not found: {path}")
return path
for candidate in DEFAULT_BROWSER_CANDIDATES:
if os.path.isfile(candidate):
return candidate
raise FileNotFoundError("No supported browser found. Pass --browser <path>.")
class NetworkTransferLedger:
"""Account for HTTP attempts using CDP completion bytes, not decoded size."""
def __init__(self):
self.reset()
def reset(self):
self.requests = []
self.active = {}
def consume(self, event):
method, params = event.get("method"), event.get("params", {})
request_id = params.get("requestId")
if method == "Network.requestWillBeSent":
redirect = params.get("redirectResponse")
previous = self.active.pop(request_id, None)
if previous is not None and redirect is not None:
previous.update(status=redirect.get("status"), finished=True,
finalBytes=redirect.get("encodedDataLength"))
url = params.get("request", {}).get("url", "")
if not url.startswith(("http://", "https://")):
return
row = {"url": url, "requestId": request_id, "finished": False,
"failed": False, "receivedBytes": 0, "headerBytes": 0,
"finalBytes": None, "fromCache": False}
self.requests.append(row)
self.active[request_id] = row
return
row = self.active.get(request_id)
if row is None:
return
if method == "Network.responseReceived":
response = params.get("response", {})
row.update(status=response.get("status"),
headerBytes=max(0, response.get("encodedDataLength", 0)),
fromCache=bool(response.get("fromDiskCache") or response.get("fromServiceWorker")),
protocol=response.get("protocol"))
row["encoding"] = next((value for key, value in response.get("headers", {}).items()
if key.lower() == "content-encoding"), "identity")
elif method == "Network.requestServedFromCache":
row["fromCache"] = True
elif method == "Network.dataReceived":
row["receivedBytes"] += max(0, params.get("encodedDataLength", 0))
elif method == "Network.loadingFinished":
row.update(finished=True, finalBytes=params.get("encodedDataLength"))
self.active.pop(request_id, None)
elif method == "Network.loadingFailed":
row.update(finished=True, failed=True, error=params.get("errorText", "download failed"))
self.active.pop(request_id, None)
def snapshot(self):
rows = [dict(row) for row in self.requests]
gaps = 0
total = 0
for row in rows:
value = row["finalBytes"]
if not isinstance(value, (int, float)) or value < 0:
gaps += 1
value = row["headerBytes"] + row["receivedBytes"]
row["transferredBytes"] = value
total += value
pending = sum(not row["finished"] for row in rows)
failed = sum(row["failed"] for row in rows)
return {"transferredBytes": total, "requests": rows, "pendingRequests": pending,
"failedRequests": failed, "unmeasuredRequests": gaps,
"complete": pending == 0 and failed == 0 and gaps == 0}
NETWORK_TRANSFERS = NetworkTransferLedger()
async def cdp_call(ws, method, params=None, recv_timeout=10, _counter=[0]):
_counter[0] += 1
msg_id = _counter[0]
await ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}}))
while True:
raw = await asyncio.wait_for(ws.recv(), timeout=recv_timeout)
resp = json.loads(raw)
NETWORK_TRANSFERS.consume(resp)
if resp.get("id") == msg_id:
if "error" in resp:
raise RuntimeError(resp["error"].get("message", "CDP error"))
return resp.get("result", {})
async def cdp_eval(ws, expr, timeout=60):
result = await cdp_call(ws, "Runtime.evaluate", {
"expression": expr,
"returnByValue": True,
"awaitPromise": True,
"timeout": timeout * 1000,
}, recv_timeout=timeout + 5)
val = result.get("result", {})
if val.get("subtype") == "error":
raise RuntimeError(val.get("description", "JS error"))
return val.get("value")
async def wait_for(ws, js_expr, timeout=30):
t0 = time.monotonic()
while time.monotonic() - t0 < timeout:
val = await cdp_eval(ws, js_expr, timeout=5)
if val:
return val
await asyncio.sleep(0.3)
raise TimeoutError(f"Timed out waiting for: {js_expr}")
def wait_for_http(url, timeout=10):
t0 = time.monotonic()
while time.monotonic() - t0 < timeout:
try:
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 VALE-SBI-Test"})
with urllib.request.urlopen(request, timeout=2) as resp:
if resp.status < 500:
return True
except Exception:
pass
time.sleep(0.2)
return False
def fetch_cdp_json(debug_port, path):
try:
raw = urllib.request.urlopen(f"http://127.0.0.1:{debug_port}{path}", timeout=1).read()
return json.loads(raw)
except Exception:
return None
def get_cdp_pages(debug_port):
return fetch_cdp_json(debug_port, "/json/list") or []
def get_sbi_ws_url(debug_port):
pages = get_cdp_pages(debug_port)
if not pages:
return None
fallback = None
for page in pages:
if page.get("type") != "page" or not page.get("webSocketDebuggerUrl"):
continue
if not fallback:
fallback = page["webSocketDebuggerUrl"]
if "/sbi/" in page.get("url", ""):
return page["webSocketDebuggerUrl"]
return fallback
def match_name(expected, got):
return expected == got
async def main():
import websockets
args = parse_args()
if args.runs <= 0:
print("ERROR: --runs must be positive")
return 1
if args.benchmark_groups is not None and (args.benchmark_groups <= 0 or not args.benchmark):
print("ERROR: --benchmark-groups requires --benchmark and a positive group count")
return 1
base = args.base_url.rstrip("/") if args.base_url else f"http://127.0.0.1:{args.port}"
sbi_url = f"{base}/sbi/?matcher={args.matcher}"
try:
browser_path = resolve_browser_path(args.browser)
except FileNotFoundError as exc:
print(f"ERROR: {exc}")
return 1
print(f"Browser: {browser_path}\n", flush=True)
if args.synthetic:
tiers = set(t.strip() for t in args.synthetic_tiers.split(",") if t.strip())
tests = filter_tests(parse_synthetic_manifest(TEST_DIR, tiers), args)
if args.limit > 0:
tests = tests[:args.limit]
else:
tests = filter_tests(parse_test_images(), args)
if not tests:
print("No test images found in", TEST_DIR)
return 1
print(f"Found {len(tests)} test image(s)\n", flush=True)
suite_t0 = time.monotonic()
server_proc = None
if not args.base_url:
server_proc = subprocess.Popen(
[sys.executable, "-m", "http.server", str(args.port), "--bind", "127.0.0.1"],
cwd=os.path.dirname(os.path.abspath(__file__)),
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == 'win32' else 0,
)
if not wait_for_http(sbi_url):
print(f"ERROR: SBI URL did not become ready at {sbi_url}")
if server_proc:
server_proc.terminate()
return 1
# Start browser with remote debugging
profile_dir = tempfile.mkdtemp(prefix="sbi-test-")
edge_log_path = os.path.join(profile_dir, "edge.log")
edge_log = open(edge_log_path, "w", encoding="utf-8", errors="replace")
edge_proc = subprocess.Popen([
browser_path,
f"--remote-debugging-port={args.debug_port}",
f"--user-data-dir={profile_dir}",
"--no-first-run", "--no-default-browser-check",
"--disable-extensions", "--disable-popup-blocking",
"--disable-gpu",
"--disable-gpu-sandbox",
"--disable-software-rasterizer",
"--disable-features=VizDisplayCompositor",
"--use-angle=swiftshader",
"--use-gl=swiftshader",
args.headless,
sbi_url,
], stdout=edge_log, stderr=edge_log,
creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == 'win32' else 0)
results = []
synthetic_records = [] # (file, packId, tier, recalled, topGroupId, expectedGroupId)
diagnostics = []
candidate_recalls = []
benchmark_samples = []
transfer_samples = []
try:
# Wait for CDP
ws_url = None
for _ in range(60):
ws_url = get_sbi_ws_url(args.debug_port)
if ws_url:
break
await asyncio.sleep(0.5)
if not ws_url:
print("ERROR: Could not connect to browser CDP")
if edge_proc.poll() is not None:
print(f"Browser exited with code {edge_proc.returncode}")
edge_log.flush()
try:
with open(edge_log_path, "r", encoding="utf-8", errors="replace") as fh:
lines = fh.readlines()[-20:]
for line in lines:
print(" " + line.rstrip())
except Exception:
pass
return 1
ws = None
for attempt in range(5):
try:
ws_url = get_sbi_ws_url(args.debug_port) or ws_url
ws = await websockets.connect(ws_url, max_size=50_000_000, proxy=None)
await cdp_call(ws, "Runtime.enable")
await cdp_call(ws, "Page.enable")
if args.benchmark == "cold":
await cdp_call(ws, "Network.enable")
break
except Exception:
if ws:
await ws.close()
ws = None
if attempt == 4:
print("ERROR: Could not attach to browser CDP target")
edge_log.flush()
try:
with open(edge_log_path, "r", encoding="utf-8", errors="replace") as fh:
lines = fh.readlines()[-20:]
for line in lines:
print(" " + line.rstrip())
except Exception:
pass
return 1
await asyncio.sleep(0.5)
async with ws:
# Wait for SBI page ready
try:
await wait_for(ws, "!!window.__sbiTest && !!document.getElementById('sbi-results')", timeout=30)
# Issue 018: headless Edge may serve a stale cached sbi.js across runs
# (the profile dir is fresh but the shared code cache is not). Force one
# cache-busting reload so the JS under test is the freshly built one.
await cdp_call(ws, "Network.enable")
await cdp_call(ws, "Network.setCacheDisabled", {"cacheDisabled": True})
await cdp_call(ws, "Page.reload", {"ignoreCache": True})
await wait_for(ws, "!!window.__sbiTest && !!document.getElementById('sbi-results')", timeout=30)
except TimeoutError:
state = await cdp_eval(ws, """
JSON.stringify({
href: location.href,
readyState: document.readyState,
hasResults: !!document.getElementById('sbi-results'),
hasHook: !!window.__sbiTest,
})
""", timeout=5)
print(f"DEBUG page state: {state}")
return 1
if args.benchmark == 'warm':
await cdp_call(ws, 'Network.setCacheDisabled', {'cacheDisabled': False})
print("SBI page ready\n", flush=True)
test_runs = []
if args.benchmark == "warm":
test_runs.append((*tests[0], True, 0))
run_count = args.runs if args.benchmark else 1
benchmark_exclusions = sorted({test[2] for test in tests})
for run_index in range(run_count):
for test in tests:
test_runs.append((*test, False, run_index + 1))
for img_name, preset, expected, is_warmup, run_index in test_runs:
if args.benchmark == "cold":
await cdp_call(ws, 'Page.navigate', {'url': 'about:blank'})
await wait_for(ws, "location.href === 'about:blank'", timeout=10)
await cdp_call(ws, 'Network.clearBrowserCache')
await cdp_call(ws, 'Network.setBypassServiceWorker', {'bypass': True})
await cdp_call(ws, "Network.setCacheDisabled", {"cacheDisabled": True})
NETWORK_TRANSFERS.reset()
await cdp_call(ws, 'Page.navigate', {'url': sbi_url})
await wait_for(ws, "!!window.__sbiTest && !!document.getElementById('sbi-results')", timeout=30)
elif args.transfer_report:
await cdp_eval(ws, 'true', timeout=5)
NETWORK_TRANSFERS.reset()
test_t0 = time.monotonic()
try:
if getattr(args, 'synthetic', False):
img_url = f"{base}/test_img/synthetic/{urllib.parse.quote(img_name)}"
else:
img_url = f"{base}/test_img/{urllib.parse.quote(img_name)}"
js_img_name = json.dumps(img_name)
js_preset = json.dumps(preset)
js_img_url = json.dumps(img_url)
js_detail = json.dumps("verbose" if args.verbose else "compact")
js_expected = json.dumps(expected)
if args.transfer_report:
source_path = os.path.join(TEST_DIR, 'synthetic', img_name) if args.synthetic else os.path.join(TEST_DIR, img_name)
with open(source_path, 'rb') as source:
encoded_image = base64.b64encode(source.read()).decode('ascii')
mime = 'image/png' if img_name.lower().endswith('.png') else 'image/jpeg'
image_script = f"const blob = new Blob([Uint8Array.from(atob({json.dumps(encoded_image)}), c => c.charCodeAt(0))], {{type:{json.dumps(mime)}}});"
else:
image_script = f"const resp = await fetch({js_img_url}); if (!resp.ok) throw new Error('Fetch ' + resp.status); const blob = await resp.blob();"
js = f"""
(async () => {{
const timings = {{}};
const t0 = performance.now();
{image_script}
timings.fetch = performance.now() - t0;
const file = new File([blob], {js_img_name}, {{ type: blob.type }});
window.__sbiTest.setForceGlobalFallback({str(bool(args.force_fallback)).lower()});
window.__sbiTest.setBenchmarkCorpusSize({args.benchmark_groups or 0}, {json.dumps(benchmark_exclusions)});
try {{
const p0 = performance.now();
await window.__sbiTest.processImage(file, {js_preset});
timings.process = performance.now() - p0;
}} catch(e) {{
return JSON.stringify({{ error: e.message || String(e) }});
}} finally {{
window.__sbiTest.setForceGlobalFallback(false);
}}
const errEl = document.querySelector('.sbi-no-results');
if (errEl) return JSON.stringify({{ error: errEl.textContent }});
const s = window.__sbiTest.getSummary({{ detail: {js_detail} }});
s.expectedResult = window.__sbiTest.getPackResult({js_expected});
if ({str(True if (args.verbose or args.diagnostics_json) else False).lower()}) {{
s.anchorEvidence = window.__sbiTest.getAnchorEvidence();
s.slotVariantImages = {{
ds: window.__sbiTest.getSlotVariantImages(0),
ep: window.__sbiTest.getSlotVariantImages(1),
}};
s.comparisons = {{}};
const names = Array.from(new Set([{js_expected}, ...s.ranked.slice(0, 3).map(row => row.name)]));
for (const name of names) {{
s.comparisons[name] = {{
DS: window.__sbiTest.comparePackSlot(name, 0, 'diamond_sword'),
EP: window.__sbiTest.comparePackSlot(name, 1, 'ender_pearl'),
}};
}}
}}
s.timings = Object.assign(timings, s.timings || {{}});
return JSON.stringify(s);
}})()
"""
raw = await cdp_eval(ws, js, timeout=90)
summary = json.loads(raw) if isinstance(raw, str) else raw
if args.transfer_report:
settle_start = time.monotonic()
stable = 0
previous_count = -1
for _ in range(60):
await asyncio.sleep(0.2)
await cdp_eval(ws, 'true', timeout=5)
snapshot = NETWORK_TRANSFERS.snapshot()
count = len(snapshot['requests'])
stable = stable + 1 if count == previous_count and snapshot['pendingRequests'] == 0 else 0
previous_count = count
if stable >= 3:
break
memory = await cdp_call(ws, 'Runtime.getHeapUsage')
transfer = NETWORK_TRANSFERS.snapshot()
transfer.update(image=img_name, matcher=args.matcher, mode=args.benchmark, run=run_index,
settleSeconds=time.monotonic() - settle_start, heap=memory,
summary=summary, inputTransport='local-file-injected')
transfer['budgetPassed'] = bool(transfer['complete'] and transfer['transferredBytes'] <= 8_000_000
and (args.benchmark != 'cold' or not any(row['fromCache'] for row in transfer['requests'])))
if not is_warmup:
transfer_samples.append(transfer)
output = os.path.abspath(args.transfer_report)
os.makedirs(os.path.dirname(output), exist_ok=True)
with open(output, 'w', encoding='utf-8') as report:
json.dump(transfer_samples, report, ensure_ascii=False)
print(f" transfer={transfer['transferredBytes']} bytes complete={transfer['complete']} budget={transfer['budgetPassed']}", flush=True)
if is_warmup:
if "error" in summary:
raise RuntimeError(f"Warm-up failed: {summary['error']}")
print("Warm cache ready\n", flush=True)
continue
if "error" in summary:
elapsed = time.monotonic() - test_t0
results.append((img_name, expected, f"ERROR: {summary['error']}", False, elapsed))
candidate_recalls.append(False)
print(f" [ERROR] {img_name}: {summary['error']} ({fmt_seconds(elapsed)})\n")
if args.fail_fast:
break
continue
ranked = summary.get("ranked", [])
debug = summary.get("debug", {})
timings = summary.get("timings", {})
top1 = ranked[0]["name"] if ranked else "(none)"
top_group = ranked[0].get("groupId") if ranked else None
expected_result = summary.get("expectedResult") or {}
expected_group = expected_result.get("groupId")
expected_member_exists = expected in (expected_result.get("groupMembers") or [])
recall_ok = bool(expected_group and expected_result.get("fullScored"))
candidate_recalls.append(recall_ok)
margin = None
if len(ranked) >= 2:
margin = ranked[0].get("score", 0) - ranked[1].get("score", 0)
ok = expected_member_exists and recall_ok and expected_group == top_group
status = "PASS" if ok else "FAIL"
elapsed = time.monotonic() - test_t0
results.append((img_name, expected, top1, ok, elapsed))
if args.synthetic:
parts = img_name.split('/')
tier = parts[1] if len(parts) >= 3 else 'unknown'
synthetic_records.append({
'image': img_name,
'expected': expected,
'tier': tier,
'recalled': recall_ok,
'expected_group': expected_group,
'top_group': top_group,
'member_exists': expected_member_exists,
})
if args.benchmark:
benchmark_samples.append({
"mode": args.benchmark,
"image": img_name,
"run": run_index,
"timings": timings,
"matchMetrics": debug.get("matchMetrics") or {},
})
if args.diagnostics_json:
diagnostics.append({
"image": img_name,
"preset": preset,
"expected": expected,
"summary": summary,
})
def fmt_row(r):
parts = [f"{r['name']}={r['score']:.4f}"]
if r.get('slotComposite') is not None:
parts.append(f"slot={r['slotComposite']:.3f}")
pts = r.get('perTypeScores') or {}
if pts:
tparts = []
for k in ('DS', 'EP', 'HL', 'SK/GC'):
v = pts.get(k)
if v is not None:
tparts.append(f"{k}={v:.2f}")
if tparts:
parts.append("(" + " ".join(tparts) + ")")
if r.get('criticalTypeScore') is not None:
parts.append(f"crit={r['criticalTypeScore']:.2f}")
if r.get('healthSim') is not None:
parts.append(f"HP={r['healthSim']:.2f}")
if r.get('hungerSim') is not None:
parts.append(f"Hu={r['hungerSim']:.2f}")
if r.get('armorSim') is not None:
parts.append(f"Ar={r['armorSim']:.2f}")
if r.get('widgetSim') is not None:
parts.append(f"wg={r['widgetSim']:.3f}")
if r.get('coverage') is not None:
parts.append(f"cov={r['coverage']:.2f}")
if r.get('distinguishability') is not None:
parts.append(f"dist={r['distinguishability']:.2f}")
if r.get('sharedness') is not None:
parts.append(f"shared={r['sharedness']:.2f}")
if r.get('anchorPenalty') is not None:
parts.append(f"apen={r['anchorPenalty']:.3f}")
gaps = r.get('anchorGaps') or {}
if gaps:
gap_parts = []
for key in ('ds', 'ep', 'widget', 'hp'):
v = gaps.get(key)
if isinstance(v, (int, float)) and v > 0.001:
gap_parts.append(f"{key}={v:.2f}")
if gap_parts:
parts.append("gap(" + " ".join(gap_parts) + ")")
return "[" + " ".join(parts) + "]"
timing_parts = []
if not args.no_timings:
for key in ("fetch", "decode", "fingerprints", "extract", "food", "inflate", "inflateSettle", "match", "render", "process"):
value = timings.get(key)
if isinstance(value, (int, float)):
timing_parts.append(f"{key}={value / 1000:.2f}s")
timing_parts.append(f"total={fmt_seconds(elapsed)}")
timing_text = (" | " + ", ".join(timing_parts)) if timing_parts else ""
margin_text = f" | margin={margin:.4f}" if isinstance(margin, (int, float)) else ""
run_text = f" [run {run_index}]" if args.benchmark else ""
print(f" [{status}]{run_text} {img_name} -> {top1}{margin_text}{timing_text}")
if not args.quiet:
print(f" Expected: {expected}")
print(f" Group: expected={expected_group or '-'}, top={top_group or '-'}, recalled={recall_ok}")
if debug:
print(f" Debug: slots={debug.get('slotCount')}, hearts={debug.get('heartCount')}, hunger={debug.get('hungerCount')}, armor={debug.get('armorCount')}, ranked={debug.get('rankedCount')}")
print(f" SlotTypes: {summary.get('slotTypes', '-')}")
if args.verbose:
expected_result = summary.get('expectedResult') or {}
print(
f" ExpectedState: rank={expected_result.get('rank')} "
f"fullScored={expected_result.get('fullScored')} group={expected_result.get('groupId')}"
)
sfs = summary.get('slotFeatures', [])
for sf in sfs:
if sf is None: continue
sig = sf.get('sig') or {}
print(f" Slot[{sf['index']}]: act={sf.get('activity',0):.2f} var={sf.get('variance',0):.0f} n={sig.get('n','?')} cov={sig.get('coverage',0):.2f} lum={sig.get('meanLum',0):.0f} R={sig.get('meanR',0):.0f} G={sig.get('meanG',0):.0f} B={sig.get('meanB',0):.0f} redF={sig.get('redFrac',0):.3f} yF={sig.get('yellowFrac',0):.3f} blueF={sig.get('blueFrac',0):.3f}")
for r in ranked[:10]:
print(f" # {fmt_row(r)}")
for pack_name, types in (summary.get('comparisons') or {}).items():
parts = []
for type_name in ('DS', 'EP'):
comparison = types.get(type_name) if isinstance(types, dict) else None
variants = (comparison or {}).get('variants') or []
if not variants:
continue
best = max(variants, key=lambda row: row.get('final') or 0)
parts.append(
f"{type_name}:final={best.get('final', 0):.3f} "
f"base={best.get('base', 0):.3f} spatial={best.get('spatial', 0):.3f} "
f"shape={best.get('spatialShape', 0):.3f} dir={best.get('spatialDirection', 0):.3f} "
f"color={best.get('spatialColor', 0):.3f} "
f"sig={best.get('signature', 0):.3f} hash={best.get('hamming', 0):.3f} "
f"hist={best.get('histogram', 0):.3f} mom={best.get('moments', 0):.3f} edge={best.get('edge', 0):.3f}"
)
if parts:
print(f" Compare {pack_name}: " + " | ".join(parts))
if not ok:
for idx, r in enumerate(ranked):
if match_name(expected, r['name']):
print(f" Expected@#{idx+1}: {fmt_row(r)}")
break
if args.fail_fast:
break
print()
except Exception as e:
elapsed = time.monotonic() - test_t0
results.append((img_name, expected, f"ERROR: {e}", False, elapsed))
candidate_recalls.append(False)
print(f" [ERROR] {img_name}: {e} ({fmt_seconds(elapsed)})\n")
if args.fail_fast:
break
finally:
edge_proc.terminate()
if server_proc:
server_proc.terminate()
try:
edge_proc.wait(timeout=5)
except Exception:
edge_proc.kill()
if server_proc:
try:
server_proc.wait(timeout=3)
except Exception:
server_proc.kill()
await asyncio.sleep(0.5)
try:
edge_log.close()
except Exception:
pass
shutil.rmtree(profile_dir, ignore_errors=True)
# Summary
passed = sum(1 for result in results if result[3])
total = len(results)
suite_elapsed = time.monotonic() - suite_t0
print("=" * 50)
print(f"Results: {passed}/{total} passed in {fmt_seconds(suite_elapsed)}")
print(f"Candidate recall: {sum(candidate_recalls)}/{len(candidate_recalls)}")
for name, expected, got, ok, elapsed in results:
print(f" {'PASS' if ok else 'FAIL'}: {name} -> {got} (expected {expected}, {fmt_seconds(elapsed)})")
if args.synthetic:
print("\n# Synthetic corpus (separate from 9-shot regression; diagnostic only, NOT acceptance)")
if not synthetic_records:
print(" (no synthetic records collected)")
else:
by_tier = {}
for rec in synthetic_records:
by_tier.setdefault(rec['tier'], []).append(rec)
for tier in sorted(by_tier):
recs = by_tier[tier]
n = len(recs)
recalled = sum(1 for r in recs if r['recalled'])
top1_group = sum(1 for r in recs if r['expected_group'] and r['expected_group'] == r['top_group'])
member_exists = sum(1 for r in recs if r['member_exists'])
print(f" [{tier}] n={n} coarse recall={recalled}/{n} ({100*recalled/n:.1f}%) "
f"group top-1={top1_group}/{n} ({100*top1_group/n:.1f}%) "
f"expected member present={member_exists}/{n} ({100*member_exists/n:.1f}%)")
# Indistinguishable pack sets: groups whose median member_exists + recalled and top1==expected
print(" \n# Indistinguishable-pack sets (groups whose every synthetic shot matched its own group as top-1):")
groups_out = {}
for rec in synthetic_records:
gid = rec['expected_group']
if not gid:
continue
ok = rec['recalled'] and rec['member_exists'] and rec['expected_group'] == rec['top_group']
groups_out.setdefault(gid, []).append(ok)
fully = [gid for gid, oks in groups_out.items() if any(oks) and all(oks)]
partial = [gid for gid, oks in groups_out.items() if any(oks) and not all(oks)]
none = [gid for gid, oks in groups_out.items() if not any(oks)]
print(f" fully indistinguishable: {len(fully)} partially: {len(partial)} none: {len(none)}")
return 0 if passed == total else 1
budget_ok = True
if benchmark_samples:
print("\nBenchmark:")
percentiles = {}
timing_metrics = (
("total", lambda sample: sample["timings"].get("process")),
("match", lambda sample: sample["timings"].get("match")),
("load", lambda sample: sample["timings"].get("fingerprints")),
("extract", lambda sample: sample["timings"].get("extract")),
("inflate", lambda sample: sample["timings"].get("inflate")),
("inflate settle", lambda sample: sample["timings"].get("inflateSettle")),
("candidate", lambda sample: sample["matchMetrics"].get("candidatePlanningMs")),
("coarse", lambda sample: sum(
sample["matchMetrics"].get(key) or 0 for key in ("coarseMs", "globalCoarseMs")
)),
("full", lambda sample: sum(
run.get("totalMs") or 0 for run in sample["matchMetrics"].get("runs", [])
)),
("refine", lambda sample: sum(
run.get("refinementMs") or 0 for run in sample["matchMetrics"].get("runs", [])
)),
("expand", lambda sample: sample["matchMetrics"].get("expansionMs")),
("render", lambda sample: sample["timings"].get("render")),
)
for label, getter in timing_metrics:
values = [getter(sample) for sample in benchmark_samples]
p50 = percentile(values, 0.50)
p95 = percentile(values, 0.95)
percentiles[label] = p95
if p50 is not None and p95 is not None:
print(f" {label}: p50={p50:.1f}ms p95={p95:.1f}ms")
count_metrics = (
("prefilter", "candidatePrefilterCount"),
("full unique", "fullScoreCount"),
("full evaluations", "fullScoreEvaluations"),
("global coarse", "globalCoarseCount"),
)
for label, key in count_metrics:
values = [sample["matchMetrics"].get(key) for sample in benchmark_samples]
p50 = percentile(values, 0.50)
p95 = percentile(values, 0.95)
if p50 is not None and p95 is not None:
print(f" {label}: p50={p50:.0f} p95={p95:.0f}")
fallback_count = sum(1 for sample in benchmark_samples if sample["matchMetrics"].get("fallback"))
print(f" samples={len(benchmark_samples)} fallback={fallback_count}/{len(benchmark_samples)}")
if args.enforce_budgets:
total_limit = 2000 if args.benchmark == "cold" else 1200
total_p95 = percentiles.get("total")
match_p95 = percentiles.get("match")
budget_ok = bool(
total_p95 is not None and total_p95 <= total_limit and
match_p95 is not None and match_p95 <= 150
)
print(f" budget={'PASS' if budget_ok else 'FAIL'} total<={total_limit}ms match<=150ms")
if args.diagnostics_json:
output_path = os.path.abspath(args.diagnostics_json)
with open(output_path, "w", encoding="utf-8") as fh:
json.dump(diagnostics, fh, ensure_ascii=False)
print(f"Diagnostics: {output_path}")
return 0 if passed == total and budget_ok else 1
if __name__ == "__main__":
sys.exit(asyncio.run(main()))