diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e3310f1..b7ffcd78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to `parse-bench` are recorded here. The format follows ## [Unreleased] +## [2.0.3] - 2026-11-16 + +### Fixed +- Add opt-in `section_matching: nested-v1` for header/footer rules; nested formatting and pagination tags no longer cause false failures. Unflagged rules retain legacy behavior. +- Add dataset metadata `sub_page_handling: transparent-v1` to ignore GT-only `` containers in text and formatting scoring while retaining their content. Required by dataset v2.02; old datasets retain their existing behavior. +- Bound oversized form-scoring inputs to avoid excessive matching work. + +### Added +- Replay saved evaluations with Metric Lab scoring policies and retain reclassifiable form-coverage events. +- Add provider configurations and published results for additional OCR pipelines, including Falcon-OCR and TeleOCR. + +## [2.0.2] - 2026-11-14 + ### Added - ParseBench 2 leaderboard accountability, as for ParseBench v1: `leaderboard/results-v2.yaml` ties every `leaderboard-v2.csv` row to its published outputs in `llamaindex/ParseBench-2-results`, `leaderboard-v2.csv` gains a diff --git a/docs/header-footer-matching.md b/docs/header-footer-matching.md index 6c01cbd3..d2019a1f 100644 --- a/docs/header-footer-matching.md +++ b/docs/header-footer-matching.md @@ -8,6 +8,6 @@ The new matcher extracts one complete `` or `` block a {"type": "is_footer", "text": "Report title", "section_matching": "nested-v1"} ``` -The local ParseBench-2 v2.02 dataset opts in on its header/footer rules. Earlier dataset snapshots remain unchanged. New-mode rule results include `details.section_matching` for auditability. Pin both the dataset revision and scorer revision for comparisons; a pre-fix scorer must not be used to evaluate opted-in rules. This flag does not implement general `` text normalization. +The local ParseBench-2 v2.02 dataset opts in on its header/footer rules. Earlier dataset snapshots remain unchanged. New-mode rule results include `details.section_matching` for auditability. Pin both the dataset revision and scorer revision for comparisons; a pre-fix scorer must not be used to evaluate opted-in rules. For GT containing ``, dataset metadata must also set `sub_page_handling: "transparent-v1"` (CLI 2.0.3+). This strips only the container tags before text and formatting scoring, preserves all content inside, and does not require providers to emit the wrappers. Without that metadata, text scoring is unchanged. Generation: `markdownToCombinedRules(md, {sectionMatching: "nested-v1"})` in the annotation tools forwards the contract to the generated header/footer rules. The default generator output omits the flag. diff --git a/src/parse_bench/__init__.py b/src/parse_bench/__init__.py index e66a5947..794c3945 100644 --- a/src/parse_bench/__init__.py +++ b/src/parse_bench/__init__.py @@ -1,3 +1,3 @@ """Document parsing evaluation system for competitive benchmarking.""" -__version__ = "2.0.2" +__version__ = "2.0.3" diff --git a/src/parse_bench/evaluation/metrics/parse/next_metrics.py b/src/parse_bench/evaluation/metrics/parse/next_metrics.py index f923ac47..5873e36d 100644 --- a/src/parse_bench/evaluation/metrics/parse/next_metrics.py +++ b/src/parse_bench/evaluation/metrics/parse/next_metrics.py @@ -49,6 +49,7 @@ from parse_bench.evaluation.metrics.parse.text_next import ( output_hypothesis, reference_markdown, + strip_sub_page_wrappers, text_next_applies, text_next_metrics, ) @@ -119,13 +120,21 @@ def _document_metrics( assert isinstance(output, ParseOutput) metrics: list[MetricValue] = [] if run_text: + sub_page_handling = metadata.get("sub_page_handling") + if sub_page_handling not in (None, "transparent-v1"): + raise ValueError(f"Unsupported sub_page_handling: {sub_page_handling!r}") + transparent_sub_pages = sub_page_handling == "transparent-v1" reference = reference_markdown(test_case.expected_markdown, rules) + if reference is not None and transparent_sub_pages: + reference = strip_sub_page_wrappers(reference) if reference is not None and text_next_applies(reference, rules, metadata): - metrics.extend(text_next_metrics(reference, output)) + metrics.extend(text_next_metrics(reference, output, transparent_sub_pages=transparent_sub_pages)) # A full-page decoration inventory takes precedence over the reference Markdown and the rules. decorations = decoration_reference(metadata) fmt_md = reference_markdown(test_case.expected_markdown, rules) + if fmt_md is not None and transparent_sub_pages: + fmt_md = strip_sub_page_wrappers(fmt_md) fmt_ref = None if fmt_md else rules_reference(rules) if fmt_md and text_next_applies(fmt_md, rules, metadata): fmt_ref = (fmt_md, FULL_SCOPE) if formatting_v2_applies(fmt_md) else None @@ -133,7 +142,11 @@ def _document_metrics( if decorations is not None: fmt_ref, neutral = (decorations[0], decorations[1]), decorations[2] if fmt_ref is not None: - metrics.extend(formatting_v2_metrics(fmt_ref[0], output_hypothesis(output), fmt_ref[1], neutral)) + fmt_reference, hypothesis = fmt_ref[0], output_hypothesis(output) + if transparent_sub_pages: + fmt_reference = strip_sub_page_wrappers(fmt_reference) + hypothesis = strip_sub_page_wrappers(hypothesis) + metrics.extend(formatting_v2_metrics(fmt_reference, hypothesis, fmt_ref[1], neutral)) forms_ref = forms_reference(metadata, rules) if forms_ref is not None and forms_v2_applies(forms_ref): diff --git a/src/parse_bench/evaluation/metrics/parse/text_next.py b/src/parse_bench/evaluation/metrics/parse/text_next.py index 9ab05f82..59f438c5 100644 --- a/src/parse_bench/evaluation/metrics/parse/text_next.py +++ b/src/parse_bench/evaluation/metrics/parse/text_next.py @@ -2003,10 +2003,21 @@ def output_hypothesis(output: ParseOutput) -> str: ) -def text_next_metrics(reference: str, output: ParseOutput) -> list[MetricValue]: +def strip_sub_page_wrappers(markdown: str) -> str: + """Remove GT-only page containers while retaining all printed content and boundaries.""" + return regex.sub(r"", "\n", markdown, flags=regex.IGNORECASE) + + +def text_next_metrics(reference: str, output: ParseOutput, *, transparent_sub_pages: bool = False) -> list[MetricValue]: """``text_next`` headline plus its lane diagnostics as metric values.""" - res = score_text_next(reference, output_hypothesis(output)) + hypothesis = output_hypothesis(output) + if transparent_sub_pages: + reference = strip_sub_page_wrappers(reference) + hypothesis = strip_sub_page_wrappers(hypothesis) + res = score_text_next(reference, hypothesis) meta = res.metadata() + if transparent_sub_pages: + meta["sub_page_handling"] = "transparent-v1" metrics = [MetricValue(metric_name="text_next", value=res.score, metadata=meta)] for lane in ("words", "critical", "integrity", "order"): value = getattr(res, lane) diff --git a/tests/parse_bench/evaluation/metrics/parse/test_text_next.py b/tests/parse_bench/evaluation/metrics/parse/test_text_next.py index 875c3751..b35fcf00 100644 --- a/tests/parse_bench/evaluation/metrics/parse/test_text_next.py +++ b/tests/parse_bench/evaluation/metrics/parse/test_text_next.py @@ -821,3 +821,45 @@ def test_short_alt_reading_does_not_exempt_unmarked_body_narration() -> None: ref = "Kite\n\n" + PROSE out = "![logo: Kite](logo.png)\n\nA bright bird perched above the distant river.\n\n" + PROSE assert score_text_next(ref, out).score < 1.0 + + +@pytest.mark.parametrize("opt_in", [False, True]) +def test_sub_page_opt_in_ignores_nested_wrappers_but_keeps_footer_content(opt_in: bool) -> None: + plain = "ACME Report\n\n" + PROSE + "\nPage 12" + wrapped = "" + plain + "" + metadata = {"scoring": "next"} + if opt_in: + metadata["sub_page_handling"] = "transparent-v1" + case = _case(metadata) + case.expected_markdown = wrapped + metrics = NextScorers().evaluate(_inference_result(_output(plain)), case).metrics + text = next(m for m in metrics if m.metric_name == "text_next") + if opt_in: + assert text.value == 1.0 + assert text.metadata["sub_page_handling"] == "transparent-v1" + else: + assert text.value == score_text_next(wrapped, plain).score < 1.0 + missing = plain.replace("Page 12", "") + metrics = NextScorers().evaluate(_inference_result(_output(missing)), case).metrics + assert next(m.value for m in metrics if m.metric_name == "text_next") < 1.0 + + +def test_sub_page_opt_in_ignores_wrappers_in_provider_output() -> None: + case = _case({"scoring": "next", "sub_page_handling": "transparent-v1"}) + md = "" + case.expected_markdown + "" + metrics = NextScorers().evaluate(_inference_result(_output(md)), case).metrics + assert next(m.value for m in metrics if m.metric_name == "text_next") == 1.0 + + +def test_unknown_sub_page_contract_is_not_silently_ignored() -> None: + case = _case({"scoring": "next", "sub_page_handling": "typo"}) + with pytest.raises(ValueError, match="Unsupported sub_page_handling"): + NextScorers().evaluate(_inference_result(_output(PROSE)), case) + + +@pytest.mark.parametrize("content", ["", "
Revenue120
"]) +def test_sub_page_wrapper_does_not_make_table_only_reference_text_applicable(content: str) -> None: + case = _case({"scoring": "next", "sub_page_handling": "transparent-v1"}) + case.expected_markdown = "" + content + "" + metrics = NextScorers().evaluate(_inference_result(_output(content)), case).metrics + assert not any(m.metric_name.startswith("text_next") for m in metrics)