From fcb04677c91af68c9deae52fd5dc13c4acc58c4e Mon Sep 17 00:00:00 2001 From: Loi Nguyen <1948922+lntutor@users.noreply.github.com> Date: Mon, 20 Jul 2026 01:56:23 +0700 Subject: [PATCH] fix: index_adjustment_after_clean_extra_whitespace maps cleaned index to wrong original offset index_adjustment_after_clean_extra_whitespace subtracts the moved distance when it must add it. moved_indices[i] is the number of characters removed before cleaned index i (per clean_extra_whitespace_ with_index_run's docstring), so the original position of a cleaned character is index + moved_indices[index]. Subtracting maps every character following a collapsed whitespace run to the wrong -- or a negative -- original index (e.g. 'B' at cleaned index 8 mapped to 4, a space, instead of 12). Change the operator and add a round-trip regression test. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N6RtoHuxrDqTUo9Mw9h4Cv --- test_unstructured/cleaners/test_core.py | 10 ++++++++++ unstructured/cleaners/core.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test_unstructured/cleaners/test_core.py b/test_unstructured/cleaners/test_core.py index 6414292d99..0dabe178c0 100644 --- a/test_unstructured/cleaners/test_core.py +++ b/test_unstructured/cleaners/test_core.py @@ -303,3 +303,13 @@ def test_clean(text, extra_whitespace, dashes, bullets, lowercase, trailing_punc def test_bytes_string_to_string(): text = "\xe6\xaf\x8f\xe6\x97\xa5\xe6\x96\xb0\xe9\x97\xbb" assert core.bytes_string_to_string(text, "utf-8") == "每日新闻" + + +def test_index_adjustment_maps_cleaned_index_back_to_original(): + text = "ITEM 1. BUSINESS" + cleaned, moved_indices = core.clean_extra_whitespace_with_index_run(text) + assert cleaned == "ITEM 1. BUSINESS" + cleaned_index = cleaned.index("B") + original_index = core.index_adjustment_after_clean_extra_whitespace(cleaned_index, moved_indices) + assert text[original_index] == cleaned[cleaned_index] == "B" + assert original_index == 12 diff --git a/unstructured/cleaners/core.py b/unstructured/cleaners/core.py index b64c7bd19c..13eb268ab4 100644 --- a/unstructured/cleaners/core.py +++ b/unstructured/cleaners/core.py @@ -487,4 +487,4 @@ def clean_extra_whitespace_with_index_run(text: str) -> Tuple[str, np.ndarray]: def index_adjustment_after_clean_extra_whitespace(index, moved_indices) -> int: - return int(index - moved_indices[index]) + return int(index + moved_indices[index])