fix(chunking): prevent infinite loop in token-based overlap splitting

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.25.2
+
+### Fixes
+
+- **Token-based chunking with overlap no longer hangs on long unbroken tokens**: chunking with `max_tokens` combined with `overlap` (or `overlap_all`) entered an unbounded CPU loop on any element containing a whitespace-free run longer than `max_tokens` — a long URL, hash, base64/JWT blob, filesystem path, or biological sequence — affecting the public `chunk_elements` (basic) and `chunk_by_title` APIs. In token mode the splitter prepended the overlap tail plus a synthetic separator space to the remainder, which could regrow the remainder back to the full input; the next split then landed on that inserted space and reproduced the input verbatim, so the driving `while remainder:` loop never terminated. The token splitter now applies overlap only when the resulting remainder is strictly shorter than its input, falling back to the un-overlapped remainder otherwise, so every split makes forward progress and chunking always terminates. The character-mode path and token mode with `overlap=0` were unaffected.
+
 ## 0.25.1
 
 ### Fixes
@@ -18,12 +24,6 @@
 
 - **Update README.md**: readme-only changes; added the Unstructured Transform MCP to the README. No library behavior changes.
 
-## 0.24.1
-
-### Fixes
-
-- **Fix stored XSS in v2 (ontology) HTML output** (GHSA-v5mq-3xhg-98m9): `partition_html(html_parser_version="v2")`, `elements_to_html()`, and `metadata.text_as_html` previously emitted untrusted document markup without output encoding, allowing attacker-controlled content (`on*` handlers, `javascript:` links, tag/attribute breakout) to execute when the HTML was viewed. Output is now sanitized — text and attribute values are HTML-escaped, event-handler attributes are dropped, tags/attributes are allowlisted, and URL schemes are filtered (`http`/`https`/`mailto`/`tel`/relative preserved; `data:` limited to raster image MIME types on `img[src]`). Legitimate formatting is unaffected.
-
 ## 0.24.0
 
 ### Enhancements

test_unstructured/chunking/test_base.py

@@ -456,6 +456,41 @@ def it_produces_correct_overlapping_splits(self, _tiktoken_installed: None):
         assert fragment2 == "seven eight nine ten eleven twelve"
         assert remainder2 == ""
 
+    def it_makes_progress_on_a_long_whitespace_free_token_with_overlap(
+        self, _tiktoken_installed: None
+    ):
+        """Regression: overlap must not spin the split loop forever on a long unbroken token.
+
+        A whitespace-free run longer than `max_tokens` (a long URL, hash, base64 blob, etc.)
+        combined with token-mode overlap used to regrow the remainder back to the full input,
+        so `PreChunk._iter_text_splits` (`while remainder: split(remainder)`) never terminated.
+        Each split must strictly shrink the remainder.
+        """
+        opts = ChunkingOptions(max_tokens=10, tokenizer="cl100k_base", overlap=5)
+        split = _TextSplitter(opts)
+
+        # -- 2000 chars, no whitespace, far more than 10 tokens --
+        text = "xyzq" * 500
+
+        # -- drive the same loop `_iter_text_splits` uses; this spins forever on the pre-fix
+        # -- splitter (there is deliberately no internal iteration cap here) --
+        fragment, remainder = split(text)
+        fragments = [fragment]
+        remainder_lengths = [len(remainder)]
+        while remainder:
+            fragment, remainder = split(remainder)
+            fragments.append(fragment)
+            remainder_lengths.append(len(remainder))
+
+        # -- the loop terminated and split into multiple fragments --
+        assert remainder == ""
+        assert len(fragments) > 1
+        # -- every split strictly shrank the remainder, which is what guarantees termination --
+        assert all(b < a for a, b in zip(remainder_lengths, remainder_lengths[1:]))
+        # -- no fragment exceeds the token limit and no original character is lost --
+        assert all(opts.measure(f) <= 10 for f in fragments if f)
+        assert "xyzq" * 3 in "".join(fragments)
+
 
 # ================================================================================================
 # PRE-CHUNKER

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.25.1"  # pragma: no cover
+__version__ = "0.25.2"  # pragma: no cover

unstructured/chunking/base.py

@@ -1490,7 +1490,15 @@ def _split_by_tokens(self, s: str) -> tuple[str, str]:
                         # -- token-based overlap: find tail with ~overlap tokens --
                         tail = self._get_token_overlap_tail(fragment, overlap)
                         overlapped_remainder = tail + " " + raw_remainder
-                        return fragment, overlapped_remainder
+                        # -- NOTE(otiscuilei): only apply the overlap when the remainder still
+                        # -- shrinks. Prepending the overlap tail (plus a synthetic separator
+                        # -- space) can regrow the remainder back to `s`; the next split then
+                        # -- lands on that inserted space and reproduces `s` verbatim, spinning
+                        # -- `_iter_text_splits` forever on a whitespace-free run longer than
+                        # -- `maxlen`. `raw_remainder` is always strictly shorter than `s`, so
+                        # -- fall back to it to guarantee forward progress.
+                        if len(overlapped_remainder) < len(s):
+                            return fragment, overlapped_remainder
                     return fragment, raw_remainder
 
         # -- fallback: split on whitespace boundary using binary search to find token limit --
@@ -1525,7 +1533,9 @@ def _split_by_tokens(self, s: str) -> tuple[str, str]:
         if overlap > 0 and fragment:
             tail = self._get_token_overlap_tail(fragment, overlap)
             overlapped_remainder = tail + " " + raw_remainder
-            return fragment, overlapped_remainder
+            # -- only overlap when it makes forward progress (see note in the separator branch) --
+            if len(overlapped_remainder) < len(s):
+                return fragment, overlapped_remainder
 
         return fragment, raw_remainder