diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f1d11f8f6..d0993513ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Fixes - **Update README.md**: readme-only changes; added a link to Unstructured Pipelines to the README. No library behavior changes. +- **Fix crash on non-UTF-8 bracket-prefixed text with no extension**: `detect_filetype()` called `json.load()` directly on the raw file bytes when probing whether `text/plain`-guessed content is actually JSON. If the content started with `[`/`{` but contained non-UTF-8 bytes, `json.load()` raised an uncaught `UnicodeDecodeError` (not the `json.JSONDecodeError` the code caught), crashing detection instead of falling back to `FileType.TXT`. Both exception types are now caught. ## 0.25.0 diff --git a/test_unstructured/file_utils/test_filetype.py b/test_unstructured/file_utils/test_filetype.py index e942714373..6981306c88 100644 --- a/test_unstructured/file_utils/test_filetype.py +++ b/test_unstructured/file_utils/test_filetype.py @@ -1342,6 +1342,15 @@ def test_json_content_type_is_disambiguated_for_ndjson(): assert predicted_type == FileType.NDJSON +def test_it_does_not_crash_on_non_utf8_bracket_prefixed_text_with_no_extension(): + non_utf8_bytes = b"{not json at all just braces \xe9 text} and more padding to fill the head" + + file_buffer = io.BytesIO(non_utf8_bytes) + predicted_type = detect_filetype(file=file_buffer) + + assert predicted_type == FileType.TXT + + def test_office_files_when_document_archive_has_non_standard_prefix(): predicted_type = detect_filetype( file_path=input_path("file_type/test_document_from_office365.docx") diff --git a/unstructured/file_utils/filetype.py b/unstructured/file_utils/filetype.py index a12a6249e9..15ef3ee8fa 100644 --- a/unstructured/file_utils/filetype.py +++ b/unstructured/file_utils/filetype.py @@ -960,7 +960,7 @@ def _is_json(self) -> bool: with self._ctx.open() as file: json.load(file) return True - except json.JSONDecodeError: + except (json.JSONDecodeError, UnicodeDecodeError): return False