feat(pdf): adopt performance-oriented core-pdfminer.six fork

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.25.2
+
+### Enhancements
+
+- **Adopt `core-pdfminer.six` for PDF extraction**: migrate the PDF extraction path to the performance-oriented `core-pdfminer.six` fork while preserving upstream parsing and extraction semantics. The original `pdfminer.six` image-extra dependency remains available for compatibility.
+
 ## 0.25.1
 
 ### Fixes

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
     "unstructured-client>=0.25.9, <1.0.0",
     "wrapt>=2.1.1, <3.0.0",
     "filelock>=3.12.0,<4.0.0",
+    "core-pdfminer-six>=0.0.0",
 ]
 
 [project.optional-dependencies]

scripts/image/test-outbound-connectivity.sh

@@ -149,7 +149,7 @@ logging.getLogger("urllib").setLevel(logging.DEBUG)
 logging.getLogger("urllib3").setLevel(logging.DEBUG)
 logging.getLogger("httpx").setLevel(logging.DEBUG)
 logging.getLogger("httpcore").setLevel(logging.DEBUG)
-logging.getLogger("pdfminer.pdfpage").setLevel(logging.CRITICAL)
+logging.getLogger("core_pdfminer_six.pdfpage").setLevel(logging.CRITICAL)
 
 for test_file in [
  "/app/example-docs/ideas-page.html",

test_unstructured/file_utils/test_model.py

@@ -96,7 +96,7 @@ def and_it_knows_which_pip_extra_needs_to_be_installed_to_get_those_dependencies
             (FileType.EMPTY, ()),
             (FileType.HTML, ()),
             (FileType.ODT, ("docx", "pypandoc")),
-            (FileType.PDF, ("pdf2image", "pdfminer", "PIL")),
+            (FileType.PDF, ("pdf2image", "core_pdfminer_six", "PIL")),
             (FileType.UNK, ()),
             (FileType.WAV, ()),  # STT agent deps validated at runtime
             (FileType.ZIP, ()),

test_unstructured/partition/pdf_image/test_pdfminer_processing.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 import pytest
-from pdfminer.layout import LAParams, LTChar, LTContainer
+from core_pdfminer_six.layout import LAParams, LTChar, LTContainer
 from PIL import Image
 from unstructured_inference.constants import IsExtracted
 from unstructured_inference.constants import Source as InferenceSource
@@ -418,7 +418,7 @@ def _build_synthetic_form_pdf(path: str) -> None:
 
 def test_get_widget_text_from_annots_extracts_filled_text_fields():
     """The widget helper recovers filled /Tx field values and skips empty ones."""
-    from pdfminer.pdfpage import PDFPage
+    from core_pdfminer_six.pdfpage import PDFPage
 
     with tempfile.TemporaryDirectory() as tmp_dir:
         pdf_path = os.path.join(tmp_dir, "form.pdf")
@@ -434,7 +434,7 @@ def test_get_widget_text_from_annots_extracts_filled_text_fields():
 
 
 def test_get_widget_text_from_annots_decodes_utf16_text_without_bom():
-    from pdfminer.psparser import PSLiteral
+    from core_pdfminer_six.psparser import PSLiteral
 
     widgets = get_widget_text_from_annots(
         [
@@ -452,7 +452,7 @@ def test_get_widget_text_from_annots_decodes_utf16_text_without_bom():
 
 
 def test_get_widget_text_from_annots_decodes_choice_field_value_arrays():
-    from pdfminer.psparser import PSLiteral
+    from core_pdfminer_six.psparser import PSLiteral
 
     widgets = get_widget_text_from_annots(
         [
@@ -475,7 +475,7 @@ def test_get_widget_text_from_annots_inherits_field_type_and_value_from_parent()
     The intermediate parent is a direct dict (the case PDFObjRef-only traversal missed) and
     the root parent carries the inherited /FT, exercising multi-level inheritance.
     """
-    from pdfminer.psparser import PSLiteral
+    from core_pdfminer_six.psparser import PSLiteral
 
     root_parent = {"FT": PSLiteral("Tx")}  # field type lives at the top of the hierarchy
     mid_parent = {"V": b"\xfe\xff\x00J\x00a\x00n\x00e", "Parent": root_parent}  # value mid-chain

test_unstructured/partition/pdf_image/test_pdfminer_utils.py

@@ -1,8 +1,8 @@
 from importlib import reload
 from unittest.mock import MagicMock, patch
 
-from pdfminer.layout import LTChar, LTContainer, LTFigure, LTLayoutContainer, LTTextLine
-from pdfminer.pdftypes import PDFStream
+from core_pdfminer_six.layout import LTChar, LTContainer, LTFigure, LTLayoutContainer, LTTextLine
+from core_pdfminer_six.pdftypes import PDFStream
 
 from test_unstructured.unit_utils import example_doc_path
 from unstructured.partition.pdf import partition_pdf
@@ -525,7 +525,7 @@ def test_oversized_flate_stream_rejected_before_materialization(self):
         without fully materializing the output, and the stream should not be mutated."""
         import zlib
 
-        from pdfminer.psparser import LIT
+        from core_pdfminer_six.psparser import LIT
 
         from unstructured.partition.pdf_image.pdfminer_utils import (
             _decode_pdfstream_with_limit,
@@ -544,7 +544,7 @@ def test_normal_stream_decodes_within_limit(self):
         """A stream that fits within the limit should decode successfully."""
         import zlib
 
-        from pdfminer.psparser import LIT
+        from core_pdfminer_six.psparser import LIT
 
         from unstructured.partition.pdf_image.pdfminer_utils import (
             _decode_pdfstream_with_limit,
@@ -593,8 +593,8 @@ def test_vertical_wmode_sets_font_vertical_flag(self):
         font.default_disp != 0 at construction time, not just on the CMap."""
         import zlib
 
-        from pdfminer.pdfinterp import PDFResourceManager
-        from pdfminer.psparser import LIT
+        from core_pdfminer_six.pdfinterp import PDFResourceManager
+        from core_pdfminer_six.psparser import LIT
 
         from unstructured.partition.pdf_image.pdfminer_utils import CustomPDFCIDFont
 
@@ -684,7 +684,7 @@ class TestCustomPDFResourceManager:
 
     def test_returns_custom_cidfont_for_cid_subtypes(self):
         """CIDFontType2 fonts should be constructed as CustomPDFCIDFont."""
-        from pdfminer.psparser import LIT
+        from core_pdfminer_six.psparser import LIT
 
         from unstructured.partition.pdf_image.pdfminer_utils import (
             CustomPDFCIDFont,
@@ -738,7 +738,7 @@ def test_returns_custom_cidfont_for_cid_subtypes(self):
 
     def test_caches_font_on_repeated_calls(self):
         """Repeated get_font calls with the same objid should return the cached instance."""
-        from pdfminer.psparser import LIT
+        from core_pdfminer_six.psparser import LIT
 
         from unstructured.partition.pdf_image.pdfminer_utils import (
             CustomPDFResourceManager,

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.25.1"  # pragma: no cover
+__version__ = "0.25.2"  # pragma: no cover

unstructured/file_utils/model.py

@@ -406,7 +406,7 @@ def partitioner_shortname(self) -> str | None:
     PDF = (
         "pdf",
         "pdf",
-        ["pdf2image", "pdfminer", "PIL"],
+        ["pdf2image", "core_pdfminer_six", "PIL"],
         "pdf",
         [".pdf"],
         "application/pdf",

unstructured/partition/pdf.py

@@ -11,8 +11,8 @@
 
 import numpy as np
 import wrapt
-from pdfminer.layout import LTContainer, LTImage, LTItem, LTTextBox
-from pdfminer.utils import open_filename
+from core_pdfminer_six.layout import LTContainer, LTImage, LTItem, LTTextBox
+from core_pdfminer_six.utils import open_filename
 from pi_heif import register_heif_opener
 from PIL import Image as PILImage
 from pypdf import PdfReader
@@ -77,20 +77,13 @@
     PartitionStrategy,
 )
 from unstructured.partition.utils.sorting import coord_has_valid_points, sort_page_elements
-from unstructured.patches.pdfminer import patch_psparser
 from unstructured.utils import first, requires_dependencies
 
 if TYPE_CHECKING:
     from unstructured_inference.inference.layout import DocumentLayout
     from unstructured_inference.inference.layoutelement import LayoutElement
 
 
-# Correct a bug that was introduced by a previous patch to
-# pdfminer.six, causing needless and unsuccessful repairing of PDFs
-# which were not actually broken.
-patch_psparser()
-
-
 RE_MULTISPACE_INCLUDING_NEWLINES = re.compile(pattern=r"\s+", flags=re.DOTALL)
 # Regex patterns for counting graphics and text operators in PDF content streams.
 GRAPHICS_OPS_PATTERN = re.compile(
@@ -476,7 +469,7 @@ def _partition_pdf_with_pdfminer(
     return elements
 
 
-@requires_dependencies("pdfminer")
+@requires_dependencies("core_pdfminer_six")
 def _process_pdfminer_pages(
     fp: IO[bytes],
     filename: str,
@@ -1268,7 +1261,10 @@ def _extract_text(item: LTItem) -> str:
 # They throw an error when we call interpreter.process_page
 # Since we don't need color info, we can just drop it in the pdfminer code
 # See #2059
-@wrapt.patch_function_wrapper("pdfminer.pdfinterp", "PDFPageInterpreter.init_resources")
+@wrapt.patch_function_wrapper(
+    "core_pdfminer_six.pdfinterp",
+    "PDFPageInterpreter.init_resources",
+)
 def pdfminer_interpreter_init_resources(wrapped, instance, args, kwargs):
     resources = args[0]
     if "ColorSpace" in resources:

unstructured/partition/pdf_image/pdfminer_processing.py

@@ -5,9 +5,9 @@
 from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Union, cast
 
 import numpy as np
-from pdfminer.layout import LAParams, LTChar, LTContainer, LTTextBox
-from pdfminer.pdftypes import PDFObjRef
-from pdfminer.utils import decode_text, open_filename
+from core_pdfminer_six.layout import LAParams, LTChar, LTContainer, LTTextBox
+from core_pdfminer_six.pdftypes import PDFObjRef
+from core_pdfminer_six.utils import decode_text, open_filename
 from unstructured_inference.config import inference_config
 from unstructured_inference.constants import FULL_PAGE_REGION_THRESHOLD, IsExtracted
 from unstructured_inference.inference.elements import Rectangle

unstructured/partition/pdf_image/pdfminer_utils.py

@@ -4,22 +4,22 @@
 import zlib
 from typing import BinaryIO, List, Mapping, Optional, Tuple, Union
 
-from pdfminer import settings as pdfminer_settings
-from pdfminer.cmapdb import CMap, CMapDB
-from pdfminer.converter import PDFPageAggregator
-from pdfminer.layout import LAParams, LTChar, LTContainer, LTImage, LTItem, LTTextLine
-from pdfminer.pdffont import PDFCIDFont, PDFFontError
-from pdfminer.pdfinterp import LITERAL_FONT, PDFPageInterpreter, PDFResourceManager
-from pdfminer.pdfpage import PDFPage
-from pdfminer.pdftypes import (
+from core_pdfminer_six import settings as pdfminer_settings
+from core_pdfminer_six.cmapdb import CMap, CMapDB
+from core_pdfminer_six.converter import PDFPageAggregator
+from core_pdfminer_six.layout import LAParams, LTChar, LTContainer, LTImage, LTItem, LTTextLine
+from core_pdfminer_six.pdffont import PDFCIDFont, PDFFontError
+from core_pdfminer_six.pdfinterp import LITERAL_FONT, PDFPageInterpreter, PDFResourceManager
+from core_pdfminer_six.pdfpage import PDFPage
+from core_pdfminer_six.pdftypes import (
     LITERALS_ASCII85_DECODE,
     LITERALS_ASCIIHEX_DECODE,
     LITERALS_FLATE_DECODE,
     PDFStream,
     resolve1,
 )
-from pdfminer.psexceptions import PSSyntaxError
-from pdfminer.psparser import literal_name
+from core_pdfminer_six.psexceptions import PSSyntaxError
+from core_pdfminer_six.psparser import literal_name
 from pydantic import BaseModel
 
 from unstructured.logger import logger
@@ -185,11 +185,11 @@ def _decode_pdfstream_with_limit(stream: PDFStream, max_decoded_bytes: int) -> O
                 return None
             data = result
         elif filt in LITERALS_ASCII85_DECODE:
-            from pdfminer.ascii85 import ascii85decode
+            from core_pdfminer_six.ascii85 import ascii85decode
 
             data = ascii85decode(data)
         elif filt in LITERALS_ASCIIHEX_DECODE:
-            from pdfminer.ascii85 import asciihexdecode
+            from core_pdfminer_six.ascii85 import asciihexdecode
 
             data = asciihexdecode(data)
         else: