fix(partition): avoid copying spooled uploads

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.25.2
+
+### Fixes
+
+- **Avoid copying spooled uploads into memory**: DOCX, PPTX, and shared partitioning paths now reuse `SpooledTemporaryFile` inputs directly on supported Python versions instead of copying their complete contents into `BytesIO`. Large uploads remain disk-backed, avoiding an additional document-sized heap allocation without changing the partition API.
+
 ## 0.25.1
 
 ### Fixes

test_unstructured/partition/common/test_common.py

@@ -1,5 +1,6 @@
 import pathlib
 from multiprocessing import Pool
+from tempfile import SpooledTemporaryFile
 
 import numpy as np
 import pytest
@@ -29,6 +30,17 @@
 from unstructured.partition.common import common
 
 
+def test_spooled_to_bytes_io_if_needed_rewinds_without_copying():
+    with SpooledTemporaryFile(max_size=1, mode="w+b") as spooled_file:
+        spooled_file.write(b"sample content")
+
+        result = common.spooled_to_bytes_io_if_needed(spooled_file)
+
+        assert result is spooled_file
+        assert result.tell() == 0
+        assert result.read() == b"sample content"
+
+
 class MockPageLayout(layout.PageLayout):
     def __init__(self, number: int, image: Image.Image):
         self.number = number

test_unstructured/partition/test_docx.py

@@ -919,19 +919,16 @@ def it_uses_the_path_to_open_the_presentation_when_file_path_is_provided(
 
         assert opts._docx_file == "l/m/n.docx"
 
-    def and_it_uses_a_BytesIO_file_to_replaces_a_SpooledTemporaryFile_provided(
-        self, opts_args: dict[str, Any]
-    ):
+    def and_it_uses_a_SpooledTemporaryFile_directly(self, opts_args: dict[str, Any]):
         with tempfile.SpooledTemporaryFile() as spooled_temp_file:
             spooled_temp_file.write(b"abcdefg")
             opts_args["file"] = spooled_temp_file
             opts = DocxPartitionerOptions(**opts_args)
 
             docx_file = opts._docx_file
 
-            assert docx_file is not spooled_temp_file
-            assert isinstance(docx_file, io.BytesIO)
-            assert docx_file.getvalue() == b"abcdefg"
+            assert docx_file is spooled_temp_file
+            assert docx_file.read() == b"abcdefg"
 
     def and_it_uses_the_provided_file_directly_when_not_a_SpooledTemporaryFile(
         self, opts_args: dict[str, Any]

test_unstructured/partition/test_pptx.py

@@ -700,19 +700,16 @@ def it_uses_the_path_to_open_the_presentation_when_file_path_is_provided(
 
         assert opts.pptx_file == "l/m/n.pptx"
 
-    def and_it_uses_a_BytesIO_file_to_replaces_a_SpooledTemporaryFile_provided(
-        self, opts_args: dict[str, Any]
-    ):
+    def and_it_uses_a_SpooledTemporaryFile_directly(self, opts_args: dict[str, Any]):
         with tempfile.SpooledTemporaryFile() as spooled_temp_file:
             spooled_temp_file.write(b"abcdefg")
             opts_args["file"] = spooled_temp_file
             opts = PptxPartitionerOptions(**opts_args)
 
             pptx_file = opts.pptx_file
 
-            assert pptx_file is not spooled_temp_file
-            assert isinstance(pptx_file, io.BytesIO)
-            assert pptx_file.getvalue() == b"abcdefg"
+            assert pptx_file is spooled_temp_file
+            assert pptx_file.read() == b"abcdefg"
 
     def and_it_uses_the_provided_file_directly_when_not_a_SpooledTemporaryFile(
         self, opts_args: dict[str, Any]

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.25.1"  # pragma: no cover
+__version__ = "0.25.2"  # pragma: no cover

unstructured/partition/common/common.py

@@ -6,7 +6,7 @@
 from io import BufferedReader, BytesIO, TextIOWrapper
 from tempfile import SpooledTemporaryFile
 from time import sleep
-from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, cast
+from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar
 
 import emoji
 import psutil
@@ -350,19 +350,20 @@ def exactly_one(**kwargs: Any) -> None:
 _T = TypeVar("_T")
 
 
-def spooled_to_bytes_io_if_needed(file: _T | SpooledTemporaryFile[bytes]) -> _T | BytesIO:
-    """Convert `file` to `BytesIO` when it is a `SpooledTemporaryFile`.
+def spooled_to_bytes_io_if_needed(file: _T) -> _T:
+    """Rewind and return a `SpooledTemporaryFile` without copying its contents.
 
     Note that `file` does not need to be IO[bytes]. It can be `None` or `bytes` and this function
     will not complain.
 
-    In Python <3.11, `SpooledTemporaryFile` does not implement `.readable()` or `.seekable()` which
-    triggers an exception when the file is loaded by certain packages. In particular, the stdlib
-    `zipfile.Zipfile` raises on opening a `SpooledTemporaryFile` as does `Pandas.read_csv()`.
+    Python 3.11 and newer provide the complete buffered-I/O interface required by consumers such
+    as `zipfile.ZipFile` and `pandas.read_csv()`. Since those are the only Python versions this
+    package supports, converting the spool to `BytesIO` only adds a document-sized allocation.
+
+    The function name is retained for compatibility with existing call sites.
     """
     if isinstance(file, SpooledTemporaryFile):
         file.seek(0)
-        return BytesIO(cast(bytes, file.read()))
 
     # -- return `file` unchanged otherwise --
     return file

unstructured/partition/docx.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import io
 import itertools
 import logging
 import os
@@ -341,12 +340,8 @@ def _docx_file(self) -> str | IO[bytes]:
         if self._file_path:
             return self._file_path
 
-        # -- In Python <3.11 SpooledTemporaryFile does not implement ".seekable" which triggers an
-        # -- exception when Zipfile tries to open it. The docx format is a zip archive so we need
-        # -- to work around that bug here.
         if isinstance(self._file, tempfile.SpooledTemporaryFile):
             self._file.seek(0)
-            return io.BytesIO(self._file.read())
 
         assert self._file is not None  # -- assured by `._validate()` --
         return self._file

unstructured/partition/pptx.py

@@ -6,7 +6,6 @@
 
 from __future__ import annotations
 
-import io
 from functools import cached_property
 from tempfile import SpooledTemporaryFile
 from typing import IO, Any, Iterator, Protocol, Sequence
@@ -447,14 +446,9 @@ def pptx_file(self) -> str | IO[bytes]:
         if self._file_path:
             return self._file_path
 
-        # -- In Python <3.11 SpooledTemporaryFile does not implement ".seekable" which triggers an
-        # -- exception when Zipfile tries to open it. The pptx format is a zip archive so we need
-        # -- to work around that bug here.
-        if isinstance(self._file, SpooledTemporaryFile):
-            self._file.seek(0)
-            return io.BytesIO(self._file.read())
-
         if self._file:
+            if isinstance(self._file, SpooledTemporaryFile):
+                self._file.seek(0)
             return self._file
 
         raise ValueError(