fix: preserve table row boundaries

CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.23.3
+
+### Fixes
+
+- **Preserve row boundaries in table text**: `HtmlTable.text` now joins cells within each
+  row while separating rows with blank lines, restoring stable row boundaries for `Table`
+  elements produced from table HTML such as XLSX, CSV, and TSV partitions.
+
 ## 0.23.2
 
 ### Enhancements

test_unstructured/common/test_html_table.py

@@ -219,7 +219,28 @@ def it_provides_access_to_the_clear_concatenated_text_of_the_table(self):
             "  <tr><td/><td> m n op\n</td><td/></tr>"
             "</table>"
         )
-        assert html_table.text == "a b c def gh i jk l m n op"
+        assert html_table.text == "a b c def\n\ngh i jk l\n\nm n op"
+
+    def and_it_suppresses_blank_rows_when_constructing_text(self):
+        html_table = HtmlTable.from_html_text(
+            "<table>"
+            "  <tr><td>alpha</td><td>beta</td></tr>"
+            "  <tr><td/><td>  </td></tr>"
+            "  <tr><td>gamma</td><td>delta</td></tr>"
+            "</table>"
+        )
+        assert html_table.text == "alpha beta\n\ngamma delta"
+
+    def it_provides_text_for_directly_instantiated_sectioned_tables(self):
+        table = fragment_fromstring(
+            "<table>"
+            "  <thead><tr><th>header</th></tr></thead>"
+            "  <tbody><tr><td>body</td></tr></tbody>"
+            "  <tfoot><tr><td>footer</td></tr></tfoot>"
+            "</table>"
+        )
+
+        assert HtmlTable(table).text == "header\n\nbody\n\nfooter"
 
 
 class DescribeHtmlRow:
@@ -260,6 +281,11 @@ def it_can_iterate_the_texts_of_the_cells_in_the_row(self):
         with pytest.raises(StopIteration):
             next(text_iter)
 
+    def it_provides_the_clear_concatenated_text_of_the_row(self):
+        row = HtmlRow(fragment_fromstring("<tr><td> a\n b </td><td/><td> c  d </td></tr>"))
+
+        assert row.text == "a b c d"
+
     def and_it_includes_descendant_inline_text_in_cell_texts(self):
         row = HtmlRow(
             fragment_fromstring(

test_unstructured/partition/test_constants.py

@@ -55,11 +55,27 @@
 
 EXPECTED_TEXT_XLSX = "Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13"
 
+EXPECTED_TEXT_WITH_ROW_BOUNDARIES = (
+    "Stanley Cups\n\n"
+    "Team Location Stanley Cups\n\n"
+    "Blues STL 1\n\n"
+    "Flyers PHI 2\n\n"
+    "Maple Leafs TOR 13"
+)
+
+EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES = (
+    "Team Location Stanley Cups\n\nBlues STL 1\n\nFlyers PHI 2\n\nMaple Leafs TOR 13"
+)
+
 EXPECTED_TEXT_WITH_EMOJI = (
     "Stanley Cups "
     "Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13 👨\\U+1F3FB🔧 TOR 15"
 )
 
+EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES = (
+    EXPECTED_TEXT_WITH_ROW_BOUNDARIES + "\n\n👨\\U+1F3FB🔧 TOR 15"
+)
+
 EXPECTED_TEXT_SEMICOLON_DELIMITER = (
     "Year Month Revenue Costs 2022 1 123 -123 2023 2 143,1 -814,38 2024 3 215,32 -11,08"
 )

test_unstructured/partition/test_csv.py

@@ -16,7 +16,7 @@
     EXPECTED_TEXT_SEMICOLON_DELIMITER,
     EXPECTED_TEXT_WITH_EMOJI,
     EXPECTED_TEXT_WITH_LINE_DELIMITER,
-    EXPECTED_TEXT_XLSX,
+    EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES,
 )
 from test_unstructured.unit_utils import (
     FixtureRequest,
@@ -207,7 +207,9 @@ def test_partition_csv_header():
     )
 
     table = elements[0]
-    assert table.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX
+    assert table.text == (
+        "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES
+    )
     assert table.metadata.text_as_html is not None
 
 

test_unstructured/partition/test_tsv.py

@@ -8,9 +8,9 @@
 from test_unstructured.partition.test_constants import (
     EXPECTED_TABLE,
     EXPECTED_TABLE_WITH_EMOJI,
-    EXPECTED_TEXT,
-    EXPECTED_TEXT_WITH_EMOJI,
-    EXPECTED_TEXT_XLSX,
+    EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES,
+    EXPECTED_TEXT_WITH_ROW_BOUNDARIES,
+    EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES,
 )
 from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path
 from unstructured.chunking.title import chunk_by_title
@@ -23,8 +23,12 @@
 @pytest.mark.parametrize(
     ("filename", "expected_text", "expected_table"),
     [
-        ("stanley-cups.tsv", EXPECTED_TEXT, EXPECTED_TABLE),
-        ("stanley-cups-with-emoji.tsv", EXPECTED_TEXT_WITH_EMOJI, EXPECTED_TABLE_WITH_EMOJI),
+        ("stanley-cups.tsv", EXPECTED_TEXT_WITH_ROW_BOUNDARIES, EXPECTED_TABLE),
+        (
+            "stanley-cups-with-emoji.tsv",
+            EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES,
+            EXPECTED_TABLE_WITH_EMOJI,
+        ),
     ],
 )
 def test_partition_tsv_from_filename(filename: str, expected_text: str, expected_table: str):
@@ -42,15 +46,19 @@ def test_partition_tsv_from_filename_with_metadata_filename():
         example_doc_path("stanley-cups.tsv"), metadata_filename="test", include_header=False
     )
 
-    assert elements[0].text == EXPECTED_TEXT
+    assert elements[0].text == EXPECTED_TEXT_WITH_ROW_BOUNDARIES
     assert all(e.metadata.filename == "test" for e in elements)
 
 
 @pytest.mark.parametrize(
     ("filename", "expected_text", "expected_table"),
     [
-        ("stanley-cups.tsv", EXPECTED_TEXT, EXPECTED_TABLE),
-        ("stanley-cups-with-emoji.tsv", EXPECTED_TEXT_WITH_EMOJI, EXPECTED_TABLE_WITH_EMOJI),
+        ("stanley-cups.tsv", EXPECTED_TEXT_WITH_ROW_BOUNDARIES, EXPECTED_TABLE),
+        (
+            "stanley-cups-with-emoji.tsv",
+            EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES,
+            EXPECTED_TABLE_WITH_EMOJI,
+        ),
     ],
 )
 def test_partition_tsv_from_file(filename: str, expected_text: str, expected_table: str):
@@ -69,7 +77,7 @@ def test_partition_tsv_from_file_with_metadata_filename():
     with open(example_doc_path("stanley-cups.tsv"), "rb") as f:
         elements = partition_tsv(file=f, metadata_filename="test", include_header=False)
 
-    assert elements[0].text == EXPECTED_TEXT
+    assert elements[0].text == EXPECTED_TEXT_WITH_ROW_BOUNDARIES
     assert all(element.metadata.filename == "test" for element in elements)
 
 
@@ -140,7 +148,9 @@ def test_partition_tsv_header():
     )
 
     table = elements[0]
-    assert table.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX
+    assert table.text == (
+        "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES
+    )
     assert table.metadata.text_as_html is not None
     assert "<table>" in table.metadata.text_as_html
 

test_unstructured/partition/test_xlsx.py

@@ -17,6 +17,7 @@
 from test_unstructured.partition.test_constants import (
     EXPECTED_TABLE_XLSX,
     EXPECTED_TEXT_XLSX,
+    EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES,
     EXPECTED_TITLE,
 )
 from test_unstructured.unit_utils import (
@@ -113,7 +114,9 @@ def test_partition_xlsx_from_filename_with_header():
     assert len(elements) == 2
     assert all(isinstance(e, Table) for e in elements)
     e = elements[0]
-    assert e.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX
+    assert e.text == (
+        "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES
+    )
     assert e.metadata.text_as_html is not None
 
 
@@ -165,7 +168,9 @@ def test_partition_xlsx_from_file_with_header():
     assert len(elements) == 2
     assert all(isinstance(e, Table) for e in elements)
     e = elements[0]
-    assert e.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX
+    assert e.text == (
+        "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES
+    )
     assert e.metadata.text_as_html is not None
 
 
@@ -236,32 +241,32 @@ def test_partition_xlsx_metadata_language_from_filename():
 
 def test_partition_xlsx_subtables():
     assert partition_xlsx("example-docs/xlsx-subtable-cases.xlsx") == [
-        Table("a b c d e"),
+        Table("a b\n\nc d e"),
         ListItem("f"),
         Title("a"),
-        Table("b c d e"),
+        Table("b c\n\nd e"),
         Title("a"),
         Title("b"),
-        Table("c d e f"),
-        Table("a b c d"),
+        Table("c d\n\ne f"),
+        Table("a b\n\nc d"),
         ListItem("2. e"),
-        Table("a b c d"),
+        Table("a b\n\nc d"),
         Title("e"),
         Title("f"),
         Title("a"),
-        Table("b c d e"),
+        Table("b c\n\nd e"),
         Title("f"),
         Title("a"),
         Title("b"),
-        Table("c d e f"),
+        Table("c d\n\ne f"),
         Title("g"),
         Title("a"),
-        Table("b c d e"),
+        Table("b c\n\nd e"),
         Title("f"),
         Title("g"),
         Title("a"),
         Title("b"),
-        Table("c d e f"),
+        Table("c d\n\ne f"),
         Title("g"),
         Title("h"),
         Table("a b c"),
@@ -309,11 +314,18 @@ def test_partition_xlsx_with_find_subtables_False_emits_one_Table_element_per_wo
     elements = partition_xlsx("example-docs/stanley-cups.xlsx", find_subtable=False)
     assert elements == [
         Table(
-            "Stanley Cups Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13"
+            "Stanley Cups\n\n"
+            "Team Location Stanley Cups\n\n"
+            "Blues STL 1\n\n"
+            "Flyers PHI 2\n\n"
+            "Maple Leafs TOR 13"
         ),
         Table(
-            "Stanley Cups Since 67 Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple"
-            " Leafs TOR 0"
+            "Stanley Cups Since 67\n\n"
+            "Team Location Stanley Cups\n\n"
+            "Blues STL 1\n\n"
+            "Flyers PHI 2\n\n"
+            "Maple Leafs TOR 0"
         ),
     ]
 
@@ -324,11 +336,18 @@ def test_partition_xlsx_with_find_subtables_False_and_infer_table_structure_Fals
     )
     assert elements == [
         Table(
-            "Stanley Cups Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13"
+            "Stanley Cups\n\n"
+            "Team Location Stanley Cups\n\n"
+            "Blues STL 1\n\n"
+            "Flyers PHI 2\n\n"
+            "Maple Leafs TOR 13"
         ),
         Table(
-            "Stanley Cups Since 67 Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple"
-            " Leafs TOR 0"
+            "Stanley Cups Since 67\n\n"
+            "Team Location Stanley Cups\n\n"
+            "Blues STL 1\n\n"
+            "Flyers PHI 2\n\n"
+            "Maple Leafs TOR 0"
         ),
     ]
     assert all(e.metadata.text_as_html is None for e in elements)

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.23.2"  # pragma: no cover
+__version__ = "0.23.3"  # pragma: no cover

unstructured/common/html_table.py

@@ -133,17 +133,17 @@ def html(self) -> str:
         return etree.tostring(self._table, encoding=str)
 
     def iter_rows(self) -> Iterator[HtmlRow]:
-        rows = cast("list[HtmlElement]", self._table.xpath("./tr"))
+        rows = cast(
+            "list[HtmlElement]", self._table.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr")
+        )
         for idx, tr in enumerate(rows):
             source_html = self._source_row_htmls[idx] if idx < len(self._source_row_htmls) else None
             yield HtmlRow(tr, is_header=(idx in self._header_row_idxs), source_html=source_html)
 
     @cached_property
     def text(self) -> str:
-        """The clean, concatenated, text for this table."""
-        table_text = " ".join(self._table.itertext())
-        # -- blank cells will introduce extra whitespace, so normalize after accumulating --
-        return " ".join(table_text.split())
+        """The clean text for this table, preserving row boundaries."""
+        return "\n\n".join(row_text for row in self.iter_rows() if (row_text := row.text))
 
 
 class HtmlRow:
@@ -184,6 +184,11 @@ def iter_cell_texts(self) -> Iterator[str]:
                 continue
             yield text
 
+    @cached_property
+    def text(self) -> str:
+        """The clean, concatenated text for this row."""
+        return " ".join(self.iter_cell_texts())
+
     @cached_property
     def text_len(self) -> int:
         """Length of the normalized text, as it would appear in `element.text`."""