From 701b2caa02600aa8ded3c2bb14aff4d169e3a723 Mon Sep 17 00:00:00 2001 From: ByteWise <156506452+2830500285@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:11:36 -0700 Subject: [PATCH 1/2] fix: preserve table row boundaries --- CHANGELOG.md | 8 +++ test_unstructured/common/test_html_table.py | 17 +++++- test_unstructured/partition/test_constants.py | 16 ++++++ test_unstructured/partition/test_csv.py | 6 ++- test_unstructured/partition/test_tsv.py | 30 +++++++---- test_unstructured/partition/test_xlsx.py | 53 +++++++++++++------ unstructured/__version__.py | 2 +- unstructured/common/html_table.py | 11 ++-- 8 files changed, 108 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ef38d9773..1abe504a67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## 0.23.3 + +### Fixes + +- **Preserve row boundaries in table text**: `HtmlTable.text` now joins cells within each + row while separating rows with blank lines, restoring stable row boundaries for `Table` + elements produced from table HTML such as XLSX, CSV, and TSV partitions. + ## 0.23.2 ### Enhancements diff --git a/test_unstructured/common/test_html_table.py b/test_unstructured/common/test_html_table.py index 50b37dc82d..8e17eb45ed 100644 --- a/test_unstructured/common/test_html_table.py +++ b/test_unstructured/common/test_html_table.py @@ -219,7 +219,17 @@ def it_provides_access_to_the_clear_concatenated_text_of_the_table(self): " m n op\n" "" ) - assert html_table.text == "a b c def gh i jk l m n op" + assert html_table.text == "a b c def\n\ngh i jk l\n\nm n op" + + def and_it_suppresses_blank_rows_when_constructing_text(self): + html_table = HtmlTable.from_html_text( + "" + " " + " " + " " + "
alphabeta
gammadelta
" + ) + assert html_table.text == "alpha beta\n\ngamma delta" class DescribeHtmlRow: @@ -260,6 +270,11 @@ def it_can_iterate_the_texts_of_the_cells_in_the_row(self): with pytest.raises(StopIteration): next(text_iter) + def it_provides_the_clear_concatenated_text_of_the_row(self): + row = HtmlRow(fragment_fromstring(" a\n b c d ")) + + assert row.text == "a b c d" + def and_it_includes_descendant_inline_text_in_cell_texts(self): row = HtmlRow( fragment_fromstring( diff --git a/test_unstructured/partition/test_constants.py b/test_unstructured/partition/test_constants.py index fb327cf481..5a5fafe4e4 100644 --- a/test_unstructured/partition/test_constants.py +++ b/test_unstructured/partition/test_constants.py @@ -55,11 +55,27 @@ EXPECTED_TEXT_XLSX = "Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13" +EXPECTED_TEXT_WITH_ROW_BOUNDARIES = ( + "Stanley Cups\n\n" + "Team Location Stanley Cups\n\n" + "Blues STL 1\n\n" + "Flyers PHI 2\n\n" + "Maple Leafs TOR 13" +) + +EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES = ( + "Team Location Stanley Cups\n\nBlues STL 1\n\nFlyers PHI 2\n\nMaple Leafs TOR 13" +) + EXPECTED_TEXT_WITH_EMOJI = ( "Stanley Cups " "Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13 👨\\U+1F3FB🔧 TOR 15" ) +EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES = ( + EXPECTED_TEXT_WITH_ROW_BOUNDARIES + "\n\n👨\\U+1F3FB🔧 TOR 15" +) + EXPECTED_TEXT_SEMICOLON_DELIMITER = ( "Year Month Revenue Costs 2022 1 123 -123 2023 2 143,1 -814,38 2024 3 215,32 -11,08" ) diff --git a/test_unstructured/partition/test_csv.py b/test_unstructured/partition/test_csv.py index b648134d08..eebb52ca8a 100644 --- a/test_unstructured/partition/test_csv.py +++ b/test_unstructured/partition/test_csv.py @@ -16,7 +16,7 @@ EXPECTED_TEXT_SEMICOLON_DELIMITER, EXPECTED_TEXT_WITH_EMOJI, EXPECTED_TEXT_WITH_LINE_DELIMITER, - EXPECTED_TEXT_XLSX, + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES, ) from test_unstructured.unit_utils import ( FixtureRequest, @@ -207,7 +207,9 @@ def test_partition_csv_header(): ) table = elements[0] - assert table.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX + assert table.text == ( + "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES + ) assert table.metadata.text_as_html is not None diff --git a/test_unstructured/partition/test_tsv.py b/test_unstructured/partition/test_tsv.py index 5276e6006d..5a029aef27 100644 --- a/test_unstructured/partition/test_tsv.py +++ b/test_unstructured/partition/test_tsv.py @@ -8,9 +8,9 @@ from test_unstructured.partition.test_constants import ( EXPECTED_TABLE, EXPECTED_TABLE_WITH_EMOJI, - EXPECTED_TEXT, - EXPECTED_TEXT_WITH_EMOJI, - EXPECTED_TEXT_XLSX, + EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES, + EXPECTED_TEXT_WITH_ROW_BOUNDARIES, + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES, ) from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path from unstructured.chunking.title import chunk_by_title @@ -23,8 +23,12 @@ @pytest.mark.parametrize( ("filename", "expected_text", "expected_table"), [ - ("stanley-cups.tsv", EXPECTED_TEXT, EXPECTED_TABLE), - ("stanley-cups-with-emoji.tsv", EXPECTED_TEXT_WITH_EMOJI, EXPECTED_TABLE_WITH_EMOJI), + ("stanley-cups.tsv", EXPECTED_TEXT_WITH_ROW_BOUNDARIES, EXPECTED_TABLE), + ( + "stanley-cups-with-emoji.tsv", + EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES, + EXPECTED_TABLE_WITH_EMOJI, + ), ], ) def test_partition_tsv_from_filename(filename: str, expected_text: str, expected_table: str): @@ -42,15 +46,19 @@ def test_partition_tsv_from_filename_with_metadata_filename(): example_doc_path("stanley-cups.tsv"), metadata_filename="test", include_header=False ) - assert elements[0].text == EXPECTED_TEXT + assert elements[0].text == EXPECTED_TEXT_WITH_ROW_BOUNDARIES assert all(e.metadata.filename == "test" for e in elements) @pytest.mark.parametrize( ("filename", "expected_text", "expected_table"), [ - ("stanley-cups.tsv", EXPECTED_TEXT, EXPECTED_TABLE), - ("stanley-cups-with-emoji.tsv", EXPECTED_TEXT_WITH_EMOJI, EXPECTED_TABLE_WITH_EMOJI), + ("stanley-cups.tsv", EXPECTED_TEXT_WITH_ROW_BOUNDARIES, EXPECTED_TABLE), + ( + "stanley-cups-with-emoji.tsv", + EXPECTED_TEXT_WITH_EMOJI_ROW_BOUNDARIES, + EXPECTED_TABLE_WITH_EMOJI, + ), ], ) def test_partition_tsv_from_file(filename: str, expected_text: str, expected_table: str): @@ -69,7 +77,7 @@ def test_partition_tsv_from_file_with_metadata_filename(): with open(example_doc_path("stanley-cups.tsv"), "rb") as f: elements = partition_tsv(file=f, metadata_filename="test", include_header=False) - assert elements[0].text == EXPECTED_TEXT + assert elements[0].text == EXPECTED_TEXT_WITH_ROW_BOUNDARIES assert all(element.metadata.filename == "test" for element in elements) @@ -140,7 +148,9 @@ def test_partition_tsv_header(): ) table = elements[0] - assert table.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX + assert table.text == ( + "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES + ) assert table.metadata.text_as_html is not None assert "" in table.metadata.text_as_html diff --git a/test_unstructured/partition/test_xlsx.py b/test_unstructured/partition/test_xlsx.py index 3baadce7ff..37f4bf8b62 100644 --- a/test_unstructured/partition/test_xlsx.py +++ b/test_unstructured/partition/test_xlsx.py @@ -17,6 +17,7 @@ from test_unstructured.partition.test_constants import ( EXPECTED_TABLE_XLSX, EXPECTED_TEXT_XLSX, + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES, EXPECTED_TITLE, ) from test_unstructured.unit_utils import ( @@ -113,7 +114,9 @@ def test_partition_xlsx_from_filename_with_header(): assert len(elements) == 2 assert all(isinstance(e, Table) for e in elements) e = elements[0] - assert e.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX + assert e.text == ( + "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES + ) assert e.metadata.text_as_html is not None @@ -165,7 +168,9 @@ def test_partition_xlsx_from_file_with_header(): assert len(elements) == 2 assert all(isinstance(e, Table) for e in elements) e = elements[0] - assert e.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX + assert e.text == ( + "Stanley Cups Unnamed: 1 Unnamed: 2\n\n" + EXPECTED_TEXT_XLSX_WITH_ROW_BOUNDARIES + ) assert e.metadata.text_as_html is not None @@ -236,32 +241,32 @@ def test_partition_xlsx_metadata_language_from_filename(): def test_partition_xlsx_subtables(): assert partition_xlsx("example-docs/xlsx-subtable-cases.xlsx") == [ - Table("a b c d e"), + Table("a b\n\nc d e"), ListItem("f"), Title("a"), - Table("b c d e"), + Table("b c\n\nd e"), Title("a"), Title("b"), - Table("c d e f"), - Table("a b c d"), + Table("c d\n\ne f"), + Table("a b\n\nc d"), ListItem("2. e"), - Table("a b c d"), + Table("a b\n\nc d"), Title("e"), Title("f"), Title("a"), - Table("b c d e"), + Table("b c\n\nd e"), Title("f"), Title("a"), Title("b"), - Table("c d e f"), + Table("c d\n\ne f"), Title("g"), Title("a"), - Table("b c d e"), + Table("b c\n\nd e"), Title("f"), Title("g"), Title("a"), Title("b"), - Table("c d e f"), + Table("c d\n\ne f"), Title("g"), Title("h"), Table("a b c"), @@ -309,11 +314,18 @@ def test_partition_xlsx_with_find_subtables_False_emits_one_Table_element_per_wo elements = partition_xlsx("example-docs/stanley-cups.xlsx", find_subtable=False) assert elements == [ Table( - "Stanley Cups Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13" + "Stanley Cups\n\n" + "Team Location Stanley Cups\n\n" + "Blues STL 1\n\n" + "Flyers PHI 2\n\n" + "Maple Leafs TOR 13" ), Table( - "Stanley Cups Since 67 Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple" - " Leafs TOR 0" + "Stanley Cups Since 67\n\n" + "Team Location Stanley Cups\n\n" + "Blues STL 1\n\n" + "Flyers PHI 2\n\n" + "Maple Leafs TOR 0" ), ] @@ -324,11 +336,18 @@ def test_partition_xlsx_with_find_subtables_False_and_infer_table_structure_Fals ) assert elements == [ Table( - "Stanley Cups Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13" + "Stanley Cups\n\n" + "Team Location Stanley Cups\n\n" + "Blues STL 1\n\n" + "Flyers PHI 2\n\n" + "Maple Leafs TOR 13" ), Table( - "Stanley Cups Since 67 Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple" - " Leafs TOR 0" + "Stanley Cups Since 67\n\n" + "Team Location Stanley Cups\n\n" + "Blues STL 1\n\n" + "Flyers PHI 2\n\n" + "Maple Leafs TOR 0" ), ] assert all(e.metadata.text_as_html is None for e in elements) diff --git a/unstructured/__version__.py b/unstructured/__version__.py index c59d9f9470..697f1b365b 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.23.2" # pragma: no cover +__version__ = "0.23.3" # pragma: no cover diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index b0c366b8fb..166af55087 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -140,10 +140,8 @@ def iter_rows(self) -> Iterator[HtmlRow]: @cached_property def text(self) -> str: - """The clean, concatenated, text for this table.""" - table_text = " ".join(self._table.itertext()) - # -- blank cells will introduce extra whitespace, so normalize after accumulating -- - return " ".join(table_text.split()) + """The clean text for this table, preserving row boundaries.""" + return "\n\n".join(row_text for row in self.iter_rows() if (row_text := row.text)) class HtmlRow: @@ -184,6 +182,11 @@ def iter_cell_texts(self) -> Iterator[str]: continue yield text + @cached_property + def text(self) -> str: + """The clean, concatenated text for this row.""" + return " ".join(self.iter_cell_texts()) + @cached_property def text_len(self) -> int: """Length of the normalized text, as it would appear in `element.text`.""" From afc45b60fad7bd10b4b0fac2b2432144c9c488fe Mon Sep 17 00:00:00 2001 From: ByteWise <156506452+2830500285@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:56:45 -0700 Subject: [PATCH 2/2] fix: support sectioned html table rows --- test_unstructured/common/test_html_table.py | 11 +++++++++++ unstructured/common/html_table.py | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/test_unstructured/common/test_html_table.py b/test_unstructured/common/test_html_table.py index 8e17eb45ed..a14c053578 100644 --- a/test_unstructured/common/test_html_table.py +++ b/test_unstructured/common/test_html_table.py @@ -231,6 +231,17 @@ def and_it_suppresses_blank_rows_when_constructing_text(self): ) assert html_table.text == "alpha beta\n\ngamma delta" + def it_provides_text_for_directly_instantiated_sectioned_tables(self): + table = fragment_fromstring( + "
" + " " + " " + " " + "
header
body
footer
" + ) + + assert HtmlTable(table).text == "header\n\nbody\n\nfooter" + class DescribeHtmlRow: """Unit-test suite for `unstructured.common.html_table.HtmlRow`.""" diff --git a/unstructured/common/html_table.py b/unstructured/common/html_table.py index 166af55087..dea99822e4 100644 --- a/unstructured/common/html_table.py +++ b/unstructured/common/html_table.py @@ -133,7 +133,9 @@ def html(self) -> str: return etree.tostring(self._table, encoding=str) def iter_rows(self) -> Iterator[HtmlRow]: - rows = cast("list[HtmlElement]", self._table.xpath("./tr")) + rows = cast( + "list[HtmlElement]", self._table.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr") + ) for idx, tr in enumerate(rows): source_html = self._source_row_htmls[idx] if idx < len(self._source_row_htmls) else None yield HtmlRow(tr, is_header=(idx in self._header_row_idxs), source_html=source_html)