Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .semversioner/next-release/patch-20260710221345787890.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"type": "patch",
"description": "Fix JSONL input loader to skip blank lines and ignore invalid JSON rows"
}
25 changes: 24 additions & 1 deletion packages/graphrag-input/graphrag_input/jsonl.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,28 @@ async def read_file(self, path: str) -> list[TextDocument]:
- output - list with a TextDocument for each row in the file.
"""
text = await self._storage.get(path, encoding=self._encoding)
rows = [json.loads(line) for line in text.splitlines()]
rows: list[dict] = []
Comment thread
natoverse marked this conversation as resolved.
for line_number, line in enumerate(text.splitlines(), start=1):
if not line.strip():
continue

try:
parsed_row = json.loads(line)
except json.JSONDecodeError:
logger.warning(
"Skipping malformed JSONL row in %s at line %s",
path,
line_number,
)
continue
Comment thread
natoverse marked this conversation as resolved.

if isinstance(parsed_row, dict):
rows.append(parsed_row)
else:
logger.warning(
"Skipping non-object JSONL row in %s at line %s",
path,
line_number,
)

return await self.process_data_columns(rows, path)
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{ "title": "Hello", "text": "Hi how are you today?"}

not json
{ "title": "Goodbye", "text": "I'm outta here"}

["not", "an", "object"]
{ "title": "Adios", "text": "See you later"}
17 changes: 17 additions & 0 deletions tests/unit/indexing/input/test_jsonl_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,20 @@ async def test_jsonl_loader_one_file_with_title():
documents = await reader.read_files()
assert len(documents) == 3
Comment thread
natoverse marked this conversation as resolved.
assert documents[0].title == "Hello"


async def test_jsonl_loader_skips_blank_and_invalid_rows():
config = InputConfig(
type=InputType.JsonLines,
title_column="title",
)
storage = create_storage(
StorageConfig(
base_dir="tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines",
)
)
reader = create_input_reader(config, storage)
documents = await reader.read_files()

assert len(documents) == 3
assert [document.title for document in documents] == ["Hello", "Goodbye", "Adios"]
Loading