Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kedro-datasets/kedro_datasets/ibis/file_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,15 +154,15 @@ def connection(self) -> BaseBackend:
return self._connection

def load(self) -> ir.Table:
load_path = self._get_load_path()
load_path = str(self._get_load_path())
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
load_path = str(self._get_load_path())
load_path = Path(self._get_load_path())

reader = getattr(self.connection, f"read_{self._file_format}")
return reader(load_path, table_name=self._table_name, **self._load_args)

def save(self, data: ir.Table) -> None:
save_path = self._get_save_path()
Path(save_path).parent.mkdir(parents=True, exist_ok=True)
Comment on lines 162 to 163
Copy link
Copy Markdown
Member

@deepyaman deepyaman Apr 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
save_path = self._get_save_path()
Path(save_path).parent.mkdir(parents=True, exist_ok=True)
save_path = Path(self._get_save_path())
save_path.parent.mkdir(parents=True, exist_ok=True)

For a temporary workaround, maybe the set of proposed suggestions is also fine?

writer = getattr(self.connection, f"to_{self._file_format}")
writer(data, save_path, **self._save_args)
writer(data, str(save_path), **self._save_args)
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
writer(data, str(save_path), **self._save_args)
writer(data, save_path, **self._save_args)


def _describe(self) -> dict[str, Any]:
return {
Expand Down
18 changes: 18 additions & 0 deletions kedro-datasets/tests/ibis/test_file_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ def filepath_csv(tmp_path):
return (tmp_path / "test.csv").as_posix()


@pytest.fixture
def filepath_parquet(tmp_path):
return (tmp_path / "test.parquet").as_posix()


@pytest.fixture
def database(tmp_path):
return (tmp_path / "file.db").as_posix()
Expand Down Expand Up @@ -65,6 +70,19 @@ def test_save_and_load(self, file_dataset, dummy_table):
reloaded = file_dataset.load()
assert_frame_equal(dummy_table.execute(), reloaded.execute())

def test_save_and_load_parquet(
self, filepath_parquet, connection_config, dummy_table
):
"""Parquet paths must be str; Ibis calls len() on the path arg and PurePosixPath has no __len__."""
ds = FileDataset(
filepath=filepath_parquet,
file_format="parquet",
connection=connection_config,
)
ds.save(dummy_table)
reloaded = ds.load()
assert_frame_equal(dummy_table.execute(), reloaded.execute())

@pytest.mark.parametrize("load_args", [{"filename": True}], indirect=True)
def test_load_extra_params(self, file_dataset, load_args, dummy_table):
"""Test overriding the default load arguments."""
Expand Down
Loading