Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 2 additions & 11 deletions protarrow/cast_to_proto.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,19 +126,18 @@ def _cast_array(

# TODO: remove when https://github.com/apache/arrow/issues/40750 is fixed
# and library is pinned to pyarrow>=17.0.0
offsets = maybe_copy_offsets(array.offsets)
keys = _cast_array(keys, key_field, config)
values = _cast_array(values, value_field, config)

if config.map_as_list:
return _map_as_list_from_arrays(
offsets=offsets,
offsets=array.offsets,
keys=keys,
values=values,
config=config,
)
else:
return pa.MapArray.from_arrays(offsets, keys, values).cast(
return pa.MapArray.from_arrays(array.offsets, keys, values).cast(
pa.map_(
keys.type,
pa.field(
Expand Down Expand Up @@ -251,11 +250,3 @@ def cast_table(
],
proto_schema,
)


def maybe_copy_offsets(offsets: pa.Array) -> pa.Array:
if offsets.offset != 0:
# TODO: this it not efficient, find a way to copy in arrow
return pa.array(offsets.to_pylist(), offsets.type)
else:
return offsets
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ classifiers = [
"Programming Language :: Python :: 3.14",
]
dynamic = [ "version" ]
dependencies = [ "googleapis-common-protos", "protobuf", "pyarrow" ]
dependencies = [ "googleapis-common-protos", "protobuf", "pyarrow>=17" ]
urls."Bug Tracker" = "https://github.com/tradewelltech/protarrow/issues"
urls.Changelog = "https://github.com/tradewelltech/protarrow/blob/main/CHANGELOG.md"
documentation = "https://protarrow.readthedocs.io/en/latest/"
Expand Down
15 changes: 2 additions & 13 deletions tests/test_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
cast_table,
get_arrow_default_value,
get_casted_array,
maybe_copy_offsets,
)
from protarrow.common import M, ProtarrowConfig, offset_values_array
from protarrow.message_extractor import MessageExtractor
Expand Down Expand Up @@ -799,6 +798,8 @@ def test_cast_map_offset():
)
assert _cast_array(from_array, field, ProtarrowConfig()) == to_array

# Test it works with offset array / slice views.
# https://github.com/apache/arrow/issues/40750
assert _cast_array(from_array[1:], field, ProtarrowConfig()) == to_array[1:]


Expand All @@ -825,18 +826,6 @@ def test_cast_list_offset():
assert _cast_array(from_array[1:], field, ProtarrowConfig()) == to_array[1:]


def test_maybe_copy_offsets():
array = pa.array([1, 2, 3], pa.int32())
assert maybe_copy_offsets(array) is array # no copy
view = array[1:]
copy = maybe_copy_offsets(array[1:])
assert copy == view
assert copy is not view # copy
assert copy.offset == 0
# Sadly pa.array is highly optimized
assert pa.array(array) is array


def test_map_cast():
array = pa.array(
[
Expand Down
19 changes: 10 additions & 9 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading