Skip to content

Commit 32b5932

Browse files
committed
Reject page_size < 1 and csv/tsv records_format in get_datastore_records
1 parent 6832186 commit 32b5932

3 files changed

Lines changed: 14 additions & 2 deletions

File tree

‎documentation/index.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1008,7 +1008,8 @@ and `include_total`, and passes any other keyword arguments to CKAN:
10081008
records = result["records"]
10091009

10101010
**get_datastore_records** takes the same query parameters but pages through
1011-
all matching rows, yielding each record (`page_size` defaults to 10000):
1011+
all matching rows, yielding each record (`page_size` defaults to 10000).
1012+
`records_format` csv and tsv are not supported; use `search_datastore` for those:
10121013

10131014
for record in resource.get_datastore_records(filters={"code": "A"}):
10141015
...

‎src/hdx/data/resource.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -930,11 +930,18 @@ def get_datastore_records(
930930
sort: Comma separated field names with optional asc/desc eg. "code desc". Defaults to None.
931931
distinct: Whether to return only distinct rows. Defaults to False.
932932
page_size: Number of rows to request per call. Defaults to 10000.
933-
**kwargs: Other parameters to pass to datastore_search eg. plain, language.
933+
**kwargs: Other parameters to pass to datastore_search eg. plain, language. records_format csv and tsv are not supported.
934934
935935
Returns:
936936
Iterator of record dictionaries
937937
"""
938+
if page_size < 1:
939+
raise HDXError("page_size must be at least 1!")
940+
records_format = kwargs.get("records_format")
941+
if records_format in ("csv", "tsv"):
942+
raise HDXError(
943+
f"records_format {records_format} is not supported! Use search_datastore instead."
944+
)
938945
offset = 0
939946
while True:
940947
result = self.search_datastore(

‎tests/hdx/data/test_resource.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1219,6 +1219,10 @@ def test_datastore_queries(self, configuration, post_datastore_query):
12191219
records = list(resource.get_datastore_records(page_size=3))
12201220
assert len(records) == 3
12211221
assert TestResource.datastore_request["offset"] == 3
1222+
with pytest.raises(HDXError):
1223+
list(resource.get_datastore_records(page_size=0))
1224+
with pytest.raises(HDXError):
1225+
list(resource.get_datastore_records(records_format="csv"))
12221226

12231227
assert resource.get_datastore_fields() == [
12241228
{"id": "code", "type": "text"},

0 commit comments

Comments
 (0)