Skip to content

Commit 49a894a

Browse files
committed
Page datastore records past HDX's row cap, warn on truncated SQL results, and document TDE
1 parent 6832186 commit 49a894a

3 files changed

Lines changed: 45 additions & 9 deletions

File tree

‎documentation/index.md‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -976,9 +976,13 @@ using the getter:
976976

977977
### Datastore
978978

979-
A resource's data can be loaded into the HDX datastore and queried. HDX
980-
requires an API key for all datastore actions and dumps, including reads of
981-
public resources. To create, update or delete the datastore for a resource:
979+
A resource's data can be loaded into the HDX datastore and queried. HDX calls this
980+
[Tabular Data Endpoints (TDE)](https://docs.humdata.org/build/hdx-apis/tabular-data-endpoints),
981+
and it is available only for resources with TDE enabled (their `datastore_active`
982+
field is true). Contact hdx@un.org to request TDE for your datasets. HDX requires an
983+
API key for all datastore actions and dumps, including reads of public resources,
984+
and limits each IP address to 60 requests per minute. To create, update or delete
985+
the datastore for a resource:
982986

983987
resource.create_datastore(
984988
[{"id": "code", "type": "text"}, {"id": "value", "type": "numeric"}],
@@ -1020,7 +1024,9 @@ internal `_id` field:
10201024

10211025
**search_datastore_sql** wraps CKAN's `datastore_search_sql`, running a
10221026
SELECT statement where tables are named by double quoted resource id and
1023-
returning the list of records:
1027+
returning the list of records. HDX returns at most 32,000 rows per query and
1028+
cuts off larger results, in which case a warning is logged (see
1029+
[paging in the TDE docs](https://docs.humdata.org/build/hdx-apis/tabular-data-endpoints/troubleshooting-and-error-handling#pagination)):
10241030

10251031
records = Resource.search_datastore_sql(
10261032
'SELECT code, SUM(value) AS total FROM "RESOURCE_ID" GROUP BY code'

‎src/hdx/data/resource.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -950,9 +950,11 @@ def get_datastore_records(
950950
)
951951
records = result["records"]
952952
yield from records
953-
if len(records) < page_size:
953+
# HDX caps limit at its maximum rows per call and returns the limit applied
954+
limit = result["limit"]
955+
if len(records) < limit:
954956
return
955-
offset += page_size
957+
offset += limit
956958

957959
def get_datastore_fields(self) -> list[dict]:
958960
"""Get the field definitions of the resource datastore excluding the
@@ -971,7 +973,8 @@ def search_datastore_sql(
971973
) -> list[dict]:
972974
"""Run a read-only SQL query against the HDX datastore using CKAN
973975
datastore_search_sql. Tables are named by resource id, which must be
974-
double quoted eg. SELECT * FROM "RESOURCE_ID".
976+
double quoted eg. SELECT * FROM "RESOURCE_ID". Logs a warning if HDX
977+
truncates the results.
975978
976979
Args:
977980
sql: SQL SELECT statement
@@ -982,7 +985,12 @@ def search_datastore_sql(
982985
"""
983986
resource = Resource(configuration=configuration)
984987
result = resource._write_to_hdx("datastore_search_sql", {"sql": sql})
985-
return result["records"]
988+
records = result["records"]
989+
if result.get("records_truncated"):
990+
logger.warning(
991+
f"SQL query results truncated by HDX to {len(records)} rows!"
992+
)
993+
return records
986994

987995
def delete_datastore(self) -> None:
988996
"""Delete a resource from the HDX datastore

‎tests/hdx/data/test_resource.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import copy
44
import json
5+
import logging
56
import os
67
from datetime import datetime, timezone
78
from os import remove
@@ -340,6 +341,7 @@ def mockresourceview(url, decodedata):
340341
class TestResource:
341342
datastore = None
342343
datastore_request = None
344+
datastore_rows_max = None
343345

344346
@pytest.fixture(scope="class")
345347
def static_yaml(self, configfolder):
@@ -604,6 +606,7 @@ def post(url, data, **kwargs):
604606

605607
@pytest.fixture(scope="function")
606608
def post_datastore_query(self):
609+
TestResource.datastore_rows_max = None
607610
all_records = [
608611
{"_id": 1, "code": "A", "value": 1.0},
609612
{"_id": 2, "code": "B", "value": 2.0},
@@ -626,6 +629,8 @@ def post(url, data, **kwargs):
626629
'{"success": false, "error": {"message": "Bad SQL", "__type": "Validation Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_search_sql"}',
627630
)
628631
result = {"records": all_records[:2], "fields": fields}
632+
if "truncated" in datadict["sql"]:
633+
result["records_truncated"] = True
629634
return MockResponse(
630635
200,
631636
json.dumps(
@@ -645,10 +650,13 @@ def post(url, data, **kwargs):
645650
TestResource.datastore_request = datadict
646651
offset = datadict["offset"]
647652
limit = datadict["limit"]
653+
if TestResource.datastore_rows_max is not None:
654+
limit = min(limit, TestResource.datastore_rows_max)
648655
result = {
649656
"resource_id": datadict["resource_id"],
650657
"fields": fields,
651658
"records": all_records[offset : offset + limit],
659+
"limit": limit,
652660
}
653661
if datadict["include_total"]:
654662
result["total"] = len(all_records)
@@ -1175,7 +1183,7 @@ def test_datastore(self, configuration, post_datastore, topline_yaml, topline_js
11751183
resource.update_datastore(records, method="insert")
11761184
assert TestResource.datastore == "upsert"
11771185

1178-
def test_datastore_queries(self, configuration, post_datastore_query):
1186+
def test_datastore_queries(self, configuration, post_datastore_query, caplog):
11791187
resource = Resource({"id": "datastore_query_resource"})
11801188
result = resource.search_datastore()
11811189
assert result["total"] == 3
@@ -1220,6 +1228,13 @@ def test_datastore_queries(self, configuration, post_datastore_query):
12201228
assert len(records) == 3
12211229
assert TestResource.datastore_request["offset"] == 3
12221230

1231+
TestResource.datastore_rows_max = 2
1232+
records = list(resource.get_datastore_records(page_size=5))
1233+
assert [record["code"] for record in records] == ["A", "B", "C"]
1234+
assert TestResource.datastore_request["offset"] == 2
1235+
assert TestResource.datastore_request["limit"] == 5
1236+
TestResource.datastore_rows_max = None
1237+
12231238
assert resource.get_datastore_fields() == [
12241239
{"id": "code", "type": "text"},
12251240
{"id": "value", "type": "numeric"},
@@ -1230,6 +1245,13 @@ def test_datastore_queries(self, configuration, post_datastore_query):
12301245
'SELECT * FROM "datastore_query_resource" LIMIT 2'
12311246
)
12321247
assert [record["code"] for record in records] == ["A", "B"]
1248+
assert "truncated" not in caplog.text
1249+
with caplog.at_level(logging.WARNING):
1250+
records = Resource.search_datastore_sql(
1251+
'SELECT * FROM "truncated_resource"'
1252+
)
1253+
assert len(records) == 2
1254+
assert "truncated by HDX to 2 rows" in caplog.text
12331255
with pytest.raises(HDXError):
12341256
Resource.search_datastore_sql("error")
12351257

0 commit comments

Comments
 (0)