Skip to content

Commit 681c722

Browse files
authored
Add DataPusher+ submission and data dictionary update to Resource (#120)
- Resource.submit_to_datapusher (datapusher_submit) and Resource.get_datapusher_status (datapusher_status) - Resource.update_hdx_data_dictionary: merges into or replaces hdx_data_dictionary via resource_patch, then updates datastore field labels/descriptions via datastore_create without reloading data. last_modified is unchanged so no dataset update notifications are sent.
1 parent b56b343 commit 681c722

3 files changed

Lines changed: 302 additions & 0 deletions

File tree

‎documentation/index.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -991,6 +991,27 @@ using the getter:
991991

992992
data_dictionary = resource.get_hdx_data_dictionary()
993993

994+
To change the data dictionary of a resource already in HDX without sending
995+
dataset update notifications, use **update_hdx_data_dictionary**. It patches
996+
only the `hdx_data_dictionary` field (so `last_modified` is unchanged), then,
997+
if the resource has a datastore, updates the datastore field labels and
998+
descriptions without reloading the data. The given columns are merged by
999+
`field` into the existing data dictionary in the resource metadata, so read
1000+
the resource from HDX first. Pass `replace=True` to replace the data
1001+
dictionary instead and `update_datastore=False` to leave the datastore alone:
1002+
1003+
resource = Resource.read_from_hdx(RESOURCE_ID)
1004+
resource.update_hdx_data_dictionary([
1005+
{
1006+
"field": "col_name",
1007+
"label": "Column Label",
1008+
"description": "New description.",
1009+
},
1010+
])
1011+
1012+
Columns are matched to datastore fields by `field`. A warning is logged for
1013+
any column with no matching datastore field.
1014+
9941015
### Datastore
9951016

9961017
A resource's data can be loaded into the HDX datastore and queried. HDX calls this
@@ -1009,6 +1030,17 @@ the datastore for a resource:
10091030
has_datastore = resource.has_datastore()
10101031
resource.delete_datastore()
10111032

1033+
HDX loads uploaded files into the datastore using DataPusher+. To reload a
1034+
resource's file into the datastore without uploading it again (so
1035+
`last_modified` is unchanged), use **submit_to_datapusher**. It returns False
1036+
if the job was not submitted, eg. because one is already pending. HDX's
1037+
datastore allowlist and format checks, which apply to automatic submissions
1038+
on upload, are not applied. **get_datapusher_status** returns the status of
1039+
the latest job or None if there is none:
1040+
1041+
submitted = resource.submit_to_datapusher(ignore_hash=True)
1042+
status = resource.get_datapusher_status() # eg. status["status"] == "complete"
1043+
10121044
To download the whole datastore (rather than the resource's file) use
10131045
**download_datastore**, which returns the URL and path like **download**.
10141046
`file_format` can be `csv` (default), `tsv`, `json` or `xml`, and `filename`

‎src/hdx/data/resource.py‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ def actions() -> dict[str, str]:
7272
"datastore_upsert": "datastore_upsert",
7373
"datastore_search": "datastore_search",
7474
"datastore_search_sql": "datastore_search_sql",
75+
"datapusher_submit": "datapusher_submit",
76+
"datapusher_status": "datapusher_status",
7577
}
7678

7779
def update_from_yaml(
@@ -319,6 +321,72 @@ def set_hdx_data_dictionary(self, data_dictionary: Sequence[dict]) -> None:
319321
data_dictionary, separators=(",", ":")
320322
)
321323

324+
def update_hdx_data_dictionary(
325+
self,
326+
data_dictionary: Sequence[dict],
327+
replace: bool = False,
328+
update_datastore: bool = True,
329+
) -> None:
330+
"""Update the resource's data dictionary (column definitions) in HDX
331+
without changing last_modified, so no dataset update notifications are
332+
sent. Column definitions are as for set_hdx_data_dictionary. Unless
333+
replace is True, they are merged by field into the existing data
334+
dictionary in the resource metadata (so call read_from_hdx first).
335+
The resource's hdx_data_dictionary field is patched, then if
336+
update_datastore is True and the resource has a datastore, the
337+
datastore field labels and descriptions are updated using
338+
datastore_create without reloading the data.
339+
340+
Args:
341+
data_dictionary: Sequence of column definition dictionaries
342+
replace: Replace rather than merge into existing data dictionary. Defaults to False.
343+
update_datastore: Update datastore field labels and descriptions. Defaults to True.
344+
345+
Returns:
346+
None
347+
"""
348+
if replace:
349+
merged = [dict(column) for column in data_dictionary]
350+
else:
351+
new_columns = {column["field"]: column for column in data_dictionary}
352+
merged = []
353+
for column in self.get_hdx_data_dictionary() or []:
354+
new_column = new_columns.pop(column.get("field"), None)
355+
if new_column:
356+
merged.append({**column, **new_column})
357+
else:
358+
merged.append(column)
359+
merged.extend(new_columns.values())
360+
self.set_hdx_data_dictionary(merged)
361+
self._write_to_hdx(
362+
"patch",
363+
{
364+
"id": self.data["id"],
365+
"hdx_data_dictionary": self.data["hdx_data_dictionary"],
366+
},
367+
"id",
368+
)
369+
if not update_datastore or not self.has_datastore():
370+
return
371+
columns = {column["field"]: column for column in merged}
372+
fields = []
373+
for field in self.get_datastore_fields():
374+
info = dict(field.get("info") or {})
375+
column = columns.pop(field["id"], None)
376+
if column:
377+
info["label"] = column["label"]
378+
info["notes"] = column["description"]
379+
fields.append({"id": field["id"], "type": field["type"], "info": info})
380+
if columns:
381+
logger.warning(
382+
f"Data dictionary fields {', '.join(columns)} not in datastore for resource {self.data['id']}!"
383+
)
384+
self._write_to_hdx(
385+
"datastore_create",
386+
{"resource_id": self.data["id"], "force": True, "fields": fields},
387+
"resource_id",
388+
)
389+
322390
def clean_format(self) -> str:
323391
"""Clean the resource's format, setting it to None if it is invalid and
324392
cannot be mapped
@@ -1015,6 +1083,41 @@ def delete_datastore(self) -> None:
10151083
if not success:
10161084
logger.debug(result)
10171085

1086+
def submit_to_datapusher(self, ignore_hash: bool = False) -> bool:
1087+
"""Submit the resource to DataPusher+ to load its file into the
1088+
datastore. This does not change last_modified. Unlike the automatic
1089+
submission on file upload, HDX's datastore allowlist and format checks
1090+
are not applied.
1091+
1092+
Args:
1093+
ignore_hash: Reload even if the file hash is unchanged. Defaults to False.
1094+
1095+
Returns:
1096+
True if submitted, False if not (eg. a job is already pending)
1097+
"""
1098+
return self._write_to_hdx(
1099+
"datapusher_submit",
1100+
{"resource_id": self.data["id"], "ignore_hash": ignore_hash},
1101+
"resource_id",
1102+
)
1103+
1104+
def get_datapusher_status(self) -> dict | None:
1105+
"""Get the status of the resource's latest DataPusher+ job
1106+
1107+
Returns:
1108+
Status dictionary with keys including status and error or None if no job
1109+
"""
1110+
success, result = self._read_from_hdx(
1111+
"datapusher",
1112+
self.data["id"],
1113+
"resource_id",
1114+
self.actions()["datapusher_status"],
1115+
)
1116+
if success:
1117+
return result
1118+
logger.debug(result)
1119+
return None
1120+
10181121
def get_resource_views(self) -> list[ResourceView]:
10191122
"""Get any resource views in the resource
10201123

‎tests/hdx/data/test_resource.py‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -747,6 +747,70 @@ def post(url, data, **kwargs):
747747

748748
Configuration.read().remoteckan().session = MockSession()
749749

750+
@pytest.fixture(scope="function")
751+
def post_datapusher(self):
752+
calls = []
753+
754+
class MockSession:
755+
@staticmethod
756+
def post(url, data, **kwargs):
757+
datadict = json.loads(data.decode("utf-8"))
758+
action = url.split("/")[-1]
759+
calls.append((action, datadict))
760+
if datadict.get("resource_id") == "NOTFOUND":
761+
return MockResponse(
762+
404,
763+
f'{{"success": false, "error": {{"message": "Not found", "__type": "Not Found Error"}}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}',
764+
)
765+
if action == "datapusher_submit":
766+
result = "true"
767+
elif action == "datapusher_status":
768+
result = '{"status": "complete", "error": {}}'
769+
elif action == "resource_patch":
770+
resultdictcopy = copy.deepcopy(resultdict)
771+
resultdictcopy["hdx_data_dictionary"] = datadict[
772+
"hdx_data_dictionary"
773+
]
774+
result = json.dumps(resultdictcopy)
775+
elif action == "datastore_search":
776+
result = json.dumps(
777+
{
778+
"fields": [
779+
{"id": "_id", "type": "int"},
780+
{
781+
"id": "code",
782+
"type": "text",
783+
"info": {
784+
"label": "Code",
785+
"notes": "Old code",
786+
"type_override": "text",
787+
},
788+
},
789+
{
790+
"id": "value",
791+
"type": "numeric",
792+
"info": {"label": "Value", "notes": "Old value"},
793+
},
794+
{"id": "notes", "type": "text"},
795+
],
796+
"records": [],
797+
}
798+
)
799+
elif action == "datastore_create":
800+
result = json.dumps(datadict)
801+
else:
802+
return MockResponse(
803+
404,
804+
f'{{"success": false, "error": {{"message": "TEST ERROR: Unexpected {action}", "__type": "TEST ERROR"}}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}',
805+
)
806+
return MockResponse(
807+
200,
808+
f'{{"success": true, "result": {result}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}',
809+
)
810+
811+
Configuration.read().remoteckan().session = MockSession()
812+
return calls
813+
750814
def test_read_from_hdx(self, configuration, read):
751815
resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811")
752816
assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5"
@@ -1343,6 +1407,109 @@ def test_get_fs_check_info(self, configuration):
13431407
resource["fs_check_info"] = events
13441408
assert resource.get_fs_check_info() == events
13451409

1410+
def test_datapusher(self, configuration, post_datapusher):
1411+
resource = Resource({"id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5"})
1412+
assert resource.submit_to_datapusher(ignore_hash=True) is True
1413+
assert post_datapusher[-1] == (
1414+
"datapusher_submit",
1415+
{
1416+
"resource_id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5",
1417+
"ignore_hash": True,
1418+
},
1419+
)
1420+
assert resource.get_datapusher_status() == {
1421+
"status": "complete",
1422+
"error": {},
1423+
}
1424+
resource = Resource({"id": "NOTFOUND"})
1425+
assert resource.get_datapusher_status() is None
1426+
with pytest.raises(HDXError):
1427+
resource.submit_to_datapusher()
1428+
1429+
def test_update_hdx_data_dictionary(self, configuration, post_datapusher):
1430+
existing = [
1431+
{
1432+
"field": "code",
1433+
"label": "Code",
1434+
"description": "Old code",
1435+
"data_type": "text",
1436+
},
1437+
{"field": "value", "label": "Value", "description": "Old value"},
1438+
]
1439+
resource = Resource(
1440+
{
1441+
"id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5",
1442+
"hdx_data_dictionary": json.dumps(existing),
1443+
}
1444+
)
1445+
resource.update_hdx_data_dictionary(
1446+
[
1447+
{"field": "code", "label": "Code", "description": "New code"},
1448+
{"field": "extra", "label": "Extra", "description": "Not in datastore"},
1449+
]
1450+
)
1451+
expected = [
1452+
{
1453+
"field": "code",
1454+
"label": "Code",
1455+
"description": "New code",
1456+
"data_type": "text",
1457+
},
1458+
{"field": "value", "label": "Value", "description": "Old value"},
1459+
{"field": "extra", "label": "Extra", "description": "Not in datastore"},
1460+
]
1461+
assert resource.get_hdx_data_dictionary() == expected
1462+
actions = [action for action, _ in post_datapusher]
1463+
assert actions == [
1464+
"resource_patch",
1465+
"datastore_search",
1466+
"datastore_search",
1467+
"datastore_create",
1468+
]
1469+
_, patch = post_datapusher[0]
1470+
assert patch == {
1471+
"id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5",
1472+
"hdx_data_dictionary": json.dumps(expected, separators=(",", ":")),
1473+
}
1474+
_, create = post_datapusher[-1]
1475+
assert create == {
1476+
"resource_id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5",
1477+
"force": True,
1478+
"fields": [
1479+
{
1480+
"id": "code",
1481+
"type": "text",
1482+
"info": {
1483+
"label": "Code",
1484+
"notes": "New code",
1485+
"type_override": "text",
1486+
},
1487+
},
1488+
{
1489+
"id": "value",
1490+
"type": "numeric",
1491+
"info": {"label": "Value", "notes": "Old value"},
1492+
},
1493+
{"id": "notes", "type": "text", "info": {}},
1494+
],
1495+
}
1496+
1497+
post_datapusher.clear()
1498+
resource.update_hdx_data_dictionary(
1499+
[{"field": "value", "label": "Value", "description": "New value"}],
1500+
replace=True,
1501+
update_datastore=False,
1502+
)
1503+
assert resource.get_hdx_data_dictionary() == [
1504+
{"field": "value", "label": "Value", "description": "New value"}
1505+
]
1506+
assert [action for action, _ in post_datapusher] == ["resource_patch"]
1507+
1508+
with pytest.raises(HDXError):
1509+
resource.update_hdx_data_dictionary(
1510+
[{"field": "value", "label": "Value"}], replace=True
1511+
)
1512+
13461513
def test_get_hdx_url(self, configuration, read):
13471514
resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811")
13481515
assert (

0 commit comments

Comments
 (0)