Skip to content

Commit 5acbd31

Browse files
committed
Replace list fields such as groups in package_revise instead of merging by position
package_revise merges an update list into the existing list by index. The existing elements keep their ids and CKAN resolves groups by id before name, so changed locations were not applied: new groups were only added past the end of the old list and removals were only sent when the removed element was last. Filter out non-resource list keys present in the existing dataset so the update replaces them.
1 parent e8b2af7 commit 5acbd31

3 files changed

Lines changed: 423 additions & 381 deletions

File tree

‎src/hdx/data/dataset.py‎

Lines changed: 6 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -674,9 +674,10 @@ def _revise_filter(
674674
resources_to_delete: Sequence[int],
675675
):
676676
"""Returns the revise filter parameter adding to it any keys in the
677-
dataset metadata and resources that are specified to be deleted. Also compare
678-
lists in original and updated metadata to see if any have had elements
679-
removed in which case these should be added to the filter
677+
dataset metadata and resources that are specified to be deleted. Also
678+
adds any non-resource list keys in the existing dataset so that
679+
package_revise replaces those lists rather than merging them by
680+
position (which keeps the existing elements' ids eg. for groups)
680681
681682
Args:
682683
dataset_data_to_update: Dataset data to be updated
@@ -690,45 +691,11 @@ def _revise_filter(
690691
for resource_index in resources_to_delete:
691692
revise_filter.append(f"-resources__{resource_index}")
692693
for key, value in dataset_data_to_update.items():
693-
if not isinstance(value, list):
694+
if key == "resources" or not isinstance(value, list):
694695
continue
695696
if key not in self._old_data:
696697
continue
697-
orig_list = self._old_data[key]
698-
elements_to_remove = []
699-
for i, orig_value in enumerate(orig_list):
700-
if isinstance(orig_value, dict) and any(
701-
x in orig_value for x in ("id", "name")
702-
):
703-
# Where lists have an id or name, we use one of those to
704-
# match elements and work out what needs to be deleted
705-
# ie. any elements in the original list that don't exist
706-
# in the updated list
707-
el_id = orig_value.get("id")
708-
el_name = orig_value.get("name")
709-
present = False
710-
for value_dict in value:
711-
if el_id and "id" in value_dict and value_dict["id"] == el_id:
712-
present = True
713-
break
714-
if (
715-
el_name
716-
and "name" in value_dict
717-
and value_dict["name"] == el_name
718-
):
719-
present = True
720-
break
721-
if not present:
722-
elements_to_remove.append(i)
723-
elif orig_value not in value:
724-
# Otherwise we do a simple exact match of list elements
725-
# and delete any elements that are in the original list but
726-
# not in the updated list
727-
elements_to_remove.append(i)
728-
for element_index in reversed(elements_to_remove):
729-
del orig_list[element_index]
730-
if element_index == len(orig_list):
731-
revise_filter.append(f"-{key}__{element_index}")
698+
revise_filter.append(f"-{key}")
732699
return revise_filter
733700

734701
def _revise_files_to_upload_resource_deletions(

‎tests/hdx/data/test_update_logic.py‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,27 @@ def check_resources(update, results):
158158
f'Mismatch at index {index}: "{rn}", file to upload is "{fu}"'
159159
)
160160

161+
def test_revise_filter_lists(self, configuration, dataset):
162+
# HDX returns list elements with ids. package_revise merges an update
163+
# list into the existing one by position and CKAN looks up groups by
164+
# id before name, so changed lists must be filtered out and replaced
165+
dataset._old_data = {
166+
"groups": [
167+
{"id": "afg-id", "name": "afg", "title": "Afghanistan"},
168+
{"id": "zmb-id", "name": "zmb", "title": "Zambia"},
169+
],
170+
"tags": [{"id": "tag-id", "name": "health"}],
171+
"notes": "old notes",
172+
}
173+
dataset_data_to_update = {
174+
"groups": [{"name": "nru"}, {"name": "zmb"}, {"name": "vut"}],
175+
"tags": [{"name": "health"}],
176+
"solr_additions": [],
177+
"notes": "new notes",
178+
}
179+
revise_filter = dataset._revise_filter(dataset_data_to_update, ["notes"], [])
180+
assert revise_filter == ["-notes", "-groups", "-tags"]
181+
161182
def test_update_logic_1(
162183
self,
163184
configuration,
@@ -237,7 +258,7 @@ def test_update_logic_1(
237258
"Urban health Indicators for Zambia": 2,
238259
"World Health Statistics Indicators for Zambia": 2,
239260
}
240-
assert results["filter"] == ["-resources__13"]
261+
assert results["filter"] == ["-resources__13", "-groups", "-tags"]
241262
update = results["update"]
242263
del update["updated_by_script"]
243264
assert update == {
@@ -947,7 +968,7 @@ def test_update_logic_2(
947968
"Urban health Indicators for Zambia": 2,
948969
"World Health Statistics Indicators for Zambia": 2,
949970
}
950-
assert results["filter"] == ["-resources__52"]
971+
assert results["filter"] == ["-resources__52", "-groups", "-tags"]
951972
update = results["update"]
952973
del update["updated_by_script"]
953974
assert update == {
@@ -1607,7 +1628,12 @@ def test_update_logic_3(
16071628
"All Health Indicators for Zambia": 2,
16081629
"Millennium Development Goals (MDGs) Indicators for Zambia": 2,
16091630
}
1610-
assert results["filter"] == ["-resources__2", "-resources__1"]
1631+
assert results["filter"] == [
1632+
"-resources__2",
1633+
"-resources__1",
1634+
"-groups",
1635+
"-tags",
1636+
]
16111637
update = results["update"]
16121638
del update["updated_by_script"]
16131639
assert update == {

0 commit comments

Comments
 (0)