Skip to content

Commit d3d3ae9

Browse files
committed
Fix list merging edge cases and split merging out of _revise_filter
- _merge_list matches by id first, then name, and no longer merges dicts without an id or name by position (fields leaked from removed entries) - A null or non-list existing value is treated as an empty list instead of raising TypeError - New _merge_list_fields does the in-place merge and returns the changed keys; _revise_filter only builds the filter - Changed list keys already in keys_to_delete are not added to the filter twice - Add return type and Returns: docstring section to _revise_filter
1 parent 836b6cf commit d3d3ae9

2 files changed

Lines changed: 88 additions & 58 deletions

File tree

‎src/hdx/data/dataset.py‎

Lines changed: 70 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -667,51 +667,38 @@ def _prepare_hdx_call(self, data: dict, kwargs: Any) -> None:
667667
if "batch_mode" not in kwargs:
668668
kwargs["batch_mode"] = "DONT_GROUP"
669669

670-
def _revise_filter(
671-
self,
672-
dataset_data_to_update: dict,
673-
keys_to_delete: Sequence[str],
674-
resources_to_delete: Sequence[int],
675-
):
676-
"""Returns the revise filter parameter adding to it any keys in the
677-
dataset metadata and resources that are specified to be deleted.
678-
679-
For each non-resource list key present in both the update and the
680-
existing dataset, the update's list is merged with the existing one
681-
(see _merge_list) in place. If the merged list differs from the
682-
existing one, the key is added to the filter so that package_revise
683-
replaces the list rather than merging it by position (which keeps the
684-
existing elements' ids eg. for groups).
670+
def _merge_list_fields(self, dataset_data_to_update: dict) -> list[str]:
671+
"""Merges each non-resource list in dataset_data_to_update with the
672+
existing list of the same key (see _merge_list) in place.
685673
686674
Args:
687675
dataset_data_to_update: Dataset data to be updated
688-
keys_to_delete: List of top level metadata keys to delete
689-
resources_to_delete: List of indexes of resources to delete
676+
677+
Returns:
678+
Keys whose merged list differs from the existing one
690679
"""
691-
revise_filter = []
692-
for key in keys_to_delete:
693-
revise_filter.append(f"-{key}")
694-
if not self.is_requestable():
695-
for resource_index in resources_to_delete:
696-
revise_filter.append(f"-resources__{resource_index}")
680+
changed_keys = []
697681
for key, value in dataset_data_to_update.items():
698-
if key == "resources" or not isinstance(value, list):
699-
continue
700-
if key not in self._old_data:
682+
if (
683+
key == "resources"
684+
or not isinstance(value, list)
685+
or key not in self._old_data
686+
):
701687
continue
702688
orig_list = self._old_data[key]
689+
if not isinstance(orig_list, list):
690+
orig_list = []
703691
merged_list = self._merge_list(orig_list, value)
704692
dataset_data_to_update[key] = merged_list
705693
if merged_list != orig_list:
706-
revise_filter.append(f"-{key}")
707-
return revise_filter
694+
changed_keys.append(key)
695+
return changed_keys
708696

709697
@staticmethod
710698
def _merge_list(orig_list: list, new_list: list) -> list:
711-
"""Merges each dict in new_list with its matching dict in orig_list so
712-
that fields not given in new_list are kept. A dict with an id or name
713-
matches the element in orig_list with the same id or name. A dict with
714-
neither matches the element at the same position in orig_list.
699+
"""Merges each dict in new_list that has an id or name with the dict in
700+
orig_list with the same id (or failing that, name) so that fields not
701+
given in new_list are kept. Other elements of new_list are kept as is.
715702
Elements of orig_list that are not matched are dropped.
716703
717704
Args:
@@ -722,31 +709,60 @@ def _merge_list(orig_list: list, new_list: list) -> list:
722709
Merged list
723710
"""
724711
merged_list = []
725-
for i, new_value in enumerate(new_list):
712+
for new_value in new_list:
726713
if not isinstance(new_value, dict):
727714
merged_list.append(new_value)
728715
continue
729-
el_id = new_value.get("id")
730-
el_name = new_value.get("name")
731-
match = None
732-
if el_id or el_name:
733-
for orig_value in orig_list:
734-
if not isinstance(orig_value, dict):
735-
continue
736-
if el_id and orig_value.get("id") == el_id:
737-
match = orig_value
738-
break
739-
if el_name and orig_value.get("name") == el_name:
740-
match = orig_value
741-
break
742-
elif i < len(orig_list) and isinstance(orig_list[i], dict):
743-
match = orig_list[i]
744-
if match:
745-
merged_list.append(match | new_value)
746-
else:
747-
merged_list.append(new_value)
716+
match = {}
717+
for field in ("id", "name"):
718+
field_value = new_value.get(field)
719+
if not field_value:
720+
continue
721+
match = next(
722+
(
723+
orig_value
724+
for orig_value in orig_list
725+
if isinstance(orig_value, dict)
726+
and orig_value.get(field) == field_value
727+
),
728+
{},
729+
)
730+
if match:
731+
break
732+
merged_list.append(match | new_value)
748733
return merged_list
749734

735+
def _revise_filter(
736+
self,
737+
keys_to_delete: Sequence[str],
738+
resources_to_delete: Sequence[int],
739+
changed_list_keys: Sequence[str],
740+
) -> list[str]:
741+
"""Returns the revise filter parameter adding to it any keys in the
742+
dataset metadata and resources that are specified to be deleted. Keys
743+
of changed lists are also added so that package_revise replaces those
744+
lists rather than merging them by position (which keeps the existing
745+
elements' ids eg. for groups).
746+
747+
Args:
748+
keys_to_delete: List of top level metadata keys to delete
749+
resources_to_delete: List of indexes of resources to delete
750+
changed_list_keys: Keys of lists that differ from the existing ones
751+
752+
Returns:
753+
Revise filter
754+
"""
755+
revise_filter = []
756+
for key in keys_to_delete:
757+
revise_filter.append(f"-{key}")
758+
if not self.is_requestable():
759+
for resource_index in resources_to_delete:
760+
revise_filter.append(f"-resources__{resource_index}")
761+
for key in changed_list_keys:
762+
if key not in keys_to_delete:
763+
revise_filter.append(f"-{key}")
764+
return revise_filter
765+
750766
def _revise_files_to_upload_resource_deletions(
751767
self,
752768
resources_to_update: Sequence["Resource"],
@@ -822,8 +838,9 @@ def _revise_dataset(
822838
): # Whether or not CKAN should perform validation steps (checking fields present)
823839
dataset_data_to_update["skip_validation"] = kwargs["skip_validation"]
824840
dataset_data_to_update["state"] = "active"
841+
changed_list_keys = self._merge_list_fields(dataset_data_to_update)
825842
revise_filter = self._revise_filter(
826-
dataset_data_to_update, keys_to_delete, resources_to_delete
843+
keys_to_delete, resources_to_delete, changed_list_keys
827844
)
828845
files_to_upload = self._revise_files_to_upload_resource_deletions(
829846
resources_to_update,
@@ -1077,7 +1094,6 @@ def _dataset_hdx_update(
10771094
break
10781095
if not found:
10791096
old_tags.append(tag)
1080-
self._old_data["tags"] = old_tags
10811097
self._prepare_hdx_call(self._old_data, kwargs)
10821098
revise_call = self._revise_dataset(
10831099
allow_no_resources,

‎tests/hdx/data/test_update_logic.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,24 +169,38 @@ def test_revise_filter_lists(self, configuration, dataset):
169169
],
170170
"tags": [{"id": "tag-id", "name": "health"}],
171171
"customviz": [{"url": "a", "extra": 1}],
172+
"solr_additions": None,
172173
"notes": "old notes",
173174
}
174175
dataset_data_to_update = {
175176
"groups": [{"name": "nru"}, {"name": "zmb"}, {"name": "vut"}],
176177
"tags": [{"name": "health"}],
177178
"customviz": [{"url": "b"}],
178-
"solr_additions": [],
179+
"solr_additions": [{"name": "x"}],
179180
"notes": "new notes",
180181
}
181-
revise_filter = dataset._revise_filter(dataset_data_to_update, ["notes"], [])
182-
assert revise_filter == ["-notes", "-groups", "-customviz"]
182+
changed_list_keys = dataset._merge_list_fields(dataset_data_to_update)
183+
assert changed_list_keys == ["groups", "customviz", "solr_additions"]
184+
revise_filter = dataset._revise_filter(["notes"], [], changed_list_keys)
185+
assert revise_filter == ["-notes", "-groups", "-customviz", "-solr_additions"]
186+
revise_filter = dataset._revise_filter(["groups"], [], changed_list_keys)
187+
assert revise_filter == ["-groups", "-customviz", "-solr_additions"]
183188
assert dataset_data_to_update["groups"] == [
184189
{"name": "nru"},
185190
{"id": "zmb-id", "name": "zmb", "title": "Zambia"},
186191
{"name": "vut"},
187192
]
188193
assert dataset_data_to_update["tags"] == [{"id": "tag-id", "name": "health"}]
189-
assert dataset_data_to_update["customviz"] == [{"url": "b", "extra": 1}]
194+
assert dataset_data_to_update["customviz"] == [{"url": "b"}]
195+
assert dataset_data_to_update["solr_additions"] == [{"name": "x"}]
196+
197+
def test_merge_list_id_before_name(self):
198+
orig_list = [
199+
{"id": "1", "name": "x", "f": "A"},
200+
{"id": "2", "name": "y", "f": "B"},
201+
]
202+
merged_list = Dataset._merge_list(orig_list, [{"id": "2", "name": "x"}])
203+
assert merged_list == [{"id": "2", "name": "x", "f": "B"}]
190204

191205
def test_update_logic_1(
192206
self,

0 commit comments

Comments
 (0)