@@ -667,51 +667,38 @@ def _prepare_hdx_call(self, data: dict, kwargs: Any) -> None:
667667 if "batch_mode" not in kwargs :
668668 kwargs ["batch_mode" ] = "DONT_GROUP"
669669
670- def _revise_filter (
671- self ,
672- dataset_data_to_update : dict ,
673- keys_to_delete : Sequence [str ],
674- resources_to_delete : Sequence [int ],
675- ):
676- """Returns the revise filter parameter adding to it any keys in the
677- dataset metadata and resources that are specified to be deleted.
678-
679- For each non-resource list key present in both the update and the
680- existing dataset, the update's list is merged with the existing one
681- (see _merge_list) in place. If the merged list differs from the
682- existing one, the key is added to the filter so that package_revise
683- replaces the list rather than merging it by position (which keeps the
684- existing elements' ids eg. for groups).
670+ def _merge_list_fields (self , dataset_data_to_update : dict ) -> list [str ]:
671+ """Merges each non-resource list in dataset_data_to_update with the
672+ existing list of the same key (see _merge_list) in place.
685673
686674 Args:
687675 dataset_data_to_update: Dataset data to be updated
688- keys_to_delete: List of top level metadata keys to delete
689- resources_to_delete: List of indexes of resources to delete
676+
677+ Returns:
678+ Keys whose merged list differs from the existing one
690679 """
691- revise_filter = []
692- for key in keys_to_delete :
693- revise_filter .append (f"-{ key } " )
694- if not self .is_requestable ():
695- for resource_index in resources_to_delete :
696- revise_filter .append (f"-resources__{ resource_index } " )
680+ changed_keys = []
697681 for key , value in dataset_data_to_update .items ():
698- if key == "resources" or not isinstance (value , list ):
699- continue
700- if key not in self ._old_data :
682+ if (
683+ key == "resources"
684+ or not isinstance (value , list )
685+ or key not in self ._old_data
686+ ):
701687 continue
702688 orig_list = self ._old_data [key ]
689+ if not isinstance (orig_list , list ):
690+ orig_list = []
703691 merged_list = self ._merge_list (orig_list , value )
704692 dataset_data_to_update [key ] = merged_list
705693 if merged_list != orig_list :
706- revise_filter .append (f"- { key } " )
707- return revise_filter
694+ changed_keys .append (key )
695+ return changed_keys
708696
709697 @staticmethod
710698 def _merge_list (orig_list : list , new_list : list ) -> list :
711- """Merges each dict in new_list with its matching dict in orig_list so
712- that fields not given in new_list are kept. A dict with an id or name
713- matches the element in orig_list with the same id or name. A dict with
714- neither matches the element at the same position in orig_list.
699+ """Merges each dict in new_list that has an id or name with the dict in
700+ orig_list with the same id (or failing that, name) so that fields not
701+ given in new_list are kept. Other elements of new_list are kept as is.
715702 Elements of orig_list that are not matched are dropped.
716703
717704 Args:
@@ -722,31 +709,60 @@ def _merge_list(orig_list: list, new_list: list) -> list:
722709 Merged list
723710 """
724711 merged_list = []
725- for i , new_value in enumerate ( new_list ) :
712+ for new_value in new_list :
726713 if not isinstance (new_value , dict ):
727714 merged_list .append (new_value )
728715 continue
729- el_id = new_value .get ("id" )
730- el_name = new_value .get ("name" )
731- match = None
732- if el_id or el_name :
733- for orig_value in orig_list :
734- if not isinstance (orig_value , dict ):
735- continue
736- if el_id and orig_value .get ("id" ) == el_id :
737- match = orig_value
738- break
739- if el_name and orig_value .get ("name" ) == el_name :
740- match = orig_value
741- break
742- elif i < len (orig_list ) and isinstance (orig_list [i ], dict ):
743- match = orig_list [i ]
744- if match :
745- merged_list .append (match | new_value )
746- else :
747- merged_list .append (new_value )
716+ match = {}
717+ for field in ("id" , "name" ):
718+ field_value = new_value .get (field )
719+ if not field_value :
720+ continue
721+ match = next (
722+ (
723+ orig_value
724+ for orig_value in orig_list
725+ if isinstance (orig_value , dict )
726+ and orig_value .get (field ) == field_value
727+ ),
728+ {},
729+ )
730+ if match :
731+ break
732+ merged_list .append (match | new_value )
748733 return merged_list
749734
735+ def _revise_filter (
736+ self ,
737+ keys_to_delete : Sequence [str ],
738+ resources_to_delete : Sequence [int ],
739+ changed_list_keys : Sequence [str ],
740+ ) -> list [str ]:
741+ """Returns the revise filter parameter adding to it any keys in the
742+ dataset metadata and resources that are specified to be deleted. Keys
743+ of changed lists are also added so that package_revise replaces those
744+ lists rather than merging them by position (which keeps the existing
745+ elements' ids eg. for groups).
746+
747+ Args:
748+ keys_to_delete: List of top level metadata keys to delete
749+ resources_to_delete: List of indexes of resources to delete
750+ changed_list_keys: Keys of lists that differ from the existing ones
751+
752+ Returns:
753+ Revise filter
754+ """
755+ revise_filter = []
756+ for key in keys_to_delete :
757+ revise_filter .append (f"-{ key } " )
758+ if not self .is_requestable ():
759+ for resource_index in resources_to_delete :
760+ revise_filter .append (f"-resources__{ resource_index } " )
761+ for key in changed_list_keys :
762+ if key not in keys_to_delete :
763+ revise_filter .append (f"-{ key } " )
764+ return revise_filter
765+
750766 def _revise_files_to_upload_resource_deletions (
751767 self ,
752768 resources_to_update : Sequence ["Resource" ],
@@ -822,8 +838,9 @@ def _revise_dataset(
822838 ): # Whether or not CKAN should perform validation steps (checking fields present)
823839 dataset_data_to_update ["skip_validation" ] = kwargs ["skip_validation" ]
824840 dataset_data_to_update ["state" ] = "active"
841+ changed_list_keys = self ._merge_list_fields (dataset_data_to_update )
825842 revise_filter = self ._revise_filter (
826- dataset_data_to_update , keys_to_delete , resources_to_delete
843+ keys_to_delete , resources_to_delete , changed_list_keys
827844 )
828845 files_to_upload = self ._revise_files_to_upload_resource_deletions (
829846 resources_to_update ,
@@ -1077,7 +1094,6 @@ def _dataset_hdx_update(
10771094 break
10781095 if not found :
10791096 old_tags .append (tag )
1080- self ._old_data ["tags" ] = old_tags
10811097 self ._prepare_hdx_call (self ._old_data , kwargs )
10821098 revise_call = self ._revise_dataset (
10831099 allow_no_resources ,
0 commit comments