Skip to content

Commit eca1fd6

Browse files
committed
Fix activity paging when limit exceeds HDX maximum and simplify recent additions
- Cap activity list limit at 100 when getting all pages, as HDX silently caps it and paging stopped after the first page - Share get_activities body via HDXObject._get_activities - Share download code between Resource.download and download_datastore - Use cls in User.get_current_token_info - Simplify data dictionary merge loop
1 parent 681c722 commit eca1fd6

8 files changed

Lines changed: 72 additions & 50 deletions

File tree

‎src/hdx/data/activity.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,9 @@ def _get_list(
129129
result = activity._write_to_hdx(action, data)
130130
return [cls(x, configuration=configuration) for x in result]
131131
data = dict(data)
132-
limit = data.setdefault("limit", 100)
132+
# HDX silently caps limit at ckan.activity_list_limit_max
133+
limit = min(data.get("limit", 100), 100)
134+
data["limit"] = limit
133135
offset = data.get("offset", 0)
134136
activities = []
135137
while True:

‎src/hdx/data/dataset.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1448,13 +1448,7 @@ def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activit
14481448
Returns:
14491449
List of Activity objects
14501450
"""
1451-
return Activity.get_activities(
1452-
"package",
1453-
self.data.get("id") or self.data.get("name"),
1454-
self.configuration,
1455-
all_pages,
1456-
**kwargs,
1457-
)
1451+
return self._get_activities("package", all_pages, **kwargs)
14581452

14591453
@classmethod
14601454
def autocomplete(

‎src/hdx/data/hdxobject.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from collections.abc import Sequence
1010
from os.path import isfile
1111
from pathlib import Path
12-
from typing import Any, Optional, Union
12+
from typing import TYPE_CHECKING, Any, Optional, Union
1313

1414
from ckanapi.errors import NotFound
1515
from hdx.utilities.dictandlist import merge_two_dictionaries
@@ -20,6 +20,9 @@
2020

2121
from hdx.api.configuration import Configuration
2222

23+
if TYPE_CHECKING:
24+
from hdx.data.activity import Activity
25+
2326
logger = logging.getLogger(__name__)
2427

2528

@@ -376,6 +379,30 @@ def _update_in_hdx(
376379
**kwargs,
377380
)
378381

382+
def _get_activities(
383+
self, object_type: str, all_pages: bool = False, **kwargs: Any
384+
) -> list["Activity"]:
385+
"""Helper method to get the activity stream of this object
386+
387+
Args:
388+
object_type: One of package, organization, group or user
389+
all_pages: Whether to get all pages of results. Defaults to False.
390+
**kwargs: Parameters to pass to Activity.get_activities
391+
392+
Returns:
393+
List of Activity objects
394+
"""
395+
# Imported here as hdx.data.activity imports this module
396+
from hdx.data.activity import Activity
397+
398+
return Activity.get_activities(
399+
object_type,
400+
self.data.get("id") or self.data.get("name"),
401+
self.configuration,
402+
all_pages,
403+
**kwargs,
404+
)
405+
379406
def _write_to_hdx(
380407
self,
381408
action: str,

‎src/hdx/data/location.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -232,13 +232,7 @@ def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activit
232232
Returns:
233233
List of Activity objects
234234
"""
235-
return Activity.get_activities(
236-
"group",
237-
self.data.get("id") or self.data.get("name"),
238-
self.configuration,
239-
all_pages,
240-
**kwargs,
241-
)
235+
return self._get_activities("group", all_pages, **kwargs)
242236

243237
@classmethod
244238
def autocomplete(

‎src/hdx/data/organization.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -326,13 +326,7 @@ def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activit
326326
Returns:
327327
List of Activity objects
328328
"""
329-
return Activity.get_activities(
330-
"organization",
331-
self.data.get("id") or self.data.get("name"),
332-
self.configuration,
333-
all_pages,
334-
**kwargs,
335-
)
329+
return self._get_activities("organization", all_pages, **kwargs)
336330

337331
@classmethod
338332
def autocomplete(

‎src/hdx/data/resource.py‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -352,10 +352,7 @@ def update_hdx_data_dictionary(
352352
merged = []
353353
for column in self.get_hdx_data_dictionary() or []:
354354
new_column = new_columns.pop(column.get("field"), None)
355-
if new_column:
356-
merged.append({**column, **new_column})
357-
else:
358-
merged.append(column)
355+
merged.append({**column, **new_column} if new_column else column)
359356
merged.extend(new_columns.values())
360357
self.set_hdx_data_dictionary(merged)
361358
self._write_to_hdx(
@@ -751,6 +748,30 @@ def search_in_hdx(
751748
logger.debug(result)
752749
return resources
753750

751+
def _download_file(
752+
self,
753+
url: str,
754+
folder: Path | str | None,
755+
filename: str,
756+
retriever: Retrieve | None,
757+
) -> Path:
758+
"""Helper method to download a file from HDX
759+
760+
Args:
761+
url: URL to download
762+
folder: Folder to download file to. None means temporary folder.
763+
filename: Filename to use
764+
retriever: Retrieve object to use. None means do not use one.
765+
766+
Returns:
767+
Path to downloaded file
768+
"""
769+
session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
770+
with Download(session=session) as downloader:
771+
if retriever:
772+
downloader = retriever.clone(downloader)
773+
return downloader.download_file(url, folder=folder, filename=filename)
774+
754775
def download(
755776
self, folder: Path | str | None = None, retriever: Retrieve | None = None
756777
) -> tuple[str, Path]:
@@ -774,12 +795,7 @@ def download(
774795
file_format = f".{self.get_format()}"
775796
if not filename.endswith(file_format):
776797
filename = f"{filename}{file_format}"
777-
session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
778-
with Download(session=session) as downloader:
779-
if retriever:
780-
downloader = retriever.clone(downloader)
781-
path = downloader.download_file(url, folder=folder, filename=filename)
782-
return url, path
798+
return url, self._download_file(url, folder, filename, retriever)
783799

784800
def get_datastore_dump_url(self, file_format: str = "csv") -> str:
785801
"""Get the URL from which the resource datastore can be downloaded in
@@ -823,19 +839,15 @@ def download_datastore(
823839
Returns:
824840
(URL downloaded, Path to downloaded file)
825841
"""
842+
file_format = file_format.lower()
826843
url = self.get_datastore_dump_url(file_format)
827844
logger.debug(f"Downloading {url}")
828845
if filename is None:
829846
filename = self.data["name"]
830-
extension = f".{file_format.lower()}"
847+
extension = f".{file_format}"
831848
if not filename.endswith(extension):
832849
filename = f"{filename}{extension}"
833-
session = hdx.api.utilities.url_utils.get_ckan_ready_session(self.configuration)
834-
with Download(session=session) as downloader:
835-
if retriever:
836-
downloader = retriever.clone(downloader)
837-
path = downloader.download_file(url, folder=folder, filename=filename)
838-
return url, path
850+
return url, self._download_file(url, folder, filename, retriever)
839851

840852
@staticmethod
841853
def get_all_resource_ids_in_datastore(

‎src/hdx/data/user.py‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ def get_current_user_organization_dicts(
335335
Returns:
336336
List of organization dicts in HDX that logged in user is a member of
337337
"""
338-
user = User(configuration=configuration)
338+
user = cls(configuration=configuration)
339339
try:
340340
return user.configuration.call_remoteckan(
341341
cls.actions()["listorgs"], {"permission": permission}
@@ -471,13 +471,7 @@ def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activit
471471
Returns:
472472
List of Activity objects
473473
"""
474-
return Activity.get_activities(
475-
"user",
476-
self.data.get("id") or self.data.get("name"),
477-
self.configuration,
478-
all_pages,
479-
**kwargs,
480-
)
474+
return self._get_activities("user", all_pages, **kwargs)
481475

482476
@classmethod
483477
def get_current_token_info(cls, configuration: Configuration | None = None) -> dict:
@@ -491,7 +485,7 @@ def get_current_token_info(cls, configuration: Configuration | None = None) -> d
491485
Returns:
492486
Dictionary with keys email_hash and token_name
493487
"""
494-
user = User(configuration=configuration)
488+
user = cls(configuration=configuration)
495489
try:
496490
return user.configuration.call_remoteckan(cls.actions()["token_info"], {})
497491
except Exception as e:

‎tests/hdx/data/test_activity.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def post(url, data, **kwargs):
6666
):
6767
return mocknotfound(action)
6868
offset = datadict.get("offset", 0)
69-
limit = datadict.get("limit", 31)
69+
limit = min(datadict.get("limit", 31), 100)
7070
return mockresponse(activities[offset : offset + limit], action)
7171
if datadict["id"] != activity_id:
7272
return mocknotfound(action)
@@ -134,6 +134,11 @@ def test_get_activities_all_pages(self, configuration, post_activity):
134134
)
135135
assert [x["id"] for x in result] == [x["id"] for x in activities[150:]]
136136
assert [x[1]["offset"] for x in TestActivity.calls] == [150, 200, 250]
137+
TestActivity.calls = []
138+
result = Activity.get_activities("package", "TEST1", all_pages=True, limit=500)
139+
assert [x["id"] for x in result] == [x["id"] for x in activities]
140+
assert [x[1]["limit"] for x in TestActivity.calls] == [100, 100, 100]
141+
assert [x[1]["offset"] for x in TestActivity.calls] == [0, 100, 200]
137142

138143
def test_get_recently_changed_packages(self, configuration, post_activity):
139144
result = Activity.get_recently_changed_packages(limit=3)

0 commit comments

Comments
 (0)