22
33import copy
44import json
5+ import logging
56import os
67from datetime import datetime , timezone
78from os import remove
@@ -340,6 +341,7 @@ def mockresourceview(url, decodedata):
340341class TestResource :
341342 datastore = None
342343 datastore_request = None
344+ datastore_rows_max = None
343345
344346 @pytest .fixture (scope = "class" )
345347 def static_yaml (self , configfolder ):
@@ -604,6 +606,7 @@ def post(url, data, **kwargs):
604606
605607 @pytest .fixture (scope = "function" )
606608 def post_datastore_query (self ):
609+ TestResource .datastore_rows_max = None
607610 all_records = [
608611 {"_id" : 1 , "code" : "A" , "value" : 1.0 },
609612 {"_id" : 2 , "code" : "B" , "value" : 2.0 },
@@ -626,6 +629,8 @@ def post(url, data, **kwargs):
626629 '{"success": false, "error": {"message": "Bad SQL", "__type": "Validation Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_search_sql"}' ,
627630 )
628631 result = {"records" : all_records [:2 ], "fields" : fields }
632+ if "truncated" in datadict ["sql" ]:
633+ result ["records_truncated" ] = True
629634 return MockResponse (
630635 200 ,
631636 json .dumps (
@@ -645,10 +650,13 @@ def post(url, data, **kwargs):
645650 TestResource .datastore_request = datadict
646651 offset = datadict ["offset" ]
647652 limit = datadict ["limit" ]
653+ if TestResource .datastore_rows_max is not None :
654+ limit = min (limit , TestResource .datastore_rows_max )
648655 result = {
649656 "resource_id" : datadict ["resource_id" ],
650657 "fields" : fields ,
651658 "records" : all_records [offset : offset + limit ],
659+ "limit" : limit ,
652660 }
653661 if datadict ["include_total" ]:
654662 result ["total" ] = len (all_records )
@@ -1175,7 +1183,7 @@ def test_datastore(self, configuration, post_datastore, topline_yaml, topline_js
11751183 resource .update_datastore (records , method = "insert" )
11761184 assert TestResource .datastore == "upsert"
11771185
1178- def test_datastore_queries (self , configuration , post_datastore_query ):
1186+ def test_datastore_queries (self , configuration , post_datastore_query , caplog ):
11791187 resource = Resource ({"id" : "datastore_query_resource" })
11801188 result = resource .search_datastore ()
11811189 assert result ["total" ] == 3
@@ -1220,6 +1228,13 @@ def test_datastore_queries(self, configuration, post_datastore_query):
12201228 assert len (records ) == 3
12211229 assert TestResource .datastore_request ["offset" ] == 3
12221230
1231+ TestResource .datastore_rows_max = 2
1232+ records = list (resource .get_datastore_records (page_size = 5 ))
1233+ assert [record ["code" ] for record in records ] == ["A" , "B" , "C" ]
1234+ assert TestResource .datastore_request ["offset" ] == 2
1235+ assert TestResource .datastore_request ["limit" ] == 5
1236+ TestResource .datastore_rows_max = None
1237+
12231238 assert resource .get_datastore_fields () == [
12241239 {"id" : "code" , "type" : "text" },
12251240 {"id" : "value" , "type" : "numeric" },
@@ -1230,6 +1245,13 @@ def test_datastore_queries(self, configuration, post_datastore_query):
12301245 'SELECT * FROM "datastore_query_resource" LIMIT 2'
12311246 )
12321247 assert [record ["code" ] for record in records ] == ["A" , "B" ]
1248+ assert "truncated" not in caplog .text
1249+ with caplog .at_level (logging .WARNING ):
1250+ records = Resource .search_datastore_sql (
1251+ 'SELECT * FROM "truncated_resource"'
1252+ )
1253+ assert len (records ) == 2
1254+ assert "truncated by HDX to 2 rows" in caplog .text
12331255 with pytest .raises (HDXError ):
12341256 Resource .search_datastore_sql ("error" )
12351257
0 commit comments