From 32b59320db7cc2ba4a4187b0ccc2ff1f6ece1aac Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 15 Oct 2026 10:15:51 +1300 Subject: [PATCH] Reject page_size < 1 and csv/tsv records_format in get_datastore_records --- documentation/index.md | 3 ++- src/hdx/data/resource.py | 9 ++++++++- tests/hdx/data/test_resource.py | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/documentation/index.md b/documentation/index.md index c1be1d6..c3c1e0b 100755 --- a/documentation/index.md +++ b/documentation/index.md @@ -1008,7 +1008,8 @@ and `include_total`, and passes any other keyword arguments to CKAN: records = result["records"] **get_datastore_records** takes the same query parameters but pages through -all matching rows, yielding each record (`page_size` defaults to 10000): +all matching rows, yielding each record (`page_size` defaults to 10000). +`records_format` csv and tsv are not supported; use `search_datastore` for those: for record in resource.get_datastore_records(filters={"code": "A"}): ... diff --git a/src/hdx/data/resource.py b/src/hdx/data/resource.py index 4a9ba7c..da18077 100755 --- a/src/hdx/data/resource.py +++ b/src/hdx/data/resource.py @@ -930,11 +930,18 @@ def get_datastore_records( sort: Comma separated field names with optional asc/desc eg. "code desc". Defaults to None. distinct: Whether to return only distinct rows. Defaults to False. page_size: Number of rows to request per call. Defaults to 10000. - **kwargs: Other parameters to pass to datastore_search eg. plain, language. + **kwargs: Other parameters to pass to datastore_search eg. plain, language. records_format csv and tsv are not supported. Returns: Iterator of record dictionaries """ + if page_size < 1: + raise HDXError("page_size must be at least 1!") + records_format = kwargs.get("records_format") + if records_format in ("csv", "tsv"): + raise HDXError( + f"records_format {records_format} is not supported! Use search_datastore instead." + ) offset = 0 while True: result = self.search_datastore( diff --git a/tests/hdx/data/test_resource.py b/tests/hdx/data/test_resource.py index 6c7fb47..72e1cf8 100755 --- a/tests/hdx/data/test_resource.py +++ b/tests/hdx/data/test_resource.py @@ -1219,6 +1219,10 @@ def test_datastore_queries(self, configuration, post_datastore_query): records = list(resource.get_datastore_records(page_size=3)) assert len(records) == 3 assert TestResource.datastore_request["offset"] == 3 + with pytest.raises(HDXError): + list(resource.get_datastore_records(page_size=0)) + with pytest.raises(HDXError): + list(resource.get_datastore_records(records_format="csv")) assert resource.get_datastore_fields() == [ {"id": "code", "type": "text"},