From f3ac261bc8ef5ffe9be6483bd79a87e9aa761e22 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Thu, 24 Sep 2026 22:15:17 +0200 Subject: [PATCH 1/2] complex_portal_download: page the search by record offset and total The Complex Portal search endpoint reports `size` as the number of elements in the page at hand and `totalNumberOfResults` as the grand total, and `first` counts records rather than pages. Reading `size` as the total ended the paging loop after the first page, so any organism with more than 100 complexes was silently truncated: S. cerevisiae (taxonomy 559292) returned 100 of its 634. Page by record offset, take the total from `totalNumberOfResults`, and warn rather than truncate silently if the page bound is ever reached. The test fixtures now use the real field semantics. --- .../databases/complex_portal_download.py | 22 +++++++--- tests/test_complex_portal_download.py | 43 +++++++++++-------- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/geckopy/databases/complex_portal_download.py b/src/geckopy/databases/complex_portal_download.py index 9d27cd8..04198c2 100644 --- a/src/geckopy/databases/complex_portal_download.py +++ b/src/geckopy/databases/complex_portal_download.py @@ -86,10 +86,11 @@ def get_complex_data( session = _make_session() - # Step 1: search to enumerate complex IDs. The search endpoint paginates - # (``elements`` is one page; ``size`` is the total), so a single request - # silently truncates organisms with more than one page of complexes. - # Page through with ``first``/``number`` until all are collected. + # Step 1: search to enumerate complex IDs. The search endpoint paginates, + # so a single request silently truncates organisms with more than one + # page of complexes. ``first`` is a record offset and ``number`` the page + # length; the grand total is ``totalNumberOfResults`` (``size`` counts + # only the elements in the page at hand). search_url = _SEARCH_URL if taxonomic_id != 0: search_url = ( @@ -104,14 +105,17 @@ def get_complex_data( for page in range(_MAX_SEARCH_PAGES): response = session.get( search_url, - params={"first": page, "number": _SEARCH_PAGE_SIZE}, + params={ + "first": page * _SEARCH_PAGE_SIZE, + "number": _SEARCH_PAGE_SIZE, + }, timeout=_TIMEOUT_SECONDS, ) response.raise_for_status() search_data = response.json() if total is None: - total = int(search_data.get("size", 0)) + total = int(search_data.get("totalNumberOfResults", 0)) if total == 0: raise ValueError( f"No complexes returned for taxonomic_id={taxonomic_id}." @@ -123,6 +127,12 @@ def get_complex_data( complex_ids.extend(elem["complexAC"] for elem in elements) if len(complex_ids) >= total: break + if total is not None and len(complex_ids) < total: + logger.warning( + "Complex Portal reported %d complexes but only %d were " + "collected before the %d-page bound.", + total, len(complex_ids), _MAX_SEARCH_PAGES, + ) logger.info( "Found %d complexes; fetching details ...", len(complex_ids) ) diff --git a/tests/test_complex_portal_download.py b/tests/test_complex_portal_download.py index 4d02870..223ec1a 100644 --- a/tests/test_complex_portal_download.py +++ b/tests/test_complex_portal_download.py @@ -55,7 +55,7 @@ def _factory(): def test_returns_list_of_entries(monkeypatch): responses = { "search/*?facets": (200, { - "size": 1, + "size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}], }), "complex/CPX-1": (200, { @@ -88,7 +88,7 @@ class _CaptureSession: def get(self, url, params=None, timeout=None): captured_urls.append(url) if "search/*" in url and "filters" not in url: - return _FakeResponse(200, {"size": 0, "elements": []}) + return _FakeResponse(200, {"size": 0, "totalNumberOfResults": 0, "elements": []}) return _FakeResponse(404, {}) from geckopy.databases import complex_portal_download as mod @@ -107,7 +107,7 @@ def test_none_taxonomic_id_raises(): def test_empty_search_response_raises(monkeypatch): _install_fake_session(monkeypatch, { - "search/*": (200, {"size": 0, "elements": []}), + "search/*": (200, {"size": 0, "totalNumberOfResults": 0, "elements": []}), }) with pytest.raises(ValueError, match="No complexes"): get_complex_data(taxonomic_id=9606) @@ -117,7 +117,7 @@ def test_404_on_individual_complex_is_skipped(monkeypatch): """If one complex's details 404, the run continues without it.""" responses = { "search/*": (200, { - "size": 2, + "size": 2, "totalNumberOfResults": 2, "elements": [{"complexAC": "CPX-1"}, {"complexAC": "CPX-MISSING"}], }), "complex/CPX-1": (200, { @@ -145,7 +145,7 @@ def test_404_on_individual_complex_is_skipped(monkeypatch): def test_missing_stoichiometry_becomes_zero(monkeypatch): """A complex with no stoichiometry info should have all zeros.""" responses = { - "search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}), + "search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}), "complex/CPX-1": (200, { "complexAc": "CPX-1", "name": "x", "species": "y", @@ -165,7 +165,7 @@ def test_only_min_value_parsed(monkeypatch): """Only minValue is parsed from a 'minValue: 2, maxValue: 4' string; maxValue is ignored.""" responses = { - "search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}), + "search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}), "complex/CPX-1": (200, { "complexAc": "CPX-1", "name": "x", "species": "y", "participants": [ @@ -187,7 +187,7 @@ def test_non_protein_participants_filtered(monkeypatch): """Participants with interactorType != 'protein' are excluded when at least one protein participant is present.""" responses = { - "search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}), + "search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}), "complex/CPX-1": (200, { "complexAc": "CPX-1", "name": "x", "species": "y", "participants": [ @@ -211,7 +211,7 @@ def test_non_protein_participants_filtered(monkeypatch): def test_writes_json_to_disk(monkeypatch, tmp_path): responses = { - "search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}), + "search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}), "complex/CPX-1": (200, { "complexAc": "CPX-1", "name": "Test", "species": "Homo sapiens", "participants": [ @@ -236,7 +236,7 @@ def test_round_trip_through_disk(monkeypatch, tmp_path): """Download, write, reload: the entries should be identical in their relevant fields.""" responses = { - "search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}), + "search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}), "complex/CPX-1": (200, { "complexAc": "CPX-1", "name": "Test", "species": "x", "participants": [ @@ -269,7 +269,7 @@ def test_complex_of_complexes_flattens(monkeypatch): proteins, with stoichiometries multiplied.""" responses = { "search/*": (200, { - "size": 3, + "size": 3, "totalNumberOfResults": 3, "elements": [ {"complexAC": "SUB-A"}, {"complexAC": "SUB-B"}, @@ -316,7 +316,7 @@ def test_subcomplex_with_ligand_ignores_ligand(monkeypatch): be treated as a sub-complex id.""" responses = { "search/*": (200, { - "size": 2, + "size": 2, "totalNumberOfResults": 2, "elements": [{"complexAC": "SUB-A"}, {"complexAC": "BIG"}], }), "complex/SUB-A": (200, { @@ -350,7 +350,7 @@ def test_ligand_only_complex_has_no_proteins(monkeypatch): """A complex whose only participants are non-protein, non-complex must not be flattened as a complex-of-complexes; it yields no proteins.""" responses = { - "search/*": (200, {"size": 1, "elements": [{"complexAC": "LIG"}]}), + "search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "LIG"}]}), "complex/LIG": (200, { "complexAc": "LIG", "name": "ligand only", "species": "x", "participants": [ @@ -369,13 +369,16 @@ def test_ligand_only_complex_has_no_proteins(monkeypatch): def test_search_paginates(monkeypatch): """The search is paged: complexes beyond the first page must still be - collected (size=3 spread over two pages).""" + collected. ``first`` is a record offset, and ``totalNumberOfResults`` + -- not ``size``, which counts only the page at hand -- is the total.""" from geckopy.databases import complex_portal_download as mod + page_size = mod._SEARCH_PAGE_SIZE pages = { - 0: {"size": 3, "elements": [{"complexAC": "CPX-1"}, - {"complexAC": "CPX-2"}]}, - 1: {"size": 3, "elements": [{"complexAC": "CPX-3"}]}, + 0: {"size": 2, "totalNumberOfResults": 3, + "elements": [{"complexAC": "CPX-1"}, {"complexAC": "CPX-2"}]}, + page_size: {"size": 1, "totalNumberOfResults": 3, + "elements": [{"complexAC": "CPX-3"}]}, } def _detail(cid): @@ -391,9 +394,11 @@ def _detail(cid): class _PagingSession: def get(self, url, params=None, timeout=None): if "search/" in url: - page = (params or {}).get("first", 0) - return _FakeResponse(200, pages.get(page, {"size": 3, - "elements": []})) + offset = (params or {}).get("first", 0) + return _FakeResponse(200, pages.get( + offset, + {"size": 0, "totalNumberOfResults": 3, "elements": []}, + )) cid = url.rsplit("/", 1)[-1] return _FakeResponse(200, _detail(cid)) From 910338506e8a3e63560cfd7b4c2d605bab080765 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Thu, 24 Sep 2026 22:17:58 +0200 Subject: [PATCH 2/2] evotune docs: corrections() does not take annotate_from_model's sources `annotate_from_model` returns names, ec_codes and sources, but `EvotuneResult.corrections()` fills sources from `result.groups` itself and has no such keyword, so the documented `result.corrections(leverage=leverage, **annotations)` raises TypeError. Spell the two keywords it does take, and say where the third one goes. --- docs/evotune_kcat_tuning.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/evotune_kcat_tuning.md b/docs/evotune_kcat_tuning.md index 94a6a40..2bc7d1f 100644 --- a/docs/evotune_kcat_tuning.md +++ b/docs/evotune_kcat_tuning.md @@ -319,10 +319,16 @@ leverage_by_rxn = dict(zip(screen["rxn_id"], screen["leverage"])) leverage = [leverage_by_rxn.get(r, 0.0) for r in result.rxns] annotations = annotate_from_model(model, result.rxns) -rows = result.corrections(leverage=leverage, **annotations) +rows = result.corrections( + leverage=leverage, + names=annotations["names"], ec_codes=annotations["ec_codes"], +) open("corrections.tsv", "w").write(corrections_tsv(rows)) ``` +`annotate_from_model` also returns `sources`, which `corrections()` +takes from `result.groups` itself rather than as a keyword. + `cumulative_share` on each row is the running fraction of total leverage the list has accounted for so far, so a reader can stop where it stops climbing rather than reading every row. This is the tuning-time