Skip to content

Fix Complex Portal search paging; correct the corrections() docs example #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
edkerk merged 2 commits into from
Sep 24, 2026
Merged

Fix Complex Portal search paging; correct the corrections() docs example #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f3ac261
complex_portal_download: page the search by record offset and total
edkerk Sep 24, 2026
9103385
evotune docs: corrections() does not take annotate_from_model's sources
edkerk Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion docs/evotune_kcat_tuning.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,10 +319,16 @@ leverage_by_rxn = dict(zip(screen["rxn_id"], screen["leverage"]))
leverage = [leverage_by_rxn.get(r, 0.0) for r in result.rxns]

annotations = annotate_from_model(model, result.rxns)
rows = result.corrections(leverage=leverage, **annotations)
rows = result.corrections(
leverage=leverage,
names=annotations["names"], ec_codes=annotations["ec_codes"],
)
open("corrections.tsv", "w").write(corrections_tsv(rows))
```

`annotate_from_model` also returns `sources`, which `corrections()`
takes from `result.groups` itself rather than as a keyword.

`cumulative_share` on each row is the running fraction of total leverage
the list has accounted for so far, so a reader can stop where it stops
climbing rather than reading every row. This is the tuning-time
Expand Down
22 changes: 16 additions & 6 deletions src/geckopy/databases/complex_portal_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,11 @@ def get_complex_data(

session = _make_session()

# Step 1: search to enumerate complex IDs. The search endpoint paginates
# (``elements`` is one page; ``size`` is the total), so a single request
# silently truncates organisms with more than one page of complexes.
# Page through with ``first``/``number`` until all are collected.
# Step 1: search to enumerate complex IDs. The search endpoint paginates,
# so a single request silently truncates organisms with more than one
# page of complexes. ``first`` is a record offset and ``number`` the page
# length; the grand total is ``totalNumberOfResults`` (``size`` counts
# only the elements in the page at hand).
search_url = _SEARCH_URL
if taxonomic_id != 0:
search_url = (
Expand All @@ -104,14 +105,17 @@ def get_complex_data(
for page in range(_MAX_SEARCH_PAGES):
response = session.get(
search_url,
params={"first": page, "number": _SEARCH_PAGE_SIZE},
params={
"first": page * _SEARCH_PAGE_SIZE,
"number": _SEARCH_PAGE_SIZE,
},
timeout=_TIMEOUT_SECONDS,
)
response.raise_for_status()
search_data = response.json()

if total is None:
total = int(search_data.get("size", 0))
total = int(search_data.get("totalNumberOfResults", 0))
if total == 0:
raise ValueError(
f"No complexes returned for taxonomic_id={taxonomic_id}."
Expand All @@ -123,6 +127,12 @@ def get_complex_data(
complex_ids.extend(elem["complexAC"] for elem in elements)
if len(complex_ids) >= total:
break
if total is not None and len(complex_ids) < total:
logger.warning(
"Complex Portal reported %d complexes but only %d were "
"collected before the %d-page bound.",
total, len(complex_ids), _MAX_SEARCH_PAGES,
)
logger.info(
"Found %d complexes; fetching details ...", len(complex_ids)
)
Expand Down
43 changes: 24 additions & 19 deletions tests/test_complex_portal_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def _factory():
def test_returns_list_of_entries(monkeypatch):
responses = {
"search/*?facets": (200, {
"size": 1,
"size": 1, "totalNumberOfResults": 1,
"elements": [{"complexAC": "CPX-1"}],
}),
"complex/CPX-1": (200, {
Expand Down Expand Up @@ -88,7 +88,7 @@ class _CaptureSession:
def get(self, url, params=None, timeout=None):
captured_urls.append(url)
if "search/*" in url and "filters" not in url:
return _FakeResponse(200, {"size": 0, "elements": []})
return _FakeResponse(200, {"size": 0, "totalNumberOfResults": 0, "elements": []})
return _FakeResponse(404, {})

from geckopy.databases import complex_portal_download as mod
Expand All @@ -107,7 +107,7 @@ def test_none_taxonomic_id_raises():

def test_empty_search_response_raises(monkeypatch):
_install_fake_session(monkeypatch, {
"search/*": (200, {"size": 0, "elements": []}),
"search/*": (200, {"size": 0, "totalNumberOfResults": 0, "elements": []}),
})
with pytest.raises(ValueError, match="No complexes"):
get_complex_data(taxonomic_id=9606)
Expand All @@ -117,7 +117,7 @@ def test_404_on_individual_complex_is_skipped(monkeypatch):
"""If one complex's details 404, the run continues without it."""
responses = {
"search/*": (200, {
"size": 2,
"size": 2, "totalNumberOfResults": 2,
"elements": [{"complexAC": "CPX-1"}, {"complexAC": "CPX-MISSING"}],
}),
"complex/CPX-1": (200, {
Expand Down Expand Up @@ -145,7 +145,7 @@ def test_404_on_individual_complex_is_skipped(monkeypatch):
def test_missing_stoichiometry_becomes_zero(monkeypatch):
"""A complex with no stoichiometry info should have all zeros."""
responses = {
"search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}),
"search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}),
"complex/CPX-1": (200, {
"complexAc": "CPX-1",
"name": "x", "species": "y",
Expand All @@ -165,7 +165,7 @@ def test_only_min_value_parsed(monkeypatch):
"""Only minValue is parsed from a 'minValue: 2, maxValue: 4' string;
maxValue is ignored."""
responses = {
"search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}),
"search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}),
"complex/CPX-1": (200, {
"complexAc": "CPX-1", "name": "x", "species": "y",
"participants": [
Expand All @@ -187,7 +187,7 @@ def test_non_protein_participants_filtered(monkeypatch):
"""Participants with interactorType != 'protein' are excluded when
at least one protein participant is present."""
responses = {
"search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}),
"search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}),
"complex/CPX-1": (200, {
"complexAc": "CPX-1", "name": "x", "species": "y",
"participants": [
Expand All @@ -211,7 +211,7 @@ def test_non_protein_participants_filtered(monkeypatch):

def test_writes_json_to_disk(monkeypatch, tmp_path):
responses = {
"search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}),
"search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}),
"complex/CPX-1": (200, {
"complexAc": "CPX-1", "name": "Test", "species": "Homo sapiens",
"participants": [
Expand All @@ -236,7 +236,7 @@ def test_round_trip_through_disk(monkeypatch, tmp_path):
"""Download, write, reload: the entries should be identical
in their relevant fields."""
responses = {
"search/*": (200, {"size": 1, "elements": [{"complexAC": "CPX-1"}]}),
"search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "CPX-1"}]}),
"complex/CPX-1": (200, {
"complexAc": "CPX-1", "name": "Test", "species": "x",
"participants": [
Expand Down Expand Up @@ -269,7 +269,7 @@ def test_complex_of_complexes_flattens(monkeypatch):
proteins, with stoichiometries multiplied."""
responses = {
"search/*": (200, {
"size": 3,
"size": 3, "totalNumberOfResults": 3,
"elements": [
{"complexAC": "SUB-A"},
{"complexAC": "SUB-B"},
Expand Down Expand Up @@ -316,7 +316,7 @@ def test_subcomplex_with_ligand_ignores_ligand(monkeypatch):
be treated as a sub-complex id."""
responses = {
"search/*": (200, {
"size": 2,
"size": 2, "totalNumberOfResults": 2,
"elements": [{"complexAC": "SUB-A"}, {"complexAC": "BIG"}],
}),
"complex/SUB-A": (200, {
Expand Down Expand Up @@ -350,7 +350,7 @@ def test_ligand_only_complex_has_no_proteins(monkeypatch):
"""A complex whose only participants are non-protein, non-complex must
not be flattened as a complex-of-complexes; it yields no proteins."""
responses = {
"search/*": (200, {"size": 1, "elements": [{"complexAC": "LIG"}]}),
"search/*": (200, {"size": 1, "totalNumberOfResults": 1, "elements": [{"complexAC": "LIG"}]}),
"complex/LIG": (200, {
"complexAc": "LIG", "name": "ligand only", "species": "x",
"participants": [
Expand All @@ -369,13 +369,16 @@ def test_ligand_only_complex_has_no_proteins(monkeypatch):

def test_search_paginates(monkeypatch):
"""The search is paged: complexes beyond the first page must still be
collected (size=3 spread over two pages)."""
collected. ``first`` is a record offset, and ``totalNumberOfResults``
-- not ``size``, which counts only the page at hand -- is the total."""
from geckopy.databases import complex_portal_download as mod

page_size = mod._SEARCH_PAGE_SIZE
pages = {
0: {"size": 3, "elements": [{"complexAC": "CPX-1"},
{"complexAC": "CPX-2"}]},
1: {"size": 3, "elements": [{"complexAC": "CPX-3"}]},
0: {"size": 2, "totalNumberOfResults": 3,
"elements": [{"complexAC": "CPX-1"}, {"complexAC": "CPX-2"}]},
page_size: {"size": 1, "totalNumberOfResults": 3,
"elements": [{"complexAC": "CPX-3"}]},
}

def _detail(cid):
Expand All @@ -391,9 +394,11 @@ def _detail(cid):
class _PagingSession:
def get(self, url, params=None, timeout=None):
if "search/" in url:
page = (params or {}).get("first", 0)
return _FakeResponse(200, pages.get(page, {"size": 3,
"elements": []}))
offset = (params or {}).get("first", 0)
return _FakeResponse(200, pages.get(
offset,
{"size": 0, "totalNumberOfResults": 3, "elements": []},
))
cid = url.rsplit("/", 1)[-1]
return _FakeResponse(200, _detail(cid))

Expand Down
Loading