From edd98302cc56d10397ac5ad8266a4fbfbddd3ebd Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Mon, 7 Sep 2026 12:07:45 +0200 Subject: [PATCH 01/15] Add CMA-ES kcat tuning (evotune) against experimental data (#51) * Add CMA-ES kcat tuning against experimental flux/growth data * Rename Bayesian kcat tuning to evolutionary (CMA-ES) tuning; drop internal process docs * Rename evolutionary_tuning to the shorter evotune --- .github/workflows/test.yml | 2 +- docs/cmaes_vs_abc_smc.md | 57 ++ docs/evotune_kcat_tuning.md | 309 ++++++ docs/internal/bayesian_tuning_plan.md | 150 --- docs/migrating_from_gecko_matlab.md | 16 +- docs/raven_integration.md | 2 +- pyproject.toml | 11 +- src/geckopy/__init__.py | 4 +- src/geckopy/adapter/params.py | 155 ++- src/geckopy/adapter/template.py | 35 +- src/geckopy/cli.py | 8 +- src/geckopy/databases/flux_data.py | 26 +- src/geckopy/gather_kcats/review.py | 267 +++++ src/geckopy/gather_kcats/select_kcat_value.py | 2 +- .../kcat_sensitivity_analysis/__init__.py | 19 - src/geckopy/kcat_tuning/__init__.py | 44 + src/geckopy/kcat_tuning/evotune/__init__.py | 57 ++ src/geckopy/kcat_tuning/evotune/data.py | 106 ++ src/geckopy/kcat_tuning/evotune/distance.py | 261 +++++ src/geckopy/kcat_tuning/evotune/parsimony.py | 248 +++++ src/geckopy/kcat_tuning/evotune/priors.py | 108 +++ src/geckopy/kcat_tuning/evotune/simulate.py | 211 ++++ src/geckopy/kcat_tuning/evotune/tuning.py | 912 ++++++++++++++++++ src/geckopy/kcat_tuning/evotune/tying.py | 98 ++ .../find_max_value.py | 4 +- .../sensitivity_tuning.py | 4 +- .../sigma_fitter.py | 4 +- .../truncate_values.py | 4 +- tests/conftest.py | 27 + tests/test_cli.py | 30 +- tests/test_deprecated_aliases.py | 2 +- tests/test_evotune.py | 425 ++++++++ tests/test_evotune_data.py | 85 ++ tests/test_evotune_parsimony.py | 162 ++++ tests/test_evotune_priors.py | 70 ++ tests/test_evotune_simulate_distance.py | 484 ++++++++++ tests/test_evotune_tying.py | 103 ++ tests/test_find_max_value.py | 2 +- tests/test_kcat_assignment_review.py | 148 +++ tests/test_load_flux_data.py | 24 +- tests/test_params.py | 54 +- tests/test_sensitivity_tuning.py | 2 +- tests/test_sigma_fitter.py | 2 +- tests/test_truncate_values.py | 4 +- tutorials/full_ecModel/code/anaerobic.py | 71 ++ .../full_ecModel/data/evotuneFluxData.tsv | 34 + .../full_ecModel/data/evotuneMaxGrowth.tsv | 9 + .../full_ecModel/data/evotuneZeroExch.tsv | 220 +++++ tutorials/full_ecModel/model_adapter.toml | 32 + tutorials/full_ecModel/protocol.py | 8 +- 50 files changed, 4790 insertions(+), 332 deletions(-) create mode 100644 docs/cmaes_vs_abc_smc.md create mode 100644 docs/evotune_kcat_tuning.md delete mode 100644 docs/internal/bayesian_tuning_plan.md create mode 100644 src/geckopy/gather_kcats/review.py delete mode 100644 src/geckopy/kcat_sensitivity_analysis/__init__.py create mode 100644 src/geckopy/kcat_tuning/__init__.py create mode 100644 src/geckopy/kcat_tuning/evotune/__init__.py create mode 100644 src/geckopy/kcat_tuning/evotune/data.py create mode 100644 src/geckopy/kcat_tuning/evotune/distance.py create mode 100644 src/geckopy/kcat_tuning/evotune/parsimony.py create mode 100644 src/geckopy/kcat_tuning/evotune/priors.py create mode 100644 src/geckopy/kcat_tuning/evotune/simulate.py create mode 100644 src/geckopy/kcat_tuning/evotune/tuning.py create mode 100644 src/geckopy/kcat_tuning/evotune/tying.py rename src/geckopy/{kcat_sensitivity_analysis => kcat_tuning}/find_max_value.py (96%) rename src/geckopy/{kcat_sensitivity_analysis => kcat_tuning}/sensitivity_tuning.py (99%) rename src/geckopy/{kcat_sensitivity_analysis => kcat_tuning}/sigma_fitter.py (98%) rename src/geckopy/{kcat_sensitivity_analysis => kcat_tuning}/truncate_values.py (94%) create mode 100644 tests/conftest.py create mode 100644 tests/test_evotune.py create mode 100644 tests/test_evotune_data.py create mode 100644 tests/test_evotune_parsimony.py create mode 100644 tests/test_evotune_priors.py create mode 100644 tests/test_evotune_simulate_distance.py create mode 100644 tests/test_evotune_tying.py create mode 100644 tests/test_kcat_assignment_review.py create mode 100644 tutorials/full_ecModel/code/anaerobic.py create mode 100644 tutorials/full_ecModel/data/evotuneFluxData.tsv create mode 100644 tutorials/full_ecModel/data/evotuneMaxGrowth.tsv create mode 100644 tutorials/full_ecModel/data/evotuneZeroExch.tsv diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9331c55..ef078fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: cache-dependency-path: pyproject.toml - name: Install - run: pip install -e ".[dev]" + run: pip install -e ".[dev,evotune]" - name: Run pytest run: pytest -q diff --git a/docs/cmaes_vs_abc_smc.md b/docs/cmaes_vs_abc_smc.md new file mode 100644 index 0000000..7fbe57c --- /dev/null +++ b/docs/cmaes_vs_abc_smc.md @@ -0,0 +1,57 @@ +# Why this package tunes kcats with CMA-ES, not ABC-SMC + +DLKcat's own Bayesian approach to kcat tuning implements ABC-SMC +(Approximate Bayesian Computation, Sequential Monte Carlo) -- +[SysBioChalmers/DLKcat/BayesianApproach](https://github.com/SysBioChalmers/DLKcat/tree/master/BayesianApproach). +geckopy's `kcat_tuning.evotune` module ported that +approach faithfully and tested it side by side with a plain optimiser, +CMA-ES, on the same objective, the same screened parameter set, and +the same experimental data (ecYeastGEM, 41 conditions). CMA-ES won on +every criterion that was checked, so it is what this package ships. + +## What ABC-SMC does, and why that's the wrong tool here + +ABC-SMC is a sampler: each generation it draws a batch of candidate +kcat vectors from a proposal distribution, scores them, and keeps the +best fraction to seed the next generation's proposals. That machinery +-- a posterior approximation, a sample schedule, truncation selection +-- earns its keep when the goal is a *distribution* over plausible +parameter values, and the model is cheap enough to sample densely +across the whole parameter space. + +Once a screen has reduced the problem to the kcats the data can +actually inform (a few dozen to a few hundred, out of thousands), that +premise stops holding. There's no longer a broad space to sample +across, and each evaluation is expensive (an FBA solve per condition). +At that point the problem is a direct search for the best-fitting +vector -- exactly what an optimiser is for, and exactly what ABC-SMC +is not. + +## What the comparison showed + +Same model, same 112-parameter screen, same objective, three seeds +each: + +| | distance | spread across seeds | +|---|---|---| +| ABC-SMC | 0.9156 | +/- 0.0549 | +| **CMA-ES** | **0.7974** | **+/- 0.0048** | + +CMA-ES fits better by 3.7 standard errors, with a spread more than ten +times tighter. The gap isn't just fit quality -- it's whether an +individual tuned kcat means anything. Both methods leave most kcats on +flat directions the data can't pin down, but ABC-SMC's posterior +sampling wanders across the whole flat region: across three ABC-SMC +seeds, lanosterol synthase (the single highest-leverage kcat in the +model) took values of 0.338, 20.3, and 0.0781 1/s -- a 260-fold spread, +at distances differing by 0.009. A number that unstable cannot be +reported as a finding. + +CMA-ES doesn't remove flat directions -- nothing can, they're a +property of the data, not the search method -- but a direct optimiser +converges to one point per flat direction instead of sampling across +it, and `prior_penalty_weight` (a Tikhonov-style penalty on moving away +from the prior, weighted by how much each source is trusted) picks out +which point: see `docs/evotune_kcat_tuning.md` for how that combines +with `screen_kcat_leverage`/`select_tunable_mask` into the method this +package actually recommends. diff --git a/docs/evotune_kcat_tuning.md b/docs/evotune_kcat_tuning.md new file mode 100644 index 0000000..83dd6b9 --- /dev/null +++ b/docs/evotune_kcat_tuning.md @@ -0,0 +1,309 @@ +# Tuning kcats against experimental data + +A first, brief protocol for fitting an ecModel's kcats to measured +growth rates and/or fluxes with `cmaes_kcat_tuning`. This covers one +recommended path end to end; it is not a full reference (see the +docstrings in `geckopy.kcat_tuning.evotune` for every +option) and it does not build an ecModel from scratch (see +[migrating_from_gecko_matlab.md](migrating_from_gecko_matlab.md) for +that). + +## What this does + +An ecModel's kcats mostly come from databases (BRENDA), predictors +(DLKcat, OpenKineticsPredictor), or manual curation, and they don't +reproduce measured growth on their own. Three functions, used in +sequence, turn measured data into corrections: + +1. **`screen_kcat_leverage`** reports which kcats the data can + actually speak to -- no optimisation, just "if this kcat were + different, how much would the fit change." Useful for curation on + its own, before any tuning run. +2. **`select_tunable_mask`** turns that report into the set of kcats a + tuning run should search over, automatically if you skip it. +3. **`cmaes_kcat_tuning`** searches that set with CMA-ES for the kcat + vector that best matches the data, weighted by how much each kcat's + source is trusted. + +A fourth function, **`review_assignment`**, is complementary rather +than part of that sequence: it checks the kcats *themselves* for +assignment mistakes -- independent of any measured data -- since the +tuner's largest corrections are usually places the assignment went +wrong rather than discoveries about biology (see Step 4). + +The result is usually a short list of corrections, each with a +before/after value and a source -- not a wholesale rewrite of the +model. A run that changes nearly every kcat by a little has not found +anything; a run that changes a handful by a lot, concentrated in the +least-trusted sources, has. + +## Before you start + +- A working ecModel and `ModelAdapter` project (an existing + `model_adapter.toml` and `models/*.yml`). +- `pip install geckopy[evotune]` -- the CMA-ES search + (`cma`) is an optional dependency, not part of the base install. +- Experimental data: at least one of a set of measured exchange fluxes + per condition, or a set of measured maximum growth rates per carbon + source. Both together works and is preferred -- flux data pins the + *shape* of metabolism, growth rate pins its *rate*. + +## Step 1: Prepare the experimental data + +Three tab-separated files go under `/data/`. All three are +optional -- a missing file is skipped, not an error -- but you need at +least one of the first two. + +| file | contents | required | +|------|----------|----------| +| `evotuneFluxData.tsv` | measured exchange fluxes, one row per condition | at least one of these two | +| `evotuneMaxGrowth.tsv` | measured maximum growth rate, one row per condition, one active carbon source per row (at `-1000`, i.e. unconstrained) | | +| `evotuneZeroExch.tsv` | reaction IDs assumed to carry zero flux in every condition | no | + +Both flux files share one column layout (the same parser geckopy uses +for regular flux data): + +``` +Condition Ptot grRate glucose (r_1714) fructose (r_1709) ... evotuneRMSEweight source +glucose NaN 0.41 -1000 NaN ... 1 DLKcat +fructose NaN 0.338 NaN -1000 ... 1 DLKcat +``` + +`Condition` names the row, `grRate` is the measured growth rate, +`Ptot` is measured protein content (`NaN` if not measured), each +` ()` column is a measured or fixed flux for that exchange +reaction, `evotuneRMSEweight` scales how much that row counts toward +the RMSE (usually `1`), and `source` is free text carried through for +your own bookkeeping. `evotuneZeroExch.tsv` is simpler -- a `Rxns` +header, then one reaction ID per line. + +## Step 2: Configure `[evotune]` in `model_adapter.toml` + +Every field has a default, so this section can be omitted entirely to +start. A configured example, in plain terms: + +```toml +[evotune] +sigma0_log_default = 0.3 # trust for any source not listed below +max_growth_weight = 2.0 # weight the growth-rate data double against flux data + +[evotune.source_groups.okp] +sources = ["OpenKineticsPredictor"] +match_okp = true + +[evotune.source_groups.brenda] +sources = ["brenda"] + +[evotune.source_groups.custom] +sources = ["custom"] + +[evotune.sigma0_log_source] +okp = 0.25 +brenda = 0.2 +custom = 0.1 +``` + +What each knob means: + +- **Trust tiers** (`source_groups` + `sigma0_log_source` + + `sigma0_log_default`). Group your kcats by how much you trust their + origin, and give each group a standard deviation in log-space: + smaller means more trusted, and the search needs stronger evidence + before it moves that kcat far. `custom` (curated by hand) at 0.1 is + three times more trusted than an unlabelled prediction at 0.3. This + is the one section with no universal default -- your model's own + source labels decide it. +- **`max_growth_weight`** (default `1.0`). The combined score is + `(rmse_flux + w * rmse_max_growth) / (w + 1)`. At `2`, growth-rate + error counts double against flux error -- useful when growth rate is + measured across many carbon sources but flux is mostly one. +- **`prior_penalty_weight`** (default `0.03`, already on). Adds + `w * mean(((log k - log k0) / sigma0)^2)` to the score: a charge for + moving a kcat, scaled by how much you trust its prior. Without this + the fit is often flat along many directions -- several very + different kcat vectors fit equally well -- and the search returns an + arbitrary point along that flat direction, which does not reproduce + if you rerun it with a different seed. The default is calibrated so + that large corrections agree in direction and land within a handful + of fold of each other across seeds, at a small cost in raw fit. Set + it to `0` to score on fit alone (also what a run reproducing GECKO + MATLAB, which has no such term, must do). The right strength is + model- and data-specific, since it depends on the scale of + `sigma0_log` and of the objective itself -- + `tune_prior_penalty_weight` runs a candidate sweep and reports fit + cost against cross-seed reproducibility so it doesn't have to be + picked blind (see its docstring; it costs one full tuning run per + candidate per seed, so treat it as an occasional calibration step, + not part of routine tuning). +- **`tie_isozymes`** (default `true`). Several ecModel reactions are + one enzymatic reaction split across isozyme copies. When copies + share a prior value and a source, no experimental condition can tell + them apart, so an untied search is free to invent a difference + between them that means nothing biologically. Tying gives such + copies one shared kcat. +- **`max_generations` / `rmse_threshold`**. Double as this search's + stopping conditions: run at most this many CMA-ES generations, or + stop earlier once the best RMSE reaches `rmse_threshold` (negative + never stops early). + +## Step 3: Find out what the data can see + +```python +from geckopy import ModelAdapter, load_ec_model +from geckopy.kcat_tuning.evotune import screen_kcat_leverage + +adapter = ModelAdapter.from_folder("path/to/project") +model = load_ec_model(adapter=adapter) # models/ecModel.yml by default + +screen = screen_kcat_leverage(model, adapter=adapter, n_proc=8) +screen.drop(columns="_positions").head(20) +``` + +This perturbs every tunable kcat (isozyme copies moved together, per +`tie_isozymes`) up and down and measures how much the fit changes -- +no tuning happens yet. The result is a table, one row per kcat or tied +group, ranked by that leverage weighted by trust: the reaction ID, how +many isozyme copies it covers, its source group, its current value, +its leverage, and `cum_leverage_share` -- the running fraction of +total leverage carried by this row and every row above it. The top +rows are what a curator should look at first, independent of whether +you ever run a tuning search: on ecYeastGEM, a handful of kcats +routinely carry the majority of what any tuning run could achieve, and +they tend to be implausible database values rather than genuine +biology. + +This costs one simulation per condition per kcat group, roughly +comparable in scale to a full tuning run's budget -- expect tens of +minutes on a genome-scale model, not a quick check. `n_proc` +parallelises it the same way tuning does. + +## Step 4: Check the assignments behind the top kcats + +Before spending a tuning run on a kcat, it is worth checking whether +the value was assigned correctly in the first place. +`review_assignment` flags source-level problems: a value with no kcat +row behind it for its EC, one number reused across many unrelated +reactions, an EC's maximum standing in for a whole distribution, an +implausible magnitude, or a curated value that just repeats what the +database already said. These are corrections to the *source* -- fix +them in whatever database or fuzzy-matching step in `gather_kcats` +produced the value -- so they propagate to every model built from the +same data, unlike a value the tuner moves for this one model alone. + +```python +from geckopy.gather_kcats.review import EcStats, review_assignment, findings_tsv + +# One EcStats per EC code queried while gathering kcats -- summarise +# whatever kcat database (usually BRENDA) the assignment came from. +ec_stats = { + "1.1.1.1": EcStats(n_kcat=12, kcat_max=340.0, kcat_median=45.0, + values=frozenset({340.0, 45.0, 12.0}), n_sa=0), + # ... +} + +findings = review_assignment( + model.ec.rxns, model.ec.kcat, model.ec.source, model.ec.eccodes, + ec_stats=ec_stats, +) +print(findings_tsv(findings)) +``` + +Each `Finding` names a reaction, its checks (`no-ec-evidence`, +`repeated-value`, `ec-maximum`, `magnitude`, `custom-duplicate`), and a +human-readable `detail`. Isozyme copies the +assignment could not tell apart (the same grouping `tie_isozymes` in +Step 2 uses) are reported once, not once per copy. Standard fallbacks +and tied groups are deliberate choices, not mistakes, so both stay +quiet unless asked for via `include`. + +Findings rank by `leverage` when you pass it in -- reuse the `screen` +from Step 3, expanded from tie-groups back to individual +`model.ec.rxns` positions via its `_positions` column -- so that +looking odd and actually mattering are not confused; without it, rows +come back in input order, and looking odd is all you get. `coverage` +(a share of flagged leverage) or `top` (a row count) then truncates the +report: on ecYeastGEM, tightening the checks themselves does not work +half as well as this does, since `coverage=0.8` cuts 2575 unfiltered +rows to 33, and the first row alone carries half. + +## Step 5: Tune + +```python +from geckopy import save_ec_model +from geckopy.kcat_tuning.evotune import cmaes_kcat_tuning + +result = cmaes_kcat_tuning( + model, adapter=adapter, screen=screen, n_proc=8, seed=0, +) + +save_ec_model(model, "ecModel_tuned.yml", adapter=adapter) +``` + +Passing the screen from Step 3 avoids recomputing it; omit `screen` +entirely and `cmaes_kcat_tuning` runs one itself. Either way, the +tunable set is built by `select_tunable_mask`, which keeps the fewest +highest-ranked kcats whose combined leverage reaches +`target_impact_share` (default `0.9`) of the total -- a cutoff +relative to the model's own achievable improvement, not a fixed count +or an absolute leverage value, so the same setting means a comparable +thing on a different model. Pass `tunable_mask` directly instead if +you want to fix the set yourself. + +`model` is mutated in place: on return, every tunable kcat carries the +best value CMA-ES found, and the model's kcat constraints are already +applied -- nothing further is needed before simulating or saving it. +Beyond `target_impact_share`, the only other knobs are `popsize` +(CMA-ES's population size, defaulting to its own dimension-scaled +choice rather than a value tuned for one particular model), `n_proc`, +and `seed`; everything else comes from the same `EvotuneParams` as +Step 2, so there is nothing new to configure once you've read it. + +If growth needs special handling for your organism -- forcing +anaerobic conditions, or scaling the protein pool with a +condition-specific biomass composition -- pass `make_anaerobic` and/or +`change_protein_biomass` callables; geckopy has no organism-agnostic +default for either. See `tutorials/full_ecModel/code/anaerobic.py` in +this repository for a worked example. + +## Step 6: Read the result + +`result` is an `EvotuneResult`: + +- `rmse_trace` / `objective_trace` -- best-so-far plain-fit RMSE and + best-so-far optimised objective, per generation (identical unless + `prior_penalty_weight` is nonzero). A flat trace for many generations + before the run ends means it stalled, not converged. +- `new_kcat` / `old_kcat` / `rxns` / `groups` -- the tuned and prior + kcats over the searched set, parallel arrays, with each kcat's trust + tier. Tied isozyme copies share one value in `new_kcat`. +- `converged` -- whether `rmse_threshold` was reached, or + `max_generations` was hit first. +- `n_generations` -- how many generations actually ran. + +Before trusting a tuned kcat, check more than the final RMSE: + +1. **How many kcats changed**, and by how much -- a result that moves + nearly everything by a little has not identified anything. + `geckopy.kcat_tuning.evotune.parsimony` has the + tools for this: `n_changed`, `fold_change`, `source_movement`. +2. **Impact share** -- `parsimony.impact_share` reports what fraction + of the total achievable improvement the changed kcats actually + carry, distinguishing "few, large, and consequential" from mere + sparsity. This is a different, unweighted quantity from + `screen_kcat_leverage`'s trust-weighted `cum_leverage_share`. +3. **Reproducibility.** Rerun with a different `seed`. Large + corrections (say, beyond two-fold) should land in the same + direction and within a small factor of each other; if they don't, + the search has found a flat direction rather than a real + correction, and `prior_penalty_weight` is the first knob to reach + for. +4. **Whether it generalises**, if you have data you didn't fit on -- + score the tuned model against it and compare to the untuned model. + A large improvement on fitted conditions with no improvement + elsewhere is a sign of overfitting to a thin dataset, not a fixed + model. + +## Where to go from here + +- `docs/cmaes_vs_abc_smc.md` -- why this method is CMA-ES rather than + the ABC-SMC sampler used elsewhere for the same problem. diff --git a/docs/internal/bayesian_tuning_plan.md b/docs/internal/bayesian_tuning_plan.md deleted file mode 100644 index 19d0b43..0000000 --- a/docs/internal/bayesian_tuning_plan.md +++ /dev/null @@ -1,150 +0,0 @@ -# Bayesian (ABC-SMC) kcat tuning — port plan (PAUSED) - -Status: investigation done, implementation not started. Paused in -favour of the OpenKineticsPredictor work. Resume from here. - -## What it is - -MATLAB source: -`GECKO/src/geckomat/kcat_sensitivity_analysis/Bayesian/` -- `bayesianSensitivityTuning.m` (856 LOC) — the main loop -- `abc_max.m` (121 LOC) — distance function (FBA + RMSE) -- `loadBayesianData.m` (19 LOC) — loads the 3 experimental TSVs -- `getrSample.m`, `updateprior.m`, `mse.m`, `addCarbonNum.m` — helpers -- ~1370 LOC total - -It is an **ABC-SMC** (Approximate Bayesian Computation, Sequential -Monte Carlo) parameter-estimation routine that searches for kcat -values letting an ecModel reproduce experimental flux / growth data. - -## Algorithm - -Vanilla ABC-SMC core: -1. Lognormal prior per kcat, with per-source initial sigma. -2. Each generation: sample N proposals -> apply each kcat set to the - ecModel -> solve FBA -> compute RMSE vs experimental data -> accept - best ~10% (`targetAccept` percentile, gated by `minKeep`/`maxKeep`). -3. Update posterior mean/sigma in log-space from accepted samples. -4. Adaptive epsilon (quantile of the RMSE distribution). -5. Stop when RMSE < `rmseThreshold` or `maxGenerations` reached. - -Distance function (`abc_max.m` -> `rmsecal`): -- RMSE between simulated FBA fluxes/growth and experimental - measurements across multiple carbon-source conditions. -- Two datasets: max growth rates, and exchange fluxes. -- Zero-flux constraints on reactions assumed inactive. -- Carbon-normalised exchange fluxes (uses `excarbon` per reaction). -- Anaerobic switch + protein-content adjustment per condition. -- Failed/infeasible solves get RMSE = 99 (high penalty). - -## GECKO-specific extensions (the load-bearing custom code) - -These sit on top of the standard SMC and are what make it work on real -models: -- **Per-source sticky priors**: each kcat source (`brenda`, `dlkcat`, - `custom`) has its own initial sigma, shrinkage threshold, variance - cap, and force-to-prior cutoff. BRENDA kcats stay near prior; - DLKcat ones drift freely. -- **Low-rank PCA proposal kernel** (`proposeLowRankMixture`): learns - the dominant directions of posterior variability (SVD in log-space) - and proposes along them. -- **Explore/exploit mixture**: alpha=0.65 PCA-guided exploit kernel + - 0.35 broad isotropic explore kernel (`cExpl` inflation). -- **Sparsity enforcement**: after each update, kcats whose change is - `< sparsityThr * sigma0` snap back to prior. -- **Variance cap per source**: caps posterior/prior variance ratio to - prevent random walk. -- **Diagnostics**: shrinkage trace, acceptance rate, epsilon, sparsity - count, diversity, per-source metrics, low-rank dim — mostly consumed - by tutorial plots. -- **Optional post-run pruning**: parameters whose change is below a - threshold revert to prior (`enablePruning`). - -Hyperparameters: ~15 in `params.bayesian` -(sigma0logDefault, kcatSources, sigma0logSource, shrinkThr*, -varianceCap*, forcePriorThr*, sparsityThreshold, scheduleGenerations, -scheduleSamples, targetAccept, minKeep, maxKeep, rmseThreshold, -maxGenerations). Fixed internals: alpha, tauResidual, sigmaFloorFrac, -adaptFracEarly, cExpl, rMax. - -## Experimental data files (per model adapter, in data/) - -- `bayesianFluxData.tsv` — carbon sources, growth rates, env - conditions, exchange rates -- `bayesianMaxGrowth.tsv` — max growth on various carbon sources -- `bayesianZeroExch.tsv` — exchange reactions assumed zero-flux - -## Library choice: pyABC (recommended) - -Compared pyABC vs ABCpy vs ELFI: - -| Library | Strength | Gap for us | -|---|---|---| -| **pyABC** | custom priors, custom Transition kernels, adaptive epsilon, dynamic-scheduled parallelism (dask/MPI/redis/mp), used in systems biology | no built-in per-source shrinkage or low-rank kernel — subclass `Transition` | -| ABCpy | widest algorithm menu | static-scheduled parallelism only | -| ELFI | BOLFI GP surrogate could cut FBA calls | research-grade; high-D kcat space may not suit BOLFI | - -pyABC chosen because (a) its `Transition` abstraction is the right -hook for the GECKO-specific kernel, and (b) dynamic scheduling matters: -each FBA solve is ~10-100 ms and the schedule is up to 500 samples x -50-100 generations, so wall-time dominates. MATLAB already uses -`parfor`; pyABC's dask backend matches that. - -Refs: -- pyABC docs: https://pyabc.readthedocs.io/en/latest/ -- pyABC JOSS paper (2022): https://arxiv.org/pdf/2203.13043 -- ABC-SMC practical guide (2025): https://arxiv.org/html/2511.21587 - -### What pyABC absorbs vs what we still write - -pyABC gives: SMC loop, adaptive epsilon (`MedianEpsilon` ~ GECKO's -quantile rule), parallel sampling, SQLite-backed posterior + per-gen -bookkeeping. ~30-40% of LOC. - -We still write (~60%): the FBA+RMSE distance (`abc_max.m` port), -per-source shrinkage / sparsity / force-to-prior / variance-cap -callback, the low-rank PCA `Transition` subclass, the 3-TSV data -loader, and the extra GECKO diagnostics. - -Without pyABC we'd write that same 60% plus a 200-400 LOC SMC core -with worse parallelism — so pyABC is worth the dependency. - -## Proposed geckopy module shape - -``` -src/geckopy/kcat_sensitivity_analysis/bayesian/ - __init__.py - data.py # load bayesian{FluxData,MaxGrowth,ZeroExch}.tsv - distance.py # FBA simulation + RMSE (port of abc_max.m + rmsecal) - transition.py # GeckoTransition(pyabc.Transition): per-source + low-rank PCA - posterior.py # per-generation post-update: shrinkage / sparsity / variance cap - tuning.py # bayesian_kcat_tuning(model, ...) orchestrator - pruning.py # optional post-run pruner - cli.py # geckopy bayesian-tune subcommand -tests/ - test_bayesian_data.py - test_bayesian_distance.py - test_bayesian_transition.py - test_bayesian_posterior.py - test_bayesian_tuning_integration.py # tiny ecTestGEM-style fixture -``` - -## Open decisions (resolve on resume) - -- **Q1 Library**: pyABC (recommended) unless reconsidered. -- **Q2 Scope**: MVP (vanilla SMC, no extensions — runs but won't match - MATLAB) vs full port (all extensions). Recommended: full port, - staged on a feature branch. -- **Q3 Parallel backend**: dask (recommended) / multiprocessing / MPI. -- **Q4 Return shape**: MATLAB returns 7 outputs - (ecModel, rmseTrace, kcatTrace, sigmaLogTrace, diagnostics, - posteriorSamples, prunedModel). geckopy convention: mutate - `model.ec.kcat` in place + return one `BayesianResult` dataclass. -- **Q5 Pruning**: port in same PR, gated behind `prune=True`. -- **Q6 Tutorial**: port the stage-4 tutorial call in a follow-up PR. - -## Dependency note - -pyABC pulls in scipy, pandas, cloudpickle, and optionally dask. Add as -an optional extra: `pip install geckopy[bayesian]` so the core install -stays lean (mirrors the proposed WILDkCAT / wildkcat optional extra). diff --git a/docs/migrating_from_gecko_matlab.md b/docs/migrating_from_gecko_matlab.md index 46aa0ef..da3278d 100644 --- a/docs/migrating_from_gecko_matlab.md +++ b/docs/migrating_from_gecko_matlab.md @@ -118,8 +118,8 @@ In MATLAB you write an adapter `classdef` and register a default with folder with a `model_adapter.toml`, loaded via `ModelAdapter.from_folder(path)`. The TOML is parsed into pydantic models (`ModelParameters` with nested `kegg`, `uniprot`, `okp`, -`bayesian`, …) and validated strictly, so config typos are caught -early. +`evotune`, …) and validated strictly, so config typos are +caught early. There is **no global default adapter**. Functions read `model.adapter` or take an explicit `adapter=` argument. Where a MATLAB function takes @@ -263,7 +263,7 @@ Each row maps a MATLAB function to its geckopy equivalent. The | `loadBRENDAdata` | `load_brenda_data` | **`[3→4]`** Reads the new TSV schema (`kcat.tsv` / `sa.tsv` / `mw.tsv`, bare EC codes, plain organism names, `references` column). **`[Py]`** Ships **both** `max` and `median` per (ec, substrate, organism) triple — see [kcat_aggregation.md](kcat_aggregation.md). Returns a `BrendaData` dataclass. | | `loadDatabases` | *(implicit)* | **`[Py]`** No combined-load entry-point; users call `load_brenda_data`, `load_uniprot_tsv`, etc. directly. | -### 5.4 Kcat sensitivity analysis (`src/geckomat/kcat_sensitivity_analysis`) +### 5.4 Kcat sensitivity analysis and tuning (`src/kcat_tuning`) | MATLAB | geckopy | Notes | | ------------------------------------- | ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -271,7 +271,8 @@ Each row maps a MATLAB function to its geckopy equivalent. The | `sigmaFitter` | `fit_sigma` | **`[3→4]`** Returns the model fitted to the **optimal** sigma (GECKO 3 returned the last trial value, sigma = 1.0). **`[Py]`** Renamed; `sigma_fitter` kept as deprecated alias. Returns a `SigmaFitterResult`. | | `findMaxValue` | `find_max_value` | **`[3→4]`** Wildcard EC branch matches real codes. | | `truncateValues` | `truncate_values` | — | -| `bayesianSensitivityTuning` + helpers | *(not ported)* | Planned. See `docs/internal/bayesian_tuning_plan.md` for the design notes. | +| *(none)* | `kcat_tuning.evotune.cmaes_kcat_tuning` + helpers | **`[Py]`** Fits kcats to experimental flux/growth data with CMA-ES rather than MATLAB's ABC-SMC `bayesianSensitivityTuning`; see [evotune_kcat_tuning.md](evotune_kcat_tuning.md) and [cmaes_vs_abc_smc.md](cmaes_vs_abc_smc.md). | +| `bayesianSensitivityTuning` + helpers | *(not ported)* | MATLAB's ABC-SMC sampler itself is not ported; geckopy uses CMA-ES instead (see above). | ### 5.5 Limit proteins (`src/geckomat/limit_proteins`) @@ -447,9 +448,10 @@ agree. The full per-function divergence list lives in the in-source ### Not yet in geckopy -- **Bayesian (ABC-SMC) kcat tuning** - (`bayesianSensitivityTuning` + helpers). Tracked in - `docs/internal/bayesian_tuning_plan.md`. +- **ABC-SMC kcat tuning** (`bayesianSensitivityTuning` + helpers). + geckopy fits kcats to experimental data with CMA-ES instead; see + [evotune_kcat_tuning.md](evotune_kcat_tuning.md) and + [cmaes_vs_abc_smc.md](cmaes_vs_abc_smc.md). - **`plotEcFVA`** — Python users plot from the `ec_fva` DataFrame with matplotlib / seaborn directly. diff --git a/docs/raven_integration.md b/docs/raven_integration.md index ede0b2f..c278b13 100644 --- a/docs/raven_integration.md +++ b/docs/raven_integration.md @@ -69,6 +69,6 @@ overlaps will be added back to this table as raven-toolbox grows. raven-toolbox has a different concern: KEGG taxonomy parsing (`raven_toolbox.reconstruction.kegg.taxonomy`). - Everything under `geckopy.limit_proteins`, - `geckopy.kcat_sensitivity_analysis`, + `geckopy.kcat_tuning`, `geckopy.ec_model.pipeline.protein_pool`, etc. — pure enzyme-constraint mechanics, no general-GEM analog in RAVEN. diff --git a/pyproject.toml b/pyproject.toml index 260519a..666f095 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,9 +44,11 @@ dependencies = [ # raven-toolbox provides the underlying model-manipulation primitives geckopy # extends, plus the ecModel YAML I/O (raven_toolbox.io.ec_data.EcData and the # ec-rxns / ec-enzymes / gecko_light sections in read/write_yaml_model). - # raven-toolbox 0.4.0 is on PyPI, but geckopy currently needs a develop-only - # addition (convert_to_irreversible(..., rxns=...), added for #42) that - # hasn't been released yet -- pinned to develop until that ships. + # geckopy needs a develop-only addition (convert_to_irreversible(..., rxns=...), + # added for #42) that isn't in any PyPI release yet -- pinned to develop until + # raven-toolbox 3.0.0b1 ships, at which point this becomes "raven-toolbox>=3.0.0b1" + # (the "b1" in the specifier itself opts pip into matching pre-releases, so no + # --pre flag is needed downstream). "raven-toolbox @ git+https://github.com/SysBioChalmers/raven-toolbox.git@develop", "requests>=2.28", "ruamel.yaml>=0.17", @@ -63,6 +65,9 @@ dev = [ tutorial = [ "matplotlib>=3.5", ] +evotune = [ + "cma>=3.3", +] [project.urls] Homepage = "https://github.com/SysBioChalmers/geckopy" diff --git a/src/geckopy/__init__.py b/src/geckopy/__init__.py index 3daccec..90d3e8a 100644 --- a/src/geckopy/__init__.py +++ b/src/geckopy/__init__.py @@ -98,8 +98,8 @@ get_ec_from_gem, # deprecated; alias of fill_eccodes_from_gem ) -# kcat_sensitivity_analysis -from geckopy.kcat_sensitivity_analysis import ( +# kcat_tuning +from geckopy.kcat_tuning import ( SigmaFitterResult, TunedKcatsResult, find_max_value, diff --git a/src/geckopy/adapter/params.py b/src/geckopy/adapter/params.py index b04b0fa..5beb777 100644 --- a/src/geckopy/adapter/params.py +++ b/src/geckopy/adapter/params.py @@ -48,60 +48,131 @@ class ComplexParams(BaseModel): ) -class BayesianParams(BaseModel): - """Hyperparameters for Bayesian kcat fitting (ABC-SMC).""" +class SourceGroupRule(BaseModel): + """One trust-tier entry in ``EvotuneParams.source_groups``. + + ``ec.source`` holds literal strings like ``"dlkcat"``, ``"brenda"``, + and -- for OpenKineticsPredictor kcats -- the raw predictor method + name (e.g. ``"CataPro"``), not a generic ``"okp"`` tag. A rule + matches a given ``ec.source`` value if it's listed in ``sources``, + or (when ``match_okp`` is True) if it equals the project's + configured OKP method (``OkpParams.method``). + """ model_config = ConfigDict(extra="forbid") - sigma0_log_default: float = 0.5 - kcat_sources: list[str] = Field( - default_factory=lambda: ["dlkcat", "brenda", "custom"] + sources: list[str] = Field( + default_factory=list, + description="Literal ec.source strings belonging to this group.", + ) + match_okp: bool = Field( + default=False, + description=( + "Also match ec.source values equal to the project's " + "configured OpenKineticsPredictor method (OkpParams.method)." + ), ) - sigma0_log_source: list[float] = Field(default_factory=lambda: [0.4, 0.2, 0.1]) - shrink_thr_default: float = 1.5 - shrink_thr_source: list[float] = Field(default_factory=lambda: [1.5, 3.5, 5.5]) - variance_cap_default: float = 10.0 - variance_cap_source: list[float] = Field(default_factory=lambda: [10.0, 4.0, 2.0]) - force_prior_thr_default: float = -1.0 - force_prior_thr_source: list[float] = Field( - default_factory=lambda: [-1.0, 4.0, 8.0] +class EvotuneParams(BaseModel): + """Hyperparameters for kcat tuning against experimental data. + + Sources not matched by any ``source_groups`` entry fall back to the + ``*_default`` fields (matching MATLAB's ``noKcatSource`` behaviour: + ``sigma0logDefault`` etc. apply to every kcat first, then only ones + with a recognised source get overridden by their group's value). + """ + model_config = ConfigDict(extra="forbid") + + sigma0_log_default: float = Field( + default=0.5, + description=( + "Prior standard deviation in log-space for kcats whose source " + "matches no source_groups entry." + ), + ) + source_groups: dict[str, SourceGroupRule] = Field( + default_factory=lambda: { + "dlkcat": SourceGroupRule(sources=["dlkcat"]), + "brenda": SourceGroupRule(sources=["brenda"]), + "custom": SourceGroupRule(sources=["custom"]), + }, + description="Trust tiers: group name -> which ec.source values it covers.", + ) + sigma0_log_source: dict[str, float] = Field( + default_factory=lambda: {"dlkcat": 0.4, "brenda": 0.2, "custom": 0.1}, + description=( + "Prior log-space standard deviation per group. Lower is more " + "trusted, and narrows that group's proposal width." + ), + ) + + rmse_threshold: float = Field( + default=0.2, + description="Stop once the best RMSE reaches this; negative never stops early.", + ) + max_generations: int = Field( + default=150, description="Hard cap on CMA-ES generations.", ) - sparsity_threshold: float = 0.3 - schedule_generations: list[int] = Field(default_factory=lambda: [1, 2, 9, 15]) - schedule_samples: list[int] = Field(default_factory=lambda: [1000, 800, 600, 400]) + max_growth_weight: float = Field( + default=1.0, + description=( + "Weight on the max-growth RMSE against the flux RMSE: " + "(rmse_flux + w * rmse_max_growth) / (w + 1). At 2 the " + "max-growth conditions count double. MATLAB weights the " + "flux term instead, so pass 0.5 to reproduce its 2." + ), + ) - target_accept: float = 10.0 - min_keep: float = 0.3 - max_keep: float = 0.6 + # Weight on a Gaussian prior term in the selection objective: + # rmse + w * mean((log(k/k0) / sigma0_log)**2). Because sigma0_log + # already encodes per-source confidence, this charges more for + # moving a trusted kcat than an unlabelled one, making parsimony + # part of what the search optimises rather than something applied + # afterwards. + # + # The default is what makes a tuned kcat reproducible. The fit is + # flat along many directions, so an unpenalised search returns an + # arbitrary point along each: on ecYeastGEM two seeds of the same + # unpenalised run disagree by up to 27 773-fold on individual + # kcats, and only 69% of their large corrections even agree on the + # direction of the change. At 0.03 the worst disagreement is 5.7x, + # every correction beyond two-fold agrees in direction, and the fit + # gives up 4.6%. Set 0 to score on RMSE alone; a run reproducing + # MATLAB, which has no such term, must do so. + prior_penalty_weight: float = Field( + default=0.03, + description=( + "Weight on a prior term in the selection objective, " + "rmse + w * mean((log(k/k0) / sigma0_log)**2). Keeps large " + "corrections reproducible across seeds. 0 scores on RMSE alone." + ), + ) - rmse_threshold: float = 0.2 - max_generations: int = 150 + tie_isozymes: bool = Field( + default=True, + description=( + "Give isozyme copies of one reaction a single kcat when they " + "share a prior value and a source, so the search cannot invent " + "a distinction the kcat assignment never made. Tying must " + "happen before the first generation is sampled, not applied " + "to an already-tuned result." + ), + ) @model_validator(mode="after") - def _check_parallel_list_lengths(self) -> "BayesianParams": - """The per-source lists must line up with ``kcat_sources``, and the - ABC-SMC schedule lists must line up with each other; otherwise a - downstream positional zip silently mismatches.""" - n = len(self.kcat_sources) - for name in ( - "sigma0_log_source", - "shrink_thr_source", - "variance_cap_source", - "force_prior_thr_source", - ): - length = len(getattr(self, name)) - if length != n: + def _check_group_keys(self) -> "EvotuneParams": + """The per-source dicts must have exactly ``source_groups``' + keys, otherwise a downstream lookup silently falls back to a + default.""" + group_names = set(self.source_groups) + for name in ("sigma0_log_source",): + keys = set(getattr(self, name)) + if keys != group_names: raise ValueError( - f"BayesianParams.{name} has length {length}, expected " - f"{n} to match kcat_sources." + f"EvotuneParams.{name} keys {sorted(keys)} must match " + f"source_groups keys {sorted(group_names)}." ) - if len(self.schedule_generations) != len(self.schedule_samples): - raise ValueError( - "BayesianParams.schedule_generations and schedule_samples " - "must have equal length." - ) return self @@ -197,5 +268,5 @@ class ModelParameters(BaseModel): kegg: KeggParams = Field(default_factory=KeggParams) uniprot: UniprotParams = Field(default_factory=UniprotParams) complex: ComplexParams = Field(default_factory=ComplexParams) - bayesian: BayesianParams = Field(default_factory=BayesianParams) + evotune: EvotuneParams = Field(default_factory=EvotuneParams) okp: OkpParams = Field(default_factory=OkpParams) diff --git a/src/geckopy/adapter/template.py b/src/geckopy/adapter/template.py index b2e80dd..1b7ce7c 100644 --- a/src/geckopy/adapter/template.py +++ b/src/geckopy/adapter/template.py @@ -18,7 +18,6 @@ from .params import ModelParameters _INJECTED_FIELDS = {"path"} # loader injects at runtime, never in template -_ADVANCED_SECTIONS = {"bayesian"} _REQUIRED_PLACEHOLDERS = { "conv_gem": '"models/REPLACE_ME.xml"', "org_name": '"REPLACE_ME"', @@ -37,11 +36,20 @@ def _format_toml_value(value: Any) -> str | None: return f'"{value.as_posix()}"' if isinstance(value, (int, float)): return repr(value) - if isinstance(value, list): + if isinstance(value, (list, tuple)): parts = [_format_toml_value(v) for v in value] if any(p is None for p in parts): return None return "[" + ", ".join(parts) + "]" + if isinstance(value, BaseModel): + value = value.model_dump() + if isinstance(value, dict): + formatted_items = [ + (k, _format_toml_value(v)) for k, v in value.items() + ] + if any(fv is None for _, fv in formatted_items): + return None + return "{ " + ", ".join(f"{k} = {fv}" for k, fv in formatted_items) + " }" return None @@ -85,17 +93,16 @@ def _is_nested_model(annotation: Any) -> bool: return inspect.isclass(annotation) and issubclass(annotation, BaseModel) -def generate_template_toml( - *, advanced: bool = False, project_name: str = "my-ecmodel", -) -> str: +def generate_template_toml(*, project_name: str = "my-ecmodel") -> str: """Generate a commented TOML template for a new ecModel project. + Every section of the schema is emitted, evotune kcat tuning + hyperparameters included; optional fields are commented out at + their defaults, so the template doubles as the reference for what + can be set. + Parameters ---------- - advanced - Include the advanced sections (currently just Bayesian - kcat-tuning hyperparameters); omitted by default since most - projects never need to touch them. project_name Name written into the template's header comment. @@ -138,11 +145,7 @@ def generate_template_toml( required=False, )) - has_omitted_advanced = False for name, model_cls in sections: - if name in _ADVANCED_SECTIONS and not advanced: - has_omitted_advanced = True - continue lines.append(f"[{name}]") instance = model_cls() for sub_name, sub_field in model_cls.model_fields.items(): @@ -153,10 +156,4 @@ def generate_template_toml( required=False, )) - if has_omitted_advanced: - lines.extend([ - "# Note: advanced sections (Bayesian kcat-tuning hyperparameters)", - "# are omitted by default. Regenerate with --advanced to include them.", - ]) - return "\n".join(lines) + "\n" diff --git a/src/geckopy/cli.py b/src/geckopy/cli.py index 1156308..16f4aca 100644 --- a/src/geckopy/cli.py +++ b/src/geckopy/cli.py @@ -49,9 +49,7 @@ def cmd_init(args: argparse.Namespace) -> int: for sub in ("data", "models", "output"): (folder / sub).mkdir(exist_ok=True) - toml_text = generate_template_toml( - advanced=args.advanced, project_name=folder.name - ) + toml_text = generate_template_toml(project_name=folder.name) (folder / "model_adapter.toml").write_text(toml_text) (folder / "adapter.py").write_text(ADAPTER_PY_STUB) @@ -111,10 +109,6 @@ def main(argv: list[str] | None = None) -> int: "folder", help="Target folder (will be created; must not already exist or be empty)", ) - init_parser.add_argument( - "--advanced", action="store_true", - help="Include advanced sections (Bayesian kcat-tuning hyperparameters)", - ) init_parser.set_defaults(func=cmd_init) brenda_parser = sub.add_parser( diff --git a/src/geckopy/databases/flux_data.py b/src/geckopy/databases/flux_data.py index 4634943..3665544 100644 --- a/src/geckopy/databases/flux_data.py +++ b/src/geckopy/databases/flux_data.py @@ -41,10 +41,10 @@ class FluxData: exch_rxn_ids Exchange reaction IDs in the model, matching the columns of ``exch_fluxes`` and ``exch_mets``. - bayesian_rmse_weight - Optional per-condition weights for the Bayesian-kcat-tuning + evotune_rmse_weight + Optional per-condition weights for the evotune-kcat-tuning RMSE term. ``None`` if the source file did not have the - ``bayesianRMSEweight`` column. + ``evotuneRMSEweight`` column. source Optional per-condition free-text describing where the data came from. ``None`` if the source file did not have a @@ -63,7 +63,7 @@ class FluxData: ) exch_mets: list[str] = field(default_factory=list) exch_rxn_ids: list[str] = field(default_factory=list) - bayesian_rmse_weight: Optional[np.ndarray] = None + evotune_rmse_weight: Optional[np.ndarray] = None source: Optional[list[str]] = None @@ -78,11 +78,11 @@ def load_flux_data(path: str | Path) -> FluxData: Optional extra columns (in any position): - * ``bayesianRMSEweight`` (float): weights for Bayesian kcat tuning. + * ``evotuneRMSEweight`` (float): weights for evotune kcat tuning. * ``source`` (str): free-text describing the data origin. Both columns are dropped from the matrix and surfaced on the - returned ``FluxData`` as ``bayesian_rmse_weight`` and ``source``. + returned ``FluxData`` as ``evotune_rmse_weight`` and ``source``. Ported from GECKO MATLAB: src/geckomat/utilities/loadFluxData.m. @@ -126,19 +126,19 @@ def load_flux_data(path: str | Path) -> FluxData: # Pull out optional columns first; collect indices to drop. optional_indices: list[int] = [] - bayesian_idx = _index_of(header, "bayesianRMSEweight") + tuning_idx = _index_of(header, "evotuneRMSEweight") source_idx = _index_of(header, "source") - bayesian_weights: Optional[np.ndarray] = None + tuning_weights: Optional[np.ndarray] = None source_values: Optional[list[str]] = None - if bayesian_idx is not None: - bayesian_weights = np.array( - [_parse_float(r[bayesian_idx]) if bayesian_idx < len(r) else float("nan") + if tuning_idx is not None: + tuning_weights = np.array( + [_parse_float(r[tuning_idx]) if tuning_idx < len(r) else float("nan") for r in data], dtype=float, ) - optional_indices.append(bayesian_idx) + optional_indices.append(tuning_idx) if source_idx is not None: source_values = [ r[source_idx] if source_idx < len(r) else "" for r in data @@ -182,7 +182,7 @@ def load_flux_data(path: str | Path) -> FluxData: exch_fluxes=exch_fluxes, exch_mets=exch_mets, exch_rxn_ids=exch_rxn_ids, - bayesian_rmse_weight=bayesian_weights, + evotune_rmse_weight=tuning_weights, source=source_values, ) diff --git a/src/geckopy/gather_kcats/review.py b/src/geckopy/gather_kcats/review.py new file mode 100644 index 0000000..177ef90 --- /dev/null +++ b/src/geckopy/gather_kcats/review.py @@ -0,0 +1,267 @@ +"""Flag kcat assignments that deserve a second look. + +The tuner's largest corrections are rarely discoveries about biology. +They are places the assignment went wrong: a value derived from one +specific-activity measurement, a database maximum standing in for a +distribution with a two-hundred-fold spread, one fallback number reused +across a whole family of reactions. Finding those directly is cheaper +than inferring them from a fit, and a corrected prior propagates to +every model built from the same databases, where a tuned kcat stays in +the model it was tuned on. + +Two design choices, both from measurement rather than taste. + +**Rank by leverage, and cut the report there.** On ecYeastGEM the +checks below flag around 1900 of 4834 kcats, which is a report nobody +reads. Tightening the checks does not fix that: raising the EC-maximum +gap from three-fold to ten-fold still returns 216 entries and drops +*all six* of the high-leverage ones, because looking odd and mattering +are unrelated properties. Filtering by leverage keeps the list short +and keeps the entries worth a curator's time. + +**Report a parameter once.** Isozyme copies the assignment could not +separate share a value and a source, so they share a finding; listing +them separately fills the report with duplicates of one decision. +""" +from __future__ import annotations + +from collections import Counter, defaultdict +from dataclasses import dataclass, fields +from typing import Iterable, Mapping, Optional, Sequence + +import numpy as np + +from ..kcat_tuning.evotune.tying import ( + base_reaction, isozyme_tie_map, +) + +#: Checks reported unless asked for explicitly. ``standard`` values are a +#: deliberate fallback and ``tied`` groups are unresolvable by design, so +#: listing every one of them is noise rather than news. +OPTIONAL_CHECKS = ("standard-fallback", "tied-isozymes") + + +@dataclass(frozen=True) +class Finding: + """One kcat assignment worth a closer look.""" + + rxn_id: str + name: str + ec_code: str + source: str + kcat: float + leverage: float + n_isozymes: int + other_reactions: int + checks: tuple[str, ...] + detail: str + + +@dataclass(frozen=True) +class EcStats: + """What the database holds for one EC number.""" + + n_kcat: int = 0 + kcat_max: float = float("nan") + kcat_median: float = float("nan") + values: frozenset = frozenset() + n_sa: int = 0 + + +def _first_ec(ec_code: str) -> str: + return str(ec_code).split(";")[0].strip() + + +def review_assignment( + rxn_ids: Sequence[str], + kcats: np.ndarray, + sources: Sequence[str], + ec_codes: Sequence[str], + *, + ec_stats: Optional[Mapping[str, EcStats]] = None, + enzymes: Optional[Sequence[Iterable[str]]] = None, + names: Optional[Sequence[str]] = None, + leverage: Optional[np.ndarray] = None, + slow: float = 1e-2, + fast: float = 1e4, + ec_max_gap: float = 3.0, + repeat_min: int = 5, + include: Sequence[str] = (), + coverage: Optional[float] = None, + top: Optional[int] = None, +) -> list[Finding]: + """Assignments worth reviewing, most consequential first. + + Parameters + ---------- + ec_stats + Per-EC database summary. Checks needing it are skipped when it + is absent, so the function still runs without a database. + enzymes + Protein ids per kcat, used to count how many other reactions + share an enzyme -- a change here may imply changes there. + leverage + Per-kcat effect on the distance from a one-at-a-time screen. + Without it the findings cannot be ranked and are returned in + input order. + coverage, top + Truncate the report. ``coverage`` keeps the rows carrying that + share of the *flagged* leverage, so 0.9 drops the tail that + together accounts for a tenth of what the findings are worth; + it adapts to the model where ``top`` does not. Measured against + all kcats instead, the threshold would be unreachable, since + flagged rows are a minority of the model's leverage. + include + Names from :data:`OPTIONAL_CHECKS` to report as well. + + Returns + ------- + list of :class:`Finding`, one per free parameter. + """ + kcats = np.asarray(kcats, dtype=float) + n = len(kcats) + for name, seq in (("sources", sources), ("ec_codes", ec_codes), + ("rxn_ids", rxn_ids)): + if len(seq) != n: + raise ValueError(f"{name} has {len(seq)} entries; kcats has {n}.") + lev = np.zeros(n) if leverage is None else np.asarray(leverage, dtype=float) + if lev.shape != kcats.shape: + raise ValueError(f"leverage has shape {lev.shape}; expected {kcats.shape}.") + + tie = isozyme_tie_map(rxn_ids, kcats, sources) + members = defaultdict(list) + for i, rep in enumerate(tie): + members[int(rep)].append(i) + + # A value reused across many distinct reactions is a fallback wearing + # a measurement's label, whatever its source says. + per_reaction = defaultdict(set) + for i, rxn_id in enumerate(rxn_ids): + per_reaction[round(float(np.log(kcats[i])), 9) if kcats[i] > 0 else None + ].add(base_reaction(str(rxn_id))) + reuse = {k: len(v) for k, v in per_reaction.items()} + + enzyme_rxns = defaultdict(set) + if enzymes is not None: + for i, prots in enumerate(enzymes): + for p in prots: + enzyme_rxns[p].add(base_reaction(str(rxn_ids[i]))) + + out: list[Finding] = [] + for rep, idx in members.items(): + i = idx[0] + ec = _first_ec(ec_codes[i]) + st = (ec_stats or {}).get(ec) + checks, detail = [], [] + + if st is not None and st.n_kcat == 0: + checks.append("no-ec-evidence") + detail.append( + f"no kcat rows for EC {ec}" + + (f"; {st.n_sa} specific-activity rows instead" if st.n_sa else "") + ) + key = round(float(np.log(kcats[i])), 9) if kcats[i] > 0 else None + if reuse.get(key, 0) >= repeat_min: + checks.append("repeated-value") + detail.append(f"{kcats[i]:g} 1/s reused across {reuse[key]} reactions") + if (st is not None and st.n_kcat and np.isfinite(st.kcat_median) + and st.kcat_median > 0 + and abs(kcats[i] - st.kcat_max) <= 1e-9 * max(kcats[i], 1.0) + and st.kcat_max / st.kcat_median > ec_max_gap): + checks.append("ec-maximum") + detail.append( + f"took the EC maximum, {st.kcat_max / st.kcat_median:.0f}x its " + f"median over {st.n_kcat} rows" + ) + if kcats[i] < slow or kcats[i] > fast: + checks.append("magnitude") + detail.append(f"{kcats[i]:g} 1/s is outside {slow:g}-{fast:g}") + if (str(sources[i]) == "custom" and st is not None + and round(float(kcats[i]), 6) in st.values): + checks.append("custom-duplicate") + detail.append("custom value equals a database value for its own EC") + if len(idx) > 1: + if "tied-isozymes" in include: + checks.append("tied-isozymes") + detail.append(f"{len(idx)} isozyme copies share this value") + if str(sources[i]) == "standard": + if "standard-fallback" not in include: + continue + checks.append("standard-fallback") + detail.append("the model's fallback value, not a measurement") + + if not checks: + continue + others = set() + if enzymes is not None: + for p in enzymes[i]: + others |= enzyme_rxns[p] + others.discard(base_reaction(str(rxn_ids[i]))) + out.append(Finding( + rxn_id=str(rxn_ids[i]), + name="" if names is None else str(names[i]), + ec_code=ec, + source=str(sources[i]), + kcat=float(kcats[i]), + leverage=float(lev[idx].max()), + n_isozymes=len(idx), + other_reactions=len(others), + checks=tuple(checks), + detail="; ".join(detail), + )) + + out.sort(key=lambda f: -f.leverage) + flagged_total = sum(f.leverage for f in out) + if coverage is not None and flagged_total > 0: + total, kept, acc = flagged_total, [], 0.0 + for f in out: + kept.append(f) + acc += f.leverage + if acc / total >= coverage: + break + out = kept + if top is not None: + out = out[:top] + return out + + +def ec_stats_from_brenda(brenda) -> dict[str, EcStats]: + """Summarise a :class:`BrendaData` per EC number.""" + stats: dict[str, EcStats] = {} + by_ec = {ec: g for ec, g in brenda.kcat_max.groupby("ec_code")} + med = {ec: g for ec, g in brenda.kcat_median.groupby("ec_code")} + sa = Counter(brenda.sa_max["ec_code"]) if len(brenda.sa_max) else Counter() + for ec in set(by_ec) | set(sa): + g = by_ec.get(ec) + vals = set() + if g is not None: + vals |= {round(float(v), 6) for v in g["kcat"]} + if ec in med: + vals |= {round(float(v), 6) for v in med[ec]["kcat"]} + stats[ec] = EcStats( + n_kcat=0 if g is None else len(g), + kcat_max=float("nan") if g is None else float(g["kcat"].max()), + kcat_median=float("nan") if ec not in med + else float(med[ec]["kcat"].median()), + values=frozenset(vals), + n_sa=int(sa.get(ec, 0)), + ) + return stats + + +def findings_tsv(rows: Sequence[Finding]) -> str: + """The findings as a TSV document, header included.""" + cols = [f.name for f in fields(Finding)] + out = ["\t".join(cols)] + for r in rows: + vals = [] + for c in cols: + v = getattr(r, c) + if isinstance(v, float): + vals.append(f"{v:.6g}") + elif isinstance(v, tuple): + vals.append(",".join(v)) + else: + vals.append(str(v)) + out.append("\t".join(vals)) + return "\n".join(out) + "\n" diff --git a/src/geckopy/gather_kcats/select_kcat_value.py b/src/geckopy/gather_kcats/select_kcat_value.py index 71d1c7b..47bd3dd 100644 --- a/src/geckopy/gather_kcats/select_kcat_value.py +++ b/src/geckopy/gather_kcats/select_kcat_value.py @@ -34,7 +34,7 @@ def format_kcat_source( ``"brenda (wc=0, origin=1)"``. Rows without fuzzy metadata (an exact OKP database hit or a prediction) render as the bare token, so the bracket's presence and the wildcard/origin numbers give downstream - consumers (notably Bayesian sensitivity tuning) a quick read on how + consumers (notably evotune kcat tuning) a quick read on how uncertain the kcat is. """ token = normalize_source(source) diff --git a/src/geckopy/kcat_sensitivity_analysis/__init__.py b/src/geckopy/kcat_sensitivity_analysis/__init__.py deleted file mode 100644 index 851154c..0000000 --- a/src/geckopy/kcat_sensitivity_analysis/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Sensitivity analysis on enzyme kcat values. - -Mirrors GECKO MATLAB's `src/geckomat/kcat_sensitivity_analysis/` -directory. -""" -from .find_max_value import find_max_value -from .sensitivity_tuning import TunedKcatsResult, sensitivity_tuning -from .sigma_fitter import SigmaFitterResult, fit_sigma, sigma_fitter -from .truncate_values import truncate_values - -__all__ = [ - "SigmaFitterResult", - "TunedKcatsResult", - "find_max_value", - "fit_sigma", - "sensitivity_tuning", - "sigma_fitter", # deprecated; alias of fit_sigma - "truncate_values", -] diff --git a/src/geckopy/kcat_tuning/__init__.py b/src/geckopy/kcat_tuning/__init__.py new file mode 100644 index 0000000..22e3136 --- /dev/null +++ b/src/geckopy/kcat_tuning/__init__.py @@ -0,0 +1,44 @@ +"""Kcat sensitivity analysis and tuning against experimental data. + +Mirrors GECKO MATLAB's `src/kcat_tuning/` +directory. +""" +from .find_max_value import find_max_value +from .sensitivity_tuning import TunedKcatsResult, sensitivity_tuning +from .sigma_fitter import SigmaFitterResult, fit_sigma, sigma_fitter +from .truncate_values import truncate_values + +try: + from . import evotune +except ImportError as exc: # pragma: no cover - exercised + # only when the optional `cma` dependency is absent. + # Bound to a module-level name: Python clears an `except ... as` + # target when the block exits, so the class below cannot close over + # `exc` itself. + _evotune_import_error = exc + + class _MissingEvotune: + """Stand-in for the `evotune` subpackage when its + `cma` dependency isn't installed. Raises only on first attribute + access, not on import, so plain `geckopy.kcat_tuning` + stays usable without the optional extra.""" + + def __getattr__(self, name: str): + raise ImportError( + "Evotune: CMA-ES kcat tuning requires the optional " + "'evotune' extra: pip install " + "geckopy[evotune]" + ) from _evotune_import_error + + evotune = _MissingEvotune() # type: ignore[assignment] + +__all__ = [ + "SigmaFitterResult", + "TunedKcatsResult", + "evotune", + "find_max_value", + "fit_sigma", + "sensitivity_tuning", + "sigma_fitter", # deprecated; alias of fit_sigma + "truncate_values", +] diff --git a/src/geckopy/kcat_tuning/evotune/__init__.py b/src/geckopy/kcat_tuning/evotune/__init__.py new file mode 100644 index 0000000..78c5a12 --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/__init__.py @@ -0,0 +1,57 @@ +"""Evotune: CMA-ES kcat tuning against experimental data. + +See ``docs/evotune_kcat_tuning.md`` for the walkthrough: +:func:`~.tuning.screen_kcat_leverage`, then +:func:`~.tuning.select_tunable_mask`, then +:func:`~.tuning.cmaes_kcat_tuning`. + +Modules in this package that need ``cma`` import it unconditionally -- +it's a hard requirement here, gated one level up instead (see +``kcat_tuning/__init__.py``) so importing plain +``geckopy.kcat_tuning`` never requires the optional +``evotune`` extra. Install it with +``pip install geckopy[evotune]``. +""" +from .data import EvotuneData, load_evotune_data +from .distance import ( + BIOMASS_CARBON_EQUIV, + INFEASIBLE_PENALTY, + evotune_distance, + compute_excarbon, + dataset_rmse, +) +from .priors import ( + UNLABELLED_GROUP, + build_sigma0_log, + classify_kcat_source, + classify_kcat_sources, +) +from .simulate import ConditionSimResult, simulate_evotune_dataset +from .tuning import ( + EvotuneResult, + cmaes_kcat_tuning, + screen_kcat_leverage, + select_tunable_mask, + tune_prior_penalty_weight, +) + +__all__ = [ + "BIOMASS_CARBON_EQUIV", + "INFEASIBLE_PENALTY", + "UNLABELLED_GROUP", + "EvotuneData", + "EvotuneResult", + "ConditionSimResult", + "evotune_distance", + "build_sigma0_log", + "classify_kcat_source", + "classify_kcat_sources", + "cmaes_kcat_tuning", + "compute_excarbon", + "dataset_rmse", + "load_evotune_data", + "screen_kcat_leverage", + "select_tunable_mask", + "simulate_evotune_dataset", + "tune_prior_penalty_weight", +] diff --git a/src/geckopy/kcat_tuning/evotune/data.py b/src/geckopy/kcat_tuning/evotune/data.py new file mode 100644 index 0000000..c8c3d08 --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/data.py @@ -0,0 +1,106 @@ +"""Load experimental data for evotune (CMA-ES) kcat tuning. + +Ported from GECKO MATLAB: +src/kcat_tuning/Bayesian/loadBayesianData.m. + +MATLAB's ``loadBayesianData`` loads ``bayesianFluxData.tsv`` and +``bayesianMaxGrowth.tsv`` (renamed here to ``evotuneFluxData.tsv`` and +``evotuneMaxGrowth.tsv``) through the *same* ``loadFluxData`` parser used +for regular flux data (verified against ``develop4``'s current source, +not assumed) -- both files share the identical +``Condition Ptot grRate () ... evotuneRMSEweight source`` +column layout. geckopy already has that parser as +:func:`~geckopy.databases.flux_data.load_flux_data`, so both files reuse +it unchanged; only ``evotuneZeroExch.tsv`` (a single ``Rxns`` id column) +needs a loader of its own. + +Unlike MATLAB -- which treats a missing/unreadable file as `[]` via a +try/catch inside ``loadFluxData`` -- all three files are treated +uniformly here as optional: a missing file yields ``None`` (or an empty +list for ``zero_flux``), never an exception. This also sidesteps a bug +flagged in the MATLAB review (``REVIEW.md`` #3): there, only the +``fluxData`` field was null-guarded before ``.biomass`` was attached, +while ``maxGrate`` wasn't, so a missing ``evotuneMaxGrowth.tsv`` there +silently produced a struct with just a ``.biomass`` field instead of a +clean "absent" signal. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from ...databases.flux_data import FluxData, load_flux_data + +if TYPE_CHECKING: + from ...adapter import ModelAdapter + +_FLUX_DATA_FILENAME = "evotuneFluxData.tsv" +_MAX_GROWTH_FILENAME = "evotuneMaxGrowth.tsv" +_ZERO_EXCH_FILENAME = "evotuneZeroExch.tsv" + + +@dataclass +class EvotuneData: + """Experimental data used by evotune kcat tuning. + + Attributes + ---------- + flux_data + Per-condition growth/exchange-flux measurements, from + ``evotuneFluxData.tsv``. ``None`` if that file isn't present. + max_grate + Per-condition maximum-growth-rate measurements (one active + carbon source per condition, at ``-1000`` i.e. unconstrained), + from ``evotuneMaxGrowth.tsv``. ``None`` if that file isn't + present. + zero_flux + Reaction IDs assumed to carry zero flux, from + ``evotuneZeroExch.tsv``. Empty if that file isn't present. + """ + + flux_data: Optional[FluxData] = None + max_grate: Optional[FluxData] = None + zero_flux: list[str] = field(default_factory=list) + + +def load_evotune_data(adapter: "ModelAdapter") -> EvotuneData: + """Load the three evotune-kcat-tuning experimental data files. + + Each file is optional and resolved as + ``/data/``; a missing file yields + ``None`` (``flux_data``/``max_grate``) or an empty list + (``zero_flux``) rather than raising. + + Parameters + ---------- + adapter + A ``ModelAdapter`` whose ``params.path`` locates the ``data/`` + folder. + + Returns + ------- + EvotuneData + """ + data_dir = Path(adapter.params.path) / "data" + return EvotuneData( + flux_data=_load_optional_flux_data(data_dir / _FLUX_DATA_FILENAME), + max_grate=_load_optional_flux_data(data_dir / _MAX_GROWTH_FILENAME), + zero_flux=_load_zero_exch(data_dir / _ZERO_EXCH_FILENAME), + ) + + +def _load_optional_flux_data(path: Path) -> Optional[FluxData]: + if not path.is_file(): + return None + return load_flux_data(path) + + +def _load_zero_exch(path: Path) -> list[str]: + """Parse ``evotuneZeroExch.tsv``: a single ``Rxns`` header, then + one reaction ID per line.""" + if not path.is_file(): + return [] + lines = path.read_text(encoding="utf-8").splitlines() + rxn_ids = [line.strip() for line in lines[1:] if line.strip()] + return rxn_ids diff --git a/src/geckopy/kcat_tuning/evotune/distance.py b/src/geckopy/kcat_tuning/evotune/distance.py new file mode 100644 index 0000000..97e30d7 --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/distance.py @@ -0,0 +1,261 @@ +"""Carbon- and condition-weighted RMSE distance for evotune kcat tuning. + +Ported from GECKO MATLAB: +src/kcat_tuning/Bayesian/abc_max.m (the +``rmsecal`` helper's RMSE half; verified against ``develop4``'s +current source). :mod:`.simulate` does the FBA half -- this module is +pure numpy (plus one small model-reading helper for the carbon +weights) so the RMSE math is directly testable against hand-computed +expected values, no ``EcModel``/FBA involved. + +Two MATLAB behaviours are intentionally *not* ported, per +``REVIEW.md``'s findings against ``develop4``: + +- ``addCarbonNum.m`` matches the biomass reaction by the literal name + ``'growth'`` (REVIEW.md #4) -- silently a no-op for any adapter that + doesn't use that exact name. Here the caller passes the real + ``bio_rxn_id`` explicitly (``adapter.params.bio_rxn``) instead. +- ``rmsecal`` recomputes ``ismember(data.conds, data.exchMets)`` (and + its validation) on every inner-loop iteration despite it not + depending on the loop variable (REVIEW.md, low severity) -- + :func:`dataset_rmse` builds its reaction-id-to-column lookup once. + +One quirk *is* ported deliberately, because it changes what the RMSE +actually measures: MATLAB's ``ecModel.excarbon(ecModel.excarbon == 0) += 1`` clamps *every* zero-carbon exchange metabolite (O2, water, ...) +up to a carbon weight of 1, not 0 -- so those reactions' flux +mismatches still count toward the RMSE at unit weight rather than +being silently zeroed out of it. See :func:`compute_excarbon`. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Iterable + +import numpy as np + +from .simulate import ConditionSimResult + +if TYPE_CHECKING: + from ...ec_model.ec_model import EcModel + from .data import EvotuneData + from ...databases.flux_data import FluxData + +#: MATLAB's hardcoded assumption of the biomass reaction's carbon +#: content, in Cmmol/gDCW, used to put growth-rate deviations on the +#: same carbon-weighted scale as exchange-flux deviations. +BIOMASS_CARBON_EQUIV = 41.0 + +#: Fixed penalty substituted for an infeasible/failed condition's RMSE +#: ("clearly worse than any real RMSE" -- MATLAB never documents why +#: 99 specifically, just that it must dominate the mean). +INFEASIBLE_PENALTY = 99.0 + + +def compute_excarbon( + model: "EcModel", rxn_ids: Iterable[str], bio_rxn_id: str, +) -> dict[str, float]: + """Per-reaction carbon-weight lookup for RMSE weighting. + + The biomass reaction gets :data:`BIOMASS_CARBON_EQUIV`. Every + other reaction gets the carbon count of its single exchanged + metabolite (via cobrapy's parsed ``Metabolite.elements``, not a + hand-rolled formula parser). A missing/unparseable formula, or a + computed carbon count of exactly 0 (e.g. O2, water), both fall + back to 1 -- matching MATLAB's ``Ematrix(isnan(Ematrix)) = 1`` and + ``excarbon(excarbon == 0) = 1`` respectively. + + Parameters + ---------- + model + Model providing the metabolite formulas. + rxn_ids + Reaction IDs to compute weights for (typically every + reaction referenced by a tuning dataset: its exchange + columns, its zero-flux list, and the biomass reaction). + bio_rxn_id + The biomass reaction ID. + + Returns + ------- + dict[str, float] + """ + excarbon: dict[str, float] = {} + for rxn_id in rxn_ids: + if rxn_id == bio_rxn_id: + excarbon[rxn_id] = BIOMASS_CARBON_EQUIV + continue + rxn = model.reactions.get_by_id(rxn_id) + mets = list(rxn.metabolites) + carbon = mets[0].elements.get("C") if len(mets) == 1 else None + value = float(carbon) if carbon is not None else 1.0 + excarbon[rxn_id] = value if value != 0.0 else 1.0 + return excarbon + + +def dataset_rmse( + flux_data: "FluxData", + sims: list[ConditionSimResult], + *, + constrain: bool, + excarbon: dict[str, float], + bio_rxn_id: str, + penalty: float = INFEASIBLE_PENALTY, +) -> tuple[float, np.ndarray]: + """RMSE (mean, per-condition array) for one tuning dataset. + + Parameters + ---------- + flux_data + The dataset being scored (``evotune_data.flux_data`` or + ``evotune_data.max_grate``). + sims + One :class:`~.simulate.ConditionSimResult` per + ``flux_data.conds`` row, e.g. from + :func:`~.simulate.simulate_evotune_dataset`. + constrain + Must match the value passed to + :func:`~.simulate.simulate_evotune_dataset` for these + ``sims``: True includes carbon-weighted exchange-flux terms + (flux data), False scores growth alone (max-growth data). + excarbon + Carbon-weight lookup from :func:`compute_excarbon`, covering + every reaction referenced by ``sims``' ``exch_fluxes`` / + ``block_fluxes`` plus ``bio_rxn_id``. A reaction missing from + the lookup falls back to a weight of 1. + bio_rxn_id + The biomass reaction ID (for reading ``flux_data.gr_rate``'s + carbon-equivalent counterpart). + penalty + RMSE substituted for an infeasible condition. + + Returns + ------- + rmse : float + Weighted (by ``flux_data.evotune_rmse_weight``, if present) + mean of the per-condition RMSEs. + rmse_list : numpy.ndarray + The per-condition RMSEs (post-penalty, pre-weighting), shape + ``(len(flux_data.conds),)``. + """ + n = len(flux_data.conds) + if len(sims) != n: + raise ValueError( + f"sims has {len(sims)} entries; expected {n} to match " + f"flux_data.conds." + ) + rxn_to_col = {r: j for j, r in enumerate(flux_data.exch_rxn_ids)} + + rmse_list = np.empty(n, dtype=float) + for i, sim in enumerate(sims): + if not sim.feasible: + rmse_list[i] = penalty + continue + + bio_meas = float(flux_data.gr_rate[i]) * BIOMASS_CARBON_EQUIV + bio_sim = sim.growth * BIOMASS_CARBON_EQUIV + + if not constrain: + rmse_list[i] = abs(bio_meas - bio_sim) + continue + + measured = [bio_meas] + simulated = [bio_sim] + for rxn_id, sim_flux in sim.exch_fluxes.items(): + w = excarbon.get(rxn_id, 1.0) + measured.append(w * float(flux_data.exch_fluxes[i, rxn_to_col[rxn_id]])) + simulated.append(w * sim_flux) + for rxn_id, sim_flux in sim.block_fluxes.items(): + w = excarbon.get(rxn_id, 1.0) + weighted_sim = w * sim_flux + if weighted_sim == 0.0: + # Already-correct zero predictions don't dilute the RMSE + # (MATLAB: `blockSim(blockSim == 0) = []`). + continue + measured.append(0.0) + simulated.append(weighted_sim) + + diff = np.asarray(measured, dtype=float) - np.asarray(simulated, dtype=float) + rmse_list[i] = float(np.sqrt(np.mean(diff ** 2))) + + weighted = rmse_list + if flux_data.evotune_rmse_weight is not None: + weighted = rmse_list * flux_data.evotune_rmse_weight + return float(np.mean(weighted)), rmse_list + + +def evotune_distance( + evotune_data: "EvotuneData", + *, + flux_sims: list[ConditionSimResult] | None, + max_grate_sims: list[ConditionSimResult] | None, + excarbon: dict[str, float], + bio_rxn_id: str, + penalty: float = INFEASIBLE_PENALTY, + max_growth_weight: float = 1.0, +) -> tuple[float, dict[str, np.ndarray]]: + """Combine both datasets' RMSE into one score, per ``abc_max.m``. + + ``flux_sims`` must be given (non-``None``) iff ``evotune_data.flux_data`` + is not ``None``, and likewise for ``max_grate_sims`` / + ``evotune_data.max_grate``. A dataset absent from ``evotune_data`` doesn't + contribute to the combined RMSE (MATLAB: a missing dataset's + ``rmse_*`` stays ``[]`` and drops out of ``validIdx``). + + Parameters + ---------- + max_growth_weight + Relative weight of the max-growth dataset against the flux + dataset, giving ``(rmse_flux + w * rmse_max_grate) / (w + 1)``. + At 2 the eight max-growth conditions carry twice the weight of + the 33 flux conditions, which is the intended emphasis here: + the max-growth set is where the carbon-source diversity lives, + while 30 of the 33 flux conditions are glucose. + + This is deliberately not MATLAB's convention. ``abc_max.m`` + applies ``weights = [maxGrowthWeight, 1]`` to + ``[rmse_flux, rmse_maxGrate]``, so there the same field scales + the *flux* term. **A run reproducing MATLAB must pass 0.5 + here** to match its ``maxGrowthWeight = 2``. + + Returns + ------- + rmse : float + Weighted mean of whichever dataset RMSEs are present. ``NaN`` + if neither dataset is present. + detail : dict[str, numpy.ndarray] + Per-condition RMSE arrays, keyed ``"flux_data"`` / + ``"max_grate"`` for whichever datasets were scored. + """ + parts: list[float] = [] + weights: list[float] = [] + detail: dict[str, np.ndarray] = {} + + if evotune_data.flux_data is not None: + if flux_sims is None: + raise ValueError("evotune_data.flux_data is set but flux_sims is None.") + rmse, per_cond = dataset_rmse( + evotune_data.flux_data, flux_sims, + constrain=True, excarbon=excarbon, bio_rxn_id=bio_rxn_id, + penalty=penalty, + ) + parts.append(rmse) + weights.append(1.0) + detail["flux_data"] = per_cond + + if evotune_data.max_grate is not None: + if max_grate_sims is None: + raise ValueError("evotune_data.max_grate is set but max_grate_sims is None.") + rmse, per_cond = dataset_rmse( + evotune_data.max_grate, max_grate_sims, + constrain=False, excarbon=excarbon, bio_rxn_id=bio_rxn_id, + penalty=penalty, + ) + parts.append(rmse) + weights.append(float(max_growth_weight)) + detail["max_grate"] = per_cond + + if parts: + rmse = float(np.dot(parts, weights) / np.sum(weights)) + else: + rmse = float("nan") + return rmse, detail diff --git a/src/geckopy/kcat_tuning/evotune/parsimony.py b/src/geckopy/kcat_tuning/evotune/parsimony.py new file mode 100644 index 0000000..7f3ffd3 --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/parsimony.py @@ -0,0 +1,248 @@ +"""Trading fit against how many kcats were moved. + +A tuned kcat vector that fits well by rewriting every parameter is not +a useful answer: changes should be few, and concentrated in the +parameters we are least confident about. Movement is measured here in +units of each kcat's own prior sigma -- MATLAB's ``devFromPrior`` -- +which weights by confidence automatically, since ``sigma0_log`` is +smaller for trusted sources. + +The tuner returns the best-RMSE particle, which on the full ecYeastGEM +model moves ~99% of kcats by ~8 prior sigmas. Reverting the least-moved +of those to their priors costs nothing up to about 3 sigma and beats +the shrunk "blend" estimate at every sparsity level, so a frontier is a +more honest report than either single vector. See +``docs/internal/matlab_replication_results.md``. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, Sequence + +import numpy as np + + +@dataclass(frozen=True) +class FrontierPoint: + """One (parsimony, fit) trade-off, from :func:`parsimony_frontier`.""" + + threshold: float + kcat: np.ndarray + n_changed: int + mean_dev: float + rmse: float + + +def fold_change(kcat: np.ndarray, kcat0: np.ndarray) -> np.ndarray: + """Per-kcat fold change from the prior, always >= 1. + + ``max(k/k0, k0/k)``. Reported in preference to + :func:`movement_in_sigma` because ``sigma0_log`` differs by source + (0.10 for ``custom`` up to 0.30 for unlabelled), so dividing by it + hides the per-source behaviour: a vector that moves every source by + the same number of sigmas has moved the trusted ones three times + *less* in fold terms. Only the *ranking* of the sigmas across + sources is meaningful, not their absolute values. + """ + return np.exp(np.abs(np.log(np.asarray(kcat, dtype=float) / kcat0))) + + +def source_movement( + kcat: np.ndarray, + kcat0: np.ndarray, + groups: np.ndarray, + order: Sequence[str], + near_tol: float = 0.1, +) -> dict[str, dict[str, float]]: + """Median fold change and near-prior fraction, per source group. + + Two medians are reported per source, separating how *many* kcats a + result moves from how *far* it moves them. ``median_fold`` is taken + over every kcat of the source; where most of a source sits at its + prior -- which a restrictive ``tunable_mask`` guarantees -- it + saturates at 1.0 and carries no information about the tier. + ``median_fold_moved`` conditions on the kcats that did move, and so + stays informative at any sparsity. + + Parameters + ---------- + order + Source names from most to least trusted. Used only to report + whether movement respects that ranking. + near_tol + A kcat within ``1 + near_tol`` fold of its prior counts as + untouched, and is excluded from ``median_fold_moved``. + + Returns + ------- + dict keyed by source name, plus ``"_ordered"``: whether + ``median_fold_moved`` is non-decreasing along ``order``, i.e. + whether the result moved trusted kcats less far than untrusted + ones. A source with nothing moved reports ``median_fold_moved`` + 1.0 and takes no part in that check. + """ + fold = fold_change(kcat, kcat0) + groups = np.asarray(groups) + out: dict[str, dict[str, float]] = {} + medians = [] + for name in order: + sel = groups == name + if not sel.any(): + continue + moved = sel & (fold >= 1.0 + near_tol) + med_moved = float(np.median(fold[moved])) if moved.any() else 1.0 + if moved.any(): + medians.append(med_moved) + out[name] = { + "median_fold": float(np.median(fold[sel])), + "median_fold_moved": med_moved, + "frac_near_prior": float(np.mean(fold[sel] < 1.0 + near_tol)), + "n": int(sel.sum()), + "n_moved": int(moved.sum()), + } + out["_ordered"] = all( + medians[i] <= medians[i + 1] for i in range(len(medians) - 1) + ) + return out + + +def impact_share( + kcat: np.ndarray, + kcat0: np.ndarray, + drmse: np.ndarray, + rel_tol: float = 0.02, +) -> float: + """Fraction of the total screened impact carried by changed kcats. + + ``drmse`` is each kcat's one-at-a-time effect on the distance. A + result that changes many parameters the data barely responds to + scores low here even when it changes few of them; one that changes + a handful of load-bearing parameters scores high. Distinguishes + "few, large, and consequential" from mere sparsity -- on the full + ecYeastGEM run the shrunk blend changes 560 kcats but carries only + 28% of the available impact, while 301 changes chosen by the screen + carry 69%. + """ + drmse = np.asarray(drmse, dtype=float) + total = drmse.sum() + if total <= 0: + return 0.0 + changed = fold_change(kcat, kcat0) > (1.0 + rel_tol) + return float(drmse[changed].sum() / total) + + +def identifiability_mask( + drmse: np.ndarray, + sigma0_log: np.ndarray, + threshold: float, +) -> np.ndarray: + """Which kcats have earned the right to be tuned. + + Keeps a kcat when ``|dRMSE| * sigma0_log > threshold``, where + ``dRMSE`` is its one-at-a-time effect on the distance. Multiplying + by ``sigma0_log`` makes a trusted source clear a proportionally + higher bar before it may move: with ``custom`` at 0.10 and + unlabelled at 0.30, a ``custom`` kcat needs three times the + measured effect of an unlabelled one to qualify. + + This uses only the *ranking* of the sigmas across sources, which is + the part that carries meaning -- their absolute values are a + convention. One global ``threshold`` sets the overall strictness; + the per-source differentiation is derived, not tuned. + """ + return (np.asarray(drmse, dtype=float) + * np.asarray(sigma0_log, dtype=float)) > threshold + + +def movement_in_sigma( + kcat: np.ndarray, kcat0: np.ndarray, sigma0_log: np.ndarray, +) -> np.ndarray: + """Per-kcat displacement from the prior, in prior standard deviations. + + ``|log(kcat / kcat0)| / sigma0_log``, matching MATLAB's + ``devFromPrior = abs(muLog - log(kcat0)) ./ sigma0log``. + """ + return np.abs(np.log(np.asarray(kcat, dtype=float) / kcat0)) / sigma0_log + + +def revert_below( + kcat: np.ndarray, + kcat0: np.ndarray, + sigma0_log: np.ndarray, + threshold: float, +) -> np.ndarray: + """Put every kcat that moved less than ``threshold`` sigma back on + its prior, leaving the rest at their tuned values.""" + keep = movement_in_sigma(kcat, kcat0, sigma0_log) >= threshold + return np.where(keep, np.asarray(kcat, dtype=float), kcat0) + + +def n_changed( + kcat: np.ndarray, kcat0: np.ndarray, rel_tol: float = 0.02, +) -> int: + """How many kcats differ from their prior by more than ``rel_tol``. + + The 2% default matches the threshold MATLAB reports its per-source + "unchanged" percentages at. + """ + return int( + np.count_nonzero( + np.abs(np.log(np.asarray(kcat, dtype=float) / kcat0)) + > np.log1p(rel_tol) + ) + ) + + +def parsimony_frontier( + score: Callable[[np.ndarray], float], + kcat: np.ndarray, + kcat0: np.ndarray, + sigma0_log: np.ndarray, + thresholds: Sequence[float] = (0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 10.0, 15.0), +) -> list[FrontierPoint]: + """Score ``kcat`` with progressively more of it reverted to prior. + + Parameters + ---------- + score + Scores one kcat vector; one call per threshold. + kcat, kcat0, sigma0_log + Tuned vector, prior vector, and per-kcat prior sigma. + thresholds + Sigma cutoffs to sweep, ascending. ``0.0`` reverts nothing and + so reproduces ``kcat``'s own score. + + Returns + ------- + list of FrontierPoint, in the order given. + """ + kcat0 = np.asarray(kcat0, dtype=float) + sigma0_log = np.asarray(sigma0_log, dtype=float) + points = [] + for t in thresholds: + vec = revert_below(kcat, kcat0, sigma0_log, t) + points.append(FrontierPoint( + threshold=float(t), + kcat=vec, + n_changed=n_changed(vec, kcat0), + mean_dev=float(movement_in_sigma(vec, kcat0, sigma0_log).mean()), + rmse=float(score(vec)), + )) + return points + + +def best_parsimonious( + points: Sequence[FrontierPoint], tolerance: float = 0.02, +) -> FrontierPoint: + """The fewest-changes point whose RMSE is within ``tolerance`` of the + best on the frontier. + + Defaults to 2%, which is the scale of the alternate-optimum noise + measured on this model -- below that, two vectors are not + meaningfully different in fit, so the sparser one wins. + """ + if not points: + raise ValueError("points is empty.") + best_rmse = min(p.rmse for p in points) + within = [p for p in points if p.rmse <= best_rmse * (1.0 + tolerance)] + return min(within, key=lambda p: p.n_changed) diff --git a/src/geckopy/kcat_tuning/evotune/priors.py b/src/geckopy/kcat_tuning/evotune/priors.py new file mode 100644 index 0000000..38558c4 --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/priors.py @@ -0,0 +1,108 @@ +"""Source classification and prior widths for evotune kcat tuning. + +Ported from GECKO MATLAB: +src/kcat_tuning/Bayesian/bayesianSensitivityTuning.m +(the ``kcatSourceIdx``/``sigma0log`` construction). Verified against +``develop4``'s current source. +""" +from __future__ import annotations + +from typing import Optional + +import numpy as np + +from ...adapter.params import EvotuneParams + +#: Sentinel group name for an ``ec.source`` value matched by no +#: ``source_groups`` entry -- MATLAB's ``noKcatSource``. +UNLABELLED_GROUP = "unlabelled" + + +def classify_kcat_source( + source: str, + params: EvotuneParams, + *, + okp_method: Optional[str] = None, +) -> str: + """Map one ``ec.source`` string to a trust-tier group name. + + Matching is case-insensitive, mirroring MATLAB's + ``strcmpi(ecModel.ec.source, kcatSources{i})``. + + Parameters + ---------- + source + One ``ec.source`` entry. + params + Supplies ``source_groups``. + okp_method + The project's configured OpenKineticsPredictor method + (``OkpParams.method``), if any. A group with ``match_okp=True`` + also matches ``source`` when it equals this (case-insensitive). + + Returns + ------- + str + A key of ``params.source_groups``, or :data:`UNLABELLED_GROUP` + if nothing matches -- callers then fall back to + ``params.sigma0_log_default`` / ``shrink_thr_default`` / + ``force_prior_thr_default``. + """ + source_lower = source.lower() + for name, rule in params.source_groups.items(): + if any(source_lower == s.lower() for s in rule.sources): + return name + if ( + rule.match_okp + and okp_method is not None + and source_lower == okp_method.lower() + ): + return name + return UNLABELLED_GROUP + + +def classify_kcat_sources( + ec_sources: list[str], + params: EvotuneParams, + *, + okp_method: Optional[str] = None, +) -> np.ndarray: + """Vectorised :func:`classify_kcat_source` over ``ec.source``. + + Returns + ------- + numpy.ndarray of str, shape ``(len(ec_sources),)`` + """ + return np.array( + [ + classify_kcat_source(s, params, okp_method=okp_method) + for s in ec_sources + ], + dtype=object, + ) + + +def build_sigma0_log(groups: np.ndarray, params: EvotuneParams) -> np.ndarray: + """Per-row prior std dev in log-space, from source-group membership. + + Mirrors MATLAB's ``sigma0log = sigma0logDefault * ones(...); + sigma0log(uniqKcatParams) = sigma0logSource(kcatSourceIdx(...))``: + every row starts at ``params.sigma0_log_default``, then rows whose + group is a real (non-:data:`UNLABELLED_GROUP`) entry get + overridden by that group's ``sigma0_log_source`` value. + + Parameters + ---------- + groups + Per-row group names from :func:`classify_kcat_sources`. + params + Supplies ``sigma0_log_default`` and ``sigma0_log_source``. + + Returns + ------- + numpy.ndarray of float, same shape as ``groups``. + """ + sigma0_log = np.full(len(groups), params.sigma0_log_default, dtype=float) + for name, value in params.sigma0_log_source.items(): + sigma0_log[groups == name] = value + return sigma0_log diff --git a/src/geckopy/kcat_tuning/evotune/simulate.py b/src/geckopy/kcat_tuning/evotune/simulate.py new file mode 100644 index 0000000..023b432 --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/simulate.py @@ -0,0 +1,211 @@ +"""FBA simulation half of the evotune kcat tuning distance function. + +Ported from GECKO MATLAB: +src/kcat_tuning/Bayesian/abc_max.m (the +``rmsecal`` helper's simulate-and-measure loop). Verified against +``develop4``'s current source, not the (superseded, per its own +``REVIEW.md``) ``fix/bayesianTuning`` branch. + +This module only does the FBA half: given an already kcat-constrained +``EcModel`` and one :class:`~geckopy.databases.flux_data.FluxData` +dataset (either ``evotune_data.flux_data`` or ``evotune_data.max_grate``), +simulate every condition and report growth + exchange fluxes. +:mod:`.distance` turns those raw numbers into an RMSE against the +matching experimental values -- kept separate so the RMSE math is +testable with hand-built inputs and no ``EcModel``/FBA at all. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable, Optional + +import numpy as np + +if TYPE_CHECKING: + from ...databases.flux_data import FluxData + from ...ec_model.ec_model import EcModel + + +@dataclass +class ConditionSimResult: + """One condition's simulated outcome. + + ``feasible=False`` (LP infeasible/unbounded, or no biomass flux) + means ``growth``/``exch_fluxes``/``block_fluxes`` are meaningless; + :mod:`.distance` applies the fixed infeasibility penalty for that + row instead of reading them. + """ + + feasible: bool + growth: float = float("nan") + exch_fluxes: dict[str, float] = field(default_factory=dict) + block_fluxes: dict[str, float] = field(default_factory=dict) + + +def simulate_evotune_dataset( + model: "EcModel", + flux_data: "FluxData", + *, + constrain: bool, + zero_flux_rxns: list[str], + bio_rxn_id: str, + make_anaerobic: Optional[Callable[["EcModel"], None]] = None, + change_protein_biomass: Optional[Callable[["EcModel", float], None]] = None, +) -> list[ConditionSimResult]: + """Simulate every condition in one tuning dataset. + + Mirrors ``abc_max.m``'s ``rmsecal`` FBA half: + + 1. Every exchange reaction that appears as a *condition name* + anywhere in ``flux_data.conds`` (matched by name against + ``flux_data.exch_mets``) is blocked (``lower_bound = 0``) up + front, across the board -- not just the reaction for the row + being solved -- so one row's carbon source can't leak flux into + another row's simulation. + 2. For row ``i``, that row's own carbon-source reaction is + unblocked: fixed at the measured uptake rate if ``constrain`` + is True (``evotune_data.flux_data``), or fully opened to ``-1000`` + if ``constrain`` is False (``evotune_data.max_grate`` -- "what's + the *best possible* growth on this carbon source", not "growth + at this measured rate"). + 3. Optional per-condition adjustments run if the corresponding + callable is supplied: anaerobic switch when the oxygen exchange + column reads exactly 0, protein-content rescaling when + ``Ptot`` is given and nonzero. + 4. Solve; record growth, plus (``constrain`` datasets only, since + ``max_grate`` has no exchange measurements to compare against) + every non-NaN measured exchange's simulated flux and every + ``zero_flux_rxns`` entry's simulated flux. + + All mutation happens inside ``with model:``, which cobra reverts + on exit -- only bounds are touched here, so this is safe to call + repeatedly against one persistent (per-worker) model instance; a + per-particle ``EcModel.copy()`` is not used (prohibitively + expensive on a real-scale model). + + Parameters + ---------- + model + EcModel with the current particle's kcats already applied + (via ``apply_kcat_constraints``). + flux_data + The dataset to simulate. + constrain + True for flux data (fix uptake at the measured rate), False + for max-growth data (open uptake fully). + zero_flux_rxns + Reaction IDs assumed zero-flux in every condition + (``evotune_data.zero_flux``); the row's own carbon source is + skipped even if it's also listed here. + bio_rxn_id + The biomass reaction ID (``adapter.params.bio_rxn``). + make_anaerobic, change_protein_biomass + Optional adapter-specific hooks (MATLAB: + ``modelAdapter.makeModelAnaerobic`` / + ``.changeProteinBiomass``). geckopy has no generic, + organism-agnostic port of these yet -- in MATLAB they're + written per model-adapter subclass (e.g. + ``YeastGEMAdapter.m``). Left as an injection point rather + than a hard requirement: data with no anaerobic/``Ptot`` rows + never needs them, and passing ``None`` for either just skips + that adjustment instead of raising. + + Returns + ------- + list[ConditionSimResult] + One entry per row of ``flux_data.conds``, in order. + + Raises + ------ + ValueError + If any ``flux_data.conds`` entry can't be matched by name to + an exchange metabolite in ``flux_data.exch_mets``. + """ + cond_to_exch_idx = _match_conditions_to_exch(flux_data) + all_condition_rxn_ids = sorted( + {flux_data.exch_rxn_ids[j] for j in set(cond_to_exch_idx)} + ) + o2_col = _find_col(flux_data.exch_mets, "oxygen") + measured_cols = [ + j for j in range(len(flux_data.exch_rxn_ids)) + ] if constrain else [] + + results: list[ConditionSimResult] = [] + for i, exch_idx in enumerate(cond_to_exch_idx): + carbon_rxn_id = flux_data.exch_rxn_ids[exch_idx] + with model: + for rxn_id in all_condition_rxn_ids: + model.reactions.get_by_id(rxn_id).lower_bound = 0.0 + + carbon_rxn = model.reactions.get_by_id(carbon_rxn_id) + if constrain: + carbon_rxn.lower_bound = float(flux_data.exch_fluxes[i, exch_idx]) + else: + carbon_rxn.lower_bound = -1000.0 + + if o2_col is not None and flux_data.exch_fluxes[i, o2_col] == 0.0: + if make_anaerobic is not None: + make_anaerobic(model) + + ptot = float(flux_data.p_tot[i]) + if not (np.isnan(ptot) or ptot == 0.0): + if change_protein_biomass is not None: + change_protein_biomass(model, ptot) + + sol = model.optimize() + if ( + sol.status != "optimal" + or sol.objective_value is None + or np.isnan(sol.objective_value) + ): + results.append(ConditionSimResult(feasible=False)) + continue + + growth = float(sol.fluxes.get(bio_rxn_id, 0.0)) + exch_fluxes: dict[str, float] = {} + block_fluxes: dict[str, float] = {} + if constrain: + for j in measured_cols: + if np.isnan(flux_data.exch_fluxes[i, j]): + continue + rxn_id = flux_data.exch_rxn_ids[j] + exch_fluxes[rxn_id] = float(sol.fluxes.get(rxn_id, 0.0)) + for rxn_id in zero_flux_rxns: + if rxn_id == carbon_rxn_id: + continue + block_fluxes[rxn_id] = float(sol.fluxes.get(rxn_id, 0.0)) + results.append( + ConditionSimResult( + feasible=True, + growth=growth, + exch_fluxes=exch_fluxes, + block_fluxes=block_fluxes, + ) + ) + return results + + +def _match_conditions_to_exch(flux_data: "FluxData") -> list[int]: + """Column index into ``exch_mets``/``exch_rxn_ids`` for each + condition, matched by name. Mirrors MATLAB's + ``ismember(data.conds, data.exchMets)`` plus its "cannot be + matched" error.""" + met_to_idx: dict[str, int] = {} + for j, met in enumerate(flux_data.exch_mets): + met_to_idx.setdefault(met, j) + missing = sorted({c for c in flux_data.conds if c not in met_to_idx}) + if missing: + raise ValueError( + 'Carbon source(s) "' + + "; ".join(missing) + + '" in the provided fluxData or maxGrowth cannot be matched ' + "by name with an exchange reaction." + ) + return [met_to_idx[c] for c in flux_data.conds] + + +def _find_col(exch_mets: list[str], name: str) -> Optional[int]: + try: + return exch_mets.index(name) + except ValueError: + return None diff --git a/src/geckopy/kcat_tuning/evotune/tuning.py b/src/geckopy/kcat_tuning/evotune/tuning.py new file mode 100644 index 0000000..f8c6fdd --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/tuning.py @@ -0,0 +1,912 @@ +"""CMA-ES kcat tuning: fitting kcats to experimental data. + +Three functions, normally called in this order: + +- :func:`screen_kcat_leverage` -- report which kcats the data can + actually speak to, with no optimisation involved. Perturbs each + tie-group by +-``fold`` and measures the resulting change in RMSE, + ranked by that leverage weighted by how much the source is trusted. + Useful on its own for curation, independent of any tuning run. +- :func:`select_tunable_mask` -- turn a screen report into a boolean + ``tunable_mask``, by keeping the fewest highest-ranked groups whose + combined leverage reaches ``target_impact_share`` of the total. A + *relative* cutoff, so the same ``target_impact_share`` selects a + comparable quality of parameter set on a model with a different + leverage scale, unlike an absolute threshold or a fixed count. +- :func:`cmaes_kcat_tuning` -- the tuning run. Screens and selects + automatically when no mask is given. + +Parallel scoring (``n_proc``) +------------------------------ +Each generation's new candidates are scored independently of each +other, so they parallelise across a process pool -- using +``cobra.util.process_pool.ProcessPool`` (the same primitive cobrapy's +own ``single_gene_deletion``/``single_reaction_deletion``/etc. use for +"repeated FBA on slightly-perturbed model copies", exactly this +module's shape of problem) rather than driving ``multiprocessing`` +directly: one persistent ``EcModel`` copy per **worker process** +(deserialised once at pool startup, not once per particle and not one +shared mutable object across processes), which ``ProcessPool`` also +gets right on Windows for free (a temp-file-based initarg handoff that +works around a real performance issue in raw +``multiprocessing.Pool(initializer=...)`` there -- see its docstring). +Each worker then scores every particle it's handed against its own +copy via incremental ``apply_kcat_constraints(update_rxns=...)`` +calls, exactly like the serial path -- a per-particle ``EcModel.copy()`` +is not used either way (~225x an FBA solve on a real-scale model). + +CMA-ES's own ``ask``/``tell`` sequence stays single-threaded in the +main process -- it's cheap relative to FBA and, more importantly, +doing it there keeps the *sequence* of candidates proposed each +generation deterministic given ``seed``, independent of ``n_proc`` or +how work happens to be chunked across workers. Only the scoring of an +already-fixed batch of candidates is parallelised, and scoring is a +pure function of the kcat vector -- each candidate's solves start from +a cold basis (:func:`_reset_solver_basis`), so a worker's result does +not depend on which candidate it scored before. A run with ``n_proc=1`` +and the same ``seed`` therefore reproduces bit-for-bit identical +results to one with ``n_proc>1``. + +``make_anaerobic``/``change_protein_biomass``, if supplied, must be +importable top-level functions (not lambdas/closures) when running +with ``n_proc>1`` on Windows, where ``multiprocessing`` needs to +pickle them into each worker; POSIX's default start method (typically +``fork``, inherited via copy-on-write with no pickling involved) has +no such restriction. +""" +from __future__ import annotations + +import logging +from contextlib import nullcontext +from dataclasses import dataclass, field +from itertools import combinations +from typing import TYPE_CHECKING, Callable, Optional, Sequence + +import cma +import cobra +import numpy as np +import pandas as pd +from cobra.util import ProcessPool + +from ...ec_model.pipeline.apply_kcat import apply_kcat_constraints +from .data import EvotuneData, load_evotune_data +from .distance import evotune_distance, compute_excarbon +from .parsimony import fold_change, n_changed +from .priors import build_sigma0_log, classify_kcat_sources +from .simulate import simulate_evotune_dataset +from .tying import isozyme_tie_map + +if TYPE_CHECKING: + from ...adapter import ModelAdapter + from ...adapter.params import EvotuneParams + from ...ec_model.ec_model import EcModel + +logger = logging.getLogger(__name__) + + +@dataclass +class EvotuneResult: + """Outcome of a :func:`cmaes_kcat_tuning` run. + + The model is mutated in place (the best kcat vector found is + applied to ``model.ec.kcat``), and this result records the run's + history for inspection. + + Attributes + ---------- + rxns + Tunable ``ec.rxns`` IDs, in the column order used throughout + (``rxns[i]`` corresponds to ``old_kcat[i]``/``new_kcat[i]``/ + ``groups[i]``). Tied isozyme copies are all listed, sharing one + value in ``new_kcat``. + old_kcat, new_kcat + kcat values before/after tuning, parallel to ``rxns``. + groups + Source-group name per tunable row (from + ``priors.classify_kcat_sources``). + rmse_trace + Best-so-far plain RMSE, per generation. Always the plain RMSE, + so runs stay comparable across penalties and against MATLAB + even when the search used a penalised objective. + objective_trace + Best-so-far value of the quantity the search actually + minimised, per generation. Identical to ``rmse_trace`` when + ``params.prior_penalty_weight`` is 0. + n_generations + Number of generations actually run. + converged + True if ``rmse_trace[-1] <= params.rmse_threshold``; False if + ``max_generations`` was reached first. + """ + + rxns: list[str] = field(default_factory=list) + old_kcat: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=float)) + new_kcat: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=float)) + groups: list[str] = field(default_factory=list) + rmse_trace: list[float] = field(default_factory=list) + objective_trace: list[float] = field(default_factory=list) + n_generations: int = 0 + converged: bool = False + + +def _resolve_context(model, adapter, params, evotune_data, bio_rxn, okp_method): + """Resolve every optional input against ``model``'s adapter, or + raise if it can't be and none was given explicitly.""" + if adapter is None: + adapter = getattr(model, "adapter", None) + if params is None: + if adapter is None: + raise ValueError( + "params not given and model has no adapter to read " + "adapter.params.evotune from." + ) + params = adapter.params.evotune + if evotune_data is None: + if adapter is None: + raise ValueError( + "evotune_data not given and model has no adapter to load it from." + ) + evotune_data = load_evotune_data(adapter) + if evotune_data.flux_data is None and evotune_data.max_grate is None: + raise ValueError( + "evotune_data has neither flux_data nor max_grate -- nothing to " + "tune against." + ) + if bio_rxn is None: + if adapter is None: + raise ValueError("bio_rxn not given and model has no adapter.") + bio_rxn = adapter.params.bio_rxn + if okp_method is None and adapter is not None: + okp_method = adapter.params.okp.method + return params, evotune_data, bio_rxn, okp_method + + +def _tunable_context( + model, params, evotune_data, bio_rxn, okp_method, tunable_mask, +): + """The tunable subset plus everything derived from it: tie groups, + trust weights, and the excarbon table. Shared by every function in + this module so a screen and the run it feeds are always looking at + exactly the same parameters.""" + is_tunable = model.ec.kcat > 0 + if tunable_mask is not None: + tunable_mask = np.asarray(tunable_mask, dtype=bool) + if tunable_mask.shape != is_tunable.shape: + raise ValueError( + f"tunable_mask has shape {tunable_mask.shape}; expected " + f"{is_tunable.shape} to match model.ec.rxns." + ) + is_tunable = is_tunable & tunable_mask + tunable_idx = np.flatnonzero(is_tunable) + if tunable_idx.size == 0: + raise ValueError( + "No tunable kcats: model.ec.kcat is all <= 0" + + (", or tunable_mask excludes every one with a kcat." + if tunable_mask is not None else ".") + ) + ec_rxn_ids_tunable = [model.ec.rxns[i] for i in tunable_idx] + kcat0 = model.ec.kcat[tunable_idx].astype(float).copy() + sources = [model.ec.source[i] for i in tunable_idx] + groups = classify_kcat_sources(sources, params, okp_method=okp_method) + sigma0_log = build_sigma0_log(groups, params) + + excarbon_rxn_ids: set[str] = {bio_rxn} + for data in (evotune_data.flux_data, evotune_data.max_grate): + if data is not None: + excarbon_rxn_ids.update(data.exch_rxn_ids) + excarbon_rxn_ids.update(evotune_data.zero_flux) + excarbon = compute_excarbon(model, excarbon_rxn_ids, bio_rxn_id=bio_rxn) + + tie_map = ( + isozyme_tie_map(ec_rxn_ids_tunable, kcat0, list(groups)) + if params.tie_isozymes else np.arange(len(kcat0)) + ) + return (tunable_idx, ec_rxn_ids_tunable, kcat0, groups, sigma0_log, + excarbon, tie_map) + + +def screen_kcat_leverage( + model: "EcModel", + *, + adapter: Optional["ModelAdapter"] = None, + params: Optional["EvotuneParams"] = None, + evotune_data: Optional[EvotuneData] = None, + okp_method: Optional[str] = None, + bio_rxn: Optional[str] = None, + make_anaerobic: Optional[Callable[["EcModel"], None]] = None, + change_protein_biomass: Optional[Callable[["EcModel", float], None]] = None, + tunable_mask: Optional[np.ndarray] = None, + fold: float = 2.0, + n_proc: Optional[int] = None, +) -> pd.DataFrame: + """Report which kcats the data can actually speak to. + + No optimisation happens here -- this only says which parameters + are worth curating or tuning, and can be read on its own well + before any tuning run. Each tie-group (isozyme copies sharing a + prior and a source move together when ``params.tie_isozymes``) is + perturbed up and down by ``fold``, and its leverage is the largest + resulting change in plain RMSE -- always the plain fit, never the + ``prior_penalty_weight``-penalised objective, since leverage asks + what the data supports, independent of how much a move is charged + for. Groups are ranked by that leverage weighted by + ``sigma0_log`` -- :func:`~.parsimony.identifiability_mask`'s + convention, where a trusted source needs proportionally more + measured effect to rank alongside an untrusted one. This + trust-weighting means the reported ``cum_leverage_share`` is a + different quantity from :func:`~.parsimony.impact_share`, which is + unweighted. + + Costs one simulation per condition per tie-group probed (two + probes each, up and down) plus one baseline -- comparable in scale + to a full tuning run's evaluation budget, not a cheap pre-check. + FVA-unreachable kcats are not filtered out beforehand: a kcat that + cannot carry flux under any condition shows zero leverage here by + construction and simply sorts to the bottom, which is a more + direct test than a separately computed static reachability mask. + + Returns + ------- + pandas.DataFrame + One row per tie-group, sorted by ``rank`` descending: + ``rxn_id`` (the group's representative), ``n_isozymes``, + ``source_group``, ``kcat0``, ``leverage``, ``sigma0_log``, + ``rank`` (``leverage * sigma0_log``), and ``cum_leverage_share`` + (the running total of ``rank``, as a fraction of its sum over + every row). An internal ``_positions`` column (a tuple of + indices into ``model.ec.rxns``) is what + :func:`select_tunable_mask` needs to expand a row back into a + mask; drop it before showing the table to a person. + """ + params, evotune_data, bio_rxn, okp_method = _resolve_context( + model, adapter, params, evotune_data, bio_rxn, okp_method) + (tunable_idx, ec_rxn_ids_tunable, kcat0, groups, sigma0_log, excarbon, + tie_map) = _tunable_context( + model, params, evotune_data, bio_rxn, okp_method, tunable_mask) + + reps = np.flatnonzero(tie_map == np.arange(len(tie_map))) + members_of = {int(r): np.flatnonzero(tie_map == r) for r in reps} + + if n_proc is None: + n_proc = cobra.Configuration().processes + n_proc = max(1, int(n_proc)) + + pairs = [] + vectors = [kcat0] + for rep in reps: + members = members_of[int(rep)] + up = kcat0.copy(); up[members] = up[members] * fold + dn = kcat0.copy(); dn[members] = dn[members] / fold + pairs.append(int(rep)) + vectors.append(up) + vectors.append(dn) + + if n_proc == 1: + rmses = [ + _score_kcat_vector( + model, tunable_idx, ec_rxn_ids_tunable, evotune_data, excarbon, + bio_rxn, v, make_anaerobic=make_anaerobic, + change_protein_biomass=change_protein_biomass, + max_growth_weight=params.max_growth_weight, + )[1] + for v in vectors + ] + else: + with ProcessPool( + n_proc, initializer=_init_worker, + initargs=(model, tunable_idx, ec_rxn_ids_tunable, evotune_data, + excarbon, bio_rxn, make_anaerobic, change_protein_biomass, + params.max_growth_weight, 0.0, None, None), + ) as pool: + chunk = max(1, len(vectors) // (n_proc * 4)) + rmses = [r for _, r in pool.map(_score_worker, vectors, chunksize=chunk)] + + # The n_proc==1 path scores directly against `model`, leaving its + # ec.kcat at whichever probe vector was scored last; a pool worker + # scores its own copy instead, so this is a no-op there. Either + # way, `model` must come out holding the prior it went in with. + model.ec.kcat[tunable_idx] = kcat0 + apply_kcat_constraints(model, update_rxns=ec_rxn_ids_tunable) + + base_rmse = rmses[0] + rows = [] + for i, rep in enumerate(pairs): + up_rmse, dn_rmse = rmses[1 + 2 * i], rmses[2 + 2 * i] + leverage = max(abs(up_rmse - base_rmse), abs(dn_rmse - base_rmse)) + members = members_of[rep] + rows.append({ + "rxn_id": ec_rxn_ids_tunable[rep], + "n_isozymes": int(len(members)), + "source_group": str(groups[rep]), + "kcat0": float(kcat0[rep]), + "leverage": float(leverage), + "sigma0_log": float(sigma0_log[rep]), + "_positions": tuple(int(tunable_idx[m]) for m in members), + }) + + df = pd.DataFrame(rows) + df["rank"] = df["leverage"] * df["sigma0_log"] + df = df.sort_values("rank", ascending=False, ignore_index=True) + total = float(df["rank"].sum()) + df["cum_leverage_share"] = ( + (df["rank"].cumsum() / total) if total > 0 else 0.0 + ) + return df + + +def select_tunable_mask( + model: "EcModel", + screen: pd.DataFrame, + *, + target_impact_share: float = 0.9, +) -> np.ndarray: + """Build a ``tunable_mask`` from a :func:`screen_kcat_leverage` report. + + Keeps the fewest highest-ranked groups whose combined leverage + reaches ``target_impact_share`` of the total -- a relative cutoff, + so the same ``target_impact_share`` selects a comparable quality of + parameter set on any model, rather than an absolute leverage value + or a fixed count calibrated on a different model's scale. + ``target_impact_share=0.9`` is a starting point, not a validated + universal constant; check the resulting mask's size against + :func:`screen_kcat_leverage`'s own curve before trusting it on a + new model. + + Pure and cheap -- no simulation, only ``screen`` is read. + """ + mask = np.zeros(len(model.ec.rxns), dtype=bool) + if screen.empty: + return mask + prev_cum = screen["cum_leverage_share"].shift(1, fill_value=0.0) + keep = prev_cum < target_impact_share + for positions in screen.loc[keep, "_positions"]: + mask[list(positions)] = True + return mask + + +def cmaes_kcat_tuning( + model: "EcModel", + *, + adapter: Optional["ModelAdapter"] = None, + params: Optional["EvotuneParams"] = None, + evotune_data: Optional[EvotuneData] = None, + okp_method: Optional[str] = None, + bio_rxn: Optional[str] = None, + make_anaerobic: Optional[Callable[["EcModel"], None]] = None, + change_protein_biomass: Optional[Callable[["EcModel", float], None]] = None, + tunable_mask: Optional[np.ndarray] = None, + screen: Optional[pd.DataFrame] = None, + target_impact_share: float = 0.9, + popsize: Optional[int] = None, + n_proc: Optional[int] = None, + seed: Optional[int] = None, + verbose: bool = True, +) -> EvotuneResult: + """Tune kcats against experimental data with CMA-ES. + + Configuration comes entirely from ``EvotuneParams``: trust tiers, + ``max_growth_weight``, ``prior_penalty_weight`` and ``tie_isozymes`` + control the search, and ``max_generations``/``rmse_threshold`` are + its stopping conditions. + + Parameters + ---------- + model + EcModel with a populated ``ec.kcat``. Mutated in place: on + return, tunable rows carry the best kcat vector CMA-ES found, + with kcat constraints already applied. + tunable_mask, screen, target_impact_share + Control which kcats are searched over, in order of precedence. + Pass ``tunable_mask`` to fix the set yourself. Otherwise, a + mask is built by :func:`select_tunable_mask` at + ``target_impact_share``, from ``screen`` if given (so a + previously computed :func:`screen_kcat_leverage` report need + not be recomputed) or from a fresh screen otherwise. + popsize + CMA-ES population size. Defaults to ``cma``'s own + dimension-scaled default (``4 + floor(3 ln n)``), so it adapts + to however many free parameters the mask/tying produced rather + than a value calibrated on one particular model. + n_proc + Number of worker processes for scoring each generation's + candidates. Defaults to ``cobra.Configuration().processes``. + ``1`` runs the original serial path (no ``Pool`` at all). See + the module docstring's "Parallel scoring" section. + seed + If given, seeds the search, for reproducible runs. + verbose + Whether per-generation progress is logged at INFO. + make_anaerobic, change_protein_biomass + Forwarded to ``simulate.simulate_evotune_dataset`` -- see its + docstring; geckopy has no generic organism-agnostic + implementation of these yet. See the module docstring's + "Parallel scoring" section for a picklability caveat when + ``n_proc>1``. + + Returns + ------- + EvotuneResult + ``rxns``/``old_kcat``/``new_kcat``/``groups`` cover the + selected tunable set, tied groups included (their members share + one value in ``new_kcat``). ``rmse_trace``/``objective_trace`` + record the best-so-far plain RMSE and optimised objective per + generation. + + Raises + ------ + ValueError + If there are no tunable kcats, or fewer than two free + parameters remain after masking and tying -- too little for + CMA-ES to search over. + """ + params, evotune_data, bio_rxn, okp_method = _resolve_context( + model, adapter, params, evotune_data, bio_rxn, okp_method) + + if tunable_mask is None: + if screen is None: + screen = screen_kcat_leverage( + model, adapter=adapter, params=params, evotune_data=evotune_data, + okp_method=okp_method, bio_rxn=bio_rxn, + make_anaerobic=make_anaerobic, + change_protein_biomass=change_protein_biomass, + n_proc=n_proc, + ) + tunable_mask = select_tunable_mask( + model, screen, target_impact_share=target_impact_share) + + (tunable_idx, ec_rxn_ids_tunable, kcat0, groups, sigma0_log, excarbon, + tie_map) = _tunable_context( + model, params, evotune_data, bio_rxn, okp_method, tunable_mask) + + reps = np.flatnonzero(tie_map == np.arange(len(tie_map))) + if len(reps) < 2: + raise ValueError( + f"Only {len(reps)} free parameter(s) after masking and tying; " + "too few for CMA-ES. Widen tunable_mask/target_impact_share." + ) + assign = np.searchsorted(reps, tie_map) + x0 = np.log(kcat0[reps]) + sig = sigma0_log[reps] + lo, hi = kcat_bounds(kcat0) + bounds = ([float(np.log(v)) for v in lo[reps]], + [float(np.log(v)) for v in hi[reps]]) + + def expand(xm: np.ndarray) -> np.ndarray: + """(n_free, n_particles) log-space matrix -> (n_tunable, + n_particles) kcat matrix, tied rows sharing their rep's value.""" + return np.exp(xm[assign, :]) + + if n_proc is None: + n_proc = cobra.Configuration().processes + n_proc = max(1, int(n_proc)) + + options = { + "verbose": -9, "maxiter": params.max_generations, + "CMA_diagonal": True, "bounds": bounds, + "scaling_of_variables": (sig / np.median(sig)).tolist(), + } + if popsize is not None: + options["popsize"] = int(popsize) + if seed is not None: + # cma treats seed=0 as falsy and falls back to an unseeded, + # time-based draw instead of raising -- verified directly + # against this cma version, not assumed. +1 keeps every + # caller-facing seed (0, 1, 2, ...) off that value. + options["seed"] = int(seed) + 1 + es = cma.CMAEvolutionStrategy(x0, float(np.median(sig)), options) + + pool_cm = ( + nullcontext(None) if n_proc == 1 else + ProcessPool( + n_proc, initializer=_init_worker, + initargs=(model, tunable_idx, ec_rxn_ids_tunable, evotune_data, + excarbon, bio_rxn, make_anaerobic, change_protein_biomass, + params.max_growth_weight, params.prior_penalty_weight, + kcat0, sigma0_log), + ) + ) + + def _score_batch(kcat_matrix, pool): + if pool is None: + return np.array([ + _score_kcat_vector( + model, tunable_idx, ec_rxn_ids_tunable, evotune_data, + excarbon, bio_rxn, kcat_matrix[:, j], + make_anaerobic=make_anaerobic, + change_protein_biomass=change_protein_biomass, + max_growth_weight=params.max_growth_weight, + prior_penalty_weight=params.prior_penalty_weight, + kcat0=kcat0, sigma0_log=sigma0_log, + ) + for j in range(kcat_matrix.shape[1]) + ]) + cols = [kcat_matrix[:, j] for j in range(kcat_matrix.shape[1])] + chunk = max(1, len(cols) // (n_proc * 4)) + return np.array(pool.map(_score_worker, cols, chunksize=chunk)) + + with pool_cm as pool: + obj0, rmse0 = _score_batch(kcat0.reshape(-1, 1), pool)[0] + best_obj, best_rmse, best_vec = float(obj0), float(rmse0), kcat0.copy() + + rmse_trace: list[float] = [] + objective_trace: list[float] = [] + generation = 0 + while not es.stop(): + asks = es.ask() + xm = np.array(asks).T + scores = _score_batch(expand(xm), pool) + es.tell(asks, list(scores[:, 0])) + generation += 1 + + j = int(np.argmin(scores[:, 0])) + if scores[j, 0] < best_obj: + best_obj = float(scores[j, 0]) + best_rmse = float(scores[j, 1]) + best_vec = expand(xm[:, [j]])[:, 0].copy() + rmse_trace.append(best_rmse) + objective_trace.append(best_obj) + if verbose: + logger.info( + "generation %d: objective %.4f, rmse %.4f", + generation, best_obj, best_rmse, + ) + if best_rmse <= params.rmse_threshold: + break + + model.ec.kcat[tunable_idx] = best_vec + apply_kcat_constraints(model, update_rxns=ec_rxn_ids_tunable) + + return EvotuneResult( + rxns=ec_rxn_ids_tunable, old_kcat=kcat0.copy(), new_kcat=best_vec, + groups=list(groups), rmse_trace=rmse_trace, + objective_trace=objective_trace, n_generations=generation, + converged=best_rmse <= params.rmse_threshold, + ) + + +def tune_prior_penalty_weight( + model: "EcModel", + *, + adapter: Optional["ModelAdapter"] = None, + params: Optional["EvotuneParams"] = None, + evotune_data: Optional[EvotuneData] = None, + okp_method: Optional[str] = None, + bio_rxn: Optional[str] = None, + make_anaerobic: Optional[Callable[["EcModel"], None]] = None, + change_protein_biomass: Optional[Callable[["EcModel", float], None]] = None, + tunable_mask: Optional[np.ndarray] = None, + screen: Optional[pd.DataFrame] = None, + target_impact_share: float = 0.9, + candidates: Sequence[float] = (0.0, 0.01, 0.03, 0.1), + seeds: Sequence[int] = (0, 1), + fold_threshold: float = 2.0, + popsize: Optional[int] = None, + n_proc: Optional[int] = None, + verbose: bool = True, +) -> pd.DataFrame: + """Sweep ``prior_penalty_weight`` and report the fit/reproducibility + trade-off, so the value doesn't have to be picked blind. + + ``prior_penalty_weight`` charges for moving a kcat away from its + prior, and mainly exists to keep large corrections reproducible + across seeds rather than landing on an arbitrary point along a flat + direction. How strong a charge is enough is model- and + data-specific: too little and the search's largest corrections + disagree across seeds; too much and the search stops finding real + corrections at all, giving up fit for nothing. This runs + :func:`cmaes_kcat_tuning` at every value in ``candidates``, each at + every seed in ``seeds``, all against the *same* tunable set (built + once, the way :func:`cmaes_kcat_tuning` would on its own if + ``tunable_mask``/``screen`` aren't given), and reports both fit and + cross-seed reproducibility per value. + + This is exactly as expensive as it sounds: ``len(candidates) * + len(seeds)`` full :func:`cmaes_kcat_tuning` runs, none of them + reusable across candidates. Reuse a ``screen`` you already have to + at least avoid recomputing that part. + + ``model`` is restored to its original kcats before every run and + left that way when this returns -- unlike :func:`cmaes_kcat_tuning`, + which mutates ``model`` with its result, this function only + reports, since there is no single "best" candidate for it to leave + the model in. Fewer than two ``seeds`` makes every reproducibility + column ``NaN``, since there is nothing to compare. + + Parameters + ---------- + candidates + ``prior_penalty_weight`` values to try. + seeds + Seeds to run each candidate at. Reproducibility columns pool + every pair of seeds, so more than two sharpens them but costs + proportionally more. + fold_threshold + A kcat counts as "moved" past this fold change from its prior. + Matches :func:`~.parsimony.fold_change`'s convention. + tunable_mask, screen, target_impact_share, popsize, n_proc, + make_anaerobic, change_protein_biomass, okp_method, bio_rxn + As in :func:`cmaes_kcat_tuning`. + + Returns + ------- + pandas.DataFrame + One row per candidate: ``prior_penalty_weight``, ``n_seeds``, + ``distance_mean`` and ``distance_spread`` (max minus min across + seeds), ``fit_cost`` (``distance_mean`` relative to the best + candidate's, as a fraction), ``n_changed_mean``, ``n_movers`` + (kcats past ``fold_threshold`` in *either* seed of a pair, summed + over every pair), ``n_both_moved`` (past it in *both* -- the + denominator for direction agreement, since a kcat only one seed + moved has no direction to agree about), ``pct_direction_agree``, + ``median_fold_spread`` and ``max_fold_spread`` (among movers, + the largest fold-change between what the two seeds landed on). + """ + params, evotune_data, bio_rxn, okp_method = _resolve_context( + model, adapter, params, evotune_data, bio_rxn, okp_method) + + if tunable_mask is None: + if screen is None: + screen = screen_kcat_leverage( + model, adapter=adapter, params=params, evotune_data=evotune_data, + okp_method=okp_method, bio_rxn=bio_rxn, + make_anaerobic=make_anaerobic, + change_protein_biomass=change_protein_biomass, + n_proc=n_proc, + ) + tunable_mask = select_tunable_mask( + model, screen, target_impact_share=target_impact_share) + + prior_kcat = model.ec.kcat.copy() + + def _reset(): + model.ec.kcat[:] = prior_kcat + apply_kcat_constraints(model, update_rxns=model.ec.rxns) + + try: + rows = [] + for lam in candidates: + lam_params = params.model_copy( + update={"prior_penalty_weight": float(lam)}) + vectors, distances, changed = [], [], [] + old_kcat = None + for seed in seeds: + _reset() + result = cmaes_kcat_tuning( + model, adapter=adapter, params=lam_params, evotune_data=evotune_data, + okp_method=okp_method, bio_rxn=bio_rxn, + make_anaerobic=make_anaerobic, + change_protein_biomass=change_protein_biomass, + tunable_mask=tunable_mask, popsize=popsize, n_proc=n_proc, + seed=seed, verbose=verbose, + ) + vectors.append(result.new_kcat) + distances.append( + result.rmse_trace[-1] if result.rmse_trace else float("nan")) + changed.append(n_changed(result.new_kcat, result.old_kcat)) + old_kcat = result.old_kcat + if verbose: + logger.info( + "prior_penalty_weight=%g seed=%d: distance %.4f, " + "%d changed", lam, seed, distances[-1], changed[-1], + ) + + n_movers = n_both = n_agree = 0 + spreads: list[float] = [] + for i, j in combinations(range(len(vectors)), 2): + fa = fold_change(vectors[i], old_kcat) + fb = fold_change(vectors[j], old_kcat) + movers = (fa > fold_threshold) | (fb > fold_threshold) + both = (fa > fold_threshold) & (fb > fold_threshold) + agree = np.sign(np.log(vectors[i][both] / old_kcat[both])) == \ + np.sign(np.log(vectors[j][both] / old_kcat[both])) + n_movers += int(movers.sum()) + n_both += int(both.sum()) + n_agree += int(agree.sum()) + if movers.any(): + ratio = vectors[i][movers] / vectors[j][movers] + spreads.extend(np.maximum(ratio, 1.0 / ratio).tolist()) + + rows.append({ + "prior_penalty_weight": float(lam), + "n_seeds": len(seeds), + "distance_mean": float(np.mean(distances)), + "distance_spread": float(np.ptp(distances)) if len(distances) > 1 else 0.0, + "n_changed_mean": float(np.mean(changed)), + "n_movers": n_movers, + "n_both_moved": n_both, + "pct_direction_agree": (n_agree / n_both) if n_both > 0 else float("nan"), + "median_fold_spread": float(np.median(spreads)) if spreads else float("nan"), + "max_fold_spread": float(np.max(spreads)) if spreads else float("nan"), + }) + finally: + _reset() + + df = pd.DataFrame(rows) + best = df["distance_mean"].min() + df["fit_cost"] = df["distance_mean"] / best - 1.0 + return df + + +# --------------------------------------------------------------------------- # +# Scoring core, shared by the screen and the search above. +# --------------------------------------------------------------------------- # + +def kcat_bounds(kcat0: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Biologically plausible bounds for proposed kcats, in 1/s. + + Proposals outside these bounds are not hypotheses worth spending an + FBA solve on. The window is 1e-2 to 1e4 for an ordinary kcat, and + always widened far enough to contain the prior with a hundred-fold + margin either side, so an unusually slow or fast enzyme keeps a + window around itself rather than being clipped towards the generic + range. + + Parameters + ---------- + kcat0 + Prior kcat per tunable row, 1/s. + + Returns + ------- + tuple of numpy.ndarray + Lower and upper bound per row. + """ + kcat0 = np.asarray(kcat0, dtype=float) + lo = np.minimum(1e-2, kcat0 / 100.0) + hi = np.maximum(1e4, kcat0 * 100.0) + return lo, hi + + +def _reset_solver_basis(model: "EcModel") -> None: + """Discard the solver's incumbent basis so the solves below start cold. + + The distance must be a function of the kcat vector alone. Solvers + warm-start from the previous solve's basis, and these LPs have + alternate optima: resuming from a basis that is already optimal + for the new problem returns that vertex in zero iterations, so the + reported exchange fluxes -- and hence the RMSE -- would otherwise + depend on which candidate this model scored before. That would + make the result depend on ``n_proc`` and on how the pool happened + to schedule candidates across workers. + + A no-op for solver interfaces whose problem object exposes no + ``reset`` (only the model's own state then determines the solve, + which is the property this guarantees for the rest). + """ + problem = getattr(model.solver, "problem", None) + reset = getattr(problem, "reset", None) + if callable(reset): + reset() + + +def _score_kcat_vector( + model: "EcModel", + tunable_idx: np.ndarray, + ec_rxn_ids_tunable: list[str], + evotune_data: EvotuneData, + excarbon: dict[str, float], + bio_rxn_id: str, + kcat_vec: np.ndarray, + *, + make_anaerobic, + change_protein_biomass, + max_growth_weight: float = 1.0, + prior_penalty_weight: float = 0.0, + kcat0: Optional[np.ndarray] = None, + sigma0_log: Optional[np.ndarray] = None, +) -> tuple[float, float]: + """Score one kcat vector against ``model`` (mutated in place). + + Writes the candidate kcat vector into ``model.ec.kcat`` and + rewrites just the tunable rows' stoichiometry via + ``apply_kcat_constraints(update_rxns=...)`` -- no per-candidate + ``EcModel.copy()`` (prohibitively expensive: ~225x an FBA solve on + a real-scale model). Called directly (looped in-process) for the + serial path, or via + :func:`_score_worker` from inside a pool worker for the parallel + path -- either way, ``model`` is one persistent object reused + across every candidate it's asked to score. + """ + model.ec.kcat[tunable_idx] = kcat_vec + apply_kcat_constraints(model, update_rxns=ec_rxn_ids_tunable) + _reset_solver_basis(model) + + flux_sims = None + if evotune_data.flux_data is not None: + flux_sims = simulate_evotune_dataset( + model, evotune_data.flux_data, + constrain=True, zero_flux_rxns=evotune_data.zero_flux, + bio_rxn_id=bio_rxn_id, + make_anaerobic=make_anaerobic, + change_protein_biomass=change_protein_biomass, + ) + max_grate_sims = None + if evotune_data.max_grate is not None: + max_grate_sims = simulate_evotune_dataset( + model, evotune_data.max_grate, + constrain=False, zero_flux_rxns=[], + bio_rxn_id=bio_rxn_id, + make_anaerobic=make_anaerobic, + change_protein_biomass=change_protein_biomass, + ) + rmse, _ = evotune_distance( + evotune_data, flux_sims=flux_sims, max_grate_sims=max_grate_sims, + excarbon=excarbon, bio_rxn_id=bio_rxn_id, + max_growth_weight=max_growth_weight, + ) + # The search may run on a penalised objective, but the plain RMSE + # is always carried alongside it so a run stays comparable to + # MATLAB's and to runs at other penalties. + objective = rmse + if prior_penalty_weight and kcat0 is not None and sigma0_log is not None: + dev = np.log(np.asarray(kcat_vec, dtype=float) / kcat0) / sigma0_log + objective = rmse + prior_penalty_weight * float(np.mean(dev ** 2)) + return objective, rmse + + +# --------------------------------------------------------------------------- # +# Parallel scoring (n_proc > 1): worker-process globals, populated once +# by _init_worker and reused for every candidate that worker scores. +# --------------------------------------------------------------------------- # + +_WORKER_MODEL: Optional["EcModel"] = None +_WORKER_TUNABLE_IDX: Optional[np.ndarray] = None +_WORKER_EC_RXN_IDS_TUNABLE: Optional[list[str]] = None +_WORKER_EVOTUNE_DATA: Optional[EvotuneData] = None +_WORKER_EXCARBON: Optional[dict[str, float]] = None +_WORKER_BIO_RXN: Optional[str] = None +_WORKER_MAKE_ANAEROBIC = None +_WORKER_CHANGE_PROTEIN_BIOMASS = None +_WORKER_MAX_GROWTH_WEIGHT: float = 1.0 +_WORKER_PRIOR_PENALTY_WEIGHT: float = 0.0 +_WORKER_KCAT0: Optional[np.ndarray] = None +_WORKER_SIGMA0_LOG: Optional[np.ndarray] = None + + +def _init_worker( + model: "EcModel", + tunable_idx: np.ndarray, + ec_rxn_ids_tunable: list[str], + evotune_data: EvotuneData, + excarbon: dict[str, float], + bio_rxn_id: str, + make_anaerobic, + change_protein_biomass, + max_growth_weight: float = 1.0, + prior_penalty_weight: float = 0.0, + kcat0: Optional[np.ndarray] = None, + sigma0_log: Optional[np.ndarray] = None, +) -> None: + """Pool initializer: stash this worker process's own EcModel copy + (deserialised by ``ProcessPool``, not by us) and everything else + needed to score a candidate.""" + global _WORKER_MODEL, _WORKER_TUNABLE_IDX, _WORKER_EC_RXN_IDS_TUNABLE + global _WORKER_EVOTUNE_DATA, _WORKER_EXCARBON, _WORKER_BIO_RXN + global _WORKER_MAKE_ANAEROBIC, _WORKER_CHANGE_PROTEIN_BIOMASS + global _WORKER_MAX_GROWTH_WEIGHT, _WORKER_PRIOR_PENALTY_WEIGHT + global _WORKER_KCAT0, _WORKER_SIGMA0_LOG + _WORKER_MODEL = model + _WORKER_TUNABLE_IDX = tunable_idx + _WORKER_EC_RXN_IDS_TUNABLE = ec_rxn_ids_tunable + _WORKER_EVOTUNE_DATA = evotune_data + _WORKER_EXCARBON = excarbon + _WORKER_BIO_RXN = bio_rxn_id + _WORKER_MAKE_ANAEROBIC = make_anaerobic + _WORKER_CHANGE_PROTEIN_BIOMASS = change_protein_biomass + _WORKER_MAX_GROWTH_WEIGHT = max_growth_weight + _WORKER_PRIOR_PENALTY_WEIGHT = prior_penalty_weight + _WORKER_KCAT0 = kcat0 + _WORKER_SIGMA0_LOG = sigma0_log + + +def _score_worker(kcat_vec: np.ndarray) -> tuple[float, float]: + assert _WORKER_MODEL is not None, "_score_worker called before _init_worker" + return _score_kcat_vector( + _WORKER_MODEL, _WORKER_TUNABLE_IDX, _WORKER_EC_RXN_IDS_TUNABLE, + _WORKER_EVOTUNE_DATA, _WORKER_EXCARBON, _WORKER_BIO_RXN, kcat_vec, + make_anaerobic=_WORKER_MAKE_ANAEROBIC, + change_protein_biomass=_WORKER_CHANGE_PROTEIN_BIOMASS, + max_growth_weight=_WORKER_MAX_GROWTH_WEIGHT, + prior_penalty_weight=_WORKER_PRIOR_PENALTY_WEIGHT, + kcat0=_WORKER_KCAT0, sigma0_log=_WORKER_SIGMA0_LOG, + ) diff --git a/src/geckopy/kcat_tuning/evotune/tying.py b/src/geckopy/kcat_tuning/evotune/tying.py new file mode 100644 index 0000000..c8fc89f --- /dev/null +++ b/src/geckopy/kcat_tuning/evotune/tying.py @@ -0,0 +1,98 @@ +"""Stop sampling distinctions the kcat assignment never made. + +A reaction catalysed by several isozymes gets one ``ec.rxns`` row per +isozyme, each with its own kcat. Where the assignment could tell them +apart -- different databases, different sequences, different measured +values -- those are genuinely different parameters. Where it could not, +every copy receives the same number from the same source, and the +tuner is then free to give them different values because no condition +can distinguish which isozyme carries the flux. + +It does exactly that. Five copies of ``r_0438`` share a prior of +54.33 1/s and come out of one run at 4.1, 8.1, 16.4 and 427 -- a +hundredfold spread that reports nothing about biology and everything +about an unidentified direction in the objective. + +Tying makes the copies one parameter. Copies must share both the prior +value and the source group to be tied: a shared value arrived at by +different routes is a coincidence, not an indistinguishability. +""" +from __future__ import annotations + +from collections import defaultdict +from typing import Optional, Sequence + +import numpy as np + +# Isozyme copies of a reaction carry this suffix; the text before it is +# the reaction they share. +ISOZYME_SUFFIX = "_EXP_" + + +def base_reaction(rxn_id: str) -> str: + """The reaction an ``ec.rxns`` entry belongs to, suffix removed.""" + return rxn_id.split(ISOZYME_SUFFIX)[0] + + +def isozyme_tie_map( + rxn_ids: Sequence[str], + kcat0: np.ndarray, + sources: Optional[Sequence[str]] = None, + *, + rel_tol: float = 1e-9, +) -> np.ndarray: + """Index of the parameter each position follows. + + Positions that are their own representative are free; the rest + follow one. Copies group together when they share a base reaction, + a prior value (to ``rel_tol``) and -- when ``sources`` is given -- + a source group. + + Returns + ------- + numpy.ndarray of int, one entry per position. ``tie_map[i] == i`` + for a free parameter, otherwise the index it follows. The + representative of a group is its lowest index, so the map is + idempotent: ``tie_map[tie_map] == tie_map``. + """ + kcat0 = np.asarray(kcat0, dtype=float) + if len(rxn_ids) != len(kcat0): + raise ValueError( + f"rxn_ids has {len(rxn_ids)} entries; kcat0 has {len(kcat0)}." + ) + tie_map = np.arange(len(kcat0)) + groups: dict[tuple, list[int]] = defaultdict(list) + for i, rxn_id in enumerate(rxn_ids): + # Rounding the prior into the key groups values equal to rel_tol + # without comparing every pair. + key = (base_reaction(str(rxn_id)), + round(float(np.log(kcat0[i])) / rel_tol) if kcat0[i] > 0 else None, + None if sources is None else str(sources[i])) + groups[key].append(i) + for members in groups.values(): + if len(members) > 1: + tie_map[members] = members[0] + return tie_map + + +def apply_ties(particles: np.ndarray, tie_map: np.ndarray) -> np.ndarray: + """Give every tied position its representative's value. + + ``particles`` is ``(n_params,)`` or ``(n_params, n_particles)``; + the array is modified in place and returned, so this can sit + directly in a sampling loop. + """ + tie_map = np.asarray(tie_map) + if particles.shape[0] != len(tie_map): + raise ValueError( + f"particles has {particles.shape[0]} rows; tie_map has " + f"{len(tie_map)}." + ) + particles[...] = particles[tie_map, ...] + return particles + + +def n_free(tie_map: np.ndarray) -> int: + """How many parameters remain free once ties are applied.""" + tie_map = np.asarray(tie_map) + return int((tie_map == np.arange(len(tie_map))).sum()) diff --git a/src/geckopy/kcat_sensitivity_analysis/find_max_value.py b/src/geckopy/kcat_tuning/find_max_value.py similarity index 96% rename from src/geckopy/kcat_sensitivity_analysis/find_max_value.py rename to src/geckopy/kcat_tuning/find_max_value.py index 88fa5b9..5aeb0e3 100644 --- a/src/geckopy/kcat_sensitivity_analysis/find_max_value.py +++ b/src/geckopy/kcat_tuning/find_max_value.py @@ -1,7 +1,7 @@ """Find the maximum kcat (or SA-derived kcat) for a set of EC numbers. Ported from GECKO MATLAB: -src/geckomat/kcat_sensitivity_analysis/findMaxValue.m. +src/kcat_tuning/findMaxValue.m. """ from __future__ import annotations @@ -28,7 +28,7 @@ def find_max_value( with ``"1.1.1."``. Ported from GECKO MATLAB: - src/geckomat/kcat_sensitivity_analysis/findMaxValue.m. + src/kcat_tuning/findMaxValue.m. Parameters ---------- diff --git a/src/geckopy/kcat_sensitivity_analysis/sensitivity_tuning.py b/src/geckopy/kcat_tuning/sensitivity_tuning.py similarity index 99% rename from src/geckopy/kcat_sensitivity_analysis/sensitivity_tuning.py rename to src/geckopy/kcat_tuning/sensitivity_tuning.py index 236d412..28164e8 100644 --- a/src/geckopy/kcat_sensitivity_analysis/sensitivity_tuning.py +++ b/src/geckopy/kcat_tuning/sensitivity_tuning.py @@ -20,7 +20,7 @@ from the adapter config). Ported from GECKO MATLAB: -src/geckomat/kcat_sensitivity_analysis/sensitivityTuning.m. +src/kcat_tuning/sensitivityTuning.m. """ from __future__ import annotations @@ -118,7 +118,7 @@ def sensitivity_tuning( 7. Re-applies kcat constraints for that single reaction. Ported from GECKO MATLAB: - src/geckomat/kcat_sensitivity_analysis/sensitivityTuning.m. + src/kcat_tuning/sensitivityTuning.m. Parameters ---------- diff --git a/src/geckopy/kcat_sensitivity_analysis/sigma_fitter.py b/src/geckopy/kcat_tuning/sigma_fitter.py similarity index 98% rename from src/geckopy/kcat_sensitivity_analysis/sigma_fitter.py rename to src/geckopy/kcat_tuning/sigma_fitter.py index 566778f..42d607e 100644 --- a/src/geckopy/kcat_sensitivity_analysis/sigma_fitter.py +++ b/src/geckopy/kcat_tuning/sigma_fitter.py @@ -12,7 +12,7 @@ difference. Ported from GECKO MATLAB: -src/geckomat/kcat_sensitivity_analysis/sigmaFitter.m. +src/kcat_tuning/sigmaFitter.m. """ from __future__ import annotations @@ -96,7 +96,7 @@ def fit_sigma( The best sigma is re-applied to ``model`` before return. Ported from GECKO MATLAB: - src/geckomat/kcat_sensitivity_analysis/sigmaFitter.m. + src/kcat_tuning/sigmaFitter.m. Parameters ---------- diff --git a/src/geckopy/kcat_sensitivity_analysis/truncate_values.py b/src/geckopy/kcat_tuning/truncate_values.py similarity index 94% rename from src/geckopy/kcat_sensitivity_analysis/truncate_values.py rename to src/geckopy/kcat_tuning/truncate_values.py index cbc33d0..b0f0f97 100644 --- a/src/geckopy/kcat_sensitivity_analysis/truncate_values.py +++ b/src/geckopy/kcat_tuning/truncate_values.py @@ -1,7 +1,7 @@ """Round numeric values to roughly 6 significant figures. Ported from GECKO MATLAB: -src/geckomat/kcat_sensitivity_analysis/truncateValues.m. +src/kcat_tuning/truncateValues.m. """ from __future__ import annotations @@ -28,7 +28,7 @@ def truncate_values( unchanged. The input is never mutated; a new array is returned. Ported from GECKO MATLAB: - src/geckomat/kcat_sensitivity_analysis/truncateValues.m. + src/kcat_tuning/truncateValues.m. Parameters ---------- diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..786d3de --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +"""Shared pytest configuration. + +Selects Gurobi as cobra's default LP solver for the whole test session +when it is importable and licensed, so every FBA a test triggers -- +including the ones inside worker processes, which inherit the solver +interface with the pickled model -- goes through the same solver. + +Falls back silently to whatever cobra would have picked (typically +GLPK) when gurobipy is absent or its license is unavailable, so the +suite still runs on a machine without Gurobi. Library code stays +solver-agnostic: geckopy reads ``cobra.Configuration()`` for this the +same way it does for ``processes``. +""" +from __future__ import annotations + +import cobra + + +def pytest_configure(): + try: + import gurobipy # noqa: F401 + except ImportError: + return + try: + cobra.Configuration().solver = "gurobi" + except Exception as exc: # unlicensed, or size-limited license + print(f"conftest: Gurobi present but unusable ({exc}); leaving cobra's default solver.") diff --git a/tests/test_cli.py b/tests/test_cli.py index 7c3151d..d043f44 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,22 +17,24 @@ def test_generated_template_parses_as_toml(): assert "sigma" not in parsed -def test_template_without_advanced_excludes_bayesian(): - text = generate_template_toml(advanced=False) - assert "[bayesian]" not in text - assert "advanced" in text.lower() - - -def test_template_with_advanced_includes_bayesian(): - text = generate_template_toml(advanced=True) - assert "[bayesian]" in text - - def test_template_includes_nested_sections(): text = generate_template_toml() assert "[kegg]" in text assert "[uniprot]" in text assert "[complex]" in text + assert "[evotune]" in text + + +def test_template_renders_dict_valued_evotune_fields(): + """The evotune section carries per-source tables, not just scalars.""" + text = generate_template_toml() + uncommented = "\n".join( + line.lstrip("# ") for line in text.splitlines() + if line.lstrip("# ").startswith(("sigma0_log_source", "source_groups")) + ) + parsed = tomllib.loads(uncommented) + assert parsed["sigma0_log_source"]["brenda"] == 0.2 + assert parsed["source_groups"]["brenda"]["sources"] == ["brenda"] def test_template_includes_field_descriptions_as_comments(): @@ -62,11 +64,11 @@ def test_cli_init_refuses_nonempty_folder(tmp_path, capsys): assert "not empty" in captured.err -def test_cli_init_accepts_advanced_flag(tmp_path): +def test_cli_init_scaffolds_the_evotune_section(tmp_path): target = tmp_path / "my-ec" - main(["init", str(target), "--advanced"]) + main(["init", str(target)]) toml_text = (target / "model_adapter.toml").read_text() - assert "[bayesian]" in toml_text + assert "[evotune]" in toml_text def test_cli_init_produces_loadable_adapter_after_filling_required(tmp_path): diff --git a/tests/test_deprecated_aliases.py b/tests/test_deprecated_aliases.py index 45f7d7e..61be3fc 100644 --- a/tests/test_deprecated_aliases.py +++ b/tests/test_deprecated_aliases.py @@ -105,7 +105,7 @@ class _MockModel: def test_sigma_fitter_alias_emits_warning(): - from geckopy.kcat_sensitivity_analysis import sigma_fitter + from geckopy.kcat_tuning import sigma_fitter class _MockModel: pass diff --git a/tests/test_evotune.py b/tests/test_evotune.py new file mode 100644 index 0000000..2b59d37 --- /dev/null +++ b/tests/test_evotune.py @@ -0,0 +1,425 @@ +"""Tests for kcat_tuning.evotune.tuning. + +Two toy EcModels: + +- ``_build_toy`` -- two independent enzymes (no isozymes), one per + carbon source, so each condition's growth depends on exactly one + kcat. Used for the basic screen/select/tune correctness checks. +- ``_build_tied_toy`` -- two isozyme copies of one reaction, sharing a + prior and a source, drawing from one shared enzyme budget. Used to + check that ``tie_isozymes`` actually changes what gets tuned. +""" +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from scipy import sparse + +import cobra +from geckopy import EcModel, ModelAdapter +from geckopy.adapter.params import EvotuneParams +from geckopy.databases.flux_data import FluxData +from geckopy.ec_model.ec_data import EcData +from geckopy.kcat_tuning.evotune.tuning import ( + cmaes_kcat_tuning, + screen_kcat_leverage, + select_tunable_mask, + tune_prior_penalty_weight, +) +from geckopy.kcat_tuning.evotune.data import EvotuneData +from geckopy.kcat_tuning.evotune.tuning import EvotuneResult + +_TRUE_KCAT = 2.0 +_START_KCAT = 1.0 +_MW = 100.0 +_TRUE_GROWTH = _TRUE_KCAT * 3600.0 / _MW + + +def _adapter(tmp_path: Path) -> ModelAdapter: + (tmp_path / "model_adapter.toml").write_text( + 'conv_gem = "dummy.xml"\n' + 'org_name = "test"\n' + 'bio_rxn = "biomass"\n' + ) + return ModelAdapter.from_folder(tmp_path) + + +def _build_toy(adapter: ModelAdapter) -> EcModel: + """Two independent branches, one kcat each.""" + model = EcModel("toy", adapter=adapter) + + glc_e = cobra.Metabolite("glc_e", compartment="e") + eth_e = cobra.Metabolite("eth_e", compartment="e") + prot_pool = cobra.Metabolite("prot_pool", compartment="c") + prot_glc = cobra.Metabolite("prot_Eglc", compartment="c") + prot_eth = cobra.Metabolite("prot_Eeth", compartment="c") + bio_met = cobra.Metabolite("bio_met", compartment="c") + model.add_metabolites([glc_e, eth_e, prot_pool, prot_glc, prot_eth, bio_met]) + + EX_glc = cobra.Reaction("EX_glc") + EX_glc.add_metabolites({glc_e: -1.0}) + EX_glc.bounds = (-1000.0, 0.0) + + EX_eth = cobra.Reaction("EX_eth") + EX_eth.add_metabolites({eth_e: -1.0}) + EX_eth.bounds = (-1000.0, 0.0) + + coeff = _MW / (_START_KCAT * 3600.0) + R_glc = cobra.Reaction("R_glc") + R_glc.add_metabolites({glc_e: -1.0, prot_glc: -coeff, bio_met: 1.0}) + R_glc.bounds = (0.0, 1000.0) + + R_eth = cobra.Reaction("R_eth") + R_eth.add_metabolites({eth_e: -1.0, prot_eth: -coeff, bio_met: 1.0}) + R_eth.bounds = (0.0, 1000.0) + + BIO = cobra.Reaction("biomass") + BIO.add_metabolites({bio_met: -1.0}) + BIO.bounds = (0.0, 1000.0) + + pool_ex = cobra.Reaction("prot_pool_exchange") + pool_ex.add_metabolites({prot_pool: 1.0}) + pool_ex.bounds = (0.0, 1.0) + + usage_glc = cobra.Reaction("usage_prot_Eglc") + usage_glc.add_metabolites({prot_pool: -1.0, prot_glc: 1.0}) + usage_glc.bounds = (0.0, 1000.0) + + usage_eth = cobra.Reaction("usage_prot_Eeth") + usage_eth.add_metabolites({prot_pool: -1.0, prot_eth: 1.0}) + usage_eth.bounds = (0.0, 1000.0) + + model.add_reactions( + [EX_glc, EX_eth, R_glc, R_eth, BIO, pool_ex, usage_glc, usage_eth] + ) + model.objective = "biomass" + + model.ec = EcData( + rxns=["R_glc", "R_eth"], + kcat=np.array([_START_KCAT, _START_KCAT]), + source=["brenda", "dlkcat"], + notes=["", ""], + eccodes=["", ""], + genes=["g_glc", "g_eth"], + enzymes=["Eglc", "Eeth"], + mw=np.array([_MW, _MW]), + sequence=["", ""], + concs=np.array([np.nan, np.nan]), + rxn_enz_mat=sparse.csr_matrix(np.eye(2)), + ) + return model + + +def _evotune_data() -> EvotuneData: + max_grate = FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_TRUE_GROWTH, _TRUE_GROWTH]), + exch_fluxes=np.array( + [[-1000.0, np.nan], [np.nan, -1000.0]] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + ) + return EvotuneData(flux_data=None, max_grate=max_grate, zero_flux=[]) + + +def _build_tied_toy(adapter: ModelAdapter) -> EcModel: + """Two isozyme copies of one reaction (sharing a prior, a source, + and one enzyme budget -- either can carry all the flux), plus one + independent branch. The independent branch exists only so that + tying still leaves two free parameters: cma's own N=1 path errors + (verified directly, not assumed), so a model that tied down to a + single parameter couldn't exercise cmaes_kcat_tuning end to end.""" + model = EcModel("tied_toy", adapter=adapter) + + glc_e = cobra.Metabolite("glc_e", compartment="e") + eth_e = cobra.Metabolite("eth_e", compartment="e") + prot_pool = cobra.Metabolite("prot_pool", compartment="c") + prot_1 = cobra.Metabolite("prot_Eiso1", compartment="c") + prot_2 = cobra.Metabolite("prot_Eiso2", compartment="c") + prot_eth = cobra.Metabolite("prot_Eeth", compartment="c") + bio_met = cobra.Metabolite("bio_met", compartment="c") + model.add_metabolites( + [glc_e, eth_e, prot_pool, prot_1, prot_2, prot_eth, bio_met] + ) + + EX_glc = cobra.Reaction("EX_glc") + EX_glc.add_metabolites({glc_e: -1.0}) + EX_glc.bounds = (-1000.0, 0.0) + + EX_eth = cobra.Reaction("EX_eth") + EX_eth.add_metabolites({eth_e: -1.0}) + EX_eth.bounds = (-1000.0, 0.0) + + coeff = _MW / (_START_KCAT * 3600.0) + R_iso1 = cobra.Reaction("R_iso_EXP_1") + R_iso1.add_metabolites({glc_e: -1.0, prot_1: -coeff, bio_met: 1.0}) + R_iso1.bounds = (0.0, 1000.0) + + R_iso2 = cobra.Reaction("R_iso_EXP_2") + R_iso2.add_metabolites({glc_e: -1.0, prot_2: -coeff, bio_met: 1.0}) + R_iso2.bounds = (0.0, 1000.0) + + R_eth = cobra.Reaction("R_eth") + R_eth.add_metabolites({eth_e: -1.0, prot_eth: -coeff, bio_met: 1.0}) + R_eth.bounds = (0.0, 1000.0) + + BIO = cobra.Reaction("biomass") + BIO.add_metabolites({bio_met: -1.0}) + BIO.bounds = (0.0, 1000.0) + + pool_ex = cobra.Reaction("prot_pool_exchange") + pool_ex.add_metabolites({prot_pool: 1.0}) + pool_ex.bounds = (0.0, 1.0) + + usage_1 = cobra.Reaction("usage_prot_iso1") + usage_1.add_metabolites({prot_pool: -1.0, prot_1: 1.0}) + usage_1.bounds = (0.0, 1000.0) + + usage_2 = cobra.Reaction("usage_prot_iso2") + usage_2.add_metabolites({prot_pool: -1.0, prot_2: 1.0}) + usage_2.bounds = (0.0, 1000.0) + + usage_eth = cobra.Reaction("usage_prot_Eeth") + usage_eth.add_metabolites({prot_pool: -1.0, prot_eth: 1.0}) + usage_eth.bounds = (0.0, 1000.0) + + model.add_reactions([ + EX_glc, EX_eth, R_iso1, R_iso2, R_eth, BIO, pool_ex, + usage_1, usage_2, usage_eth, + ]) + model.objective = "biomass" + + model.ec = EcData( + rxns=["R_iso_EXP_1", "R_iso_EXP_2", "R_eth"], + kcat=np.array([_START_KCAT, _START_KCAT, _START_KCAT]), + source=["brenda", "brenda", "dlkcat"], + notes=["", "", ""], + eccodes=["", "", ""], + genes=["g_iso1", "g_iso2", "g_eth"], + enzymes=["Eiso1", "Eiso2", "Eeth"], + mw=np.array([_MW, _MW, _MW]), + sequence=["", "", ""], + concs=np.array([np.nan, np.nan, np.nan]), + rxn_enz_mat=sparse.csr_matrix(np.eye(3)), + ) + return model + + +def _tied_evotune_data() -> EvotuneData: + max_grate = FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_TRUE_GROWTH, _TRUE_GROWTH]), + exch_fluxes=np.array( + [[-1000.0, np.nan], [np.nan, -1000.0]] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + ) + return EvotuneData(flux_data=None, max_grate=max_grate, zero_flux=[]) + + +def test_screen_ranks_both_load_bearing_kcats(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + + screen = screen_kcat_leverage( + model, adapter=adapter, evotune_data=evotune_data, n_proc=1, + ) + + assert set(screen["rxn_id"]) == {"R_glc", "R_eth"} + assert (screen["leverage"] > 0).all() + assert (screen["n_isozymes"] == 1).all() + # Sorted by rank descending, cumulative share reaches 1.0. + assert list(screen["rank"]) == sorted(screen["rank"], reverse=True) + assert screen["cum_leverage_share"].iloc[-1] == pytest.approx(1.0) + + +def test_select_tunable_mask_is_relative_and_monotone(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + screen = screen_kcat_leverage( + model, adapter=adapter, evotune_data=evotune_data, n_proc=1, + ) + + tiny = select_tunable_mask(model, screen, target_impact_share=0.01) + full = select_tunable_mask(model, screen, target_impact_share=1.0) + + # A minimal target still keeps at least the top-ranked kcat. + assert tiny.sum() == 1 + assert tiny[list(model.ec.rxns).index(screen["rxn_id"].iloc[0])] + # A target of 1.0 keeps everything with nonzero leverage. + assert full.sum() == 2 + # Raising the target never drops something a lower target kept. + assert np.all(full[tiny]) + + +def test_select_tunable_mask_empty_screen_returns_empty_mask(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + empty = select_tunable_mask(model, pd.DataFrame()) + assert empty.sum() == 0 + assert empty.shape == model.ec.kcat.shape + + +def test_cmaes_tuning_moves_kcats_toward_truth(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + params = EvotuneParams(max_generations=15, rmse_threshold=1e-6) + before = model.ec.kcat.copy() + + result = cmaes_kcat_tuning( + model, adapter=adapter, params=params, evotune_data=evotune_data, + n_proc=1, seed=0, verbose=False, + ) + + assert isinstance(result, EvotuneResult) + assert result.rxns == ["R_glc", "R_eth"] + assert result.groups == ["brenda", "dlkcat"] + assert np.array_equal(result.old_kcat, before) + assert np.array_equal(model.ec.kcat, result.new_kcat) + assert len(result.rmse_trace) == result.n_generations > 0 + + # Both kcats should end up much closer to the true value than they + # started (both start at exactly half of it). + for i in range(2): + assert abs(result.new_kcat[i] - _TRUE_KCAT) < abs(before[i] - _TRUE_KCAT) + + +def test_cmaes_tuning_n_proc_matches_serial(tmp_path): + params = EvotuneParams(max_generations=5, rmse_threshold=-1.0) + + def _run(n_proc): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + return cmaes_kcat_tuning( + model, adapter=adapter, params=params, evotune_data=evotune_data, + n_proc=n_proc, seed=0, popsize=6, verbose=False, + ) + + serial = _run(1) + parallel = _run(2) + + assert np.array_equal(serial.new_kcat, parallel.new_kcat) + assert serial.rmse_trace == parallel.rmse_trace + assert serial.n_generations == parallel.n_generations + + +def test_tie_isozymes_forces_equal_values(tmp_path): + adapter = _adapter(tmp_path) + model = _build_tied_toy(adapter) + evotune_data = _tied_evotune_data() + params = EvotuneParams(max_generations=8, rmse_threshold=-1.0, + tie_isozymes=True) + + screen = screen_kcat_leverage( + model, adapter=adapter, params=params, evotune_data=evotune_data, n_proc=1, + ) + # Two rows: the tied isozyme pair as one group, R_eth as another. + assert len(screen) == 2 + assert sorted(screen["n_isozymes"]) == [1, 2] + + result = cmaes_kcat_tuning( + model, adapter=adapter, params=params, evotune_data=evotune_data, + n_proc=1, seed=0, verbose=False, + ) + assert result.rxns == ["R_iso_EXP_1", "R_iso_EXP_2", "R_eth"] + assert result.new_kcat[0] == result.new_kcat[1] + + +def test_tie_isozymes_false_keeps_groups_separate(tmp_path): + adapter = _adapter(tmp_path) + model = _build_tied_toy(adapter) + evotune_data = _tied_evotune_data() + params = EvotuneParams(tie_isozymes=False) + + screen = screen_kcat_leverage( + model, adapter=adapter, params=params, evotune_data=evotune_data, n_proc=1, + ) + assert len(screen) == 3 + assert (screen["n_isozymes"] == 1).all() + + +def test_no_tunable_kcats_raises(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + model.ec.kcat[:] = 0.0 + evotune_data = _evotune_data() + with pytest.raises(ValueError, match="No tunable kcats"): + cmaes_kcat_tuning(model, adapter=adapter, evotune_data=evotune_data, n_proc=1) + + +def test_too_few_free_parameters_raises(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + mask = np.array([True, False]) + with pytest.raises(ValueError, match="free parameter"): + cmaes_kcat_tuning( + model, adapter=adapter, evotune_data=evotune_data, + tunable_mask=mask, n_proc=1, + ) + + +def test_tune_prior_penalty_weight_shape_and_fit_cost(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + params = EvotuneParams(max_generations=6, rmse_threshold=-1.0) + + report = tune_prior_penalty_weight( + model, adapter=adapter, params=params, evotune_data=evotune_data, + candidates=(0.0, 1000.0), seeds=(0, 1), n_proc=1, verbose=False, + ) + + assert len(report) == 2 + assert list(report["prior_penalty_weight"]) == [0.0, 1000.0] + assert (report["n_seeds"] == 2).all() + # The best (lowest-distance) candidate has zero fit cost by construction. + assert report["fit_cost"].min() == pytest.approx(0.0) + assert (report["fit_cost"] >= 0.0).all() + # A crushing penalty should move fewer kcats than none at all. + unpenalised = report.loc[report["prior_penalty_weight"] == 0.0].iloc[0] + crushed = report.loc[report["prior_penalty_weight"] == 1000.0].iloc[0] + assert crushed["n_changed_mean"] <= unpenalised["n_changed_mean"] + + +def test_tune_prior_penalty_weight_restores_model(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + before = model.ec.kcat.copy() + params = EvotuneParams(max_generations=4, rmse_threshold=-1.0) + + tune_prior_penalty_weight( + model, adapter=adapter, params=params, evotune_data=evotune_data, + candidates=(0.0,), seeds=(0,), n_proc=1, verbose=False, + ) + + # Unlike cmaes_kcat_tuning, this only reports -- it leaves the model + # exactly as it found it. + assert np.array_equal(model.ec.kcat, before) + + +def test_tune_prior_penalty_weight_single_seed_gives_nan_reproducibility(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + evotune_data = _evotune_data() + params = EvotuneParams(max_generations=4, rmse_threshold=-1.0) + + report = tune_prior_penalty_weight( + model, adapter=adapter, params=params, evotune_data=evotune_data, + candidates=(0.0,), seeds=(0,), n_proc=1, verbose=False, + ) + + row = report.iloc[0] + assert row["n_both_moved"] == 0 + assert np.isnan(row["pct_direction_agree"]) + assert np.isnan(row["median_fold_spread"]) diff --git a/tests/test_evotune_data.py b/tests/test_evotune_data.py new file mode 100644 index 0000000..94e1422 --- /dev/null +++ b/tests/test_evotune_data.py @@ -0,0 +1,85 @@ +"""Tests for kcat_tuning.evotune.data.""" +from pathlib import Path + +from geckopy import ModelAdapter +from geckopy.kcat_tuning.evotune import ( + EvotuneData, + load_evotune_data, +) + + +def _adapter(tmp_path: Path) -> ModelAdapter: + (tmp_path / "model_adapter.toml").write_text( + 'conv_gem = "dummy.xml"\n' + 'org_name = "test"\n' + ) + return ModelAdapter.from_folder(tmp_path) + + +_FLUX_DATA_TSV = ( + "Condition\tPtot\tgrRate\tglucose (r_1714)\tethanol (r_1761)\t" + "evotuneRMSEweight\tsource\n" + "glucose\tNaN\t0.36\t-13.33\t19.83\t1\tDLKcat\n" + "ethanol\tNaN\t0.12\tNaN\t-8.5\t1\tbrenda\n" +) + +_MAX_GROWTH_TSV = ( + "Condition\tPtot\tgrRate\tglucose (r_1714)\tfructose (r_1709)\t" + "evotuneRMSEweight\tsource\n" + "glucose\tNaN\t0.41\t-1000\tNaN\t1\tDLKcat\n" + "fructose\tNaN\t0.338\tNaN\t-1000\t1\tDLKcat\n" +) + +_ZERO_EXCH_TSV = "Rxns\nr_1542\nr_1545\nr_1546\n" + + +def test_load_evotune_data_all_files_present(tmp_path): + adapter = _adapter(tmp_path) + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "evotuneFluxData.tsv").write_text(_FLUX_DATA_TSV) + (data_dir / "evotuneMaxGrowth.tsv").write_text(_MAX_GROWTH_TSV) + (data_dir / "evotuneZeroExch.tsv").write_text(_ZERO_EXCH_TSV) + + evotune_data = load_evotune_data(adapter) + + assert isinstance(evotune_data, EvotuneData) + + assert evotune_data.flux_data is not None + assert evotune_data.flux_data.conds == ["glucose", "ethanol"] + assert evotune_data.flux_data.exch_rxn_ids == ["r_1714", "r_1761"] + assert evotune_data.flux_data.source == ["DLKcat", "brenda"] + assert list(evotune_data.flux_data.evotune_rmse_weight) == [1.0, 1.0] + + assert evotune_data.max_grate is not None + assert evotune_data.max_grate.conds == ["glucose", "fructose"] + assert evotune_data.max_grate.exch_rxn_ids == ["r_1714", "r_1709"] + + assert evotune_data.zero_flux == ["r_1542", "r_1545", "r_1546"] + + +def test_load_evotune_data_all_files_missing(tmp_path): + adapter = _adapter(tmp_path) + + evotune_data = load_evotune_data(adapter) + + assert evotune_data.flux_data is None + assert evotune_data.max_grate is None + assert evotune_data.zero_flux == [] + + +def test_load_evotune_data_partial(tmp_path): + """Only flux_data present: max_grate stays None (not a bogus + partial struct), matching MATLAB's *intended* behaviour for a + missing optional file -- unlike the asymmetric-guard bug flagged + in the MATLAB REVIEW.md, both fields are treated identically.""" + adapter = _adapter(tmp_path) + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "evotuneFluxData.tsv").write_text(_FLUX_DATA_TSV) + + evotune_data = load_evotune_data(adapter) + + assert evotune_data.flux_data is not None + assert evotune_data.max_grate is None + assert evotune_data.zero_flux == [] diff --git a/tests/test_evotune_parsimony.py b/tests/test_evotune_parsimony.py new file mode 100644 index 0000000..ddb3e5a --- /dev/null +++ b/tests/test_evotune_parsimony.py @@ -0,0 +1,162 @@ +"""Tests for kcat_tuning.evotune.parsimony.""" +import numpy as np +import pytest + +from geckopy.kcat_tuning.evotune.parsimony import ( + best_parsimonious, + movement_in_sigma, + n_changed, + parsimony_frontier, + revert_below, +) + + +def test_movement_is_scaled_by_each_kcats_own_sigma(): + """The same fractional change counts more for a trusted kcat.""" + kcat0 = np.array([10.0, 10.0]) + kcat = kcat0 * np.exp(0.3) + sigma0 = np.array([0.1, 0.3]) # custom-like, unlabelled-like + dev = movement_in_sigma(kcat, kcat0, sigma0) + assert dev == pytest.approx([3.0, 1.0]) + + +def test_revert_below_keeps_only_the_large_movers(): + kcat0 = np.array([1.0, 1.0, 1.0]) + sigma0 = np.array([0.1, 0.1, 0.1]) + kcat = kcat0 * np.exp([0.05, 0.2, 0.5]) # 0.5, 2.0, 5.0 sigma + out = revert_below(kcat, kcat0, sigma0, threshold=1.0) + assert out[0] == pytest.approx(kcat0[0]) # reverted + assert out[1] == pytest.approx(kcat[1]) # kept + assert out[2] == pytest.approx(kcat[2]) # kept + # A zero threshold reverts nothing. + assert revert_below(kcat, kcat0, sigma0, 0.0) == pytest.approx(kcat) + + +def test_n_changed_uses_the_two_percent_convention(): + kcat0 = np.array([1.0, 1.0, 1.0]) + kcat = np.array([1.0, 1.01, 1.5]) # unchanged, under 2%, over + assert n_changed(kcat, kcat0) == 1 + + +def test_frontier_reverts_progressively_and_scores_each_point(): + kcat0 = np.ones(4) + sigma0 = np.full(4, 0.1) + kcat = kcat0 * np.exp([0.05, 0.15, 0.35, 0.55]) # 0.5, 1.5, 3.5, 5.5 sigma + calls = [] + + def score(vec): + calls.append(vec.copy()) + return float(np.abs(np.log(vec / kcat0)).sum()) + + pts = parsimony_frontier(score, kcat, kcat0, sigma0, + thresholds=(0.0, 2.0, 4.0, 6.0)) + assert len(pts) == len(calls) == 4 + # Reverting more can only reduce the number of changed kcats. + assert [p.n_changed for p in pts] == sorted( + [p.n_changed for p in pts], reverse=True + ) + assert pts[0].n_changed == 4 and pts[-1].n_changed == 0 + assert pts[0].threshold == 0.0 + + +def test_best_parsimonious_prefers_fewer_changes_within_tolerance(): + from geckopy.kcat_tuning.evotune.parsimony import FrontierPoint + + pts = [ + FrontierPoint(0.0, np.array([1.0]), 4000, 7.0, 0.900), + FrontierPoint(3.0, np.array([1.0]), 3000, 6.0, 0.905), # within 2% + FrontierPoint(6.0, np.array([1.0]), 1000, 3.0, 1.400), # too costly + ] + assert best_parsimonious(pts, tolerance=0.02).n_changed == 3000 + # A tight tolerance falls back to the best-fitting point. + assert best_parsimonious(pts, tolerance=0.0).n_changed == 4000 + + +def test_fold_change_is_symmetric_and_at_least_one(): + from geckopy.kcat_tuning.evotune.parsimony import fold_change + + kcat0 = np.array([10.0, 10.0, 10.0]) + kcat = np.array([20.0, 5.0, 10.0]) # doubled, halved, unchanged + assert fold_change(kcat, kcat0) == pytest.approx([2.0, 2.0, 1.0]) + + +def test_source_movement_flags_whether_trust_order_is_respected(): + from geckopy.kcat_tuning.evotune.parsimony import source_movement + + kcat0 = np.ones(6) + groups = np.array(["custom", "custom", "brenda", "brenda", "okp", "okp"]) + order = ["custom", "brenda", "okp"] + + # Trusted sources moved least: the ordering holds. + good = np.array([1.05, 1.05, 1.5, 1.5, 3.0, 3.0]) + rep = source_movement(good, kcat0, groups, order) + assert rep["_ordered"] is True + assert rep["custom"]["median_fold"] == pytest.approx(1.05) + assert rep["custom"]["frac_near_prior"] == pytest.approx(1.0) + assert rep["okp"]["frac_near_prior"] == pytest.approx(0.0) + + # The most trusted source moved most: flagged. + bad = np.array([3.0, 3.0, 1.5, 1.5, 1.05, 1.05]) + assert source_movement(bad, kcat0, groups, order)["_ordered"] is False + + +def test_source_movement_reads_the_tier_when_most_kcats_are_untouched(): + from geckopy.kcat_tuning.evotune.parsimony import source_movement + + kcat0 = np.ones(10) + groups = np.array(["custom"] * 5 + ["brenda"] * 5) + order = ["custom", "brenda"] + + # One kcat moved per source, custom four-fold against brenda's two. + kcat = np.array([1.0, 1.0, 1.0, 1.0, 4.0, 1.0, 1.0, 1.0, 1.0, 2.0]) + rep = source_movement(kcat, kcat0, groups, order) + + # Over the whole source both medians sit at the prior and say + # nothing; conditioning on what moved shows custom moving furthest. + assert rep["custom"]["median_fold"] == pytest.approx(1.0) + assert rep["brenda"]["median_fold"] == pytest.approx(1.0) + assert rep["custom"]["median_fold_moved"] == pytest.approx(4.0) + assert rep["brenda"]["median_fold_moved"] == pytest.approx(2.0) + assert rep["custom"]["n_moved"] == 1 + assert rep["_ordered"] is False + + # A source with nothing moved reports 1.0 and does not decide the + # ordering on its own. + still = np.array([1.0] * 5 + [1.0, 1.0, 1.0, 1.0, 2.0]) + rep = source_movement(still, kcat0, groups, order) + assert rep["custom"]["n_moved"] == 0 + assert rep["custom"]["median_fold_moved"] == pytest.approx(1.0) + assert rep["_ordered"] is True + + +def test_identifiability_mask_asks_more_of_trusted_sources(): + from geckopy.kcat_tuning.evotune.parsimony import identifiability_mask + + # Identical measured effect, different trust: custom (0.1) must clear + # a bar three times higher than unlabelled (0.3). + drmse = np.array([1e-3, 1e-3]) + sigma0 = np.array([0.1, 0.3]) + assert list(identifiability_mask(drmse, sigma0, 1.5e-4)) == [False, True] + # A large enough effect qualifies whatever the source. + assert list(identifiability_mask(np.array([1e-2, 1e-2]), sigma0, 1.5e-4)) == [True, True] + # And nothing qualifies under an unreachable bar. + assert not identifiability_mask(drmse, sigma0, 1.0).any() + + +def test_impact_share_rewards_changing_the_kcats_that_matter(): + from geckopy.kcat_tuning.evotune.parsimony import impact_share + + kcat0 = np.ones(4) + drmse = np.array([1.0, 1.0, 0.01, 0.01]) # two matter, two barely + + # Two changes, both load-bearing. + few_good = np.array([2.0, 2.0, 1.0, 1.0]) + assert impact_share(few_good, kcat0, drmse) == pytest.approx(2.0 / 2.02) + + # Two changes, neither load-bearing: same count, far less impact. + few_bad = np.array([1.0, 1.0, 2.0, 2.0]) + assert impact_share(few_bad, kcat0, drmse) == pytest.approx(0.02 / 2.02) + assert impact_share(few_bad, kcat0, drmse) < impact_share(few_good, kcat0, drmse) + + # Nothing changed carries nothing. + assert impact_share(kcat0, kcat0, drmse) == 0.0 diff --git a/tests/test_evotune_priors.py b/tests/test_evotune_priors.py new file mode 100644 index 0000000..a7e33c0 --- /dev/null +++ b/tests/test_evotune_priors.py @@ -0,0 +1,70 @@ +"""Tests for kcat_tuning.evotune.priors.""" +import numpy as np +import pytest + +from geckopy.adapter.params import EvotuneParams, SourceGroupRule +from geckopy.kcat_tuning.evotune.priors import ( + UNLABELLED_GROUP, + build_sigma0_log, + classify_kcat_source, + classify_kcat_sources, +) + + +def _params() -> EvotuneParams: + return EvotuneParams( + sigma0_log_default=0.5, + source_groups={ + "dlkcat": SourceGroupRule(sources=["dlkcat"]), + "brenda": SourceGroupRule(sources=["brenda"]), + "okp": SourceGroupRule(match_okp=True), + }, + sigma0_log_source={"dlkcat": 0.4, "brenda": 0.2, "okp": 0.3}, + ) + + +# --------------------------------------------------------------------------- # +# classify_kcat_source(s) +# --------------------------------------------------------------------------- # + +def test_classify_matches_explicit_source_case_insensitively(): + params = _params() + assert classify_kcat_source("DLKcat", params) == "dlkcat" + assert classify_kcat_source("brenda", params) == "brenda" + assert classify_kcat_source("BRENDA", params) == "brenda" + + +def test_classify_unmatched_source_is_unlabelled(): + params = _params() + assert classify_kcat_source("some_other_source", params) == UNLABELLED_GROUP + + +def test_classify_matches_okp_method(): + params = _params() + assert classify_kcat_source("CataPro", params, okp_method="CataPro") == "okp" + # Case-insensitive here too. + assert classify_kcat_source("catapro", params, okp_method="CataPro") == "okp" + # No okp_method configured -> falls through to unlabelled. + assert classify_kcat_source("CataPro", params, okp_method=None) == UNLABELLED_GROUP + + +def test_classify_kcat_sources_vectorised(): + params = _params() + groups = classify_kcat_sources( + ["dlkcat", "brenda", "CataPro", "unknown"], params, okp_method="CataPro", + ) + assert list(groups) == ["dlkcat", "brenda", "okp", UNLABELLED_GROUP] + + +# --------------------------------------------------------------------------- # +# build_sigma0_log +# --------------------------------------------------------------------------- # + +def test_build_sigma0_log_uses_group_values_and_default_fallback(): + params = _params() + groups = np.array(["dlkcat", "brenda", UNLABELLED_GROUP, "okp"], dtype=object) + + sigma0_log = build_sigma0_log(groups, params) + + assert sigma0_log.tolist() == pytest.approx([0.4, 0.2, 0.5, 0.3]) + diff --git a/tests/test_evotune_simulate_distance.py b/tests/test_evotune_simulate_distance.py new file mode 100644 index 0000000..bef6404 --- /dev/null +++ b/tests/test_evotune_simulate_distance.py @@ -0,0 +1,484 @@ +"""Tests for kcat_tuning.evotune.simulate / .distance. + +Styled like test_sensitivity_tuning.py's hand-built toy EcModel: a +tiny two-carbon-source model with a known enzyme cap, so every +simulated growth/exchange-flux value used below is hand-predictable +from the model's stoichiometry rather than treated as a black box. +""" +from pathlib import Path + +import cobra +import numpy as np +import pytest + +from geckopy import EcModel, ModelAdapter +from geckopy.databases.flux_data import FluxData +from geckopy.kcat_tuning.evotune.data import EvotuneData +from geckopy.kcat_tuning.evotune.distance import ( + BIOMASS_CARBON_EQUIV, + evotune_distance, + compute_excarbon, + dataset_rmse, +) +from geckopy.kcat_tuning.evotune.simulate import ( + ConditionSimResult, + simulate_evotune_dataset, +) + + +# --------------------------------------------------------------------------- # +# Toy two-carbon-source ecModel fixture +# --------------------------------------------------------------------------- # +# +# glucose and ethanol each feed a shared pool metabolite A_c, which the +# single enzyme-gated reaction R turns into biomass at an enzyme-capped +# rate of kcat*3600/mw = 1*3600/100 = 36 (pool size 1 mg/gDW). R also +# co-produces a small fixed amount of a "byproduct" the experimental +# data assumes is never made (byp_c, 0.1 mol per mol of R flux) -- this +# exercises `zero_flux_rxns`' nonzero-simulated-flux case. +# +# - "glucose" condition: uptake unconstrained relative to the enzyme +# cap (measured flux -36 exactly matches TR's capacity), so growth +# is enzyme-limited: 36. +# - "ethanol" condition: uptake capped at -20 (below the enzyme cap), +# so growth is carbon-limited instead: 20. + +_KCAT = 1.0 +_MW = 100.0 +_ENZYME_CAP = _KCAT * 3600.0 / _MW # 36.0 +_BYPRODUCT_YIELD = 0.1 + + +def _adapter(tmp_path: Path) -> ModelAdapter: + (tmp_path / "model_adapter.toml").write_text( + 'conv_gem = "dummy.xml"\n' + 'org_name = "test"\n' + 'bio_rxn = "biomass"\n' + ) + return ModelAdapter.from_folder(tmp_path) + + +def _build_toy(adapter: ModelAdapter) -> EcModel: + model = EcModel("toy", adapter=adapter) + + glc_e = cobra.Metabolite("glc_e", compartment="e", formula="C6H12O6") + eth_e = cobra.Metabolite("eth_e", compartment="e", formula="C2H6O") + byp_c = cobra.Metabolite("byp_c", compartment="c", formula="C1") + A_c = cobra.Metabolite("A_c", compartment="c") + pool = cobra.Metabolite("prot_pool", compartment="c") + prot_E = cobra.Metabolite("prot_E", compartment="c") + bio_met = cobra.Metabolite("bio_met", compartment="c") + model.add_metabolites([glc_e, eth_e, byp_c, A_c, pool, prot_E, bio_met]) + + EX_glc = cobra.Reaction("EX_glc") + EX_glc.add_metabolites({glc_e: -1.0}) + EX_glc.bounds = (-1000.0, 0.0) + + EX_eth = cobra.Reaction("EX_eth") + EX_eth.add_metabolites({eth_e: -1.0}) + EX_eth.bounds = (-1000.0, 0.0) + + EX_byp = cobra.Reaction("EX_byp") + EX_byp.add_metabolites({byp_c: -1.0}) + EX_byp.bounds = (0.0, 1000.0) + + TR_glc = cobra.Reaction("TR_glc") + TR_glc.add_metabolites({glc_e: -1.0, A_c: 1.0}) + TR_glc.bounds = (0.0, 1000.0) + + TR_eth = cobra.Reaction("TR_eth") + TR_eth.add_metabolites({eth_e: -1.0, A_c: 1.0}) + TR_eth.bounds = (0.0, 1000.0) + + coeff = _MW / (_KCAT * 3600.0) # mg/mmol per unit flux + R = cobra.Reaction("R") + R.add_metabolites( + {A_c: -1.0, prot_E: -coeff, bio_met: 1.0, byp_c: _BYPRODUCT_YIELD} + ) + R.bounds = (0.0, 1000.0) + + BIO = cobra.Reaction("biomass") + BIO.add_metabolites({bio_met: -1.0}) + BIO.bounds = (0.0, 1000.0) + + pool_ex = cobra.Reaction("prot_pool_exchange") + pool_ex.add_metabolites({pool: 1.0}) + pool_ex.bounds = (0.0, 1.0) # 1 mg/gDW pool + + usage_E = cobra.Reaction("usage_prot_E") + usage_E.add_metabolites({pool: -1.0, prot_E: 1.0}) + usage_E.bounds = (0.0, 1000.0) + + model.add_reactions( + [EX_glc, EX_eth, EX_byp, TR_glc, TR_eth, R, BIO, pool_ex, usage_E] + ) + model.objective = "biomass" + return model + + +def _flux_data(*, ethanol_grrate: float) -> FluxData: + """Two conditions: glucose (enzyme-limited, everything matches + exactly) and ethanol (carbon-limited; ``ethanol_grrate`` controls + whether the measured growth matches the simulated 20.0 exactly or + deliberately deviates from it).""" + return FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_ENZYME_CAP, ethanol_grrate]), + exch_fluxes=np.array( + [ + [-_ENZYME_CAP, np.nan], + [np.nan, -20.0], + ] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + evotune_rmse_weight=np.array([1.0, 1.0]), + source=["test", "test"], + ) + + +def _max_grate_data() -> FluxData: + """Max-growth dataset: uptake fully opened, so both conditions hit + the same enzyme-capped growth of 36.""" + return FluxData( + conds=["glucose", "ethanol"], + p_tot=np.array([np.nan, np.nan]), + gr_rate=np.array([_ENZYME_CAP, _ENZYME_CAP]), + exch_fluxes=np.array( + [ + [-1000.0, np.nan], + [np.nan, -1000.0], + ] + ), + exch_mets=["glucose", "ethanol"], + exch_rxn_ids=["EX_glc", "EX_eth"], + ) + + +# --------------------------------------------------------------------------- # +# simulate_evotune_dataset +# --------------------------------------------------------------------------- # + +def test_simulate_flux_data_enzyme_vs_carbon_limited(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + flux_data = _flux_data(ethanol_grrate=20.0) + + results = simulate_evotune_dataset( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + + assert len(results) == 2 + glucose, ethanol = results + + # Glucose: enzyme-limited growth of 36, full carbon uptake used. + assert glucose.feasible + assert glucose.growth == pytest.approx(_ENZYME_CAP) + assert glucose.exch_fluxes["EX_glc"] == pytest.approx(-_ENZYME_CAP) + assert glucose.block_fluxes["EX_byp"] == pytest.approx( + _ENZYME_CAP * _BYPRODUCT_YIELD + ) + + # Ethanol: carbon-limited to the -20 uptake cap, growth = 20. + assert ethanol.feasible + assert ethanol.growth == pytest.approx(20.0) + assert ethanol.exch_fluxes["EX_eth"] == pytest.approx(-20.0) + assert ethanol.block_fluxes["EX_byp"] == pytest.approx(20.0 * _BYPRODUCT_YIELD) + + # Bounds fully revert after the call (both `with model:` blocks exited). + assert model.reactions.get_by_id("EX_glc").bounds == (-1000.0, 0.0) + assert model.reactions.get_by_id("EX_eth").bounds == (-1000.0, 0.0) + + +def test_simulate_max_grate_opens_uptake_fully(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + max_grate = _max_grate_data() + + results = simulate_evotune_dataset( + model, max_grate, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + + assert [r.growth for r in results] == pytest.approx([_ENZYME_CAP, _ENZYME_CAP]) + # max_grate rows carry no exchange-flux measurements to report back. + assert results[0].exch_fluxes == {} + assert results[0].block_fluxes == {} + + +def test_simulate_unmatched_condition_name_raises(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + flux_data = _flux_data(ethanol_grrate=20.0) + flux_data.conds = ["glucose", "propionate"] + + with pytest.raises(ValueError, match="propionate"): + simulate_evotune_dataset( + model, flux_data, + constrain=True, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + + +# --------------------------------------------------------------------------- # +# compute_excarbon +# --------------------------------------------------------------------------- # + +def test_compute_excarbon(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + + excarbon = compute_excarbon( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + assert excarbon["EX_glc"] == pytest.approx(6.0) # C6H12O6 + assert excarbon["EX_eth"] == pytest.approx(2.0) # C2H6O + assert excarbon["EX_byp"] == pytest.approx(1.0) # C1 + assert excarbon["biomass"] == pytest.approx(BIOMASS_CARBON_EQUIV) + + +def test_compute_excarbon_zero_carbon_clamps_to_one(tmp_path): + """A genuinely zero-carbon exchanged metabolite (e.g. O2) still + gets weight 1, not 0 -- MATLAB's `excarbon(excarbon==0)=1` quirk, + ported deliberately so those reactions' mismatches aren't silently + excluded from the RMSE.""" + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + o2_e = cobra.Metabolite("o2_e", compartment="e", formula="O2") + model.add_metabolites([o2_e]) + EX_o2 = cobra.Reaction("EX_o2") + EX_o2.add_metabolites({o2_e: -1.0}) + EX_o2.bounds = (-1000.0, 0.0) + model.add_reactions([EX_o2]) + + excarbon = compute_excarbon(model, ["EX_o2"], bio_rxn_id="biomass") + + assert excarbon["EX_o2"] == pytest.approx(1.0) + + +# --------------------------------------------------------------------------- # +# dataset_rmse / evotune_distance +# --------------------------------------------------------------------------- # + +def test_dataset_rmse_measured_terms_match_only_byproduct_deviates(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + # ethanol_grrate=20.0 matches the ethanol condition's actual + # carbon-limited growth exactly, and both conditions' own measured + # exchange fluxes match the simulation exactly too -- the only + # nonzero term left is the unmeasured byproduct (assumed + # zero-flux, but nonzero in the simulation). + flux_data = _flux_data(ethanol_grrate=20.0) + sims = simulate_evotune_dataset( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + excarbon = compute_excarbon( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, excarbon=excarbon, bio_rxn_id="biomass", + ) + + # The byproduct term is nonzero-simulated-vs-zero-measured, so RMSE + # is not zero overall -- but every *measured* exchange/growth term + # matches exactly, and the byproduct term is hand-computable. + glucose_byp = _ENZYME_CAP * _BYPRODUCT_YIELD * 1.0 # carbon weight 1 + ethanol_byp = 20.0 * _BYPRODUCT_YIELD * 1.0 + # 3 terms per condition: growth (matches exactly -> 0), the + # condition's own exchange flux (matches exactly -> 0), and the + # byproduct block term (measured 0 vs the nonzero simulated value). + expected_glucose_rmse = np.sqrt(np.mean([0.0, 0.0, glucose_byp ** 2])) + expected_ethanol_rmse = np.sqrt(np.mean([0.0, 0.0, ethanol_byp ** 2])) + + assert per_cond[0] == pytest.approx(expected_glucose_rmse) + assert per_cond[1] == pytest.approx(expected_ethanol_rmse) + assert rmse == pytest.approx(np.mean([expected_glucose_rmse, expected_ethanol_rmse])) + + +def test_dataset_rmse_growth_mismatch_is_hand_computable(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + # Deliberately wrong measured growth for the ethanol condition + # (18.0 vs the actual carbon-limited 20.0) -- a known, hand-provable + # deviation on top of the byproduct term. + flux_data = _flux_data(ethanol_grrate=18.0) + sims = simulate_evotune_dataset( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + excarbon = compute_excarbon( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + _, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, excarbon=excarbon, bio_rxn_id="biomass", + ) + + growth_diff = (18.0 - 20.0) * BIOMASS_CARBON_EQUIV # bio_meas - bio_sim + byp_diff = 0.0 - (20.0 * _BYPRODUCT_YIELD) # measured 0 vs simulated + expected = np.sqrt(np.mean([growth_diff ** 2, 0.0, byp_diff ** 2])) + assert per_cond[1] == pytest.approx(expected) + + +def test_dataset_rmse_infeasible_condition_gets_penalty(tmp_path): + flux_data = _flux_data(ethanol_grrate=20.0) + sims = [ConditionSimResult(feasible=False), ConditionSimResult(feasible=False)] + excarbon = {} + + rmse, per_cond = dataset_rmse( + flux_data, sims, + constrain=True, excarbon=excarbon, bio_rxn_id="biomass", penalty=99.0, + ) + + assert list(per_cond) == [99.0, 99.0] + assert rmse == pytest.approx(99.0) + + +def test_dataset_rmse_max_grate_scores_growth_only(): + max_grate = _max_grate_data() + sims = [ + ConditionSimResult(feasible=True, growth=_ENZYME_CAP), + ConditionSimResult(feasible=True, growth=30.0), # deliberately off + ] + + rmse, per_cond = dataset_rmse( + max_grate, sims, + constrain=False, excarbon={}, bio_rxn_id="biomass", + ) + + assert per_cond[0] == pytest.approx(0.0) + expected_second = abs((_ENZYME_CAP - 30.0) * BIOMASS_CARBON_EQUIV) + assert per_cond[1] == pytest.approx(expected_second) + assert rmse == pytest.approx(np.mean([0.0, expected_second])) + + +def test_evotune_distance_combines_both_datasets(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + flux_data = _flux_data(ethanol_grrate=20.0) + max_grate = _max_grate_data() + evotune_data = EvotuneData(flux_data=flux_data, max_grate=max_grate, zero_flux=["EX_byp"]) + + flux_sims = simulate_evotune_dataset( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + max_grate_sims = simulate_evotune_dataset( + model, max_grate, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + excarbon = compute_excarbon( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, detail = evotune_distance( + evotune_data, + flux_sims=flux_sims, max_grate_sims=max_grate_sims, + excarbon=excarbon, bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, excarbon=excarbon, bio_rxn_id="biomass", + ) + expected_max_grate_rmse, _ = dataset_rmse( + max_grate, max_grate_sims, constrain=False, excarbon=excarbon, + bio_rxn_id="biomass", + ) + assert rmse == pytest.approx( + np.mean([expected_flux_rmse, expected_max_grate_rmse]) + ) + assert "flux_data" in detail and "max_grate" in detail + + +def test_evotune_distance_max_growth_weight_scales_the_max_growth_term(tmp_path): + """``max_growth_weight`` scales the max-growth term, so 2 makes the + eight max-growth conditions count double against the 33 flux ones. + + MATLAB's ``abc_max.m`` weights the flux term with the same field, + so reproducing its ``maxGrowthWeight = 2`` needs 0.5 here.""" + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + flux_data = _flux_data(ethanol_grrate=20.0) + max_grate = _max_grate_data() + evotune_data = EvotuneData(flux_data=flux_data, max_grate=max_grate, zero_flux=["EX_byp"]) + + flux_sims = simulate_evotune_dataset( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + max_grate_sims = simulate_evotune_dataset( + model, max_grate, + constrain=False, zero_flux_rxns=[], bio_rxn_id="biomass", + ) + excarbon = compute_excarbon( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, excarbon=excarbon, bio_rxn_id="biomass", + ) + expected_max_grate_rmse, _ = dataset_rmse( + max_grate, max_grate_sims, constrain=False, excarbon=excarbon, + bio_rxn_id="biomass", + ) + + rmse, _ = evotune_distance( + evotune_data, + flux_sims=flux_sims, max_grate_sims=max_grate_sims, + excarbon=excarbon, bio_rxn_id="biomass", max_growth_weight=2.0, + ) + assert rmse == pytest.approx( + (expected_flux_rmse + 2.0 * expected_max_grate_rmse) / 3.0 + ) + + # 0.5 reproduces MATLAB's maxGrowthWeight = 2, which weights flux. + rmse_matlab, _ = evotune_distance( + evotune_data, + flux_sims=flux_sims, max_grate_sims=max_grate_sims, + excarbon=excarbon, bio_rxn_id="biomass", max_growth_weight=0.5, + ) + assert rmse_matlab == pytest.approx( + (2.0 * expected_flux_rmse + expected_max_grate_rmse) / 3.0 + ) + + # A lone dataset carries the whole score whatever the weight. + solo = EvotuneData(flux_data=flux_data, max_grate=None, zero_flux=["EX_byp"]) + rmse_solo, _ = evotune_distance( + solo, + flux_sims=flux_sims, max_grate_sims=None, + excarbon=excarbon, bio_rxn_id="biomass", max_growth_weight=2.0, + ) + assert rmse_solo == pytest.approx(expected_flux_rmse) + + +def test_evotune_distance_missing_dataset_contributes_nothing(tmp_path): + adapter = _adapter(tmp_path) + model = _build_toy(adapter) + flux_data = _flux_data(ethanol_grrate=20.0) + evotune_data = EvotuneData(flux_data=flux_data, max_grate=None, zero_flux=["EX_byp"]) + + flux_sims = simulate_evotune_dataset( + model, flux_data, + constrain=True, zero_flux_rxns=["EX_byp"], bio_rxn_id="biomass", + ) + excarbon = compute_excarbon( + model, ["EX_glc", "EX_eth", "EX_byp", "biomass"], bio_rxn_id="biomass", + ) + + rmse, detail = evotune_distance( + evotune_data, + flux_sims=flux_sims, max_grate_sims=None, + excarbon=excarbon, bio_rxn_id="biomass", + ) + + expected_flux_rmse, _ = dataset_rmse( + flux_data, flux_sims, constrain=True, excarbon=excarbon, bio_rxn_id="biomass", + ) + assert rmse == pytest.approx(expected_flux_rmse) + assert set(detail) == {"flux_data"} diff --git a/tests/test_evotune_tying.py b/tests/test_evotune_tying.py new file mode 100644 index 0000000..c594654 --- /dev/null +++ b/tests/test_evotune_tying.py @@ -0,0 +1,103 @@ +"""Tests for tying isozyme copies the kcat assignment could not tell apart.""" +import numpy as np +import pytest + +from geckopy.kcat_tuning.evotune.tying import ( + apply_ties, base_reaction, isozyme_tie_map, n_free, +) + + +def test_base_reaction_strips_the_isozyme_suffix(): + assert base_reaction("r_0438_EXP_3") == "r_0438" + assert base_reaction("r_0698") == "r_0698" + + +def test_copies_sharing_prior_and_source_are_tied(): + rxns = ["r_1_EXP_1", "r_1_EXP_2", "r_1_EXP_3", "r_2"] + kcat0 = np.array([54.33, 54.33, 54.33, 7.0]) + sources = ["okp", "okp", "okp", "brenda"] + + tie = isozyme_tie_map(rxns, kcat0, sources) + + assert list(tie) == [0, 0, 0, 3] + assert n_free(tie) == 2 # one for r_1's three copies, one for r_2 + + +def test_copies_with_different_priors_stay_free(): + """A distinction the assignment made is a distinction worth keeping.""" + rxns = ["r_1_EXP_1", "r_1_EXP_2"] + kcat0 = np.array([54.33, 12.0]) + + tie = isozyme_tie_map(rxns, kcat0, ["okp", "okp"]) + + assert list(tie) == [0, 1] + assert n_free(tie) == 2 + + +def test_a_shared_value_from_different_sources_is_a_coincidence(): + """Equal priors reached by different routes are not indistinguishable.""" + rxns = ["r_1_EXP_1", "r_1_EXP_2"] + kcat0 = np.array([230.0, 230.0]) + + assert list(isozyme_tie_map(rxns, kcat0, ["brenda", "custom"])) == [0, 1] + # Without source information there is nothing to separate them by. + assert list(isozyme_tie_map(rxns, kcat0)) == [0, 0] + + +def test_copies_of_different_reactions_are_never_tied(): + rxns = ["r_1_EXP_1", "r_2_EXP_1"] + kcat0 = np.array([5.0, 5.0]) + + assert list(isozyme_tie_map(rxns, kcat0, ["okp", "okp"])) == [0, 1] + + +def test_the_map_is_idempotent(): + """Representatives follow themselves, so applying twice changes nothing.""" + rxns = ["r_1_EXP_1", "r_1_EXP_2", "r_1_EXP_3"] + tie = isozyme_tie_map(rxns, np.full(3, 2.0), ["okp"] * 3) + + assert list(tie[tie]) == list(tie) + + +def test_apply_ties_gives_copies_their_representatives_value(): + tie = np.array([0, 0, 0, 3]) + + one = apply_ties(np.array([1.0, 9.0, 99.0, 5.0]), tie) + assert list(one) == [1.0, 1.0, 1.0, 5.0] + + # A batch of particles is projected column by column. + batch = np.array([[1.0, 2.0], [9.0, 8.0], [99.0, 88.0], [5.0, 6.0]]) + out = apply_ties(batch, tie) + assert out.shape == (4, 2) + assert list(out[:, 0]) == [1.0, 1.0, 1.0, 5.0] + assert list(out[:, 1]) == [2.0, 2.0, 2.0, 6.0] + + +def test_apply_ties_rejects_a_mismatched_map(): + with pytest.raises(ValueError, match="tie_map has"): + apply_ties(np.ones((3, 2)), np.array([0, 0])) + + +def test_tie_map_rejects_mismatched_inputs(): + with pytest.raises(ValueError, match="rxn_ids has"): + isozyme_tie_map(["a"], np.ones(2)) + + +def test_tying_reduces_the_dimension_a_search_sees(): + """A screen or optimiser working on free parameters sees fewer.""" + from geckopy.kcat_tuning.evotune import tying + + rxns = ["r_1_EXP_1", "r_1_EXP_2", "r_2_EXP_1", "r_2_EXP_2", "r_3"] + kcat0 = np.array([10.0, 10.0, 4.0, 9.0, 1.0]) + sources = ["brenda"] * 5 + tie = tying.isozyme_tie_map(rxns, kcat0, sources) + + # r_1's copies share a prior and tie; r_2's do not; r_3 stands alone. + assert n_free(tie) == 4 + + rng = np.random.default_rng(0) + particles = rng.lognormal(size=(5, 50)) + tying.apply_ties(particles, tie) + + assert np.array_equal(particles[0], particles[1]) + assert not np.array_equal(particles[2], particles[3]) diff --git a/tests/test_find_max_value.py b/tests/test_find_max_value.py index dc076a1..d064d3c 100644 --- a/tests/test_find_max_value.py +++ b/tests/test_find_max_value.py @@ -2,7 +2,7 @@ import pandas as pd from geckopy.databases import BrendaData -from geckopy.kcat_sensitivity_analysis import find_max_value +from geckopy.kcat_tuning import find_max_value # --------------------------------------------------------------------------- # diff --git a/tests/test_kcat_assignment_review.py b/tests/test_kcat_assignment_review.py new file mode 100644 index 0000000..017b63b --- /dev/null +++ b/tests/test_kcat_assignment_review.py @@ -0,0 +1,148 @@ +"""Tests for flagging kcat assignments that deserve a second look.""" +import numpy as np +import pytest + +from geckopy.gather_kcats.review import ( + EcStats, Finding, findings_tsv, review_assignment, +) + + +def _stats(**kw): + base = dict(n_kcat=5, kcat_max=100.0, kcat_median=50.0, + values=frozenset({100.0, 50.0}), n_sa=0) + base.update(kw) + return EcStats(**base) + + +def test_no_database_evidence_for_the_ec_is_flagged(): + """The highest-yield check: the value did not come from a kcat.""" + rows = review_assignment( + ["r_1"], np.array([0.0019]), ["brenda"], ["5.4.99.7"], + ec_stats={"5.4.99.7": _stats(n_kcat=0, kcat_max=float("nan"), + kcat_median=float("nan"), + values=frozenset(), n_sa=4)}, + ) + assert [c for c in rows[0].checks if c == "no-ec-evidence"] + assert "specific-activity" in rows[0].detail + + +def test_a_value_reused_across_reactions_is_a_fallback(): + ids = [f"r_{i}" for i in range(6)] + kcats = np.full(6, 0.0918) + rows = review_assignment(ids, kcats, ["brenda"] * 6, ["1.1.1.1"] * 6, + ec_stats={"1.1.1.1": _stats()}, repeat_min=5) + assert all("repeated-value" in r.checks for r in rows) + assert "6 reactions" in rows[0].detail + + # Below the threshold it is not remarkable. + quiet = review_assignment(ids[:3], kcats[:3], ["brenda"] * 3, + ["1.1.1.1"] * 3, + ec_stats={"1.1.1.1": _stats()}, repeat_min=5) + assert all("repeated-value" not in r.checks for r in quiet) + + +def test_taking_the_ec_maximum_is_flagged_only_when_the_spread_is_wide(): + st_wide = {"1.1.1.1": _stats(kcat_max=230.0, kcat_median=37.0, n_kcat=7)} + wide = review_assignment(["r_1"], np.array([230.0]), ["brenda"], + ["1.1.1.1"], ec_stats=st_wide) + assert "ec-maximum" in wide[0].checks + assert "6x its median" in wide[0].detail + + # A maximum close to the median says nothing. + st_tight = {"1.1.1.1": _stats(kcat_max=60.0, kcat_median=50.0)} + tight = review_assignment(["r_1"], np.array([60.0]), ["brenda"], + ["1.1.1.1"], ec_stats=st_tight) + assert tight == [] + + +def test_custom_duplicating_its_own_ec_value_is_flagged(): + """Curation that copies the database adds no evidence but tightens + the prior, since custom carries a narrower sigma than brenda.""" + st = {"4.2.1.11": _stats(kcat_max=230.0, kcat_median=37.0, + values=frozenset({230.0, 37.0}))} + rows = review_assignment(["r_0366"], np.array([230.0]), ["custom"], + ["4.2.1.11"], ec_stats=st) + assert "custom-duplicate" in rows[0].checks + + # A curated value the database does not hold is doing real work. + rows = review_assignment(["r_0366"], np.array([12.0]), ["custom"], + ["4.2.1.11"], ec_stats=st) + assert all("custom-duplicate" not in r.checks for r in rows) + + +def test_standard_fallbacks_and_tied_groups_are_quiet_by_default(): + """Both are deliberate, so listing every one is noise.""" + st = {"1.1.1.1": _stats()} + assert review_assignment(["r_1"], np.array([13.0]), ["standard"], + ["1.1.1.1"], ec_stats=st) == [] + + rows = review_assignment(["r_1"], np.array([13.0]), ["standard"], + ["1.1.1.1"], ec_stats=st, + include=("standard-fallback",)) + assert "standard-fallback" in rows[0].checks + + +def test_isozyme_copies_are_reported_once(): + ids = ["r_1_EXP_1", "r_1_EXP_2", "r_1_EXP_3"] + kcats = np.full(3, 0.001) + rows = review_assignment(ids, kcats, ["okp"] * 3, ["1.1.1.1"] * 3, + ec_stats={"1.1.1.1": _stats()}) + assert len(rows) == 1 + assert rows[0].n_isozymes == 3 + assert "3 isozyme copies" in rows[0].detail + + +def test_findings_rank_by_leverage_and_truncate_by_coverage(): + ids = [f"r_{i}" for i in range(4)] + kcats = np.array([0.001, 0.002, 0.003, 0.004]) # all flagged: magnitude + lev = np.array([0.1, 5.0, 3.0, 1.9]) + st = {"1.1.1.1": _stats()} + + rows = review_assignment(ids, kcats, ["brenda"] * 4, ["1.1.1.1"] * 4, + ec_stats=st, leverage=lev) + assert [r.rxn_id for r in rows] == ["r_1", "r_2", "r_3", "r_0"] + + # 80% of 10.0 total is reached after 5.0 + 3.0. + short = review_assignment(ids, kcats, ["brenda"] * 4, ["1.1.1.1"] * 4, + ec_stats=st, leverage=lev, coverage=0.8) + assert [r.rxn_id for r in short] == ["r_1", "r_2"] + + assert len(review_assignment(ids, kcats, ["brenda"] * 4, ["1.1.1.1"] * 4, + ec_stats=st, leverage=lev, top=1)) == 1 + + +def test_shared_enzymes_are_counted_not_filtered(): + """A protein used elsewhere makes a correction a wider question.""" + ids = ["r_1", "r_2", "r_3"] + rows = review_assignment( + ids, np.array([0.001, 50.0, 50.0]), ["brenda"] * 3, ["1.1.1.1"] * 3, + ec_stats={"1.1.1.1": _stats()}, + enzymes=[["P1", "P2"], ["P1"], ["P2"]], + ) + assert len(rows) == 1 # only r_1 trips a check + assert rows[0].other_reactions == 2 # P1 in r_2, P2 in r_3 + + +def test_checks_needing_a_database_are_skipped_without_one(): + rows = review_assignment(["r_1"], np.array([0.001]), ["brenda"], + ["1.1.1.1"]) + assert rows[0].checks == ("magnitude",) + + +def test_review_rejects_mismatched_inputs(): + with pytest.raises(ValueError, match="sources has"): + review_assignment(["a", "b"], np.ones(2), ["brenda"], ["1.1.1.1"] * 2) + with pytest.raises(ValueError, match="leverage has shape"): + review_assignment(["a"], np.ones(1), ["brenda"], ["1.1.1.1"], + leverage=np.ones(2)) + + +def test_tsv_carries_every_field(): + rows = [Finding("r_0698", "lanosterol synthase", "5.4.99.7", "brenda", + 0.0019, 1.45, 1, 3, ("no-ec-evidence", "magnitude"), + "no kcat rows for EC 5.4.99.7")] + text = findings_tsv(rows) + header, body = text.splitlines() + assert header.split("\t")[0] == "rxn_id" + assert len(body.split("\t")) == len(header.split("\t")) + assert "no-ec-evidence,magnitude" in body diff --git a/tests/test_load_flux_data.py b/tests/test_load_flux_data.py index b7e6d43..baa5188 100644 --- a/tests/test_load_flux_data.py +++ b/tests/test_load_flux_data.py @@ -159,22 +159,22 @@ def test_no_optional_columns_yields_none(tmp_path): ["c1", "0.5", "0.4", "-10.0"], ]) fd = load_flux_data(p) - assert fd.bayesian_rmse_weight is None + assert fd.evotune_rmse_weight is None assert fd.source is None -def test_bayesian_rmse_weight_column_loaded(tmp_path): +def test_evotune_rmse_weight_column_loaded(tmp_path): p = tmp_path / "fd.tsv" _write_tsv(p, [ ["condition", "Ptot", "grRate", - "glc (EX_glc)", "bayesianRMSEweight"], + "glc (EX_glc)", "evotuneRMSEweight"], ["c1", "0.5", "0.4", "-10.0", "1.5"], ["c2", "0.6", "0.5", "-8.5", "0.7"], ]) fd = load_flux_data(p) - assert fd.bayesian_rmse_weight is not None - np.testing.assert_array_equal(fd.bayesian_rmse_weight, [1.5, 0.7]) - # Bayesian column dropped from exch_fluxes. + assert fd.evotune_rmse_weight is not None + np.testing.assert_array_equal(fd.evotune_rmse_weight, [1.5, 0.7]) + # Tuning-weight column dropped from exch_fluxes. assert fd.exch_fluxes.shape == (2, 1) assert fd.exch_rxn_ids == ["EX_glc"] @@ -195,12 +195,12 @@ def test_both_optional_columns(tmp_path): p = tmp_path / "fd.tsv" _write_tsv(p, [ ["condition", "Ptot", "grRate", - "glc (EX_glc)", "bayesianRMSEweight", "source"], + "glc (EX_glc)", "evotuneRMSEweight", "source"], ["c1", "0.5", "0.4", "-10.0", "1.5", "PMID:12345"], ]) fd = load_flux_data(p) - assert fd.bayesian_rmse_weight is not None - np.testing.assert_array_equal(fd.bayesian_rmse_weight, [1.5]) + assert fd.evotune_rmse_weight is not None + np.testing.assert_array_equal(fd.evotune_rmse_weight, [1.5]) assert fd.source == ["PMID:12345"] assert fd.exch_fluxes.shape == (1, 1) @@ -211,13 +211,13 @@ def test_optional_columns_in_arbitrary_position(tmp_path): p = tmp_path / "fd.tsv" _write_tsv(p, [ ["condition", "Ptot", "grRate", - "source", "glc (EX_glc)", "bayesianRMSEweight"], + "source", "glc (EX_glc)", "evotuneRMSEweight"], ["c1", "0.5", "0.4", "PMID:1", "-10.0", "1.5"], ]) fd = load_flux_data(p) assert fd.source == ["PMID:1"] - assert fd.bayesian_rmse_weight is not None - np.testing.assert_array_equal(fd.bayesian_rmse_weight, [1.5]) + assert fd.evotune_rmse_weight is not None + np.testing.assert_array_equal(fd.evotune_rmse_weight, [1.5]) np.testing.assert_array_equal(fd.exch_fluxes, [[-10.0]]) assert fd.exch_rxn_ids == ["EX_glc"] diff --git a/tests/test_params.py b/tests/test_params.py index c8304da..a7b3e72 100644 --- a/tests/test_params.py +++ b/tests/test_params.py @@ -2,37 +2,43 @@ import pytest from pydantic import ValidationError -from geckopy.adapter.params import BayesianParams +from geckopy.adapter.params import EvotuneParams, SourceGroupRule def test_defaults_are_consistent(): - # The shipped defaults must satisfy the parallel-list validator. - bp = BayesianParams() - assert len(bp.sigma0_log_source) == len(bp.kcat_sources) + # The shipped defaults must satisfy the group-key validator. + bp = EvotuneParams() + assert set(bp.sigma0_log_source) == set(bp.source_groups) -def test_mismatched_source_list_raises(): +def test_mismatched_source_dict_raises(): with pytest.raises(ValidationError, match="sigma0_log_source"): - BayesianParams( - kcat_sources=["dlkcat", "brenda", "custom"], - sigma0_log_source=[0.4, 0.2], # one short + EvotuneParams( + source_groups={ + "dlkcat": SourceGroupRule(sources=["dlkcat"]), + "brenda": SourceGroupRule(sources=["brenda"]), + }, + sigma0_log_source={"dlkcat": 0.4}, # missing "brenda" ) -def test_mismatched_schedule_lengths_raises(): - with pytest.raises(ValidationError, match="schedule_generations"): - BayesianParams( - schedule_generations=[1, 2, 9], - schedule_samples=[1000, 800], - ) - - -def test_consistent_custom_lists_ok(): - bp = BayesianParams( - kcat_sources=["a", "b"], - sigma0_log_source=[0.1, 0.2], - shrink_thr_source=[1.0, 2.0], - variance_cap_source=[5.0, 6.0], - force_prior_thr_source=[0.0, 1.0], +def test_consistent_custom_groups_ok(): + bp = EvotuneParams( + source_groups={ + "a": SourceGroupRule(sources=["src_a"]), + "b": SourceGroupRule(sources=["src_b"], match_okp=True), + }, + sigma0_log_source={"a": 0.1, "b": 0.2}, ) - assert bp.kcat_sources == ["a", "b"] + assert set(bp.source_groups) == {"a", "b"} + assert bp.source_groups["b"].match_okp is True + + +def test_dropped_fields_are_rejected(): + """target_accept and variance_cap_* were confirmed dead/cosmetic + in MATLAB and are not part of the schema; extra="forbid" should + reject them outright rather than silently ignoring them.""" + with pytest.raises(ValidationError): + EvotuneParams(target_accept=10.0) + with pytest.raises(ValidationError): + EvotuneParams(variance_cap_default=10.0) diff --git a/tests/test_sensitivity_tuning.py b/tests/test_sensitivity_tuning.py index 27a0c42..5d7f779 100644 --- a/tests/test_sensitivity_tuning.py +++ b/tests/test_sensitivity_tuning.py @@ -8,7 +8,7 @@ from geckopy import EcModel, ModelAdapter from geckopy.ec_model.ec_data import EcData -from geckopy.kcat_sensitivity_analysis import ( +from geckopy.kcat_tuning import ( TunedKcatsResult, sensitivity_tuning, ) diff --git a/tests/test_sigma_fitter.py b/tests/test_sigma_fitter.py index 80f234c..ef9dd8c 100644 --- a/tests/test_sigma_fitter.py +++ b/tests/test_sigma_fitter.py @@ -8,7 +8,7 @@ from geckopy import EcModel, ModelAdapter from geckopy.ec_model.ec_data import EcData -from geckopy.kcat_sensitivity_analysis import ( +from geckopy.kcat_tuning import ( SigmaFitterResult, fit_sigma, ) diff --git a/tests/test_truncate_values.py b/tests/test_truncate_values.py index 1e5e538..622668c 100644 --- a/tests/test_truncate_values.py +++ b/tests/test_truncate_values.py @@ -2,8 +2,8 @@ import numpy as np import pytest -from geckopy.kcat_sensitivity_analysis import truncate_values -from geckopy.kcat_sensitivity_analysis.truncate_values import _truncate_scalar +from geckopy.kcat_tuning import truncate_values +from geckopy.kcat_tuning.truncate_values import _truncate_scalar # --------------------------------------------------------------------------- # diff --git a/tutorials/full_ecModel/code/anaerobic.py b/tutorials/full_ecModel/code/anaerobic.py new file mode 100644 index 0000000..7d41d28 --- /dev/null +++ b/tutorials/full_ecModel/code/anaerobic.py @@ -0,0 +1,71 @@ +"""Anaerobic switch for the yeast-GEM ecModel. + +Port of `GECKO/tutorials/full_ecModel/code/anaerobicModel_GECKO.m`, +which `YeastGEMAdapter.makeModelAnaerobic` calls for every evotune +condition whose oxygen exchange is measured as exactly 0. + +The changes are organism-specific, so they live with the tutorial +rather than in geckopy, mirroring where GECKO keeps them. Pass +:func:`make_anaerobic` to `cmaes_kcat_tuning(make_anaerobic=...)`; +it is a module-level function so it survives pickling to the scoring +pool's workers. + +Every change goes through the cobra API while the caller holds the +model's context (`with model:`), so all of them revert when the +condition's simulation ends. +""" +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from geckopy import EcModel + +#: Uptakes opened for anaerobic growth: sterols and fatty acids cannot +#: be synthesised without oxygen, and the vitamins feed NAD(P)H and CoA +#: synthesis. +_OPEN_UPTAKE = ( + "r_1757", # ergosterol + "r_1915", # lanosterol + "r_2106", # zymosterol + "r_2134", # 14-demethyllanosterol + "r_1994", # palmitoleate + "r_2189", # oleate + "r_1967", # nicotinate + "r_1548", # (R)-pantothenate +) + +#: Blocked outright. MDH2 and IDP2 are repressed on glucose and absent +#: from the proteome; the sterol uptake is blocked because it recycles +#: NADH to ergosterol. +_BLOCKED = ( + "r_2137", # ergosta-5,7,22,24(28)-tetraen-3beta-ol + "r_0714", # MDH2, malate dehydrogenase (cytoplasmic) + "r_0659", # IDP2, isocitrate dehydrogenase (NADP) +) + +#: FADH2 turnover from disulphide bond formation via Ero1, added to the +#: biomass pseudoreaction. +_FADH2_PROD = 0.08 + + +def make_anaerobic(model: "EcModel") -> None: + """Constrain a yeast-GEM ecModel to anaerobic conditions, in place.""" + # Heme a leaves the cofactor pseudoreaction: its synthesis needs O2. + cofactor = model.reactions.get_by_id("r_4598") + heme = model.metabolites.get_by_id("s_3714") + if heme in cofactor.metabolites: + cofactor.add_metabolites({heme: -cofactor.metabolites[heme]}) + + model.reactions.get_by_id("r_1992").lower_bound = 0.0 # O2 uptake + for rxn_id in _OPEN_UPTAKE: + model.reactions.get_by_id(rxn_id).lower_bound = -1000.0 + for rxn_id in _BLOCKED: + # MATLAB's setParam('eq', ...) matches the reaction ID exactly, + # so the `_REV` half of a split reversible pair is left alone. + model.reactions.get_by_id(rxn_id).bounds = (0.0, 0.0) + + biomass = model.reactions.get_by_id("r_4041") + biomass.add_metabolites({ + model.metabolites.get_by_id("s_0689"): _FADH2_PROD, # FADH2[c] + model.metabolites.get_by_id("s_0687"): -_FADH2_PROD, # FAD[c] + model.metabolites.get_by_id("s_0794"): -2 * _FADH2_PROD, # H+[c] + }) diff --git a/tutorials/full_ecModel/data/evotuneFluxData.tsv b/tutorials/full_ecModel/data/evotuneFluxData.tsv new file mode 100644 index 0000000..7364bea --- /dev/null +++ b/tutorials/full_ecModel/data/evotuneFluxData.tsv @@ -0,0 +1,34 @@ +Condition Ptot grRate glucose (r_1714) fructose (r_1709) sucrose (r_2058) xylose (r_1718) CO2 (r_1672) oxygen (r_1992) pyruvate (r_2033) ethyl acetate (r_1765) glycerol (r_1808) acetate (r_1634) ethanol (r_1761) succinate (r_2056) evotuneRMSEweight source +glucose NaN 0.36 -13.33333333 0 0 0 NaN NaN 0 0 0.38 NaN 19.82608696 NaN 1 DLKcat +glucose NaN 0.39 -15.44444444 0 0 0 NaN NaN 0 0 0.37 NaN 22.4068323 NaN 1 DLKcat +glucose NaN 0.37 -19.55555556 0 0 0 NaN NaN 0 0 NaN NaN NaN NaN 1 DLKcat +glucose NaN 0.37 -14.8 0 0 0 NaN NaN 0 0 1.1 0.4 22 NaN 1 DLKcat +glucose NaN 0.38 -15.05555556 0 0 0 NaN NaN 0 0 NaN NaN NaN NaN 1 DLKcat +glucose NaN 0.35 -17.66666667 0 0 0 NaN NaN 0 0 0.34 NaN 23.51778656 NaN 1 DLKcat +glucose NaN 0.37 -11.2 0 0 0 19.58 NaN 0 0 0.12 0.43 15.74 NaN 1 DLKcat +glucose NaN 0.37 -13.66666667 0 0 0 NaN NaN 0 0 NaN NaN NaN NaN 1 DLKcat +glucose NaN 0.38 -17.55555556 0 0 0 NaN NaN 0 0 0.41 NaN 28.22463768 NaN 1 DLKcat +glucose NaN 0.32 -25.38888889 0 0 0 NaN NaN 0 0 0.43 NaN 42.73291925 NaN 1 DLKcat +fructose NaN 0.1 0 -6.888888889 0 0 NaN 0 0 0 0.41 NaN 11.14130435 NaN 1 DLKcat +fructose NaN 0.25 0 -12.44444444 0 0 NaN 0 0 0 0.41 NaN 20.256917 NaN 1 DLKcat +sucrose NaN 0.32 0 0 -6.076541046 0 NaN 0 0 0 NaN NaN 19.94202899 NaN 1 DLKcat +glucose NaN 0.025 -0.3 0 0 0 0.75198 -0.73102 0 0 0 0 0 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.05 -0.6 0 0 0 1.3697 -1.3251 0 0 0 0 0 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.1 -1.1 0 0 0 2.4007 -2.3167 0 0 0 0 0 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.15 -1.7 0 0 0 3.6379 -3.5066 0 0 0 0 0 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.2 -2.3 0 0 0 4.8751 -4.6966 0 0 0 0 0 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.25 -2.8 0 0 0 5.906 -5.6881 0 0 0 0 0 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.28 -3.4 0 0 0 7.0413 -6.6773 0 0 0 0.08 0.11 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.3 -4.5 0 0 0 8.5667 -6.012 0 0 0 0.41 2.3 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.35 -8.6 0 0 0 15.2091 -5.3809 0 0 0 0.62 9.5 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.4 -11.1 0 0 0 19.3207 -5.0171 0 0 0 0.6 13.9 NaN 1 doi:10.1128/AEM.64.11.4226-4233.1998 +glucose NaN 0.4 -20.2 0 0 0 34.4 NaN 0 0 0 1 30 NaN 1 DLKcat +glucose NaN 0.41 -14.22222222 0 0 0 NaN NaN 0 0 0 NaN 25.625 NaN 1 DLKcat +glucose NaN 0.1 -6.44 0 0 0 NaN 0 0 0 1.08 0 9.26 NaN 1 doi:10.1186/1752-0509-2-60 +glucose NaN 0.2449 -10.173 0 0 0 19.109 NaN 0 0 1.2908 0.212 15.494 NaN 1 doi:10.1002/bit.25447 +glucose NaN 0.2287 -13.492 0 0 0 27.31 0 0 0 2.9849 0.328 20.636 NaN 1 doi:10.1002/bit.25447 +glucose NaN 0.1 -6.0184 0 0 0 9.8220288 0 0.0120368 0 1.0351648 0.0240736 8.9734344 0.0270828 1 doi:10.1099/00221287-143-1-203 +glucose NaN 0.2 -11.5927 0 0 0 18.9888426 0 0.0463708 0 2.1098714 0.0695562 17.2499376 0.05216715 1 doi:10.1099/00221287-143-1-203 +glucose NaN 0.3 -17.0704 0 0 0 27.3467808 0 0.1024224 0 3.243376 0.2048448 25.2983328 0.0512112 1 doi:10.1099/00221287-143-1-203 +glucose NaN 0.4 -22.5483 0 0 0 35.3106378 0 0.1803864 0 4.9155294 0.450966 33.6195153 0.0676449 1 doi:10.1099/00221287-143-1-203 +glucose 0.46 0.1 -1.0709285 0 0 0 2.627555 -2.446473 NaN NaN 0.0008575 0 0 NaN 1 doi:10.1016/j.cels.2017.03.003 diff --git a/tutorials/full_ecModel/data/evotuneMaxGrowth.tsv b/tutorials/full_ecModel/data/evotuneMaxGrowth.tsv new file mode 100644 index 0000000..9f67bd4 --- /dev/null +++ b/tutorials/full_ecModel/data/evotuneMaxGrowth.tsv @@ -0,0 +1,9 @@ +Condition Ptot grRate glucose (r_1714) fructose (r_1709) sucrose (r_2058) acetate (r_1634) ethanol (r_1761) galactose (r_1710) maltose (r_1931) sucrose (r_2058) mannose (r_1715) evotuneRMSEweight source +fructose NaN 0.338 NaN -1000 NaN NaN NaN NaN NaN NaN NaN 1 DLKcat +glucose NaN 0.41 -1000 NaN NaN NaN NaN NaN NaN NaN NaN 1 DLKcat +ethanol NaN 0.12 NaN NaN NaN NaN -1000 NaN NaN NaN NaN 1 DLKcat +mannose NaN 0.33 NaN NaN NaN NaN NaN NaN NaN NaN -1000 1 DLKcat +galactose NaN 0.28 NaN NaN NaN NaN NaN -1000 NaN NaN NaN 1 DLKcat +acetate NaN 0.17 NaN NaN NaN -1000 NaN NaN NaN NaN NaN 1 DLKcat +maltose NaN 0.4 NaN NaN NaN NaN NaN NaN -1000 NaN NaN 1 DLKcat +sucrose NaN 0.39 NaN NaN -1000 NaN NaN NaN NaN -1000 NaN 1 DLKcat diff --git a/tutorials/full_ecModel/data/evotuneZeroExch.tsv b/tutorials/full_ecModel/data/evotuneZeroExch.tsv new file mode 100644 index 0000000..fabf909 --- /dev/null +++ b/tutorials/full_ecModel/data/evotuneZeroExch.tsv @@ -0,0 +1,220 @@ +Rxns +r_1542 +r_1545 +r_1546 +r_1547 +r_1548 +r_1549 +r_1550 +r_1551 +r_1552 +r_1553 +r_1554 +r_1563 +r_1564 +r_1565 +r_1566 +r_1572 +r_1577 +r_1580 +r_1581 +r_1586 +r_1589 +r_1598 +r_1604 +r_1621 +r_1625 +r_1627 +r_1629 +r_1630 +r_1631 +r_1639 +r_1641 +r_1643 +r_1648 +r_1649 +r_1650 +r_1651 +r_1663 +r_1671 +r_1683 +r_1687 +r_1702 +r_1705 +r_1706 +r_1710 +r_1711 +r_1712 +r_1714 +r_1715 +r_1716 +r_1718 +r_1727 +r_1730 +r_1749 +r_1753 +r_1757 +r_1764 +r_1765 +r_1788 +r_1791 +r_1792 +r_1793 +r_1798 +r_1800 +r_1806 +r_1807 +r_1810 +r_1814 +r_1815 +r_1818 +r_1820 +r_1834 +r_1841 +r_1843 +r_1847 +r_1862 +r_1865 +r_1866 +r_1867 +r_1870 +r_1873 +r_1875 +r_1878 +r_1879 +r_1880 +r_1881 +r_1883 +r_1886 +r_1889 +r_1891 +r_1893 +r_1896 +r_1897 +r_1899 +r_1900 +r_1902 +r_1903 +r_1904 +r_1906 +r_1909 +r_1911 +r_1912 +r_1913 +r_1914 +r_1915 +r_1916 +r_1931 +r_1947 +r_1952 +r_1967 +r_1968 +r_1984 +r_1987 +r_1989 +r_1993 +r_1994 +r_1999 +r_2000 +r_2001 +r_2024 +r_2028 +r_2038 +r_2043 +r_2044 +r_2046 +r_2051 +r_2052 +r_2055 +r_2056 +r_2058 +r_2062 +r_2066 +r_2067 +r_2068 +r_2073 +r_2083 +r_2090 +r_2091 +r_2092 +r_2102 +r_2104 +r_2106 +r_2134 +r_2137 +r_2187 +r_2188 +r_2189 +r_2193 +r_4043 +r_4044 +r_4442 +r_4494 +r_4496 +r_4497 +r_4498 +r_4499 +r_4500 +r_4501 +r_4502 +r_4503 +r_4504 +r_4505 +r_4506 +r_4507 +r_4508 +r_4509 +r_4510 +r_4511 +r_4512 +r_4513 +r_4514 +r_4515 +r_4516 +r_4517 +r_4518 +r_4519 +r_4520 +r_4521 +r_4522 +r_4523 +r_4524 +r_4525 +r_4526 +r_4529 +r_4530 +r_4531 +r_4532 +r_4533 +r_4534 +r_4535 +r_4536 +r_4537 +r_4538 +r_4539 +r_4540 +r_4541 +r_4542 +r_4543 +r_4544 +r_4545 +r_4546 +r_4547 +r_4548 +r_4549 +r_4550 +r_4551 +r_4552 +r_4553 +r_4554 +r_4555 +r_4556 +r_4557 +r_4558 +r_4559 +r_4560 +r_4561 +r_4562 +r_4563 +r_4564 +r_4565 diff --git a/tutorials/full_ecModel/model_adapter.toml b/tutorials/full_ecModel/model_adapter.toml index f2c7ceb..9fa0a59 100644 --- a/tutorials/full_ecModel/model_adapter.toml +++ b/tutorials/full_ecModel/model_adapter.toml @@ -26,3 +26,35 @@ reviewed = true [complex] taxonomic_id = 559292 + +[evotune] +sigma0_log_default = 0.3 +rmse_threshold = 0.2 +# CMA-ES generations. 212 approximates a 6800-evaluation budget at +# cmaes_kcat_tuning's own default population size for ~110 free +# parameters; a different tunable-set size changes that population +# size and so the evaluation count this actually spends. +max_generations = 212 +# Weights the max-growth term: the eight max-growth conditions count double +# against the 33 flux conditions, which are 30 parts glucose. MATLAB weights +# the flux term with this field instead, so its maxGrowthWeight = 2 is 0.5 here. +max_growth_weight = 2.0 + +# Trust tiers, least to most trusted. This model labels predicted kcats +# "OpenKineticsPredictor"; match_okp additionally catches models that label +# them with the predictor method name (OkpParams.method) instead. +[evotune.source_groups.okp] +sources = ["OpenKineticsPredictor"] +match_okp = true + +[evotune.source_groups.brenda] +sources = ["brenda"] + +[evotune.source_groups.custom] +sources = ["custom"] + +[evotune.sigma0_log_source] +okp = 0.25 +brenda = 0.2 +custom = 0.1 + diff --git a/tutorials/full_ecModel/protocol.py b/tutorials/full_ecModel/protocol.py index b59f20b..99e6649 100644 --- a/tutorials/full_ecModel/protocol.py +++ b/tutorials/full_ecModel/protocol.py @@ -259,9 +259,11 @@ # %% [markdown] # Note: `bayesianSensitivityTuning` (the ABC-SMC variant introduced -# in GECKO 3.3.0) is not yet ported. The MATLAB tutorial also -# skips it for the protocol walkthrough; the maintained ecYeastGEM -# in the yeast-GEM repository uses it instead. +# in GECKO 3.3.0) is not ported; geckopy fits kcats to experimental +# data with CMA-ES instead (see docs/evotune_kcat_tuning.md), +# not wired into this protocol walkthrough. The MATLAB tutorial also +# skips it here; the maintained ecYeastGEM in the yeast-GEM repository +# uses it instead. # %% [markdown] # **STEP 45-51** Curate kcat values based on the tuning result. From 48e9610c04b3227f21ea3b274a5f00ee165dd8c3 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Mon, 7 Sep 2026 15:18:14 +0200 Subject: [PATCH 02/15] Report a tuning result as the corrections it makes (#52) * Add EvotuneResult.corrections(): report a tuning result as a ranked table * Move corrections to kcat_tuning; give sensitivity_tuning its own corrections() --- docs/evotune_kcat_tuning.md | 27 +++ src/geckopy/kcat_tuning/__init__.py | 10 ++ src/geckopy/kcat_tuning/corrections.py | 163 ++++++++++++++++++ src/geckopy/kcat_tuning/evotune/tuning.py | 24 +++ src/geckopy/kcat_tuning/sensitivity_tuning.py | 25 ++- tests/test_corrections.py | 127 ++++++++++++++ tests/test_evotune.py | 17 ++ tests/test_sensitivity_tuning.py | 18 ++ 8 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 src/geckopy/kcat_tuning/corrections.py create mode 100644 tests/test_corrections.py diff --git a/docs/evotune_kcat_tuning.md b/docs/evotune_kcat_tuning.md index 83dd6b9..94a6a40 100644 --- a/docs/evotune_kcat_tuning.md +++ b/docs/evotune_kcat_tuning.md @@ -303,6 +303,33 @@ Before trusting a tuned kcat, check more than the final RMSE: elsewhere is a sign of overfitting to a thin dataset, not a fixed model. +`new_kcat`/`old_kcat` is thousands of numbers, most of them unchanged -- +not something a reviewer can read. `result.corrections()` turns it into +the actual review artifact: one `Correction` row per changed kcat +(reaction, EC code, source, prior/tuned value, fold change), ranked by +*leverage* rather than by how far a kcat moved, so the parameters the +data cannot see (free to drift furthest, least worth trusting) sort to +the bottom instead of the top: + +```python +from geckopy.kcat_tuning import annotate_from_model, corrections_tsv + +# screen is Step 3's table -- reuse it rather than recomputing leverage. +leverage_by_rxn = dict(zip(screen["rxn_id"], screen["leverage"])) +leverage = [leverage_by_rxn.get(r, 0.0) for r in result.rxns] + +annotations = annotate_from_model(model, result.rxns) +rows = result.corrections(leverage=leverage, **annotations) +open("corrections.tsv", "w").write(corrections_tsv(rows)) +``` + +`cumulative_share` on each row is the running fraction of total leverage +the list has accounted for so far, so a reader can stop where it stops +climbing rather than reading every row. This is the tuning-time +counterpart to `review_assignment` in Step 4: that flags kcats worth +curating *before* a run; this reports which ones the run actually +changed, and how much each change is worth trusting. + ## Where to go from here - `docs/cmaes_vs_abc_smc.md` -- why this method is CMA-ES rather than diff --git a/src/geckopy/kcat_tuning/__init__.py b/src/geckopy/kcat_tuning/__init__.py index 22e3136..59eac2e 100644 --- a/src/geckopy/kcat_tuning/__init__.py +++ b/src/geckopy/kcat_tuning/__init__.py @@ -3,6 +3,12 @@ Mirrors GECKO MATLAB's `src/kcat_tuning/` directory. """ +from .corrections import ( + Correction, + annotate_from_model, + corrections, + corrections_tsv, +) from .find_max_value import find_max_value from .sensitivity_tuning import TunedKcatsResult, sensitivity_tuning from .sigma_fitter import SigmaFitterResult, fit_sigma, sigma_fitter @@ -33,8 +39,12 @@ def __getattr__(self, name: str): evotune = _MissingEvotune() # type: ignore[assignment] __all__ = [ + "Correction", "SigmaFitterResult", "TunedKcatsResult", + "annotate_from_model", + "corrections", + "corrections_tsv", "evotune", "find_max_value", "fit_sigma", diff --git a/src/geckopy/kcat_tuning/corrections.py b/src/geckopy/kcat_tuning/corrections.py new file mode 100644 index 0000000..79ed417 --- /dev/null +++ b/src/geckopy/kcat_tuning/corrections.py @@ -0,0 +1,163 @@ +"""Report a tuning result as the corrections it makes. + +A tuned kcat vector is one number per enzyme-reaction pair, and on a +genome-scale model that is thousands of numbers of which a few dozen +are supported by the data. Read as a vector it cannot be checked. Read +as a ranked list of changes, each carrying where its old value came +from and how much of the model's measured sensitivity it accounts for, +it can: a reviewer can look up whether one enzyme is really that slow. + +The ranking is by *leverage* -- how far the distance moves when that +kcat alone is perturbed -- rather than by how far the tuning moved it. +Sorting by movement puts the parameters the data cannot see at the top, +since those are free to drift furthest. ``cumulative_share`` says how +much of the total leverage the list has accounted for by each row, so a +reader can stop where it stops climbing. +""" +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import Optional, Sequence + +import numpy as np + +# Isozyme copies of one reaction are stored as separate ec rows with a +# suffix; the underlying reaction carries the name. +_ISOZYME_SUFFIX = "_EXP_" + + +@dataclass(frozen=True) +class Correction: + """One kcat the tuning changed, with its provenance.""" + + rxn_id: str + name: str + ec_code: str + source: str + kcat_prior: float + kcat_tuned: float + fold_change: float + leverage: float + leverage_share: float + cumulative_share: float + + +def _fold(new: np.ndarray, old: np.ndarray) -> np.ndarray: + return np.exp(np.abs(np.log(np.asarray(new, dtype=float) / old))) + + +def corrections( + kcat: np.ndarray, + kcat0: np.ndarray, + rxn_ids: Sequence[str], + *, + sources: Optional[Sequence[str]] = None, + names: Optional[Sequence[str]] = None, + ec_codes: Optional[Sequence[str]] = None, + leverage: Optional[np.ndarray] = None, + rel_tol: float = 0.02, +) -> list[Correction]: + """The changed kcats, most consequential first. + + Parameters + ---------- + kcat, kcat0 + Tuned and prior kcat vectors, same length as ``rxn_ids``. + sources, names, ec_codes + Per-kcat provenance. Missing entries are reported as ``""`` + rather than dropped, so a row never silently disappears. + leverage + Per-kcat effect on the distance from a one-at-a-time screen (see + :func:`~.tuning.screen_kcat_leverage`). If given, rows are + ordered by it and the share columns are filled; otherwise rows + are ordered by fold change and the shares are 0. + rel_tol + A kcat within ``1 + rel_tol`` fold of its prior is unchanged. + + Returns + ------- + list of :class:`Correction`, ordered most consequential first. + """ + kcat = np.asarray(kcat, dtype=float) + kcat0 = np.asarray(kcat0, dtype=float) + if kcat.shape != kcat0.shape: + raise ValueError( + f"kcat has shape {kcat.shape}; kcat0 has {kcat0.shape}." + ) + if len(rxn_ids) != len(kcat): + raise ValueError( + f"rxn_ids has {len(rxn_ids)} entries; kcat has {len(kcat)}." + ) + + fold = _fold(kcat, kcat0) + changed = np.flatnonzero(fold > 1.0 + rel_tol) + + if leverage is None: + lev = np.zeros(len(kcat)) + order = changed[np.argsort(fold[changed])[::-1]] + total = 0.0 + else: + lev = np.asarray(leverage, dtype=float) + if lev.shape != kcat.shape: + raise ValueError( + f"leverage has shape {lev.shape}; expected {kcat.shape}." + ) + order = changed[np.argsort(lev[changed])[::-1]] + total = float(lev.sum()) + + def at(seq, i): + return "" if seq is None else str(seq[i]) + + rows: list[Correction] = [] + running = 0.0 + for i in order: + share = lev[i] / total if total > 0 else 0.0 + running += share + rows.append(Correction( + rxn_id=str(rxn_ids[i]), + name=at(names, i), + ec_code=at(ec_codes, i), + source=at(sources, i), + kcat_prior=float(kcat0[i]), + kcat_tuned=float(kcat[i]), + fold_change=float(fold[i]), + leverage=float(lev[i]), + leverage_share=float(share), + cumulative_share=float(running), + )) + return rows + + +def corrections_tsv(rows: Sequence[Correction]) -> str: + """The corrections as a TSV document, header included.""" + cols = [f.name for f in fields(Correction)] + out = ["\t".join(cols)] + for r in rows: + vals = [] + for c in cols: + v = getattr(r, c) + vals.append(f"{v:.6g}" if isinstance(v, float) else str(v)) + out.append("\t".join(vals)) + return "\n".join(out) + "\n" + + +def annotate_from_model(model, rxn_ids: Sequence[str]) -> dict[str, list[str]]: + """Names, EC codes and sources for ``rxn_ids``, read off an ecModel. + + An isozyme copy carries a ``_EXP_n`` suffix that the underlying + reaction does not, so the name is looked up under the stripped id. + Anything absent is reported as ``""``. + """ + ec = model.ec + by_id = {r: i for i, r in enumerate(ec.rxns)} + names, ec_codes, sources = [], [], [] + for rxn_id in rxn_ids: + i = by_id.get(rxn_id) + ec_codes.append("" if i is None or ec.eccodes is None else str(ec.eccodes[i])) + sources.append("" if i is None or ec.source is None else str(ec.source[i])) + base = rxn_id.split(_ISOZYME_SUFFIX)[0] + try: + names.append(model.reactions.get_by_id(base).name or "") + except (KeyError, AttributeError): + names.append("") + return {"names": names, "ec_codes": ec_codes, "sources": sources} diff --git a/src/geckopy/kcat_tuning/evotune/tuning.py b/src/geckopy/kcat_tuning/evotune/tuning.py index f8c6fdd..e646d19 100644 --- a/src/geckopy/kcat_tuning/evotune/tuning.py +++ b/src/geckopy/kcat_tuning/evotune/tuning.py @@ -69,6 +69,7 @@ from cobra.util import ProcessPool from ...ec_model.pipeline.apply_kcat import apply_kcat_constraints +from ..corrections import Correction, corrections as build_corrections from .data import EvotuneData, load_evotune_data from .distance import evotune_distance, compute_excarbon from .parsimony import fold_change, n_changed @@ -128,6 +129,29 @@ class EvotuneResult: n_generations: int = 0 converged: bool = False + def corrections( + self, + *, + leverage: Optional[np.ndarray] = None, + names: Optional[Sequence[str]] = None, + ec_codes: Optional[Sequence[str]] = None, + rel_tol: float = 0.02, + ) -> list["Correction"]: + """The changes this run makes, most consequential first. + + The vector in ``new_kcat`` is the answer; this is the answer in + a form that can be checked. Pass ``leverage`` from a + one-at-a-time screen (:func:`screen_kcat_leverage`) to rank by + how much each change matters rather than by how far it moved, + and to fill the share columns. ``annotate_from_model`` supplies + ``names``/``ec_codes``. + """ + return build_corrections( + self.new_kcat, self.old_kcat, self.rxns, + sources=self.groups, names=names, ec_codes=ec_codes, + leverage=leverage, rel_tol=rel_tol, + ) + def _resolve_context(model, adapter, params, evotune_data, bio_rxn, okp_method): """Resolve every optional input against ``model``'s adapter, or diff --git a/src/geckopy/kcat_tuning/sensitivity_tuning.py b/src/geckopy/kcat_tuning/sensitivity_tuning.py index 28164e8..83cff05 100644 --- a/src/geckopy/kcat_tuning/sensitivity_tuning.py +++ b/src/geckopy/kcat_tuning/sensitivity_tuning.py @@ -26,7 +26,7 @@ import logging from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Sequence import numpy as np @@ -35,6 +35,7 @@ USAGE_PREFIX, ) from ..ec_model.pipeline.apply_kcat import apply_kcat_constraints +from .corrections import Correction, corrections as build_corrections if TYPE_CHECKING: from ..ec_model.ec_model import EcModel @@ -87,6 +88,28 @@ class TunedKcatsResult: ) source: list[str] = field(default_factory=list) + def corrections( + self, + *, + leverage: Optional[np.ndarray] = None, + ec_codes: Optional[Sequence[str]] = None, + rel_tol: float = 0.02, + ) -> list["Correction"]: + """The changes this run makes, most consequential first. + + Unlike a screen-backed result, sensitivity_tuning has no + leverage of its own to offer -- every bump was, by + construction, the bottleneck at the time it was made -- so + without ``leverage`` rows fall back to ordering by fold change. + ``rxn_names`` already carries resolved names, so there is no + need for ``annotate_from_model`` here. + """ + return build_corrections( + self.new_kcat, self.old_kcat, self.rxns, + sources=self.source, names=self.rxn_names, ec_codes=ec_codes, + leverage=leverage, rel_tol=rel_tol, + ) + def sensitivity_tuning( model: "EcModel", diff --git a/tests/test_corrections.py b/tests/test_corrections.py new file mode 100644 index 0000000..d7f3c6c --- /dev/null +++ b/tests/test_corrections.py @@ -0,0 +1,127 @@ +"""Tests for reporting a tuning result as its corrections.""" +import numpy as np +import pytest + +from geckopy.kcat_tuning.corrections import ( + Correction, annotate_from_model, corrections, corrections_tsv, +) + + +def test_corrections_lists_only_changed_kcats(): + kcat0 = np.array([1.0, 1.0, 1.0, 1.0]) + kcat = np.array([1.0, 1.01, 4.0, 0.5]) # unchanged, within tol, up, down + rows = corrections(kcat, kcat0, ["a", "b", "c", "d"]) + + assert [r.rxn_id for r in rows] == ["c", "d"] + # Fold change is direction-free: quartered and quadrupled both read 4. + assert rows[0].fold_change == pytest.approx(4.0) + assert rows[1].fold_change == pytest.approx(2.0) + assert rows[0].kcat_prior == 1.0 and rows[0].kcat_tuned == 4.0 + + +def test_corrections_rank_by_leverage_not_by_movement(): + """Sorting by how far a kcat moved puts the unidentifiable first.""" + kcat0 = np.ones(3) + kcat = np.array([100.0, 3.0, 2.0]) + leverage = np.array([0.001, 1.0, 0.5]) # the big mover barely matters + + by_leverage = corrections(kcat, kcat0, ["drifter", "big", "mid"], + leverage=leverage) + assert [r.rxn_id for r in by_leverage] == ["big", "mid", "drifter"] + + # Without leverage there is nothing to rank by but movement. + by_movement = corrections(kcat, kcat0, ["drifter", "big", "mid"]) + assert [r.rxn_id for r in by_movement] == ["drifter", "big", "mid"] + assert all(r.leverage_share == 0.0 for r in by_movement) + + +def test_cumulative_share_shows_where_the_list_stops_mattering(): + kcat0 = np.ones(4) + kcat = np.array([2.0, 2.0, 2.0, 1.0]) # three changed, one not + leverage = np.array([0.6, 0.3, 0.1, 5.0]) # the unchanged one dominates + + rows = corrections(kcat, kcat0, ["a", "b", "c", "untouched"], + leverage=leverage) + # Shares are of *total* leverage, so changing everything that matters + # is distinguishable from changing everything. + assert rows[0].leverage_share == pytest.approx(0.1) + assert rows[-1].cumulative_share == pytest.approx(0.16666, abs=1e-4) + assert [r.cumulative_share for r in rows] == sorted( + r.cumulative_share for r in rows) + + +def test_missing_provenance_is_reported_not_dropped(): + kcat0 = np.ones(2) + kcat = np.array([2.0, 3.0]) + rows = corrections(kcat, kcat0, ["a", "b"], sources=["brenda", "custom"]) + + assert [r.source for r in rows] == ["custom", "brenda"] # ordered by fold + assert all(r.name == "" and r.ec_code == "" for r in rows) + + +def test_corrections_reject_mismatched_inputs(): + with pytest.raises(ValueError, match="rxn_ids has"): + corrections(np.ones(3), np.ones(3), ["a"]) + with pytest.raises(ValueError, match="kcat0 has"): + corrections(np.ones(3), np.ones(2), ["a", "b", "c"]) + with pytest.raises(ValueError, match="leverage has shape"): + corrections(np.ones(3), np.ones(3), ["a", "b", "c"], + leverage=np.ones(2)) + + +def test_tsv_round_trips_every_field(): + rows = corrections(np.array([4.0]), np.array([1.0]), ["r_0698"], + sources=["brenda"], names=["lanosterol synthase"], + ec_codes=["5.4.99.7"], leverage=np.array([2.0])) + text = corrections_tsv(rows) + header, body = text.splitlines() + + assert header.split("\t")[0] == "rxn_id" + assert len(body.split("\t")) == len(header.split("\t")) + assert "lanosterol synthase" in body and "5.4.99.7" in body + assert text.endswith("\n") + + +def test_tsv_of_no_corrections_is_a_header_alone(): + rows = corrections(np.ones(2), np.ones(2), ["a", "b"]) + assert rows == [] + assert corrections_tsv(rows).strip().startswith("rxn_id") + + +class _FakeEc: + rxns = ["r_1", "r_2_EXP_1", "r_2_EXP_2"] + eccodes = ["1.1.1.1", "2.2.2.2", "2.2.2.2"] + source = ["brenda", "custom", "custom"] + + +class _FakeReaction: + def __init__(self, name): + self.name = name + + +class _FakeReactions: + def __init__(self, mapping): + self._m = mapping + + def get_by_id(self, rxn_id): + return _FakeReaction(self._m[rxn_id]) + + +class _FakeModel: + ec = _FakeEc() + reactions = _FakeReactions({"r_1": "alcohol dehydrogenase", + "r_2": "lanosterol synthase"}) + + +def test_annotations_strip_the_isozyme_suffix_to_find_the_name(): + got = annotate_from_model(_FakeModel(), ["r_2_EXP_2", "r_1"]) + + # The name lives on the reaction, which has no _EXP_ suffix. + assert got["names"] == ["lanosterol synthase", "alcohol dehydrogenase"] + assert got["ec_codes"] == ["2.2.2.2", "1.1.1.1"] + assert got["sources"] == ["custom", "brenda"] + + +def test_annotations_report_unknown_ids_as_blank(): + got = annotate_from_model(_FakeModel(), ["r_absent"]) + assert got == {"names": [""], "ec_codes": [""], "sources": [""]} diff --git a/tests/test_evotune.py b/tests/test_evotune.py index 2b59d37..796bbf4 100644 --- a/tests/test_evotune.py +++ b/tests/test_evotune.py @@ -423,3 +423,20 @@ def test_tune_prior_penalty_weight_single_seed_gives_nan_reproducibility(tmp_pat assert row["n_both_moved"] == 0 assert np.isnan(row["pct_direction_agree"]) assert np.isnan(row["median_fold_spread"]) + + +def test_result_reports_its_own_corrections(): + """The result already carries everything the table needs.""" + res = EvotuneResult( + rxns=["r_slow", "r_untouched", "r_drift"], + old_kcat=np.array([1.0, 5.0, 2.0]), + new_kcat=np.array([10.0, 5.0, 40.0]), + groups=["brenda", "custom", "okp"], + ) + rows = res.corrections(leverage=np.array([3.0, 1.0, 0.01])) + + # Ranked by leverage: the big drifter is last despite moving furthest. + assert [r.rxn_id for r in rows] == ["r_slow", "r_drift"] + assert rows[0].source == "brenda" + assert rows[0].fold_change == pytest.approx(10.0) + assert rows[0].cumulative_share == pytest.approx(3.0 / 4.01) diff --git a/tests/test_sensitivity_tuning.py b/tests/test_sensitivity_tuning.py index 5d7f779..b28446f 100644 --- a/tests/test_sensitivity_tuning.py +++ b/tests/test_sensitivity_tuning.py @@ -241,3 +241,21 @@ def test_result_has_expected_fields_populated(tmp_path): assert result.old_kcat[0] == pytest.approx(1.0) assert result.new_kcat[0] >= 10.0 assert result.source[0] == "initial" + + +def test_result_reports_its_own_corrections(): + """rxn_names is already resolved, so no annotate_from_model needed.""" + result = TunedKcatsResult( + rxns=["r_slow", "r_untouched", "r_drift"], + rxn_names=["slow enzyme", "untouched enzyme", "drifting enzyme"], + old_kcat=np.array([1.0, 5.0, 2.0]), + new_kcat=np.array([10.0, 5.0, 40.0]), + source=["brenda", "custom", "okp"], + ) + rows = result.corrections() + + # No leverage screen of its own, so rows fall back to fold-change order. + assert [r.rxn_id for r in rows] == ["r_drift", "r_slow"] + assert rows[0].name == "drifting enzyme" + assert rows[1].source == "brenda" + assert rows[1].fold_change == pytest.approx(10.0) From 0d3fddcffd843aa23a4e79f1b6ac1842297651c4 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 09:15:56 +0200 Subject: [PATCH 03/15] pyproject: pin raven-toolbox to the PyPI 3.0.0b1 release Switches off the git develop pin now that 3.0.0b1 carries the convert_to_irreversible(rxns=...) addition make_ec_model needs (#42). Drops allow-direct-references along with it, since nothing in dependencies needs a direct URL reference anymore -- PyPI would have rejected an sdist containing one. Full suite passes against the PyPI release (1410 passed, 1 skipped, 1 xfailed). --- pyproject.toml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 666f095..c08edad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,12 +44,10 @@ dependencies = [ # raven-toolbox provides the underlying model-manipulation primitives geckopy # extends, plus the ecModel YAML I/O (raven_toolbox.io.ec_data.EcData and the # ec-rxns / ec-enzymes / gecko_light sections in read/write_yaml_model). - # geckopy needs a develop-only addition (convert_to_irreversible(..., rxns=...), - # added for #42) that isn't in any PyPI release yet -- pinned to develop until - # raven-toolbox 3.0.0b1 ships, at which point this becomes "raven-toolbox>=3.0.0b1" - # (the "b1" in the specifier itself opts pip into matching pre-releases, so no - # --pre flag is needed downstream). - "raven-toolbox @ git+https://github.com/SysBioChalmers/raven-toolbox.git@develop", + # >=3.0.0b1: first release carrying convert_to_irreversible(..., rxns=...), + # which make_ec_model needs (#42). The "b1" in the specifier opts pip into + # matching pre-releases, so no --pre flag is needed downstream. + "raven-toolbox>=3.0.0b1", "requests>=2.28", "ruamel.yaml>=0.17", "scipy>=1.10", @@ -79,14 +77,6 @@ Issues = "https://github.com/SysBioChalmers/geckopy/issues" [project.scripts] geckopy = "geckopy.cli:main" -[tool.hatch.metadata] -# Allow `raven-toolbox @ git+https://...` (PEP 508 direct URL) in -# project.dependencies. hatchling refuses direct references by default -# because PyPI rejects sdists that contain them. Re-evaluate once -# raven-toolbox ships the `convert_to_irreversible(rxns=...)` addition -# in a release and geckopy can pin a version instead. -allow-direct-references = true - [tool.hatch.build.targets.wheel] packages = ["src/geckopy"] From f0c67272e2ea8d8677f6f4f884ae0e386dda17b5 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 09:33:48 +0200 Subject: [PATCH 04/15] Prepare 4.0.0b1 release --- CHANGELOG.md | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 44 ++++++++++++----------- pyproject.toml | 4 +-- 3 files changed, 123 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170e983..b7ac16f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,94 @@ All notable changes to **geckopy** are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses [PEP 440](https://peps.python.org/pep-0440/) pre-release versioning. +## [4.0.0b1] — 2026-09-08 + +Version realignment: geckopy adopts GECKO's own toolbox-generation numbering +(MATLAB GECKO 1–3, geckopy = 4.x) instead of its own from-zero semver — see +the README for why. First release published to PyPI, and the first to ship +kcat tuning against experimental data. + +### Added + +- **`geckopy.kcat_tuning.evotune`** — new subpackage fitting `ec.kcat` to + measured growth rates and exchange fluxes with CMA-ES (evaluated against + MATLAB's ABC-SMC approach, which it replaces): `screen_kcat_leverage`, + `select_tunable_mask`, `cmaes_kcat_tuning`, `tune_prior_penalty_weight`. + `EvotuneResult.corrections()` turns a tuning run into a ranked, + human-readable table of what changed and why (reaction, EC code, source, + prior/tuned value, fold change, leverage) instead of a raw kcat vector. + Gated behind the new `evotune` extra (`cma`); the base install is + unaffected. See [`docs/evotune_kcat_tuning.md`](docs/evotune_kcat_tuning.md). + (#51, #52) +- **`geckopy.gather_kcats.review.review_assignment`** — flags kcat-assignment + problems (repeated values, EC-maximum fallback, implausible magnitude) + worth curating before spending a tuning run on them. (#51) + +### Fixed + +- **`flexibilize_enz_concs`**: the post-loop refinement pinned + `bio_rxn.lower_bound` to `exp_growth` exactly; MATLAB's + `flexibilizeEnzConcs.m` uses a soft ±0.25% band instead, whose minimiser + settles at the band's low edge rather than at `exp_growth`. geckopy now + uses the same soft band, matching MATLAB's growth rate and flexed enzyme + bounds. (#41) +- **`make_ec_model`** was splitting exchange reactions in the + irreversibility stage; MATLAB's `makeEcModel.m` excludes them + (`nonExchRxns`). Latent divergence exposed once raven-toolbox's + `convert_to_irreversible` stopped special-casing exchange reactions by + default. (#42) +- **`get_reactions_from_enzyme`**: `protein_id` matching against + `ec.enzymes` was case-sensitive; MATLAB's `getReactionsFromEnzyme` uses + `strcmpi`, so `"p4"` now finds what `"P4"` finds. (#43) +- **`read_dlkcat_output`**: the summary log lumped unrecognized-substrate + drops into "dropped N row(s) with non-numeric kcat" even when the row's + kcat was numeric — drop reasons are now counted separately. New + `strict=True` option (default `False`) raises on any unrecognized + substrate, matching MATLAB's fail-fast behavior for callers who want it. + (#44) +- **`parse_okp_output`**: per-row provenance wasn't tagged with the `OKP-` + prefix MATLAB's `readOpenKineticsPredictorOutput.m` applies, so the same + prediction ended up unprefixed after `normalize_source`. `merge_kcats` + now strips a recognized `okp_` prefix before tiering/`source_priority` + matching, so a tagged row still ranks like its bare-sourced counterpart. + (#46) +- **`set_kcat_for_reactions`** rejected a per-isozyme kcat list passed + against an un-suffixed base reaction name; MATLAB's `setKcatForReactions` + accepts this and assigns positionally. Also wrote `ec.source = "manual"` + where MATLAB's actual (undocumented) value is `"setKcatForReactions"`. + Both now match. (#47) +- **sdist packaging** bundled the entire local `.venv-win` virtualenv + (including a compiled scipy wheel), ballooning it from ~6MB to ~60MB: + `.gitignore` listed `.venv/` but not `.venv-win/`, and hatchling's default + sdist file selection only reads the top-level `.gitignore`, not git's + nested per-directory ignore rules. + +### Changed + +- **`ec_fva`**'s parallel path now uses `cobra.util.process_pool.ProcessPool` + instead of a hand-rolled `multiprocessing.Pool`, matching the pattern + cobrapy's own deletion/FVA helpers use: fixes a slow initializer-arg + handoff on Windows and switches pool teardown from `terminate()` to a + graceful `close()` + `join()`. Public `ec_fva(...)` signature unchanged. + (#45) +- **`kcat_sensitivity_analysis` renamed to `kcat_tuning`**, with the CMA-ES + subpackage named `evotune` rather than `bayesian` — the shipped method is + CMA-ES, not Bayesian inference. (#51) + +### Dependencies + +- **`raven-toolbox` now pins `>=3.0.0b1`** (was a `git+...@develop` + reference) — that release carries `convert_to_irreversible(..., + rxns=...)`, which `make_ec_model` needs (#42). PyPI rejects direct URL + references in package metadata, so this also drops the + `allow-direct-references` hatchling override that existed only for it. + +### Documentation + +- README: disambiguation note simplified, and repointed at the specific old + `geckopy` release (`2.0.2` and earlier) now that this project has taken + over the PyPI project name itself. + ## [0.3.0] — 2026-08-30 Parity pass against the MATLAB GECKO Toolbox: geckopy's core functions now @@ -360,6 +448,7 @@ ecModel build is ported; the yeast-GEM tutorial runs end-to-end. [`docs/raven_integration.md`](docs/raven_integration.md) for the current delegation and the planned future migrations. +[4.0.0b1]: https://github.com/SysBioChalmers/geckopy/releases/tag/v4.0.0b1 [0.3.0]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.3.0 [0.2.1]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.2.1 [0.2.0]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.2.0 @@ -394,3 +483,12 @@ ecModel build is ported; the yeast-GEM tutorial runs end-to-end. [#37]: https://github.com/SysBioChalmers/geckopy/pull/37 [#38]: https://github.com/SysBioChalmers/geckopy/pull/38 [#39]: https://github.com/SysBioChalmers/geckopy/pull/39 +[#41]: https://github.com/SysBioChalmers/geckopy/pull/41 +[#42]: https://github.com/SysBioChalmers/geckopy/pull/42 +[#43]: https://github.com/SysBioChalmers/geckopy/pull/43 +[#44]: https://github.com/SysBioChalmers/geckopy/pull/44 +[#45]: https://github.com/SysBioChalmers/geckopy/pull/45 +[#46]: https://github.com/SysBioChalmers/geckopy/pull/46 +[#47]: https://github.com/SysBioChalmers/geckopy/pull/47 +[#51]: https://github.com/SysBioChalmers/geckopy/pull/51 +[#52]: https://github.com/SysBioChalmers/geckopy/pull/52 diff --git a/README.md b/README.md index 4a86e49..7694e5f 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ Enzyme-constrained genome-scale metabolic modelling in Python. -> **Disambiguation — this is not "geckopy 3.0".** The current geckopy is a new, -> from-scratch port of the MATLAB [GECKO Toolbox](https://github.com/SysBioChalmers/GECKO). -> It is unrelated to, and shares no code with, the earlier and separate `geckopy` -> Python package ([on PyPI as `geckopy`](https://pypi.org/project/geckopy/)) described in -> Carrasco Muriel, Long & Sonnenschein, *Microbiology Spectrum* 11(6):e01705-23 (2023), -> [doi:10.1128/spectrum.01705-23](https://doi.org/10.1128/spectrum.01705-23). The two -> projects share only the name. +> **Disambiguation.** This is a new, from-scratch port of the +> [GECKO Toolbox](https://github.com/SysBioChalmers/GECKO) (MATLAB). It is unrelated to +> the earlier and separate `geckopy` Python package +> ([on PyPI as release 2.0.2 and earlier](https://pypi.org/project/geckopy/2.0.2/)) +> described in Carrasco Muriel, Long & Sonnenschein, *Microbiology Spectrum* +> 11(6):e01705-23 (2023), [doi:10.1128/spectrum.01705-23](https://doi.org/10.1128/spectrum.01705-23). ## What this is @@ -27,33 +26,36 @@ the standard modelling layer, then layers GECKO's enzyme-constraint machinery on top. It fetches kcat values from BRENDA, predicts them with DLKcat, applies custom curations, and integrates proteomics measurements. -> **Status: alpha.** All the MATLAB GECKO 3.2.5 functions used in the -> standard ecModel build are ported. The yeast-GEM tutorial runs -> end-to-end. Not yet on PyPI; install from GitHub for now. +> **Status: beta (`4.0.0b1`).** All the MATLAB GECKO 3.2.5 functions used in +> the standard ecModel build are ported, plus CMA-ES-based kcat tuning +> against experimental data. The yeast-GEM tutorial runs end-to-end. +> geckopy's version tracks the GECKO project's own toolbox-generation +> numbering (MATLAB GECKO 1–3, geckopy = 4.x), not a semantic-versioning +> API-stability claim. ## Install -geckopy depends on [raven-toolbox](https://github.com/SysBioChalmers/raven-toolbox) -(the Python port of the RAVEN Toolbox), which is not yet on PyPI. Install both -from GitHub: - ```bash -pip install \ - git+https://github.com/SysBioChalmers/raven-toolbox.git@develop \ - git+https://github.com/SysBioChalmers/geckopy.git@develop +pip install --pre geckopy ``` -Once raven-toolbox and geckopy are published to PyPI, this collapses to -`pip install geckopy` (raven-toolbox will be pulled in transitively). +geckopy is currently published as a pre-release, so `--pre` (or an exact +pin, e.g. `geckopy==4.0.0b1`) is required until the `4.0.0` final release — +plain `pip install geckopy` won't pick it up before then. +[raven-toolbox](https://github.com/SysBioChalmers/raven-toolbox) (the Python +port of the RAVEN Toolbox) is pulled in automatically. Optional extras: ```bash # Adds matplotlib (needed to render plots from the tutorial) -pip install "geckopy[tutorial] @ git+https://github.com/SysBioChalmers/geckopy.git@develop" +pip install --pre "geckopy[tutorial]" + +# Adds cma (needed for CMA-ES kcat tuning, geckopy.kcat_tuning.evotune) +pip install --pre "geckopy[evotune]" # Adds pytest + ruff (only needed if you're contributing) -pip install "geckopy[dev] @ git+https://github.com/SysBioChalmers/geckopy.git@develop" +pip install --pre "geckopy[dev]" ``` Requires Python 3.11 or newer. diff --git a/pyproject.toml b/pyproject.toml index c08edad..bbaa3f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "geckopy" -version = "0.3.0" +version = "4.0.0b1" description = "Enzyme-constrained genome-scale metabolic modeling in Python, inspired by the GECKO Toolbox" readme = "README.md" requires-python = ">=3.11" @@ -22,7 +22,7 @@ keywords = [ "constraint-based-modeling", ] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", From 7c64386a72fb4d9dbe54f7d874817bfd9596b9ed Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 10:41:54 +0200 Subject: [PATCH 05/15] CHANGELOG: restyle 4.0.0b1 entry to match raven-toolbox, add #49/#50 Flattens the Keep-a-Changelog ### Added/Fixed/Changed subsections into one bullet list (matching raven-toolbox's CHANGELOG.md format) and drops the version header's brackets to match. Also folds in #49 and #50, which had merged straight to main since 0.3.0 and were missing from develop until the merge two commits back. --- CHANGELOG.md | 129 ++++++++++++++++++++------------------------------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7ac16f..3cf4e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,92 +4,64 @@ All notable changes to **geckopy** are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses [PEP 440](https://peps.python.org/pep-0440/) pre-release versioning. -## [4.0.0b1] — 2026-09-08 +## 4.0.0b1 — 2026-09-08 Version realignment: geckopy adopts GECKO's own toolbox-generation numbering (MATLAB GECKO 1–3, geckopy = 4.x) instead of its own from-zero semver — see the README for why. First release published to PyPI, and the first to ship kcat tuning against experimental data. -### Added - -- **`geckopy.kcat_tuning.evotune`** — new subpackage fitting `ec.kcat` to - measured growth rates and exchange fluxes with CMA-ES (evaluated against - MATLAB's ABC-SMC approach, which it replaces): `screen_kcat_leverage`, - `select_tunable_mask`, `cmaes_kcat_tuning`, `tune_prior_penalty_weight`. - `EvotuneResult.corrections()` turns a tuning run into a ranked, - human-readable table of what changed and why (reaction, EC code, source, - prior/tuned value, fold change, leverage) instead of a raw kcat vector. - Gated behind the new `evotune` extra (`cma`); the base install is - unaffected. See [`docs/evotune_kcat_tuning.md`](docs/evotune_kcat_tuning.md). - (#51, #52) -- **`geckopy.gather_kcats.review.review_assignment`** — flags kcat-assignment +* New: `kcat_tuning.evotune` fits `ec.kcat` to measured growth rates and + exchange fluxes with CMA-ES (`screen_kcat_leverage`, `select_tunable_mask`, + `cmaes_kcat_tuning`, `tune_prior_penalty_weight`), replacing the earlier + ABC-SMC approach. `EvotuneResult.corrections()` reports a tuning run as a + ranked table of what changed and why, rather than a raw kcat vector. Gated + behind the new `evotune` extra. (#51, #52) +* New: `gather_kcats.review.review_assignment` flags kcat-assignment problems (repeated values, EC-maximum fallback, implausible magnitude) - worth curating before spending a tuning run on them. (#51) - -### Fixed - -- **`flexibilize_enz_concs`**: the post-loop refinement pinned - `bio_rxn.lower_bound` to `exp_growth` exactly; MATLAB's - `flexibilizeEnzConcs.m` uses a soft ±0.25% band instead, whose minimiser - settles at the band's low edge rather than at `exp_growth`. geckopy now - uses the same soft band, matching MATLAB's growth rate and flexed enzyme - bounds. (#41) -- **`make_ec_model`** was splitting exchange reactions in the - irreversibility stage; MATLAB's `makeEcModel.m` excludes them - (`nonExchRxns`). Latent divergence exposed once raven-toolbox's - `convert_to_irreversible` stopped special-casing exchange reactions by - default. (#42) -- **`get_reactions_from_enzyme`**: `protein_id` matching against - `ec.enzymes` was case-sensitive; MATLAB's `getReactionsFromEnzyme` uses - `strcmpi`, so `"p4"` now finds what `"P4"` finds. (#43) -- **`read_dlkcat_output`**: the summary log lumped unrecognized-substrate - drops into "dropped N row(s) with non-numeric kcat" even when the row's - kcat was numeric — drop reasons are now counted separately. New - `strict=True` option (default `False`) raises on any unrecognized - substrate, matching MATLAB's fail-fast behavior for callers who want it. - (#44) -- **`parse_okp_output`**: per-row provenance wasn't tagged with the `OKP-` - prefix MATLAB's `readOpenKineticsPredictorOutput.m` applies, so the same - prediction ended up unprefixed after `normalize_source`. `merge_kcats` - now strips a recognized `okp_` prefix before tiering/`source_priority` - matching, so a tagged row still ranks like its bare-sourced counterpart. - (#46) -- **`set_kcat_for_reactions`** rejected a per-isozyme kcat list passed - against an un-suffixed base reaction name; MATLAB's `setKcatForReactions` - accepts this and assigns positionally. Also wrote `ec.source = "manual"` - where MATLAB's actual (undocumented) value is `"setKcatForReactions"`. - Both now match. (#47) -- **sdist packaging** bundled the entire local `.venv-win` virtualenv - (including a compiled scipy wheel), ballooning it from ~6MB to ~60MB: - `.gitignore` listed `.venv/` but not `.venv-win/`, and hatchling's default - sdist file selection only reads the top-level `.gitignore`, not git's - nested per-directory ignore rules. - -### Changed - -- **`ec_fva`**'s parallel path now uses `cobra.util.process_pool.ProcessPool` - instead of a hand-rolled `multiprocessing.Pool`, matching the pattern - cobrapy's own deletion/FVA helpers use: fixes a slow initializer-arg - handoff on Windows and switches pool teardown from `terminate()` to a - graceful `close()` + `join()`. Public `ec_fva(...)` signature unchanged. - (#45) -- **`kcat_sensitivity_analysis` renamed to `kcat_tuning`**, with the CMA-ES + worth curating before a tuning run. (#51) +* `kcat_sensitivity_analysis` renamed to `kcat_tuning`, with the CMA-ES subpackage named `evotune` rather than `bayesian` — the shipped method is CMA-ES, not Bayesian inference. (#51) - -### Dependencies - -- **`raven-toolbox` now pins `>=3.0.0b1`** (was a `git+...@develop` - reference) — that release carries `convert_to_irreversible(..., - rxns=...)`, which `make_ec_model` needs (#42). PyPI rejects direct URL - references in package metadata, so this also drops the - `allow-direct-references` hatchling override that existed only for it. - -### Documentation - -- README: disambiguation note simplified, and repointed at the specific old - `geckopy` release (`2.0.2` and earlier) now that this project has taken +* `flexibilize_enz_concs`: the post-loop refinement now uses MATLAB's soft + ±0.25% growth band instead of pinning to `exp_growth` exactly, matching + `flexibilizeEnzConcs.m`. (#41) +* `make_ec_model` no longer splits exchange reactions in the irreversibility + stage, matching `makeEcModel.m`'s `nonExchRxns` exclusion. (#42) +* `get_reactions_from_enzyme` now matches `protein_id` case-insensitively, + matching MATLAB's `strcmpi`. (#43) +* `read_dlkcat_output` gained an opt-in `strict=True` fail-fast on + unrecognized substrates, matching MATLAB, and now reports drop reasons + separately instead of lumping them together. (#44) +* `assign_standard_kcat`: a NaN-kcat reaction no longer counts toward a + subsystem's threshold eligibility or gets folded into the subsystem + median; unset kcats now get filled for NaN too, not just literal zeros. + (#49) +* `parse_okp_output` now tags provenance with the `OKP-` prefix MATLAB + applies; `merge_kcats` strips it again before source-priority matching so + tagged rows still rank correctly. (#46) +* `set_kcat_for_reactions` now accepts a per-isozyme kcat list against an + un-suffixed base name (assigned positionally, matching MATLAB), and writes + `ec.source = "setKcatForReactions"` (MATLAB's actual value) instead of + `"manual"`. (#47) +* `ec_fseof`: scan floor now anchors to the production target's flux at + biomass-optimum instead of a fixed fraction of the theoretical maximum, + and candidates are restricted to gene-associated, non-standard reactions + before classification — both matching `ecFSEOF.m`. (#50) +* `ec_fva`'s parallel path now uses `cobra.util.process_pool.ProcessPool` + instead of a hand-rolled `multiprocessing.Pool` — fixes a slow Windows + initializer handoff and gives graceful pool teardown. Public signature + unchanged. (#45) +* `raven-toolbox` dependency switched from a `git+...@develop` pin to the + PyPI `>=3.0.0b1` release, which carries the `convert_to_irreversible(..., + rxns=...)` addition #42 needs. +* Packaging: the sdist was bundling the entire local `.venv-win` + virtualenv (~60MB, including a compiled scipy wheel) because `.gitignore` + listed `.venv/` but not `.venv-win/`, and hatchling's default sdist + selection only reads the top-level `.gitignore`. +* `release.yml`: PyPI publish job enabled (trusted publishing). +* README: disambiguation note simplified and repointed at the specific old + `geckopy` release (`2.0.2` and earlier), now that this project has taken over the PyPI project name itself. ## [0.3.0] — 2026-08-30 @@ -448,7 +420,6 @@ ecModel build is ported; the yeast-GEM tutorial runs end-to-end. [`docs/raven_integration.md`](docs/raven_integration.md) for the current delegation and the planned future migrations. -[4.0.0b1]: https://github.com/SysBioChalmers/geckopy/releases/tag/v4.0.0b1 [0.3.0]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.3.0 [0.2.1]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.2.1 [0.2.0]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.2.0 @@ -490,5 +461,7 @@ ecModel build is ported; the yeast-GEM tutorial runs end-to-end. [#45]: https://github.com/SysBioChalmers/geckopy/pull/45 [#46]: https://github.com/SysBioChalmers/geckopy/pull/46 [#47]: https://github.com/SysBioChalmers/geckopy/pull/47 +[#49]: https://github.com/SysBioChalmers/geckopy/pull/49 +[#50]: https://github.com/SysBioChalmers/geckopy/pull/50 [#51]: https://github.com/SysBioChalmers/geckopy/pull/51 [#52]: https://github.com/SysBioChalmers/geckopy/pull/52 From 2e808fdac0e4b0b32d8670c950366f852e0d8800 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 11:21:45 +0200 Subject: [PATCH 06/15] CHANGELOG: drop PR citations, restyle 0.x entries to match raven-toolbox Removes every (#NN) / ([#NN]) citation and the now-unused version/PR link-reference block at the bottom of the file. Flattens each 0.x.y entry's Keep-a-Changelog ### Added/Fixed/Changed/... subsections into one bullet list per raven-toolbox's CHANGELOG.md format, matching what the 4.0.0b1 entry already does. --- CHANGELOG.md | 591 ++++++++++++++++++++------------------------------- 1 file changed, 233 insertions(+), 358 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cf4e50..777a750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,6 @@ # Changelog -All notable changes to **geckopy** are documented here. The format follows -[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses +All notable changes to **geckopy** are documented here. The project uses [PEP 440](https://peps.python.org/pep-0440/) pre-release versioning. ## 4.0.0b1 — 2026-09-08 @@ -16,45 +15,44 @@ kcat tuning against experimental data. `cmaes_kcat_tuning`, `tune_prior_penalty_weight`), replacing the earlier ABC-SMC approach. `EvotuneResult.corrections()` reports a tuning run as a ranked table of what changed and why, rather than a raw kcat vector. Gated - behind the new `evotune` extra. (#51, #52) + behind the new `evotune` extra. * New: `gather_kcats.review.review_assignment` flags kcat-assignment problems (repeated values, EC-maximum fallback, implausible magnitude) - worth curating before a tuning run. (#51) + worth curating before a tuning run. * `kcat_sensitivity_analysis` renamed to `kcat_tuning`, with the CMA-ES subpackage named `evotune` rather than `bayesian` — the shipped method is - CMA-ES, not Bayesian inference. (#51) + CMA-ES, not Bayesian inference. * `flexibilize_enz_concs`: the post-loop refinement now uses MATLAB's soft ±0.25% growth band instead of pinning to `exp_growth` exactly, matching - `flexibilizeEnzConcs.m`. (#41) + `flexibilizeEnzConcs.m`. * `make_ec_model` no longer splits exchange reactions in the irreversibility - stage, matching `makeEcModel.m`'s `nonExchRxns` exclusion. (#42) + stage, matching `makeEcModel.m`'s `nonExchRxns` exclusion. * `get_reactions_from_enzyme` now matches `protein_id` case-insensitively, - matching MATLAB's `strcmpi`. (#43) + matching MATLAB's `strcmpi`. * `read_dlkcat_output` gained an opt-in `strict=True` fail-fast on unrecognized substrates, matching MATLAB, and now reports drop reasons - separately instead of lumping them together. (#44) + separately instead of lumping them together. * `assign_standard_kcat`: a NaN-kcat reaction no longer counts toward a subsystem's threshold eligibility or gets folded into the subsystem median; unset kcats now get filled for NaN too, not just literal zeros. - (#49) * `parse_okp_output` now tags provenance with the `OKP-` prefix MATLAB applies; `merge_kcats` strips it again before source-priority matching so - tagged rows still rank correctly. (#46) + tagged rows still rank correctly. * `set_kcat_for_reactions` now accepts a per-isozyme kcat list against an un-suffixed base name (assigned positionally, matching MATLAB), and writes `ec.source = "setKcatForReactions"` (MATLAB's actual value) instead of - `"manual"`. (#47) + `"manual"`. * `ec_fseof`: scan floor now anchors to the production target's flux at biomass-optimum instead of a fixed fraction of the theoretical maximum, and candidates are restricted to gene-associated, non-standard reactions - before classification — both matching `ecFSEOF.m`. (#50) + before classification — both matching `ecFSEOF.m`. * `ec_fva`'s parallel path now uses `cobra.util.process_pool.ProcessPool` instead of a hand-rolled `multiprocessing.Pool` — fixes a slow Windows initializer handoff and gives graceful pool teardown. Public signature - unchanged. (#45) + unchanged. * `raven-toolbox` dependency switched from a `git+...@develop` pin to the PyPI `>=3.0.0b1` release, which carries the `convert_to_irreversible(..., - rxns=...)` addition #42 needs. + rxns=...)` addition `make_ec_model` needs. * Packaging: the sdist was bundling the entire local `.venv-win` virtualenv (~60MB, including a compiled scipy wheel) because `.gitignore` listed `.venv/` but not `.venv-win/`, and hatchling's default sdist @@ -64,404 +62,281 @@ kcat tuning against experimental data. `geckopy` release (`2.0.2` and earlier), now that this project has taken over the PyPI project name itself. -## [0.3.0] — 2026-08-30 +## 0.3.0 — 2026-08-30 Parity pass against the MATLAB GECKO Toolbox: geckopy's core functions now reproduce every value GECKO's own unit-test suite pins down. -### Added - -- **`tests/test_gecko_matlab_parity.py` — a 1:1 port of GECKO's - `test/unit_tests/geckoCoreFunctionTests.m`** (47 tests, MATLAB test cases +* New: `tests/test_gecko_matlab_parity.py`, a 1:1 port of GECKO's + `test/unit_tests/geckoCoreFunctionTests.m` (47 tests, MATLAB test cases `tc0001`–`tc0013`), run against the same `ecTestGEM` fixture MATLAB uses. Every expected value is copied verbatim from the MATLAB sources rather than - re-derived from geckopy's output, so the file is the executable definition of - "geckopy builds the same ecModel as GECKO". It needs no MATLAB to run and - covers `makeEcModel` (full and light), `applyComplexData`, `setProtPoolSize`, - `getECfromGEM`, `getECfromDatabase`, save/load round-trip, - `fuzzyKcatMatching`, `writeDLKcatInput`, `mergeDLKcatAndFuzzyKcats`, - `selectKcatValue`, `applyKcatConstraints`, `getKcatAcrossIsozymes`, - `applyCustomKcats`, `findMetSmiles` and the proteomics-integration chain. - -### Fixed - -- **`make_ec_model` sorts identifiers for full models**, as MATLAB - `makeEcModel` does (`sortIdentifiers`, line 203, full models only, before the - protein pseudoreactions are appended). geckopy kept the input GEM's reaction + re-derived from geckopy's output, so the file is the executable definition + of "geckopy builds the same ecModel as GECKO". Covers `makeEcModel` (full + and light), `applyComplexData`, `setProtPoolSize`, `getECfromGEM`, + `getECfromDatabase`, save/load round-trip, `fuzzyKcatMatching`, + `writeDLKcatInput`, `mergeDLKcatAndFuzzyKcats`, `selectKcatValue`, + `applyKcatConstraints`, `getKcatAcrossIsozymes`, `applyCustomKcats`, + `findMetSmiles` and the proteomics-integration chain. +* `make_ec_model` now sorts identifiers for full models, as MATLAB + `makeEcModel` does (`sortIdentifiers`, full models only, before the protein + pseudoreactions are appended) — geckopy kept the input GEM's reaction order, so a full ecModel came out with the same reactions in a different - order from MATLAB's — visible in the saved YAML and in every downstream table - that follows `ec.rxns` order. -- **`write_dlkcat_input` emits rows in reaction order**, matching MATLAB's - column-major `find(clearedRedS < 0)`. numpy's `where` is row-major, so + order from MATLAB's. +* `write_dlkcat_input` now emits rows in reaction order, matching MATLAB's + column-major `find(clearedRedS < 0)` — numpy's `where` is row-major, so geckopy grouped the DLKcat input by substrate instead of by reaction. -- **`EcModel.copy()` clones the `ec` substructure.** `cobra.Model.copy` - rebuilds the network but carries every other attribute over by reference, - so `model.copy().ec` *was* the same `EcData` object as `model.ec`: writing +* `EcModel.copy()` now clones the `ec` substructure. `cobra.Model.copy` + rebuilds the network but carried every other attribute over by reference, + so `model.copy().ec` was the same `EcData` object as `model.ec` — writing a kcat, eccode, source, conc or coupling coefficient on the copy silently - changed the model it was copied from. The copy's `enzymes` view was - likewise still bound to the original model, so reads and writes through - `model.enzymes` went to the wrong model. `copy()` now clones `ec` and - rebinds `enzymes`, giving the value semantics MATLAB GECKO has throughout. - `adapter` stays a shared reference (project configuration, not model - state). `copy.deepcopy(model)` was already correct and is unchanged. -- **Reverse (`_REV`) reactions keep their EC code**, via a fix in + changed the model it was copied from. `adapter` stays a shared reference + (project configuration, not model state); `copy.deepcopy(model)` was + already correct and is unchanged. +* Reverse (`_REV`) reactions now keep their EC code, via a fix in raven-toolbox's `convert_to_irreversible` (it now copies annotations, - subsystem and notes onto the reverse reaction, as MATLAB's `convertToIrrev` - copies `eccodes` / `rxnMiriams` / `subSystems` / `rxnNotes`). Without it - `fill_eccodes_from_gem` returned `''` for every `_REV` reaction, fuzzy BRENDA - matching found no kcat, and reverse directions were left unconstrained. - **Requires raven-toolbox at or after that fix.** -- **`copy.deepcopy(ec_model)` (and of any `cobra.Model`) no longer - `RecursionError`s on Python 3.14.** The cause was in cobra, not geckopy: - `Reaction.__copy__`/`__deepcopy__` delegated to `super()` as a way to avoid - infinitely recursing into themselves, but that trick relies on `copy` - memoizing an object before recursing into its state, and Python 3.14's - `copy` module no longer does so for that code path. Every reaction holds - its metabolites and every metabolite holds the reactions it participates - in, so the resulting cycle exists in any real model, full ecModels - included — deep-copying one recursed until the interpreter's recursion - limit gave out. Fixed upstream in cobra 0.31.0 by deleting the - overrides; **requires cobra at or after 0.31.1**. -- **`examples/yeast-GEM`'s adapter config used a rejected pydantic field.** + subsystem and notes onto the reverse reaction, as MATLAB's + `convertToIrrev` copies `eccodes`/`rxnMiriams`/`subSystems`/`rxnNotes`). + Requires raven-toolbox at or after that fix. +* `copy.deepcopy(ec_model)` (and of any `cobra.Model`) no longer + `RecursionError`s on Python 3.14. The cause was in cobra: + `Reaction.__copy__`/`__deepcopy__` delegated to `super()` to avoid + infinitely recursing into themselves, relying on `copy` memoizing an + object before recursing into its state — Python 3.14's `copy` module + stopped doing so for that code path. Fixed upstream in cobra 0.31.0 by + deleting the overrides; requires cobra at or after 0.31.1. +* `examples/yeast-GEM`'s adapter config used a rejected pydantic field: `model_adapter.toml` set `uniprot.tax_id`, but `ModelParameters`'s schema - (`additionalProperties=false`) has no such field — it's called `id`, and - already defaults to the taxonomy id — so `ModelAdapter.from_folder` raised a - validation error on this folder unconditionally. A parametrized regression - test now runs over every `examples/` subfolder so schema drift like this is - caught for any future adapter config too. (#34) - -### Documentation - -- Docstrings across geckopy's source and tests reviewed for clarity and + has no such field (it's `id`, already defaulting to the taxonomy id), so + `ModelAdapter.from_folder` raised a validation error unconditionally. A + parametrized regression test now runs over every `examples/` subfolder to + catch schema drift like this for any future adapter config too. +* Docstrings across geckopy's source and tests reviewed for clarity and accuracy: `MATLAB-COMPAT` comparison paragraphs replaced with plain statements of current behavior, other historical/regression narration - dropped, and several docstrings that no longer matched the code corrected. - Comment/docstring-only; no functional changes. (#36) -- **`protein_pool.py`'s module docstring no longer claims `prot_pool_exchange`'s - direction diverges from MATLAB pending a "future I/O layer".** GECKO - MATLAB's `makeEcModel.m` already writes the same forward (`lb=0`, `ub=1000`) - convention as geckopy; the reverse convention only applies to legacy - pre-GECKO-4 files, which `load_ec_model.py` already normalizes on load. (#38) -- **`mw.py`'s stale `MATLAB-COMPAT` comments removed** now that `calculate_mw` + dropped, stale docstrings corrected. Comment/docstring-only, no functional + changes. +* `protein_pool.py`'s module docstring no longer claims `prot_pool_exchange`'s + direction diverges from MATLAB pending a "future I/O layer" — GECKO + MATLAB's `makeEcModel.m` already writes the same forward (`lb=0`, + `ub=1000`) convention as geckopy; the reverse convention only applies to + legacy pre-GECKO-4 files, which `load_ec_model.py` already normalizes on + load. +* `mw.py`'s stale `MATLAB-COMPAT` comments removed now that `calculate_mw` matches GECKO MATLAB's current water-mass constant (18.01528) and mean X-residue mass (118.885), closed upstream as GECKO#459. No behavioral - change — the code already computed the now-matching values. (#39) - -### Internal - -- Removed the dead `_run_preprocess` stub from `test_populate_ec`: an empty - helper (docstring and comments, no body) that was never called. The - preprocessing stages it described are already exercised elsewhere. (#37) - -### Notes - -- `merge_kcats` is n-ary and concatenates surviving rows list by list, so the - equivalent of MATLAB's `mergeDLKcatAndFuzzyKcats(dlkcat, fuzzy)` — which - always emits fuzzy rows first regardless of argument order — is + change. +* Removed the dead `_run_preprocess` stub from `test_populate_ec`: an empty + helper that was never called. +* `merge_kcats` is n-ary and concatenates surviving rows list by list, so + the equivalent of MATLAB's `mergeDLKcatAndFuzzyKcats(dlkcat, fuzzy)` — + which always emits fuzzy rows first regardless of argument order — is `merge_kcats(fuzzy, dlkcat, ...)`. The deprecated `merge_dlkcat_and_fuzzy_kcats` alias is `merge_kcats`, so it follows the geckopy convention, not MATLAB's signature. Row order has no numerical effect: `apply_kcat_list` aggregates per reaction. -## [0.2.1] — 2026-07-16 +## 0.2.1 — 2026-07-16 Bugfix release. `make_ec_model` no longer mutates the GEM it is given. -### Fixed - -- **`make_ec_model` leaves the input GEM unchanged.** Stages 1–5 +* `make_ec_model` now leaves the input GEM unchanged. Stages 1–5 (`remove_pseudoreaction_gprs`, `invert_backwards_only_reactions`, `convert_to_irreversible`, `expand_model`) ran in place on the caller's - model, leaving it with `_REV` / `_EXP_` reaction splits, `prot_` - pseudometabolites, and stripped pseudoreaction GPRs. The pipeline now works - on an internal `model.copy()`, matching MATLAB GECKO's `makeEcModel` value - semantics. (#32) -- `test_loads_sbml_gem` on Windows: the test wrote a backslash path into the + model, leaving it with `_REV`/`_EXP_` reaction splits, `prot_` + pseudometabolites, and stripped pseudoreaction GPRs. The pipeline now + works on an internal `model.copy()`, matching MATLAB GECKO's + `makeEcModel` value semantics. +* `test_loads_sbml_gem` on Windows: the test wrote a backslash path into the adapter TOML, which TOML reads as an escape sequence. - -### Documentation - -- README: disambiguation note stating that this geckopy is a from-scratch port - of the MATLAB GECKO Toolbox, unrelated to and sharing no code with the - separate PyPI `geckopy` package described in Carrasco Muriel et al. (2023). - -### Internal - -- **CI: leaner test matrix** — full Python range (3.11–3.13) on Linux plus a +* README: disambiguation note stating that this geckopy is a from-scratch + port of the MATLAB GECKO Toolbox, unrelated to and sharing no code with + the separate PyPI `geckopy` package described in Carrasco Muriel et al. + (2023). +* CI: leaner test matrix — full Python range (3.11–3.13) on Linux plus a single Python (3.12) on macOS and Windows, instead of the full 3×3 - OS × Python grid. Same OS coverage at roughly half the runner cost. (#31) -- Removed unused test imports flagged by ruff (F401). - -## [0.2.0] — 2026-06-14 - -Dependency rename + CI hardening. geckopy now targets **raven-toolbox** -(the renamed raven-python) and runs its test matrix on every pull request. - -### Changed - -- **Depend on `raven-toolbox`** (renamed from `raven-python`): all imports are - now `raven_toolbox.*`, and the `pyproject.toml` git-URL dependency and project - URL point at `SysBioChalmers/raven-toolbox`. (#29) -- Use the public `raven_toolbox.manipulation.expand.gpr_to_dnf` instead of the - former private `_gpr_to_dnf`. (#26) -- **CI: run the test matrix on every pull request**, not only PRs into - `main` / `master`. -- **CI: bump GitHub Actions to Node 24 runtimes** — `checkout@v5`, - `setup-python@v6`, `upload-artifact@v7`, `download-artifact@v8`. (#27) - -### Documentation - -- Rewrote `migrating_from_gecko_matlab.md` as a complete reference (#25); moved - internal planning notes into `docs/internal/` (#28). - -## [0.1.0a3] — 2026-05-30 - -Build-config hotfix. v0.1.0a2 was tagged but its release.yml build -failed: hatchling refuses PEP 508 direct-URL dependencies -(`raven-toolbox @ git+https://...`) by default. The v0.1.0a2 tag and -release were deleted; v0.1.0a3 is the published artifact carrying the -0.1.0a2 content described below. No code changes vs. v0.1.0a2. - -### Fixed - -- `pyproject.toml`: `[tool.hatch.metadata] allow-direct-references = - true` so hatchling accepts the `raven-toolbox` git URL dependency. - PyPI publish stays disabled (geckopy's name is taken, raven-toolbox - isn't on PyPI yet); the direct reference is acceptable for the - git-install path the alpha series uses. - -## [0.1.0a2] — 2026-05-30 + OS × Python grid. Same OS coverage at roughly half the runner cost. +* Removed unused test imports flagged by ruff (F401). + +## 0.2.0 — 2026-06-14 + +Dependency rename + CI hardening. geckopy now targets raven-toolbox (the +renamed raven-python) and runs its test matrix on every pull request. + +* Depend on `raven-toolbox` (renamed from `raven-python`): all imports are + now `raven_toolbox.*`, and the `pyproject.toml` git-URL dependency and + project URL point at `SysBioChalmers/raven-toolbox`. +* Use the public `raven_toolbox.manipulation.expand.gpr_to_dnf` instead of + the former private `_gpr_to_dnf`. +* CI: run the test matrix on every pull request, not only PRs into + `main`/`master`. +* CI: bump GitHub Actions to Node 24 runtimes — `checkout@v5`, + `setup-python@v6`, `upload-artifact@v7`, `download-artifact@v8`. +* Rewrote `migrating_from_gecko_matlab.md` as a complete reference; moved + internal planning notes into `docs/internal/`. + +## 0.1.0a3 — 2026-05-30 + +Build-config hotfix. v0.1.0a2 was tagged but its release.yml build failed: +hatchling refuses PEP 508 direct-URL dependencies (`raven-toolbox @ +git+https://...`) by default. The v0.1.0a2 tag and release were deleted; +v0.1.0a3 is the published artifact carrying the 0.1.0a2 content described +below. No code changes vs. v0.1.0a2. + +* `pyproject.toml`: `[tool.hatch.metadata] allow-direct-references = true` + so hatchling accepts the `raven-toolbox` git URL dependency. PyPI publish + stays disabled (geckopy's name is taken, raven-toolbox isn't on PyPI yet); + the direct reference is acceptable for the git-install path the alpha + series uses. + +## 0.1.0a2 — 2026-05-30 Second public alpha. Layering pass: the bits of geckopy that aren't GECKO-specific moved into raven-toolbox, and `gecko-light` ecModels are now buildable end-to-end. Repo housekeeping for a discoverable first GitHub release (LICENSE, planning docs out of the user-facing tree). -### Added - -- **gecko-light end-to-end.** `make_ec_model(gecko_light=True)` now produces - a working light ecModel: cobra reactions stay singular; per-isozyme +* gecko-light end-to-end: `make_ec_model(gecko_light=True)` now produces a + working light ecModel — cobra reactions stay singular; per-isozyme coupling lives in `ec.rxns` as duplicate rows with a `###_` counter - prefix; only the shared `prot_pool` constrains enzyme usage. `apply_kcat_constraints` - picks the lowest-cost (smallest `MW_sum/kcat`) isozyme per cobra reaction - and writes one `prot_pool` stoichiometric coefficient. `set_kcat_for_reactions` - recognises both base names and the `###_` prefix; the `Enzyme` proxy's - read paths work on light models. New `tutorials/light_ecModel/protocol.py` - mirrors MATLAB GECKO's light tutorial; opt-in Human-GEM smoke test - (`pytest -m smoke`) builds an ecModel from the unmodified - Human-GEM YAML in ~2.5 min. See [`docs/gecko_light_status.md`](docs/gecko_light_status.md). ([#20]) - -### Changed - -- **YAML I/O delegated to raven-toolbox.** `save_ec_model` / `load_ec_model` - are now ~80-LOC wrappers around `raven_toolbox.io.read_yaml_model` / - `write_yaml_model`. raven-toolbox owns the typed `EcData` (now re-exported - as `geckopy.EcData`), the `ec-rxns` / `ec-enzymes` / `gecko_light` YAML - schema, and the three legacy GECKO normalisations (top-level `smiles` → - `annotation`, reverse-direction `usage_prot_*` flip, bare-`-` document - root). Geckopy keeps file-extension dispatch, adapter-aware path + prefix; only the shared `prot_pool` constrains enzyme usage. + `apply_kcat_constraints` picks the lowest-cost (smallest `MW_sum/kcat`) + isozyme per cobra reaction and writes one `prot_pool` stoichiometric + coefficient. `set_kcat_for_reactions` recognises both base names and the + `###_` prefix; the `Enzyme` proxy's read paths work on light models. New + `tutorials/light_ecModel/protocol.py` mirrors MATLAB GECKO's light + tutorial; opt-in Human-GEM smoke test (`pytest -m smoke`) builds an + ecModel from the unmodified Human-GEM YAML in ~2.5 min. See + [`docs/gecko_light_status.md`](docs/gecko_light_status.md). +* YAML I/O delegated to raven-toolbox: `save_ec_model`/`load_ec_model` are + now ~80-LOC wrappers around `raven_toolbox.io.read_yaml_model`/ + `write_yaml_model`. raven-toolbox owns the typed `EcData` (now + re-exported as `geckopy.EcData`), the `ec-rxns`/`ec-enzymes`/`gecko_light` + YAML schema, and the three legacy GECKO normalisations (top-level + `smiles` → `annotation`, reverse-direction `usage_prot_*` flip, bare-`-` + document root). Geckopy keeps file-extension dispatch, adapter-aware path resolution, and provenance/diagnostics. SBML ecModel I/O removed (`geckopy.io.sbml` deleted; `cobra.io.read_sbml_model` is still used for - loading the conventional starting GEM). See [`docs/raven_integration.md`](docs/raven_integration.md). ([#19]) - -- **`ec_fseof` re-aligned with `raven_toolbox.analysis.fseof`.** Thin - wrapper over raven's regression-based FSEOF: drops `usage_prot_*` from - the scan + targets, resolves `bio_rxn` from the adapter, and emits an + loading the conventional starting GEM). See + [`docs/raven_integration.md`](docs/raven_integration.md). +* `ec_fseof` re-aligned with `raven_toolbox.analysis.fseof`: thin wrapper + over raven's regression-based FSEOF, dropping `usage_prot_*` from the + scan + targets, resolving `bio_rxn` from the adapter, and emitting an optional carbon-source consistency warning. Selection moves from MATLAB GECKO's strict-monotonicity + top-25%-by-slope to raven's `|correlation| ≥ threshold` with pFBA per step. Result type is raven's `FSEOFResult` (replaces `EcFseofResult`); action labels are - `amplify` / `knockdown` / `knockout` (replaces `OE` / `KD` / `KO`); the - per-gene `essentiality` column drops (use - `cobra.flux_analysis.single_gene_deletion`). ([#21]) - -- **`raven-toolbox` pin** moved from `@main` to `@develop`. raven-toolbox's + `amplify`/`knockdown`/`knockout` (replaces `OE`/`KD`/`KO`); the per-gene + `essentiality` column drops (use + `cobra.flux_analysis.single_gene_deletion`). +* `raven-toolbox` pin moved from `@main` to `@develop`. raven-toolbox's `main` is empty in the current iteration; all releaseable work lives on `develop`. - -### Fixed - -- **`get_conc_control_coeffs` solver-state guard.** Both `_shadow_price_coeffs` +* `get_conc_control_coeffs` solver-state guard: both `_shadow_price_coeffs` and `_finite_difference_coeffs` previously accepted any solve whose `objective_value` was non-`None` and non-NaN, including infeasible glpk-via-optlang solves (which return `status="infeasible"` but a - non-NaN objective from the binding-constraint rhs). New `_solution_is_optimal` - helper requires `sol.status == "optimal"` at every solve point. - -### Removed - -- `geckopy.io.sbml.{read,write}_sbml_ec_model` and the `geckopy.io` package - (SBML ecModel I/O dropped; YAML is the supported on-disk format). -- `geckopy.EcFseofResult` (use `raven_toolbox.analysis.fseof.FSEOFResult`). -- Top-level `requirements.txt` (a `pip freeze` snapshot that duplicated - `pyproject.toml`'s declared dependencies). pyproject is the source of - truth. - -### Internal - -- Internal planning docs (`brenda_refresh_plan`, `openkineticspredictor_plan`, + non-NaN objective from the binding-constraint rhs). New + `_solution_is_optimal` helper requires `sol.status == "optimal"` at every + solve point. +* Removed `geckopy.io.sbml.{read,write}_sbml_ec_model` and the `geckopy.io` + package (SBML ecModel I/O dropped; YAML is the supported on-disk format). +* Removed `geckopy.EcFseofResult` (use + `raven_toolbox.analysis.fseof.FSEOFResult`). +* Removed the top-level `requirements.txt` (a `pip freeze` snapshot that + duplicated `pyproject.toml`'s declared dependencies); pyproject is the + source of truth. +* Internal planning docs (`brenda_refresh_plan`, `openkineticspredictor_plan`, `porting_plan`, `raven_inventory`, `code_review`) moved from `docs/` to `docs/internal/` so the user-facing docs tree only carries reference documentation. -- `LICENSE` file added (MIT was already declared in `pyproject.toml` and +* `LICENSE` file added (MIT was already declared in `pyproject.toml` and classifier; the text file was missing). -## [0.1.0a1] — 2026-05-30 +## 0.1.0a1 — 2026-05-30 First public alpha. Every MATLAB GECKO 3.2.5 function used in the standard ecModel build is ported; the yeast-GEM tutorial runs end-to-end. -### Foundations - -- **Project skeleton** — `EcModel` / `EcData` / `Enzyme` core classes built on - cobrapy; `ModelParameters` / `ModelAdapter` scaffolding (TOML-driven config, - template generator, `geckopy init` CLI). ([#1]) -- **`make_ec_model` pipeline** — 9-stage MATLAB-port pipeline that builds an - ecModel from a conventional GEM + adapter. Fixes a unit bug where MW values - were treated as kDa instead of Da. ([#2]) -- **Kcat constraints + protein-complex / MW** — `apply_kcat_constraints`, - `set_kcat_for_reactions`, `apply_custom_kcats`, isozyme averaging, - Complex Portal loader, `find_met_smiles`, `calculate_mw`. ([#3]) - -### Kcat sources - -- **BRENDA + DLKcat pipelines** — EC-code resolution helpers, `fuzzy_kcat_matching` - MATLAB-port against BRENDA, DLKcat wrappers (`write_dlkcat_input`, `run_dlkcat`, - `read_dlkcat_output`), standard-kcat fallback. ([#4]) -- **Wide BRENDA snapshot** — `kcat.tsv` / `sa.tsv` / `mw.tsv` ship both `max` - and `median` per `(ec, substrate, organism)` triple (plus `n` = - measurement count). `BrendaData.kcat_for("max" | "median")` picks the - view; three adapter fields (`kcat_aggregate_brenda` / - `kcat_aggregate_candidates` / `kcat_aggregate_isozymes`) let projects - flip aggregation defaults globally. See - [`docs/kcat_aggregation.md`](docs/kcat_aggregation.md). ([#18]) -- **Generalised kcat merge** — `merge_kcats` records per-row provenance and - accepts any combination of BRENDA / DLKcat / OpenKineticsPredictor / - manual sources. ([#13]) -- **Data-source downloaders** — `geckopy brenda-refresh` CLI (BRENDA bulk - JSON), KEGG as an alternative protein/EC source (with a warning when - fallback uses a bare KEGG gene id), OpenKineticsPredictor REST-API - submit/fetch wrappers, YAML I/O aligned with cobrapy + GECKO keys. ([#12]) - -### Proteomics integration & analysis - -- **Protein concentrations + sigma + sensitivity** — `calculate_f_factor` + +* Project skeleton: `EcModel`/`EcData`/`Enzyme` core classes built on + cobrapy; `ModelParameters`/`ModelAdapter` scaffolding (TOML-driven config, + template generator, `geckopy init` CLI). +* `make_ec_model` pipeline: 9-stage MATLAB-port pipeline that builds an + ecModel from a conventional GEM + adapter. Fixes a unit bug where MW + values were treated as kDa instead of Da. +* Kcat constraints + protein-complex/MW: `apply_kcat_constraints`, + `set_kcat_for_reactions`, `apply_custom_kcats`, isozyme averaging, Complex + Portal loader, `find_met_smiles`, `calculate_mw`. +* BRENDA + DLKcat pipelines: EC-code resolution helpers, + `fuzzy_kcat_matching` MATLAB-port against BRENDA, DLKcat wrappers + (`write_dlkcat_input`, `run_dlkcat`, `read_dlkcat_output`), standard-kcat + fallback. +* Wide BRENDA snapshot: `kcat.tsv`/`sa.tsv`/`mw.tsv` ship both `max` and + `median` per `(ec, substrate, organism)` triple (plus `n` = measurement + count). `BrendaData.kcat_for("max" | "median")` picks the view; three + adapter fields (`kcat_aggregate_brenda`/`kcat_aggregate_candidates`/ + `kcat_aggregate_isozymes`) let projects flip aggregation defaults + globally. See [`docs/kcat_aggregation.md`](docs/kcat_aggregation.md). +* Generalised kcat merge: `merge_kcats` records per-row provenance and + accepts any combination of BRENDA/DLKcat/OpenKineticsPredictor/manual + sources. +* Data-source downloaders: `geckopy brenda-refresh` CLI (BRENDA bulk JSON), + KEGG as an alternative protein/EC source (with a warning when fallback + uses a bare KEGG gene id), OpenKineticsPredictor REST-API submit/fetch + wrappers, YAML I/O aligned with cobrapy + GECKO keys. +* Protein concentrations + sigma + sensitivity: `calculate_f_factor` + `load_pax_db`, `fill_enz_concs`, `constrain_enz_concs`, `get_conc_control_coeffs`, `flexibilize_enz_concs`, `constrain_flux_data`; - sigma fitting and `sensitivity_tuning`. ([#5]) -- **Analysis utilities** — `load_flux_data`, `load_prot_data`, - `enzyme_usage` + `report_enzyme_usage`, `map_rxns_to_conv`, `ec_fva`, - `ec_fseof`, `add_new_rxns_to_ec`, `get_subset_ec_model`. ([#6]) -- **Persistence** — `load_ec_model` + `save_ec_model` (YAML + SBML - round-trip with `ec_*` metadata), `merge_dlkcat_and_fuzzy_kcats`, - `load_conventional_gem`. ([#7]) -- **Enzyme accessor + analysis utilities + SBML I/O** — - `Enzyme` / `EnzymeView` accessor, `pfba_enzymes`, - `get_enzyme_bottlenecks`, parallel `ec_fva` via multiprocessing, - `relax_proteomics_greedy`, SBML reader/writer with MW_KCAT encoding, - flatter top-level imports. ([#9]) - -### Refactors - -- **Dedup, renames, splits** — de-duped ec-layer constants across 12 - files, renamed 7 functions for verb-style consistency (with deprecated - aliases), flattened top-level API, `resolve_adapter` helper, - `fuzzy_kcat_matching` split into a 4-file subpackage, SBML name - aliases, auto-flip of legacy reverse-direction protein reactions, - `ec.kcat` sentinel switched from `NaN` to `0`. ([#11]) - -### Solver / numerics - -- **Bisect sigma, shadow-price control coeffs, diagonal ecFVA, median - kcat** — `fit_sigma(method="bisect")` exploiting monotonicity; - `calculate_mw` returns NaN on empty / all-skipped sequence; - `get_conc_control_coeffs` switches to LP shadow-price with scipy - auto-fallback; `ec_fva` reports the exact per-reaction range - (diagonal); `median` option for kcat aggregation. ([#17]) - -### Code review hardening - -- **§1 silent-data-loss guards** — `genes`-list length validation, - numpy-scalar coercion, kcat / relaxation loop non-termination guards, - enzyme-provenance round-trip across SBML, UniProt-accession dedup, - Complex Portal sub-complex classification, downloader hardening, + sigma fitting and `sensitivity_tuning`. +* Analysis utilities: `load_flux_data`, `load_prot_data`, `enzyme_usage` + + `report_enzyme_usage`, `map_rxns_to_conv`, `ec_fva`, `ec_fseof`, + `add_new_rxns_to_ec`, `get_subset_ec_model`. +* Persistence: `load_ec_model` + `save_ec_model` (YAML + SBML round-trip + with `ec_*` metadata), `merge_dlkcat_and_fuzzy_kcats`, + `load_conventional_gem`. +* Enzyme accessor + analysis utilities + SBML I/O: `Enzyme`/`EnzymeView` + accessor, `pfba_enzymes`, `get_enzyme_bottlenecks`, parallel `ec_fva` via + multiprocessing, `relax_proteomics_greedy`, SBML reader/writer with + MW_KCAT encoding, flatter top-level imports. +* Dedup, renames, splits: de-duped ec-layer constants across 12 files, + renamed 7 functions for verb-style consistency (with deprecated aliases), + flattened top-level API, `resolve_adapter` helper, `fuzzy_kcat_matching` + split into a 4-file subpackage, SBML name aliases, auto-flip of legacy + reverse-direction protein reactions, `ec.kcat` sentinel switched from + `NaN` to `0`. +* Bisect sigma, shadow-price control coeffs, diagonal ecFVA, median kcat: + `fit_sigma(method="bisect")` exploiting monotonicity; `calculate_mw` + returns NaN on empty/all-skipped sequence; `get_conc_control_coeffs` + switches to LP shadow-price with scipy auto-fallback; `ec_fva` reports + the exact per-reaction range (diagonal); `median` option for kcat + aggregation. +* Code review hardening — silent-data-loss guards: `genes`-list length + validation, numpy-scalar coercion, kcat/relaxation loop non-termination + guards, enzyme-provenance round-trip across SBML, UniProt-accession + dedup, Complex Portal sub-complex classification, downloader hardening, NaN-aware kcat overwrite, non-fatal DLKcat unknown-substrate handling, - `ec_fseof` / `pfba_enzymes` feasibility checks, OKP GET retries, - current PubChem SMILES property, perf hoists, BayesianParams length - validation, robust BRENDA protein lookup. ([#14]) -- **§2 adapter decoupling** — adapter-free read / inspect of ecModels; - per-parameter overrides decouple analysis functions from the adapter; - `bio_rxn` overrides + loader robustness; writable BRENDA data kept out - of the install tree. ([#15]) -- **§4 / §5 cleanup** — sanity guards + DLKcat ignore-list header - tolerance; dedup helpers + standardised infeasibility check; + `ec_fseof`/`pfba_enzymes` feasibility checks, OKP GET retries, current + PubChem SMILES property, perf hoists, BayesianParams length validation, + robust BRENDA protein lookup. +* Code review hardening — adapter decoupling: adapter-free read/inspect of + ecModels; per-parameter overrides decouple analysis functions from the + adapter; `bio_rxn` overrides + loader robustness; writable BRENDA data + kept out of the install tree. +* Code review hardening — cleanup: sanity guards + DLKcat ignore-list + header tolerance; dedup helpers + standardised infeasibility check; dead-code removal; tuned-kcat capping + negative-concentration guards; - dropped private back-compat aliases in `fuzzy_kcat_matching`. ([#16]) - -### Docs, tutorial, packaging - -- **Tutorial + packaging** — full_ecModel notebook tutorial walking - through Stages 0–5 (build → proteomics → simulation), - `ec_fva` tqdm progress bar, ruff baseline, `.gitattributes` LF - enforcement, PyPI-ready packaging metadata, GitHub Actions for - tests / lint / release. ([#8]) -- **Docs sweep** — README rewrite for non-specialist readability, - friendlier docstrings on the public-API modules, Carrasco et al. (2023) - citation, third-party library gotchas (SBML / multiprocessing / ruamel), - RAVEN function inventory + ravenpy scope, gecko-light status. ([#10]) - -### Dependencies - -- **raven-toolbox is now a hard dependency** — geckopy delegates - `expand_model` and `convert_to_irreversible` to raven-toolbox (the - functions originated in geckopy and were adopted upstream as their - canonical home). raven-toolbox is not yet on PyPI; install both via - the git URLs in the README. See - [`docs/raven_integration.md`](docs/raven_integration.md) for the - current delegation and the planned future migrations. - -[0.3.0]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.3.0 -[0.2.1]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.2.1 -[0.2.0]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.2.0 -[0.1.0a3]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.1.0a3 -[0.1.0a2]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.1.0a2 -[0.1.0a1]: https://github.com/SysBioChalmers/geckopy/releases/tag/v0.1.0a1 -[#1]: https://github.com/SysBioChalmers/geckopy/pull/1 -[#2]: https://github.com/SysBioChalmers/geckopy/pull/2 -[#3]: https://github.com/SysBioChalmers/geckopy/pull/3 -[#4]: https://github.com/SysBioChalmers/geckopy/pull/4 -[#5]: https://github.com/SysBioChalmers/geckopy/pull/5 -[#6]: https://github.com/SysBioChalmers/geckopy/pull/6 -[#7]: https://github.com/SysBioChalmers/geckopy/pull/7 -[#8]: https://github.com/SysBioChalmers/geckopy/pull/8 -[#9]: https://github.com/SysBioChalmers/geckopy/pull/9 -[#10]: https://github.com/SysBioChalmers/geckopy/pull/10 -[#11]: https://github.com/SysBioChalmers/geckopy/pull/11 -[#12]: https://github.com/SysBioChalmers/geckopy/pull/12 -[#13]: https://github.com/SysBioChalmers/geckopy/pull/13 -[#14]: https://github.com/SysBioChalmers/geckopy/pull/14 -[#15]: https://github.com/SysBioChalmers/geckopy/pull/15 -[#16]: https://github.com/SysBioChalmers/geckopy/pull/16 -[#17]: https://github.com/SysBioChalmers/geckopy/pull/17 -[#18]: https://github.com/SysBioChalmers/geckopy/pull/18 -[#19]: https://github.com/SysBioChalmers/geckopy/pull/19 -[#20]: https://github.com/SysBioChalmers/geckopy/pull/20 -[#21]: https://github.com/SysBioChalmers/geckopy/pull/21 -[#31]: https://github.com/SysBioChalmers/geckopy/pull/31 -[#32]: https://github.com/SysBioChalmers/geckopy/pull/32 -[#34]: https://github.com/SysBioChalmers/geckopy/pull/34 -[#36]: https://github.com/SysBioChalmers/geckopy/pull/36 -[#37]: https://github.com/SysBioChalmers/geckopy/pull/37 -[#38]: https://github.com/SysBioChalmers/geckopy/pull/38 -[#39]: https://github.com/SysBioChalmers/geckopy/pull/39 -[#41]: https://github.com/SysBioChalmers/geckopy/pull/41 -[#42]: https://github.com/SysBioChalmers/geckopy/pull/42 -[#43]: https://github.com/SysBioChalmers/geckopy/pull/43 -[#44]: https://github.com/SysBioChalmers/geckopy/pull/44 -[#45]: https://github.com/SysBioChalmers/geckopy/pull/45 -[#46]: https://github.com/SysBioChalmers/geckopy/pull/46 -[#47]: https://github.com/SysBioChalmers/geckopy/pull/47 -[#49]: https://github.com/SysBioChalmers/geckopy/pull/49 -[#50]: https://github.com/SysBioChalmers/geckopy/pull/50 -[#51]: https://github.com/SysBioChalmers/geckopy/pull/51 -[#52]: https://github.com/SysBioChalmers/geckopy/pull/52 + dropped private back-compat aliases in `fuzzy_kcat_matching`. +* Tutorial + packaging: full_ecModel notebook tutorial walking through + Stages 0–5 (build → proteomics → simulation), `ec_fva` tqdm progress bar, + ruff baseline, `.gitattributes` LF enforcement, PyPI-ready packaging + metadata, GitHub Actions for tests/lint/release. +* Docs sweep: README rewrite for non-specialist readability, friendlier + docstrings on the public-API modules, Carrasco et al. (2023) citation, + third-party library gotchas (SBML/multiprocessing/ruamel), RAVEN function + inventory + ravenpy scope, gecko-light status. +* raven-toolbox is now a hard dependency: geckopy delegates `expand_model` + and `convert_to_irreversible` to raven-toolbox (the functions originated + in geckopy and were adopted upstream as their canonical home). + raven-toolbox is not yet on PyPI; install both via the git URLs in the + README. See [`docs/raven_integration.md`](docs/raven_integration.md) for + the current delegation and the planned future migrations. From b106e21497960e234e29b5b20c3022a24a51c944 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 11:24:36 +0200 Subject: [PATCH 07/15] test_corrections: remove unused Correction import --- tests/test_corrections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_corrections.py b/tests/test_corrections.py index d7f3c6c..77feaf9 100644 --- a/tests/test_corrections.py +++ b/tests/test_corrections.py @@ -3,7 +3,7 @@ import pytest from geckopy.kcat_tuning.corrections import ( - Correction, annotate_from_model, corrections, corrections_tsv, + annotate_from_model, corrections, corrections_tsv, ) From e9984c223cfbe5378fe093b03d90766d0ecdc2cc Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 11:30:40 +0200 Subject: [PATCH 08/15] update changelog --- CHANGELOG.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 777a750..58ac5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,9 @@ # Changelog -All notable changes to **geckopy** are documented here. The project uses -[PEP 440](https://peps.python.org/pep-0440/) pre-release versioning. - ## 4.0.0b1 — 2026-09-08 -Version realignment: geckopy adopts GECKO's own toolbox-generation numbering -(MATLAB GECKO 1–3, geckopy = 4.x) instead of its own from-zero semver — see -the README for why. First release published to PyPI, and the first to ship -kcat tuning against experimental data. +Version realignment: geckopy numbering will be aligned with GECKO (MATLAB), +justifying a jump to 4.x. This is the first release published to PyPI. * New: `kcat_tuning.evotune` fits `ec.kcat` to measured growth rates and exchange fluxes with CMA-ES (`screen_kcat_leverage`, `select_tunable_mask`, From a6aee4adce3bf7448fac123045a8a6ce4af0fa01 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 12:31:57 +0200 Subject: [PATCH 09/15] test_evotune: tolerant comparison for serial-vs-parallel rmse_trace Exact list equality between the serial and 2-worker-pool runs is fragile: rmse_trace is summed in a different order across worker processes than in-process, so it can differ by a couple of ULPs even when the optimization result (new_kcat) is bit-identical. Failed on macOS CI for exactly that reason. --- tests/test_evotune.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_evotune.py b/tests/test_evotune.py index 796bbf4..b926671 100644 --- a/tests/test_evotune.py +++ b/tests/test_evotune.py @@ -308,7 +308,10 @@ def _run(n_proc): parallel = _run(2) assert np.array_equal(serial.new_kcat, parallel.new_kcat) - assert serial.rmse_trace == parallel.rmse_trace + # rmse_trace values are summed in a different order across worker + # processes than in-process, so they can differ by a couple of ULPs + # even when the optimization itself (new_kcat) is bit-identical. + assert np.allclose(serial.rmse_trace, parallel.rmse_trace, rtol=1e-9, atol=1e-12) assert serial.n_generations == parallel.n_generations From bbab544f78b275f3869b464ea145cac1072e4310 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 12:46:12 +0200 Subject: [PATCH 10/15] README: rewrite install section for two pre-release packages Explains both a plain pip install --pre geckopy (which resolves raven-toolbox's own pre-release transitively, verified against pip's actual resolver) and an explicit dual-pin for exact reproducibility. Also drops the under-active-development title suffix now that Status already covers that, and points at the evotune docs from Quick start. --- README.md | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7694e5f..ecde2b3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# geckopy - under active development +# geckopy [![tests](https://github.com/SysBioChalmers/geckopy/actions/workflows/test.yml/badge.svg)](https://github.com/SysBioChalmers/geckopy/actions/workflows/test.yml) @@ -23,27 +23,36 @@ adds those limits to the network. geckopy is a Python port of the [GECKO Toolbox](https://github.com/SysBioChalmers/GECKO) (MATLAB). It builds on [cobrapy](https://github.com/opencobra/cobrapy) for the standard modelling layer, then layers GECKO's enzyme-constraint machinery -on top. It fetches kcat values from BRENDA, predicts them with DLKcat, -applies custom curations, and integrates proteomics measurements. +on top: it fetches kcat values from BRENDA, predicts them with DLKcat, tunes +them against experimental growth/flux data with CMA-ES, applies custom +curations, and integrates proteomics measurements. > **Status: beta (`4.0.0b1`).** All the MATLAB GECKO 3.2.5 functions used in -> the standard ecModel build are ported, plus CMA-ES-based kcat tuning -> against experimental data. The yeast-GEM tutorial runs end-to-end. -> geckopy's version tracks the GECKO project's own toolbox-generation -> numbering (MATLAB GECKO 1–3, geckopy = 4.x), not a semantic-versioning -> API-stability claim. +> the standard ecModel build are ported, and the yeast-GEM tutorial runs +> end-to-end. geckopy's version number tracks the GECKO project's own +> toolbox-generation numbering (MATLAB GECKO 1–3, geckopy = 4.x), rather than +> making a semantic-versioning API-stability claim. ## Install +geckopy depends on [raven-toolbox](https://github.com/SysBioChalmers/raven-toolbox) +(the Python port of the RAVEN Toolbox). Both are currently pre-releases on +PyPI, so a plain `pip install geckopy` won't find anything — `--pre` is +required: + ```bash pip install --pre geckopy ``` -geckopy is currently published as a pre-release, so `--pre` (or an exact -pin, e.g. `geckopy==4.0.0b1`) is required until the `4.0.0` final release — -plain `pip install geckopy` won't pick it up before then. -[raven-toolbox](https://github.com/SysBioChalmers/raven-toolbox) (the Python -port of the RAVEN Toolbox) is pulled in automatically. +`raven-toolbox` doesn't need to be named separately: geckopy's own dependency +on it already specifies `>=3.0.0b1`, and a specifier that names a pre-release +opts pip into matching pre-releases for that package too, `--pre` or not. To +pin the exact versions this was written against instead of "whatever's +newest," name both explicitly: + +```bash +pip install raven-toolbox==3.0.0b1 geckopy==4.0.0b1 +``` Optional extras: @@ -103,6 +112,10 @@ proteomics integration, and Crabtree-effect simulation) is in That script reproduces the MATLAB GECKO Nature Protocols tutorial in Python, step by step, and is the easiest way to see geckopy in action. +Fitting kcats against measured growth rates and exchange fluxes (rather +than looking them up) is a separate workflow, `geckopy.kcat_tuning.evotune` +— see [`docs/evotune_kcat_tuning.md`](docs/evotune_kcat_tuning.md). + ## On-disk format geckopy reads and writes ecModels as **YAML**: a canonical, From ca23c9d4941988a40f9c7c08bbf0fe55df9577b2 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 14:34:48 +0200 Subject: [PATCH 11/15] Remove the MATLAB-COMPAT comment convention, add gecko-docs references Only one MATLAB-COMPAT: comment remained in src/ (query.py) -- the 0.3.0 docstring cleanup already replaced the rest with plain statements of current behavior, so the convention future_improvements.md and the README described as active was stale. Rewrites the two remaining source notes (query.py, _brenda_query.py) as plain statements, drops the now-false every-divergence-carries-a- MATLAB-COMPAT-comment claims from README.md and migrating_from_gecko_matlab.md, and points readers at the actively maintained MATLAB to Python translation guide on gecko-docs.readthedocs.io instead. Also cleans up docs/future_improvements.md: removes three MATLAB GECKO changes items that PRs #43 and #47 resolved in the opposite direction (geckopy changed to match MATLAB, rather than MATLAB adopting geckopy old behavior), and the drop-the-raven-toolbox- git-URL deferred-migration item, done a few commits back. --- README.md | 11 +++---- docs/future_improvements.md | 29 +++++-------------- docs/migrating_from_gecko_matlab.md | 19 +++++++----- src/geckopy/ec_model/pipeline/query.py | 7 ++--- .../fuzzy_kcat_matching/_brenda_query.py | 5 ++-- 5 files changed, 31 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index ecde2b3..4cba6d8 100644 --- a/README.md +++ b/README.md @@ -134,11 +134,12 @@ specification and how it differs from the legacy MATLAB / RAVEN format. geckopy is a function-by-function port: algorithmic fidelity comes first, Pythonic idiom second. Each ported source file carries a -`Ported from GECKO MATLAB: ` docstring header. Where the Python -version intentionally differs (different direction conventions, fixed -MATLAB bugs, etc.), the divergence is tagged with a `MATLAB-COMPAT:` -comment in source. The cumulative list of MATLAB-side improvements -the port surfaced lives in +`Ported from GECKO MATLAB: ` docstring header. For current +MATLAB ↔ Python behavioural differences, see the +[MATLAB ↔ Python translation guide](https://gecko-docs.readthedocs.io/en/latest/api/translation/) +on [gecko-docs.readthedocs.io](https://gecko-docs.readthedocs.io/), the +shared documentation site for both toolboxes. The cumulative list of +MATLAB-side bugs and rough edges the port surfaced lives in [`docs/future_improvements.md`](docs/future_improvements.md). ## License diff --git a/docs/future_improvements.md b/docs/future_improvements.md index 2030638..21a1988 100644 --- a/docs/future_improvements.md +++ b/docs/future_improvements.md @@ -7,10 +7,11 @@ weren't blocking. Three audiences: model", "Third-party library gotchas", and "Deferred migrations". - **MATLAB GECKO maintainers** — section "MATLAB GECKO changes". Each item is a bug or rough edge in MATLAB GECKO that the port - spotted while comparing the two implementations side by side. - Every item is also flagged with a `MATLAB-COMPAT:` comment in - the geckopy source, so `grep -rn "MATLAB-COMPAT:" src/` gives - the full list with file references. + spotted while comparing the two implementations side by side. For + the current, maintained list of MATLAB ↔ Python behavioural + differences (broader than just what's tracked here), see the + [MATLAB ↔ Python translation guide](https://gecko-docs.readthedocs.io/en/latest/api/translation/) + on [gecko-docs.readthedocs.io](https://gecko-docs.readthedocs.io/). - **anyone hitting a strange dependency bug** — section "Third-party library gotchas". @@ -68,32 +69,17 @@ keep an in-flight release on schedule. would make every layer consistent. Small effect (only rows reported as ranges) — deferred. -- **Drop the git URL from `pyproject.toml`'s raven-toolbox - dependency once raven-toolbox publishes to PyPI.** Currently - pinned to `raven-toolbox @ git+https://github.com/SysBioChalmers/raven-toolbox.git@main`, - which forces an internet round-trip on every install and means - no install reproducibility per release. Switch to a plain - `raven-toolbox>=` pin (and collapse the README install - block to `pip install geckopy`) the moment raven-toolbox is on - PyPI. - ## MATLAB GECKO changes Aimed at MATLAB GECKO maintainers. Each item is something the geckopy port found while comparing the two implementations: bugs, -dead code, unit-comment slip-ups, or unclear behaviour. The -in-source `MATLAB-COMPAT:` comments in geckopy carry the same -notes alongside the Python implementation. +dead code, unit-comment slip-ups, or unclear behaviour. Notable items: - Implement gene-cell splitting in the UniProt loader. - Apply the `stoicho` column from `customKcats.tsv`, or drop it from the schema. -- Change source-string convention for `setKcatForReactions` to `'manual'`. -- Make `getReactionsFromEnzyme` case-sensitive. -- Forbid length-N kcat lists for un-suffixed `rxn_ids` in - `setKcatForReactions` (strict matching rule). - Rename `getKcatAcrossIsozymes` to `fillKcatsFromIsozymes` — the verb `get` is misleading, the function mutates `ec.kcat` in place. geckopy renamed its Python counterpart; the MATLAB side is the @@ -148,7 +134,8 @@ Notable items: the docstring's origin ranking implies origin 4 should win. Either swap the search order so any-no-subs-kcat is tried before org-SA, or update the docstring to match the search order. geckopy - replicates the current MATLAB behavior with a MATLAB-COMPAT note. + replicates the current MATLAB behavior; see the note in + `fuzzy_kcat_matching/_brenda_query.py`. - Delete `updateProtPool` from MATLAB GECKO. The function has been obsolete since GECKO 3.2.0 (all enzymes, measured and unmeasured, draw from the protein pool); its sole runtime behaviour on a diff --git a/docs/migrating_from_gecko_matlab.md b/docs/migrating_from_gecko_matlab.md index da3278d..2683fd7 100644 --- a/docs/migrating_from_gecko_matlab.md +++ b/docs/migrating_from_gecko_matlab.md @@ -415,10 +415,10 @@ See [kcat_aggregation.md](kcat_aggregation.md). The port found and fixed several GECKO 3 bugs (EC-code assignment, sigma fitting, standard-kcat subsystem means, duplicate EC codes, …); **`[3→4]`** GECKO 4 carries the same fixes, so the two toolboxes -agree. The full per-function divergence list lives in the in-source -`MATLAB-COMPAT:` comments -(`grep -rn "MATLAB-COMPAT:" src/geckopy/`) and in -[future_improvements.md](future_improvements.md). +agree. The full per-function divergence list lives in the +[MATLAB ↔ Python translation guide](https://gecko-docs.readthedocs.io/en/latest/api/translation/) +on gecko-docs.readthedocs.io, plus [future_improvements.md](future_improvements.md) +for the MATLAB-side items the port surfaced. ### Available in both (added during the port) @@ -459,10 +459,13 @@ agree. The full per-function divergence list lives in the in-source ## 9. Where to look when something differs -- Every intentional divergence in geckopy carries a - **`MATLAB-COMPAT:`** comment in source. Each ported function's - docstring also opens with `Ported from GECKO MATLAB: .` so - you can find the exact MATLAB original. +- Each ported function's docstring opens with + `Ported from GECKO MATLAB: .` so you can find the exact + MATLAB original. +- For current MATLAB ↔ Python behavioural differences, see the + [MATLAB ↔ Python translation guide](https://gecko-docs.readthedocs.io/en/latest/api/translation/) + on [gecko-docs.readthedocs.io](https://gecko-docs.readthedocs.io/), the + shared documentation site for both toolboxes. - MATLAB-side bugs and rough edges the port found are tracked in [future_improvements.md](future_improvements.md). - For the GECKO 3→4 changes as they apply on the MATLAB side, see diff --git a/src/geckopy/ec_model/pipeline/query.py b/src/geckopy/ec_model/pipeline/query.py index f4152d1..140e1f3 100644 --- a/src/geckopy/ec_model/pipeline/query.py +++ b/src/geckopy/ec_model/pipeline/query.py @@ -45,10 +45,9 @@ def get_reactions_from_enzyme( ------ ValueError If ``protein_id`` is not found in ec.enzymes (case-insensitive). - MATLAB-COMPAT: MATLAB's getReactionsFromEnzyme returns five empty - outputs instead of raising on an unmatched proteinId. Raising here - is a deliberate Python-API choice, not a parity gap to close -- - see raven-gecko-parity#70. + A deliberate Python-API choice, not a parity gap to close: MATLAB's + ``getReactionsFromEnzyme`` returns five empty outputs instead of + raising on an unmatched proteinId. See raven-gecko-parity#70. """ protein_id_lower = protein_id.lower() try: diff --git a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py index 449c49c..3cfa520 100644 --- a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py +++ b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py @@ -39,8 +39,9 @@ # 1: org+subs+kcat 2: any+subs+kcat 3: org+!subs+kcat # 4: any+!subs+kcat 5: org+SA 6: any+SA # but the SEARCH order tries org-SA (output 5) BEFORE any-no-subs-kcat -# (output 4), matching MATLAB exactly. See the `D5` MATLAB-COMPAT note -# in docs/future_improvements.md. +# (output 4), matching MATLAB exactly -- see "Fix the search-order/ +# output-ranking inconsistency in fuzzyKcatMatching" in +# docs/future_improvements.md. SEARCH_LEVELS: list[tuple[bool, bool, bool, int]] = [ (True, True, False, 1), (False, True, False, 2), From 8a615fa0ff2c42b491345eea112e3fad91292088 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 14:45:47 +0200 Subject: [PATCH 12/15] docs: add and fix Ported-from-GECKO-MATLAB headers Audits the `Ported from GECKO MATLAB:` docstring convention across src/geckopy: adds the header to files that port real GECKO MATLAB functionality but were missing it, and fixes three headers that cited the wrong path. - adapter/adapter.py, adapter/resolve.py, adapter/template.py, adapter/params.py: port GECKO MATLAB's model_adapter/ (ModelAdapter.m, ModelAdapterManager.m, adapterTemplate.m), not geckopy-only scaffolding. - cli.py: the `init` subcommand ports startGECKOproject.m's project-scaffolding logic. - ec_model/pipeline/preprocess.py, populate_ec.py, protein_pool.py: split-out stages of makeEcModel.m; protein_pool.py's set_prot_pool_size also cites setProtPoolSize.m specifically. - ec_model/pipeline/query.py: ports getReactionsFromEnzyme.m. - gather_kcats/fuzzy_kcat_matching/_brenda_query.py, _escalation.py, _organism.py: split-out helpers of fuzzyKcatMatching.m, already cited by the parent package's __init__.py. - databases/uniprot_loader.py: cited src/geckomat/utilities/loadDatabases.m; loadDatabases.m lives under get_enzyme_data/, not utilities/. - databases/kegg_download.py, uniprot_download.py: cited nonexistent downloadKEGG.m / downloadUniProt.m paths; point at the real source, loadDatabases.m, with the extraction noted in prose. - databases/phyl_dist.py: normalized to the standard "Ported from GECKO MATLAB: ." wording (was missing the colon, so it didn't match the repo convention or the audit grep). --- src/geckopy/adapter/adapter.py | 2 ++ src/geckopy/adapter/params.py | 10 +++++++++- src/geckopy/adapter/resolve.py | 2 ++ src/geckopy/adapter/template.py | 2 ++ src/geckopy/cli.py | 12 +++++++++++- src/geckopy/databases/kegg_download.py | 4 ++-- src/geckopy/databases/phyl_dist.py | 4 ++-- src/geckopy/databases/uniprot_download.py | 4 ++-- src/geckopy/databases/uniprot_loader.py | 2 +- src/geckopy/ec_model/pipeline/populate_ec.py | 2 ++ src/geckopy/ec_model/pipeline/preprocess.py | 2 ++ src/geckopy/ec_model/pipeline/protein_pool.py | 5 +++++ src/geckopy/ec_model/pipeline/query.py | 2 ++ .../fuzzy_kcat_matching/_brenda_query.py | 2 ++ .../gather_kcats/fuzzy_kcat_matching/_escalation.py | 2 ++ .../gather_kcats/fuzzy_kcat_matching/_organism.py | 2 ++ 16 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/geckopy/adapter/adapter.py b/src/geckopy/adapter/adapter.py index 4fea4bb..d9042d5 100644 --- a/src/geckopy/adapter/adapter.py +++ b/src/geckopy/adapter/adapter.py @@ -15,6 +15,8 @@ - ``data/`` — cached databases (UniProt, BRENDA, ComplexPortal, proteomics, etc.). +Ported from GECKO MATLAB: src/geckomat/model_adapter/ModelAdapter.m. + Then: .. code-block:: python diff --git a/src/geckopy/adapter/params.py b/src/geckopy/adapter/params.py index 5beb777..becd688 100644 --- a/src/geckopy/adapter/params.py +++ b/src/geckopy/adapter/params.py @@ -1,4 +1,12 @@ -"""Parameter schemas for ecModel adapters.""" +"""Parameter schemas for ecModel adapters. + +``KeggParams``, ``UniprotParams``, ``ComplexParams``, and the core fields +of ``ModelParameters`` mirror the ``obj.params.*`` fields set in GECKO +MATLAB's adapter template. ``BayesianParams`` and ``OkpParams`` are +geckopy-only extensions with no MATLAB equivalent. + +Ported from GECKO MATLAB: src/geckomat/model_adapter/adapterTemplate.m. +""" from pathlib import Path from typing import Literal, Optional diff --git a/src/geckopy/adapter/resolve.py b/src/geckopy/adapter/resolve.py index 3639997..eac9b4a 100644 --- a/src/geckopy/adapter/resolve.py +++ b/src/geckopy/adapter/resolve.py @@ -11,6 +11,8 @@ ``resolve_adapter`` consolidates the "explicit-arg-or-fallback-to- ``model.adapter``-or-raise" pattern that 30+ functions duplicated inline, with inconsistent error messages. + +Ported from GECKO MATLAB: src/geckomat/model_adapter/ModelAdapterManager.m. """ from __future__ import annotations diff --git a/src/geckopy/adapter/template.py b/src/geckopy/adapter/template.py index 1b7ce7c..43ab882 100644 --- a/src/geckopy/adapter/template.py +++ b/src/geckopy/adapter/template.py @@ -6,6 +6,8 @@ uncommented with a placeholder (for required fields), or commented out showing its default value (for optional fields). Field descriptions are emitted as TOML comments above each field. + +Ported from GECKO MATLAB: src/geckomat/model_adapter/adapterTemplate.m. """ from __future__ import annotations diff --git a/src/geckopy/cli.py b/src/geckopy/cli.py index 16f4aca..8ec8007 100644 --- a/src/geckopy/cli.py +++ b/src/geckopy/cli.py @@ -1,4 +1,14 @@ -"""Command-line interface for geckopy.""" +"""Command-line interface for geckopy. + +The `init` subcommand ports GECKO MATLAB's project-scaffolding logic +(`src/geckomat/utilities/startGECKOproject.m`): create the project +folder tree and write out a template adapter. The `brenda-refresh`, +`uniprot-download`, and `kegg-download` subcommands are new tooling +with no MATLAB equivalent. + +Ported from GECKO MATLAB: src/geckomat/utilities/startGECKOproject.m +(the `init` subcommand only). +""" from __future__ import annotations import argparse diff --git a/src/geckopy/databases/kegg_download.py b/src/geckopy/databases/kegg_download.py index 9b41d47..91233c4 100644 --- a/src/geckopy/databases/kegg_download.py +++ b/src/geckopy/databases/kegg_download.py @@ -1,8 +1,8 @@ """Download organism-specific protein information from KEGG REST. Ported from GECKO MATLAB: -src/geckomat/get_enzyme_data/downloadKEGG.m -(extracted from loadDatabases.m). +src/geckomat/get_enzyme_data/loadDatabases.m +(the KEGG-download branch, extracted into its own module). Two REST endpoints are used: diff --git a/src/geckopy/databases/phyl_dist.py b/src/geckopy/databases/phyl_dist.py index 820a3d8..ec09959 100644 --- a/src/geckopy/databases/phyl_dist.py +++ b/src/geckopy/databases/phyl_dist.py @@ -3,8 +3,8 @@ Used by fuzzy_kcat_matching to find the evolutionarily closest organism when an exact match is not available in BRENDA. -Ported from GECKO MATLAB (the inline `KEGG_struct` helper in -src/geckomat/gather_kcats/fuzzyKcatMatching.m). +Ported from GECKO MATLAB: +src/geckomat/gather_kcats/fuzzyKcatMatching.m (the inline `KEGG_struct` helper). """ from __future__ import annotations diff --git a/src/geckopy/databases/uniprot_download.py b/src/geckopy/databases/uniprot_download.py index 5f47683..aea08db 100644 --- a/src/geckopy/databases/uniprot_download.py +++ b/src/geckopy/databases/uniprot_download.py @@ -1,8 +1,8 @@ """Download a per-organism UniProt TSV from the UniProt REST API. Ported from GECKO MATLAB: -src/geckomat/get_enzyme_data/downloadUniProt.m -(extracted from loadDatabases.m). +src/geckomat/get_enzyme_data/loadDatabases.m +(the UniProt-download branch, extracted into its own module). Hits ``https://rest.uniprot.org/uniprotkb/stream`` with a query built from ``id_type`` + ``uniprot_id`` and writes the resulting diff --git a/src/geckopy/databases/uniprot_loader.py b/src/geckopy/databases/uniprot_loader.py index a1fb0b0..c7a63a0 100644 --- a/src/geckopy/databases/uniprot_loader.py +++ b/src/geckopy/databases/uniprot_loader.py @@ -1,6 +1,6 @@ """Parse uniprot.tsv into a UniprotDB struct. -Ported from GECKO MATLAB: src/geckomat/utilities/loadDatabases.m +Ported from GECKO MATLAB: src/geckomat/get_enzyme_data/loadDatabases.m (the UniProt-parsing branch; the download branch is deferred to a separate future module). diff --git a/src/geckopy/ec_model/pipeline/populate_ec.py b/src/geckopy/ec_model/pipeline/populate_ec.py index 2bf9e8f..b05fc89 100644 --- a/src/geckopy/ec_model/pipeline/populate_ec.py +++ b/src/geckopy/ec_model/pipeline/populate_ec.py @@ -18,6 +18,8 @@ All three functions mutate the ec substructure in place. None of them talk to the network. + +Ported from GECKO MATLAB: src/geckomat/change_model/makeEcModel.m. """ from __future__ import annotations diff --git a/src/geckopy/ec_model/pipeline/preprocess.py b/src/geckopy/ec_model/pipeline/preprocess.py index dcb35a4..f9e4f30 100644 --- a/src/geckopy/ec_model/pipeline/preprocess.py +++ b/src/geckopy/ec_model/pipeline/preprocess.py @@ -3,6 +3,8 @@ Corresponds to stages 1 to 4 of makeEcModel in GECKO MATLAB. These stages do not touch the `ec` substructure; they only reshape the underlying model so subsequent stages operate on a clean irreversible form. + +Ported from GECKO MATLAB: src/geckomat/change_model/makeEcModel.m. """ from __future__ import annotations from typing import TYPE_CHECKING diff --git a/src/geckopy/ec_model/pipeline/protein_pool.py b/src/geckopy/ec_model/pipeline/protein_pool.py index 04193af..2637446 100644 --- a/src/geckopy/ec_model/pipeline/protein_pool.py +++ b/src/geckopy/ec_model/pipeline/protein_pool.py @@ -9,6 +9,8 @@ files used the reverse-direction convention (`lb=-1000, ub=0`); those are normalized to the forward convention on load (see `load_ec_model.py`), mirroring MATLAB's `loadEcModel.m`. + +Ported from GECKO MATLAB: src/geckomat/change_model/makeEcModel.m. """ from __future__ import annotations @@ -256,6 +258,9 @@ def set_prot_pool_size( ) -> float: """Set the upper bound of ``prot_pool_exchange`` (the protein budget). + Ported from GECKO MATLAB: + src/geckomat/change_model/setProtPoolSize.m. + The protein pool exchange represents the cell's total enzyme supply. Its upper bound is what limits how much protein the model can deploy at once. The formula is: diff --git a/src/geckopy/ec_model/pipeline/query.py b/src/geckopy/ec_model/pipeline/query.py index 140e1f3..e1a4b7c 100644 --- a/src/geckopy/ec_model/pipeline/query.py +++ b/src/geckopy/ec_model/pipeline/query.py @@ -2,6 +2,8 @@ These functions do not mutate the model; they pull subsets of the ec data for inspection, reporting, or downstream pipeline use. + +Ported from GECKO MATLAB: src/geckomat/change_model/getReactionsFromEnzyme.m. """ from __future__ import annotations diff --git a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py index 3cfa520..7e0528f 100644 --- a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py +++ b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_brenda_query.py @@ -9,6 +9,8 @@ org+!subs+kcat, org+SA, any+!subs+kcat, any+SA) are tried in the MATLAB-compat order. See the orchestrator's docstring for the full background. + +Ported from GECKO MATLAB: src/geckomat/gather_kcats/fuzzyKcatMatching.m. """ from __future__ import annotations diff --git a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_escalation.py b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_escalation.py index 2c319c4..e8955d0 100644 --- a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_escalation.py +++ b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_escalation.py @@ -4,6 +4,8 @@ replacing its rightmost numeric level with ``-`` and trying again. These helpers do just that string manipulation -- no BRENDA, no model. + +Ported from GECKO MATLAB: src/geckomat/gather_kcats/fuzzyKcatMatching.m. """ from __future__ import annotations diff --git a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_organism.py b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_organism.py index 7543db5..fed08cb 100644 --- a/src/geckopy/gather_kcats/fuzzy_kcat_matching/_organism.py +++ b/src/geckopy/gather_kcats/fuzzy_kcat_matching/_organism.py @@ -4,6 +4,8 @@ is looked up in the KEGG phylogenetic-distance struct. When BRENDA has no exact organism match, the matcher uses this struct to keep the rows for the phylogenetically closest organism(s). + +Ported from GECKO MATLAB: src/geckomat/gather_kcats/fuzzyKcatMatching.m. """ from __future__ import annotations From a447813c8fa297046fcfdb7dc04d9d7af9f09a51 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 14:56:26 +0200 Subject: [PATCH 13/15] docs: fix mangled GECKO MATLAB paths in kcat_tuning headers The kcat_tuning package (moved here from kcat_sensitivity_analysis, which no longer exists) cited `src/kcat_tuning/...` for its ported functions. That path doesn't resolve on the default GECKO MATLAB branch, and only partially resolves on `develop4` (sensitivityTuning.m, sigmaFitter.m, truncateValues.m exist there under `src/kcat_tuning/`, but findMaxValue.m and the whole `Bayesian/` folder do not). The real, resolvable source for all eight citations is `src/geckomat/kcat_sensitivity_analysis/`, matching every other header in the codebase and confirmed via the GECKO repo's actual tree (including a stale `doc/` snapshot on `develop4` that still shows `loadBayesianData.m` at that path). - kcat_tuning/find_max_value.py, sensitivity_tuning.py, sigma_fitter.py, truncate_values.py: `src/kcat_tuning/X.m` -> `src/geckomat/kcat_sensitivity_analysis/X.m`. - kcat_tuning/evotune/priors.py, distance.py, simulate.py, data.py: `src/kcat_tuning/Bayesian/X.m` -> `src/geckomat/kcat_sensitivity_analysis/Bayesian/X.m`. --- src/geckopy/kcat_tuning/evotune/data.py | 2 +- src/geckopy/kcat_tuning/evotune/distance.py | 2 +- src/geckopy/kcat_tuning/evotune/priors.py | 2 +- src/geckopy/kcat_tuning/evotune/simulate.py | 2 +- src/geckopy/kcat_tuning/find_max_value.py | 4 ++-- src/geckopy/kcat_tuning/sensitivity_tuning.py | 4 ++-- src/geckopy/kcat_tuning/sigma_fitter.py | 4 ++-- src/geckopy/kcat_tuning/truncate_values.py | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/geckopy/kcat_tuning/evotune/data.py b/src/geckopy/kcat_tuning/evotune/data.py index c8c3d08..6ecf36c 100644 --- a/src/geckopy/kcat_tuning/evotune/data.py +++ b/src/geckopy/kcat_tuning/evotune/data.py @@ -1,7 +1,7 @@ """Load experimental data for evotune (CMA-ES) kcat tuning. Ported from GECKO MATLAB: -src/kcat_tuning/Bayesian/loadBayesianData.m. +src/geckomat/kcat_sensitivity_analysis/Bayesian/loadBayesianData.m. MATLAB's ``loadBayesianData`` loads ``bayesianFluxData.tsv`` and ``bayesianMaxGrowth.tsv`` (renamed here to ``evotuneFluxData.tsv`` and diff --git a/src/geckopy/kcat_tuning/evotune/distance.py b/src/geckopy/kcat_tuning/evotune/distance.py index 97e30d7..410935f 100644 --- a/src/geckopy/kcat_tuning/evotune/distance.py +++ b/src/geckopy/kcat_tuning/evotune/distance.py @@ -1,7 +1,7 @@ """Carbon- and condition-weighted RMSE distance for evotune kcat tuning. Ported from GECKO MATLAB: -src/kcat_tuning/Bayesian/abc_max.m (the +src/geckomat/kcat_sensitivity_analysis/Bayesian/abc_max.m (the ``rmsecal`` helper's RMSE half; verified against ``develop4``'s current source). :mod:`.simulate` does the FBA half -- this module is pure numpy (plus one small model-reading helper for the carbon diff --git a/src/geckopy/kcat_tuning/evotune/priors.py b/src/geckopy/kcat_tuning/evotune/priors.py index 38558c4..19f1e61 100644 --- a/src/geckopy/kcat_tuning/evotune/priors.py +++ b/src/geckopy/kcat_tuning/evotune/priors.py @@ -1,7 +1,7 @@ """Source classification and prior widths for evotune kcat tuning. Ported from GECKO MATLAB: -src/kcat_tuning/Bayesian/bayesianSensitivityTuning.m +src/geckomat/kcat_sensitivity_analysis/Bayesian/bayesianSensitivityTuning.m (the ``kcatSourceIdx``/``sigma0log`` construction). Verified against ``develop4``'s current source. """ diff --git a/src/geckopy/kcat_tuning/evotune/simulate.py b/src/geckopy/kcat_tuning/evotune/simulate.py index 023b432..0117c06 100644 --- a/src/geckopy/kcat_tuning/evotune/simulate.py +++ b/src/geckopy/kcat_tuning/evotune/simulate.py @@ -1,7 +1,7 @@ """FBA simulation half of the evotune kcat tuning distance function. Ported from GECKO MATLAB: -src/kcat_tuning/Bayesian/abc_max.m (the +src/geckomat/kcat_sensitivity_analysis/Bayesian/abc_max.m (the ``rmsecal`` helper's simulate-and-measure loop). Verified against ``develop4``'s current source, not the (superseded, per its own ``REVIEW.md``) ``fix/bayesianTuning`` branch. diff --git a/src/geckopy/kcat_tuning/find_max_value.py b/src/geckopy/kcat_tuning/find_max_value.py index 5aeb0e3..88fa5b9 100644 --- a/src/geckopy/kcat_tuning/find_max_value.py +++ b/src/geckopy/kcat_tuning/find_max_value.py @@ -1,7 +1,7 @@ """Find the maximum kcat (or SA-derived kcat) for a set of EC numbers. Ported from GECKO MATLAB: -src/kcat_tuning/findMaxValue.m. +src/geckomat/kcat_sensitivity_analysis/findMaxValue.m. """ from __future__ import annotations @@ -28,7 +28,7 @@ def find_max_value( with ``"1.1.1."``. Ported from GECKO MATLAB: - src/kcat_tuning/findMaxValue.m. + src/geckomat/kcat_sensitivity_analysis/findMaxValue.m. Parameters ---------- diff --git a/src/geckopy/kcat_tuning/sensitivity_tuning.py b/src/geckopy/kcat_tuning/sensitivity_tuning.py index 83cff05..386567d 100644 --- a/src/geckopy/kcat_tuning/sensitivity_tuning.py +++ b/src/geckopy/kcat_tuning/sensitivity_tuning.py @@ -20,7 +20,7 @@ from the adapter config). Ported from GECKO MATLAB: -src/kcat_tuning/sensitivityTuning.m. +src/geckomat/kcat_sensitivity_analysis/sensitivityTuning.m. """ from __future__ import annotations @@ -141,7 +141,7 @@ def sensitivity_tuning( 7. Re-applies kcat constraints for that single reaction. Ported from GECKO MATLAB: - src/kcat_tuning/sensitivityTuning.m. + src/geckomat/kcat_sensitivity_analysis/sensitivityTuning.m. Parameters ---------- diff --git a/src/geckopy/kcat_tuning/sigma_fitter.py b/src/geckopy/kcat_tuning/sigma_fitter.py index 42d607e..566778f 100644 --- a/src/geckopy/kcat_tuning/sigma_fitter.py +++ b/src/geckopy/kcat_tuning/sigma_fitter.py @@ -12,7 +12,7 @@ difference. Ported from GECKO MATLAB: -src/kcat_tuning/sigmaFitter.m. +src/geckomat/kcat_sensitivity_analysis/sigmaFitter.m. """ from __future__ import annotations @@ -96,7 +96,7 @@ def fit_sigma( The best sigma is re-applied to ``model`` before return. Ported from GECKO MATLAB: - src/kcat_tuning/sigmaFitter.m. + src/geckomat/kcat_sensitivity_analysis/sigmaFitter.m. Parameters ---------- diff --git a/src/geckopy/kcat_tuning/truncate_values.py b/src/geckopy/kcat_tuning/truncate_values.py index b0f0f97..cbc33d0 100644 --- a/src/geckopy/kcat_tuning/truncate_values.py +++ b/src/geckopy/kcat_tuning/truncate_values.py @@ -1,7 +1,7 @@ """Round numeric values to roughly 6 significant figures. Ported from GECKO MATLAB: -src/kcat_tuning/truncateValues.m. +src/geckomat/kcat_sensitivity_analysis/truncateValues.m. """ from __future__ import annotations @@ -28,7 +28,7 @@ def truncate_values( unchanged. The input is never mutated; a new array is returned. Ported from GECKO MATLAB: - src/kcat_tuning/truncateValues.m. + src/geckomat/kcat_sensitivity_analysis/truncateValues.m. Parameters ---------- From b0a93645602d7e83508904a7622981343f764660 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 16:41:37 +0200 Subject: [PATCH 14/15] README: put the gecko-docs.readthedocs.io pointer at the top Adds a callout right under the tagline, before Disambiguation -- that site carries the full protocol, install, troubleshooting, and API reference for both toolboxes, well beyond what this README covers, so it belongs as the first thing a reader sees rather than buried in Relationship to MATLAB GECKO near the bottom. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 4cba6d8..defb0ff 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,12 @@ Enzyme-constrained genome-scale metabolic modelling in Python. +> **Full documentation for both MATLAB GECKO and geckopy — the six-stage +> protocol, installation, troubleshooting, and complete API reference — +> lives at [gecko-docs.readthedocs.io](https://gecko-docs.readthedocs.io/).** +> This README only covers install and a quick orientation; that site is +> the place to actually learn or look things up. + > **Disambiguation.** This is a new, from-scratch port of the > [GECKO Toolbox](https://github.com/SysBioChalmers/GECKO) (MATLAB). It is unrelated to > the earlier and separate `geckopy` Python package From 798ab0cdd8a1d25c95415b6698112b5888b0aeb2 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Tue, 8 Sep 2026 22:44:59 +0200 Subject: [PATCH 15/15] minor edits to README --- README.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index defb0ff..e6e67f4 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,14 @@ curations, and integrates proteomics measurements. > **Status: beta (`4.0.0b1`).** All the MATLAB GECKO 3.2.5 functions used in > the standard ecModel build are ported, and the yeast-GEM tutorial runs -> end-to-end. geckopy's version number tracks the GECKO project's own -> toolbox-generation numbering (MATLAB GECKO 1–3, geckopy = 4.x), rather than -> making a semantic-versioning API-stability claim. +> end-to-end. geckopy's major and minor version number (x.y.-) will follow +> the MATLAB GECKO numbering, justifying a jump to version 4. ## Install geckopy depends on [raven-toolbox](https://github.com/SysBioChalmers/raven-toolbox) (the Python port of the RAVEN Toolbox). Both are currently pre-releases on -PyPI, so a plain `pip install geckopy` won't find anything — `--pre` is +PyPI, so a plain `pip install geckopy` will install the wrong version — `--pre` is required: ```bash @@ -141,13 +140,8 @@ specification and how it differs from the legacy MATLAB / RAVEN format. geckopy is a function-by-function port: algorithmic fidelity comes first, Pythonic idiom second. Each ported source file carries a `Ported from GECKO MATLAB: ` docstring header. For current -MATLAB ↔ Python behavioural differences, see the -[MATLAB ↔ Python translation guide](https://gecko-docs.readthedocs.io/en/latest/api/translation/) -on [gecko-docs.readthedocs.io](https://gecko-docs.readthedocs.io/), the -shared documentation site for both toolboxes. The cumulative list of -MATLAB-side bugs and rough edges the port surfaced lives in -[`docs/future_improvements.md`](docs/future_improvements.md). - +MATLAB ↔ Python behavioural differences, see [gecko-docs.readthedocs.io](https://gecko-docs.readthedocs.io/), the +shared documentation site for both toolboxes. ## License MIT. See [`LICENSE`](LICENSE), or the `license` field of