From 9bbab5a0313a9abdb66a3ed1589328eda8e8111a Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:07:38 +0000 Subject: [PATCH 1/4] feat(challenges): load docker challenges and move bounty out The master now runs any challenge that implements docs/CHALLENGES.md as a Docker container: it reads authenticated get_weights once per epoch, signs exact leaves, seals, and proxies public routes under /challenge//. The new challenge-supervisor is the only process with the Docker socket. It pulls stable/edge/pinned GHCR images, checks labels and GitHub build provenance, canaries without secrets, and rolls back. Trust root version >= 3 enables algorithm 3: 1..64 challenges, each paying share * min(sum(leaves), 10^12) / 10^12, with the rest burned. Validators keep verifying signed leaves only. Bounty under algorithm 3 with full_share_mass = 10 pays exactly the algorithm 2 amount. The in-process bounty service is removed; it now lives in CortexLM/bounty. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.example | 8 +- .greptile/rules.md | 18 +- AGENTS.md | 71 +-- CHANGELOG.md | 17 + CONTRIBUTING.md | 3 +- README.md | 43 +- SECURITY.md | 2 +- config/challenges-v3.example.toml | 25 + deploy/README.md | 88 ++-- deploy/challenges/registry.toml | 29 ++ deploy/compose/role-master.yml | 62 ++- deploy/env/master.env.example | 7 +- deploy/secrets/README.md | 18 +- docs/AGENTS.md | 4 +- docs/ARCHITECTURE.md | 44 +- docs/BOUNTY.md | 151 ------ docs/CHALLENGES.md | 161 +++++++ docs/NAMING.md | 23 +- docs/OPERATOR_SECURITY.md | 39 +- docs/THREAT_MODEL.md | 32 +- docs/external-miner/README.md | 18 +- docs/external-miner/bounty.md | 197 +------- docs/external-miner/proof.md | 5 +- docs/external-miner/troubleshoot.md | 7 +- docs/external-miner/validators.md | 18 +- docs/how-to/trust-root.md | 45 +- docs/index.md | 4 +- docs/reference/configuration.md | 37 +- scripts/check_deploy.py | 92 +++- scripts/check_repo.py | 50 +- src/cortex/bounty/__init__.py | 16 - src/cortex/bounty/api.py | 224 --------- src/cortex/bounty/backend.py | 470 ------------------- src/cortex/bounty/scoring.py | 110 ----- src/cortex/bounty/service.py | 167 ------- src/cortex/bounty/store.py | 264 ----------- src/cortex/challenges/__init__.py | 1 + src/cortex/challenges/__main__.py | 59 +++ src/cortex/challenges/client.py | 113 +++++ src/cortex/challenges/proxy.py | 74 +++ src/cortex/challenges/registry.py | 139 ++++++ src/cortex/challenges/supervisor.py | 255 +++++++++++ src/cortex/cli.py | 10 +- src/cortex/config.py | 47 +- src/cortex/gateway/api.py | 7 + src/cortex/gateway/projection.py | 16 +- src/cortex/gateway/service.py | 19 +- src/cortex/master.py | 135 +++--- src/cortex/miner.py | 11 +- src/cortex/protocol/aggregate.py | 30 +- src/cortex/protocol/bundle.py | 6 +- src/cortex/protocol/models.py | 19 +- src/cortex/state.py | 2 +- tests/bounty/test_api.py | 684 ---------------------------- tests/bounty/test_backend.py | 674 --------------------------- tests/bounty/test_scoring.py | 68 --- tests/bounty/test_store.py | 119 ----- tests/challenges/test_contract.py | 123 +++++ tests/challenges/test_supervisor.py | 211 +++++++++ tests/test_deploy_contract.py | 114 ++++- tests/test_master.py | 84 +--- tests/test_master_config.py | 2 - tests/test_miner_wallet.py | 3 +- tests/test_network_e2e.py | 302 ++++++------ tests/test_repo_contract.py | 15 + tests/test_state_security.py | 4 +- 66 files changed, 2245 insertions(+), 3670 deletions(-) create mode 100644 config/challenges-v3.example.toml create mode 100644 deploy/challenges/registry.toml delete mode 100644 docs/BOUNTY.md create mode 100644 docs/CHALLENGES.md delete mode 100644 src/cortex/bounty/__init__.py delete mode 100644 src/cortex/bounty/api.py delete mode 100644 src/cortex/bounty/backend.py delete mode 100644 src/cortex/bounty/scoring.py delete mode 100644 src/cortex/bounty/service.py delete mode 100644 src/cortex/bounty/store.py create mode 100644 src/cortex/challenges/__init__.py create mode 100644 src/cortex/challenges/__main__.py create mode 100644 src/cortex/challenges/client.py create mode 100644 src/cortex/challenges/proxy.py create mode 100644 src/cortex/challenges/registry.py create mode 100644 src/cortex/challenges/supervisor.py delete mode 100644 tests/bounty/test_api.py delete mode 100644 tests/bounty/test_backend.py delete mode 100644 tests/bounty/test_scoring.py delete mode 100644 tests/bounty/test_store.py create mode 100644 tests/challenges/test_contract.py create mode 100644 tests/challenges/test_supervisor.py diff --git a/.env.example b/.env.example index 8ac31f5cf..a25738a14 100644 --- a/.env.example +++ b/.env.example @@ -8,9 +8,12 @@ BASE_CHALLENGES_FILE=config/challenges.toml BASE_MEASUREMENTS_FILE=config/measurements.toml BASE_GATEWAY_SK_FILE=.local/secrets/gateway.seed BASE_GATEWAY_ADMIN_TOKEN_FILE=.local/secrets/operator.token -BOUNTY_SK_FILE=.local/secrets/bounty.seed PROOF_SK_FILE=.local/secrets/proof.seed -BOUNTY_SESSION_SECRET_FILE=.local/secrets/bounty-session.secret +# Leaf seeds of container challenges: /.key for every trusted id but proof. +BASE_CHALLENGE_KEYS_DIR=.local/secrets +# Optional operator registry; see docs/CHALLENGES.md. Unset runs no container challenge. +# BASE_CHALLENGE_REGISTRY_FILE=deploy/challenges/registry.toml +BASE_CHALLENGE_SECRETS_DIR=.local/challenge-secrets # Shared topic/experiment VM shape; see deploy/README.md for host capacity. PROOF_RLM_VM_VCPUS=1 @@ -19,7 +22,6 @@ PROOF_RLM_VM_DISK_MIB=16384 # Live integrations remain disabled until the operator provides verified endpoints # and locally computed artifact pins. -# BOUNTY_BACKEND_PUBLIC_URL=https://backend.example # PROOF_VM_ORCHESTRATOR_URL=https://proof-vm.internal:9443 # PROOF_VM_ORCHESTRATOR_TOKEN_FILE=.local/secrets/proof-vm.token # PROOF_VM_ORCHESTRATOR_CA_FILE=.local/secrets/proof-vm-ca.pem diff --git a/.greptile/rules.md b/.greptile/rules.md index 52074a4a6..9f0ab318e 100644 --- a/.greptile/rules.md +++ b/.greptile/rules.md @@ -1,16 +1,18 @@ # Cortex review rules -Cortex is a Python Bittensor research subnet with exactly two live challenges: -`bounty` at 3,000 basis points and `proof` at 7,000 under algorithm 2. The -owner-signed legacy 2,000/8,000 profile retains algorithm 1. The trust-root sum -is always 10,000; the new profile requires challenge-document version >=2. -Design, Prism and Relearn are historical only. +Cortex is a Python Bittensor research subnet. The owner-signed trust root lists +the live challenges: legacy bounty/proof 2,000/8,000 (algorithm 1), 3,000/7,000 +(algorithm 2, version >=2) or 1..64 unique ids (algorithm 3, version >=3). The +sum is always 10,000. Proof is built in; every other challenge is a Docker +container per docs/CHALLENGES.md. Design, Prism and Relearn are historical only. - Preserve frozen SCALE encodings, Merkle construction, aggregation and every `base-*-v1` signature preimage. Cross-language vectors must remain green. Algorithm 2 counts each valid Bounty report once, distributes proportionally, scales its 30% share by min(total_valid/10, 1), and burns unused/quarantined mass. The total includes only the signed expected participant population. + Algorithm 3 pays each challenge share * min(sum(leaves), 10^12) / 10^12 and + burns the rest; unpaid mass never moves to another challenge. Never allow an algorithm 1 body under the new profile or activate before the owner-signed epoch; historical seals and journals stay immutable. - Preserve existing `BASE_*` names and deployed compatibility paths. New master @@ -26,8 +28,10 @@ Design, Prism and Relearn are historical only. - Recursive children share the parent's hard call, token, tool, depth and wall budgets. External side effects require durable intent and idempotent recovery. - Shared observations are untrusted and private until owner-signed approval. -- Bounty scoring reads only the stable CortexLM/backend public feed. An outage - returns 503 at intake and explicit `ChallengeInternal` leaves at emission. +- A trusted challenge container that is unregistered, failing or returns invalid + weights yields explicit `ChallengeInternal` leaves; never a stale or guessed + score. Containers never receive a leaf seed; only `challenge-supervisor` + mounts the Docker socket; the proxy never forwards `internal/` paths. - Validators independently fetch historical chain state, recompute exact u16 weights and refuse the unsealed UID0 fallback. A sealed UID0 burn is valid. - Production images are digest-pinned. Secrets are private files and must never diff --git a/AGENTS.md b/AGENTS.md index 72cb2ce1c..e36ea2e78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,23 +5,28 @@ canonical documentation instead of duplicating runbooks. ## Product -Cortex is an autonomous research subnet on Bittensor. It has exactly two live -challenge IDs: - -| Challenge | Algorithm 2 share | Purpose | -| --- | ---: | --- | -| `bounty` | 3,000 bps | useful vulnerability reports, scored from the CortexLM/backend public feed | -| `proof` | 7,000 bps | operator-created research topics evaluated by Cortex's recursive language-model engine | - -The sum is always 10,000 basis points. The owner-signed legacy 2,000/8,000 -profile retains algorithm 1. The 3,000/7,000 profile requires challenge-document -version >=2 and algorithm 2; activation is an offline owner ceremony. Design, -Prism and Relearn are retired products. Their frozen specifications and historical miner pointers remain for +Cortex is an autonomous research subnet on Bittensor. Proof runs inside the +master. Every other challenge is a Docker container that implements +[the challenge contract](docs/CHALLENGES.md); the master loads it from an +operator registry and signs its leaves. + +| Challenge | Source | Purpose | +| --- | --- | --- | +| `proof` | this repository | operator-created research topics evaluated by Cortex's recursive language-model engine | +| `bounty` | [CortexLM/bounty](https://github.com/CortexLM/bounty) container | useful vulnerability reports, scored from the CortexLM/backend public feed | +| `opentype` | [OpentypeAI/challenge](https://github.com/OpentypeAI/challenge) container | exact-gold typed-decision duels on DiffusionGemma weights | + +Shares always sum to 10,000 basis points. Algorithm 1 is the legacy signed +bounty/proof 2,000/8,000 profile; algorithm 2 is bounty/proof 3,000/7,000 +(document version >=2); algorithm 3 (document version >=3) admits any 1..64 +unique ids and pays each challenge `share * min(sum(leaves), 10^12) / 10^12`. +Activation is an offline owner ceremony. Design, Prism and Relearn are retired +products. Their frozen specifications and historical miner pointers remain for compatibility; no active code, service, trust-root row or leaf may register them. Start with [the architecture](docs/ARCHITECTURE.md), -[Proof](docs/PROOF.md), [Bounty](docs/BOUNTY.md) and +[Proof](docs/PROOF.md), [challenge containers](docs/CHALLENGES.md) and [the threat model](docs/THREAT_MODEL.md). Do not describe a fake-boundary test, one model call or a pinned image as proof of live KVM execution, scientific reproduction or on-chain payment. @@ -33,7 +38,7 @@ reproduction or on-chain payment. | `src/cortex/protocol/` | frozen SCALE, signatures, Merkle and aggregation | | `src/cortex/gateway/` | durable leaves, immutable seals and burn fallback | | `src/cortex/validator/` | independent recomputation, root consensus and chain dispatch | -| `src/cortex/bounty/` | pairing, report intake, external-feed scoring and adjudication | +| `src/cortex/challenges/` | container registry, weights client, public proxy and auto-updating supervisor | | `src/cortex/proof/` | topics, submissions, setup, executor offers and reward allocation | | `src/cortex/rlm/` | recursive agent, budgets, compaction, journals and shared knowledge | | `src/cortex/vm/` | Firecracker host, guest protocol and measured experiment lifecycle | @@ -68,35 +73,33 @@ or permit UID is not a submit path. | --- | --- | | owner seed | signs challenge and measurement trust documents offline | | gateway seed | signs immutable epoch bundles | -| Bounty seed | signs Bounty leaves and must match the trust root | +| challenge seed | `.key`; signs that container challenge's leaves and must match the trust root | | Proof seed | signs Proof topics and leaves and must match the trust root | | operator bearer | protects master administrative routes | | VM orchestrator bearer | authenticates master to the dedicated KVM host; not a wallet | | validator hotkey | signs root/dissent evidence and Bittensor submissions | +| challenge internal token | master bearer for a container's `get_weights`; never a signing key | | miner hotkey | signs Bounty pairing or Proof submission payloads | Do not conflate gateway sealing, master ownership and validator chain signing. Follow [the trust-root ceremony](docs/how-to/trust-root.md). -## Bounty contracts - -- `/v1/pair` verifies a Substrate-context hotkey signature, explicit terms and - a single-use nonce. Re-pairing an account revokes its prior session. -- `/v1/reports` reads the external feed before storing anything. Missing, - moving or malformed feed data returns 503 with no row. -- Local adjudication supports only `valid`, `already_fixed_not_prod`, - `invalid_malicious` and `duplicate`. A valid report without severity is not - creditable. -- Scores come only from `BOUNTY_BACKEND_PUBLIC_URL`. Never add an offline live - scorer. On feed failure, cover every expected participant with - `NoScore(ChallengeInternal)` so the Bounty share burns without blocking Proof. -- Under algorithm 2, each valid report contributes one point regardless of severity. - Reward authors proportionally; Bounty pays `0.30 * min(total_valid / 10, 1)`. - Count cumulative published reports only for the expected participant set. - Burn unused or unmapped mass to UID0; never increase Proof or surviving - challenge shares. Keep algorithm 1 and its frozen vectors unchanged. -- A public API, quota or scoring change must update - `docs/external-miner/bounty.md` in the same change. +## Challenge container contracts + +- The owner-signed trust root decides emission; the unsigned registry + (`deploy/challenges/registry.toml`) only decides what runs. A container never + receives a leaf-signing seed. +- `challenge-supervisor` is the only process with the Docker socket. It verifies + image labels and GitHub build provenance, canaries without secrets, rolls back + on failure and never reads a secret. +- A missing, failing or invalid `get_weights` answer covers every expected + participant with `NoScore(ChallengeInternal)`: that share burns to UID0 and + never moves to another challenge. +- The public proxy `/challenge//` never forwards `internal/` paths or + credentials other than `authorization`. +- A contract change must update `docs/CHALLENGES.md`, both challenge + repositories and the E2E tests in the same change. Bounty API changes belong + to CortexLM/bounty; keep `docs/external-miner/bounty.md` pointing there. ## Proof contracts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a9831846..2f8377a7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Challenge containers (`docs/CHALLENGES.md`): the master loads any Docker + challenge from an operator registry, polls its authenticated `get_weights` + once per epoch, signs its leaves and proxies its public routes under + `/challenge//`. `cortex challenge-supervisor` pulls `stable`/`edge`/pinned + GHCR images, checks labels and GitHub build provenance, canaries without + secrets, updates and rolls back automatically. It alone holds the Docker socket. +- Owner-activated algorithm 3 (trust document version >=3): 1..64 challenges, + each paying `share * min(sum(leaves), 10^12) / 10^12`, remainder burned. + Validators need no change beyond the release; they recompute signed leaves. +- `GET /v1/metagraph/latest` serves the sealed hotkey map to challenges. + +### Removed + +- The in-process Bounty service (`src/cortex/bounty`, `docs/BOUNTY.md` and the + `BOUNTY_*` master settings). Bounty now runs as the CortexLM/bounty container; + its operator guide migrates `bounty.sqlite3` and the session key unchanged. + - Owner-activated algorithm 2: each valid Bounty report earns one point, all authors share proportionally, and ten valid reports unlock its full 30% of emission. Proof retains 70%; unused Bounty mass burns to UID0. The legacy diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c5b2b6072..8469cb1f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,7 @@ # Contributing to Cortex -Cortex is a Python Bittensor subnet with two live challenges: Bounty and Proof. +Cortex is a Python Bittensor subnet with Proof built in and further challenges loaded as Docker containers +(see docs/CHALLENGES.md). Read [AGENTS.md](AGENTS.md), the [architecture](docs/ARCHITECTURE.md), and the [naming contract](docs/NAMING.md) before changing protocol or deployment code. diff --git a/README.md b/README.md index 3db5b6c92..a95491317 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,26 @@ # Cortex -Cortex research subnet: Bounty and agentic Proof on Bittensor. +Cortex research subnet on Bittensor: agentic Proof plus Docker challenge containers. -The Python implementation runs the gateway and both challenges on a master, -verifies sealed rewards in independent validators, and isolates research work -in Firecracker guests. Algorithm 2 assigns up to 30% of emission to Bounty -and 70% to Proof; activating it requires a new owner-signed trust root. -Proof uses Cortex's own recursive language-model engine with persistent memory, -context compaction and bounded tool execution. +The master runs the gateway, Proof and every challenge container, then signs one +leaf per miner and seals each epoch. Validators only verify the sealed bundle +from the gateway API and submit the weights. + +| Role | Runs | Command | +| --- | --- | --- | +| validator master | gateway, Proof, [challenge containers](docs/CHALLENGES.md), auto-updater | `cortex master` + `cortex challenge-supervisor` | +| validator | verification and weight submission, nothing else | `cortex validator` | + +Challenges live in their own repositories and are loaded automatically: +[CortexLM/bounty](https://github.com/CortexLM/bounty) (vulnerability reports) and +[OpentypeAI/challenge](https://github.com/OpentypeAI/challenge) (exact-gold +DiffusionGemma duels). The owner-signed trust root sets each challenge's emission +share. The shortfall of a challenge burns to UID 0 and never moves to another one. Cortex is experimental research software. Offline tests exercise submission, scoring, sealing and validator dispatch through fake external boundaries. A live model smoke is distinct from live KVM execution or confirmed on-chain payment. -## Current launch mode - -The initial production mode enables Bounty against the configured -`CortexLM/backend` public feed and leaves Proof execution unwired. The signed -trust root still contains `bounty = 2000` and `proof = 8000`: Proof emits -`ChallengeInternal` absences and its share burns to UID 0. Never renormalize -Bounty to 100%. Production pairing, report intake and adjudication stay in -`CortexLM/backend`; Cortex reads its immutable public scoring snapshots. -Algorithm 2 pays one point per valid report, proportionally across authors; -ten valid reports across expected participants unlock the full Bounty share. -The unsigned [30/70 template](config/challenges-v2.example.toml) changes nothing -until the [trust-root migration](docs/how-to/trust-root.md#activate-proportional-bounty) -is completed on the gateway and validators. - ## Installation Linux, Python 3.12 or 3.13, `uv`, and libsodium 1.0.18 or newer are required. @@ -42,6 +36,7 @@ uv run cortex --help ```bash uv run cortex master --help +uv run cortex challenge-supervisor --help uv run cortex validator --help uv run cortex vm-host --help uv run cortex miner --help @@ -56,6 +51,8 @@ receipts and signs the resulting document. No research task catalog is built in. - [Operator configuration](docs/reference/configuration.md) - [Architecture and trust boundaries](docs/ARCHITECTURE.md) +- [Challenge container contract](docs/CHALLENGES.md) +- [Deployment](deploy/README.md) - [Proof miner guide](docs/external-miner/proof.md) - [Bounty miner guide](docs/external-miner/bounty.md) - [Validator guide](docs/external-miner/validators.md) @@ -72,8 +69,8 @@ uv run python scripts/check_deploy.py --check-examples uv build --no-build-isolation ``` -Tests cover signatures and Rust wire vectors, replay protection, feed outages, -artifact validation, topic setup, rejected submissions, VM lifecycle failures, +Tests cover signatures and Rust wire vectors, replay protection, challenge +container outages, auto-update rollback, artifact validation, topic setup, rejected submissions, VM lifecycle failures, RLM recursion/compaction, reward allocation and sealed-weight submission. CI runs offline and never rents a GPU or boots Firecracker. diff --git a/SECURITY.md b/SECURITY.md index 13107deef..be346cfc4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,7 +11,7 @@ reproduction. State whether the issue is already public. ## Scope -The gateway, validator, Bounty/Proof services, RLM, Firecracker host, miner CLI, +The gateway, validator, Proof service, challenge supervisor and proxy, RLM, Firecracker host, miner CLI, deployment definitions, signature formats, and sealed-weight path are in scope. Third-party model/GPU providers and miner artifacts remain untrusted external boundaries, but failures in Cortex's validation of them are in scope. diff --git a/config/challenges-v3.example.toml b/config/challenges-v3.example.toml new file mode 100644 index 000000000..f7b6ccbb2 --- /dev/null +++ b/config/challenges-v3.example.toml @@ -0,0 +1,25 @@ +# UNSIGNED algorithm 3 template (document version >= 3). Choose the shares, the +# next owner document version and the activation epoch, replace development +# public keys, then sign offline using docs/how-to/trust-root.md. +# Each challenge pays share * min(sum(leaves), 10^12) / 10^12; the rest burns. +# Container ids also need a row in deploy/challenges/registry.toml to run. +version = 3 +introduced_epoch = "CHOOSE_ACTIVATION_EPOCH" + +[[challenges]] +id = "bounty" +public_key = "743688a1e1b2848b309205706b4dcae54bffe4233a5d7018053471e1dce45c21" +emission_share_bps = 3000 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "opentype" +public_key = "ac6cad384e46122a7000a6efa71d084a9a2d6407ab22e86aa2b2e5e9dcb2e43f" +emission_share_bps = 3500 +policy = "all_metagraph_hotkeys" + +[[challenges]] +id = "proof" +public_key = "3e7f70f09165e265ab89ab04a4fc91dc0531d54a100c538fb14c6f008421c375" +emission_share_bps = 3500 +policy = "all_metagraph_hotkeys" diff --git a/deploy/README.md b/deploy/README.md index 4eb77a67b..53c94afa0 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -4,7 +4,7 @@ Cortex uses three independently operated roles: | Role | Runtime | Responsibility | | --- | --- | --- | -| master | Docker Compose | public gateway, Bounty, Proof and durable epoch emission | +| master | Docker Compose | public gateway, Proof, challenge containers and durable epoch emission | | validator | Docker Compose | independent bundle verification, peer roots and Bittensor submission | | Proof VM host | systemd on a KVM machine | Firecracker topic and experiment VMs | @@ -15,7 +15,7 @@ dedicated machine reachable from the master over a private network and HTTPS. - [Validate the deployment source](#validate-the-deployment-source) - [Build and publish the Python image](#build-and-publish-the-python-image) - [Master](#master) -- [Bounty-only launch](#bounty-only-launch) +- [Challenge containers](#challenge-containers) - [Validator](#validator) - [Proof VM host](#proof-vm-host) - [Promotion and rollback](#promotion-and-rollback) @@ -73,7 +73,8 @@ network plus optional bounded `wss://` fallback RPC origins, then set: - `CORTEX_IMAGE` to the published `repository@sha256:<64 hex>`; - the master VPC bind address and netuid; -- the external Bounty feed when Bounty should accept reports; +- `BASE_CHALLENGE_SECRETS_HOST_DIR`, the absolute host directory of challenge + bearers, and `BASE_DOCKER_GID`, the group owning `/var/run/docker.sock`; - the Proof VM URL, CA, exact rootfs digest, signed inference-offer commitment and registered custom IDs when Proof custom topics should open; - explicit `PROOF_RLM_VM_VCPUS`, `PROOF_RLM_VM_MEM_MIB` and @@ -84,16 +85,14 @@ Prepare a private directory, owned so container UID 65532 can read it: ```text deploy/secrets/master/ gateway.key - bounty.key proof.key - bounty-session.key + bounty.key # one .key per trusted container challenge operator.token proof-vm.token proof-vm-ca.pem ``` -Seeds are raw 32 bytes or 64 hexadecimal characters. `bounty-session.key` has at -least 32 random bytes. Bearers are nonempty opaque values. Files are regular, +Seeds are raw 32 bytes or 64 hexadecimal characters. Bearers are nonempty opaque values. Files are regular, not hardlinks or symlinks, and mode 0400 or 0600. The Proof VM token and CA are needed only when `PROOF_VM_ORCHESTRATOR_URL` is set. @@ -106,36 +105,59 @@ docker compose --project-directory . --env-file deploy/env/master.env \ -f deploy/compose/role-master.yml --profile master up -d ``` -The state volume contains gateway seals, Bounty state, Proof jobs/topics and the -epoch journal. Back it up as SQLite state, including WAL, through a quiesced copy +The state volume contains gateway seals, Proof jobs/topics and the epoch +journal. Each challenge keeps its own state in the `cortex-challenge--data` +volume. Back it up as SQLite state, including WAL, through a quiesced copy or SQLite's backup API. A filesystem copy of only the main database while the service writes is not a valid backup. -## Bounty-only launch +## Challenge containers -Set `BOUNTY_BACKEND_PUBLIC_URL` to the reviewed HTTPS `CortexLM/backend` origin -and leave `PROOF_VM_ORCHESTRATOR_URL` empty. Keep `proof.key` mounted. Legacy -deployments retain the signed `bounty = 2000`, `proof = 8000` profile until -[algorithm 2 activation](../docs/how-to/trust-root.md#activate-proportional-bounty) -coordinates the gateway and validators with a signed 3000/7000 profile. -With no open Proof topic, its entire configured share burns to UID0 through -signed `ChallengeInternal` leaves. Algorithm 2 pays up to 30% for Bounty, -proportionally to valid reports with a global ten-report ramp; Bounty is never -scaled to 100%. +The master role runs a second service, `challenge-supervisor`, which is the only +Cortex process holding the Docker socket. It reads +`deploy/challenges/registry.toml`, and for each entry it: -Keep `BOUNTY_GATEWAY_URL` empty in CortexLM/backend unless an authenticated, -idempotent delivery contract is deployed. The production dependency for this -mode is the public leaderboard/report feed, including severity, pagination and -one shared snapshot revision. +1. pulls the channel or pin; +2. checks the slug, contract and source labels and the GitHub build provenance; +3. runs a secretless canary; +4. replaces the container `cortex-challenge-` on the private + `cortex-challenges` network; +5. rolls back when the new digest does not answer `/version`. -Production miners pair and file reports through CortexLM/backend. Do not expose -the Python compatibility intake as an automatic payment path: it does not write -to the backend publication. +The gateway joins that network as `cortex-master`, polls `get_weights` once per +completed epoch and proxies public routes under `/challenge//`. The full +contract is in [docs/CHALLENGES.md](../docs/CHALLENGES.md). -Before enabling validators, run a real Bounty intake failure probe, then a valid -intake and public-feed publication. Confirm the completed epoch has signed leaves, -a `sealed: true` latest response, the expected Bounty/Proof burn split and a -successful validator `--verify-only --once` recomputation. +Per challenge, on the master host: + +```bash +dir="$BASE_CHALLENGE_SECRETS_HOST_DIR/bounty" +install -d -m 0700 -o 65532 -g 65532 "$dir" +(umask 077; openssl rand -hex 32 >"$dir/internal.token"; openssl rand -hex 32 >"$dir/admin.token") +chown 65532:65532 "$dir"/*.token +``` + +Put the matching leaf seed at `deploy/secrets/master/.key` (its public key +is the trust-root row) and any challenge settings in the registry `env` table, +for example `BOUNTY_BACKEND_PUBLIC_URL`. Challenge-specific secrets, such as +Bounty's `session.key`, go next to the tokens; see each challenge's operator +guide ([Bounty](https://github.com/CortexLM/bounty/blob/main/docs/operator.md), +[OpenType](https://github.com/OpentypeAI/challenge/blob/main/docs/operator.md)). + +A registered id that the trust root does not list runs without emission. Use this +burn-in to check `GET /challenge//version` and the logs. Then activate it with +a signed [algorithm 3 profile](../docs/how-to/trust-root.md#activate-container-challenges-algorithm-3). +A trusted id that is missing, unhealthy or invalid burns its share. + +Operator switches: + +- `channel = "edge"` follows `main`; +- `pin = "sha256:..."` freezes a digest; +- `attestation = false` is for local images only; +- deleting a registry row stops that container and keeps its volume. + +The master re-reads the registry when the file changes, and the supervisor +re-reads it every 15 seconds. ## Validator @@ -285,8 +307,10 @@ The protected `production` environment verifies provenance and emits evidence, changing `CORTEX_IMAGE` to its immutable digest, rendering Compose, backing up state, pulling that digest and recreating one role. -There is deliberately no `git pull`, Docker-socket watcher, CI SSH deployment or -unattended host updater. Safe automation must preserve the same operator gate, +The Cortex image itself has deliberately no `git pull`, Docker-socket watcher, +CI SSH deployment or unattended host updater; only challenge containers +auto-update, through the supervisor gates above, and they never hold a leaf +seed. Safe automation of the Cortex image must preserve the same operator gate, take a consistent SQLite backup, verify role health plus a fresh sealed bundle, and restore both the prior digest and schema-compatible state on failure. diff --git a/deploy/challenges/registry.toml b/deploy/challenges/registry.toml new file mode 100644 index 000000000..af2d17a36 --- /dev/null +++ b/deploy/challenges/registry.toml @@ -0,0 +1,29 @@ +# Unsigned operator registry: which challenge containers the master runs. +# Emission is decided only by the owner-signed trust root (config/challenges.toml); +# a registered id absent from it runs without emission. See docs/CHALLENGES.md. +# The supervisor re-reads this file every tick and the master on every change. +version = 1 + +[[challenge]] +id = "bounty" +image = "ghcr.io/cortexlm/bounty" +channel = "stable" +source = "https://github.com/CortexLM/bounty" +poll_seconds = 300 +cpus = 1.0 +memory_mib = 512 +pids = 128 + +[challenge.env] +BOUNTY_BACKEND_PUBLIC_URL = "" + +[[challenge]] +id = "opentype" +image = "ghcr.io/opentypeai/challenge" +channel = "stable" +source = "https://github.com/OpentypeAI/challenge" +poll_seconds = 300 +cpus = 2.0 +memory_mib = 2048 +pids = 256 +proxy_body_limit = 5242880 diff --git a/deploy/compose/role-master.yml b/deploy/compose/role-master.yml index a0a1fa3e9..355fdd084 100644 --- a/deploy/compose/role-master.yml +++ b/deploy/compose/role-master.yml @@ -21,10 +21,11 @@ services: BASE_CHALLENGES_FILE: /etc/base/config/challenges.toml BASE_MEASUREMENTS_FILE: /etc/base/config/measurements.toml BASE_GATEWAY_SK_FILE: /run/secrets/gateway.key - BOUNTY_SK_FILE: /run/secrets/bounty.key PROOF_SK_FILE: /run/secrets/proof.key - BOUNTY_SESSION_SECRET_FILE: /run/secrets/bounty-session.key BASE_GATEWAY_ADMIN_TOKEN_FILE: /run/secrets/operator.token + BASE_CHALLENGE_KEYS_DIR: /run/secrets + BASE_CHALLENGE_REGISTRY_FILE: /etc/base/challenges/registry.toml + BASE_CHALLENGE_SECRETS_DIR: /run/challenge-secrets ports: - "${BASE_GATEWAY_BIND_ADDRESS:?Set the master VPC address}:8080:8080" volumes: @@ -40,7 +41,23 @@ services: read_only: true bind: create_host_path: false + - type: bind + source: ${BASE_CHALLENGE_REGISTRY_DIR:-deploy/challenges} + target: /etc/base/challenges + read_only: true + bind: + create_host_path: false + - type: bind + source: ${BASE_CHALLENGE_SECRETS_HOST_DIR:?Set the absolute host directory of challenge secrets} + target: /run/challenge-secrets + read_only: true + bind: + create_host_path: false - base-master-state:/var/lib/base + networks: + default: {} + challenges: + aliases: [cortex-master] tmpfs: - /tmp:size=64m,mode=1777 healthcheck: @@ -49,5 +66,46 @@ services: timeout: 10s retries: 3 start_period: 90s + # The only service with Docker control: pulls, verifies, canaries and updates + # challenge containers listed in the registry. It never reads a secret. + challenge-supervisor: + profiles: [master] + image: ${CORTEX_IMAGE:?Set the published image repository@sha256:digest} + command: + - challenge-supervisor + - --registry + - /etc/base/challenges/registry.toml + - --secrets-host-dir + - ${BASE_CHALLENGE_SECRETS_HOST_DIR:?Set the absolute host directory of challenge secrets} + - --network + - cortex-challenges + - --master-url + - http://cortex-master:8080 + restart: unless-stopped + init: true + read_only: true + user: "65532:65532" + group_add: ["${BASE_DOCKER_GID:?Set the host docker group id}"] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + read_only: true + bind: + create_host_path: false + - type: bind + source: ${BASE_CHALLENGE_REGISTRY_DIR:-deploy/challenges} + target: /etc/base/challenges + read_only: true + bind: + create_host_path: false + networks: [challenges] + tmpfs: + - /tmp:size=16m,mode=1777 volumes: base-master-state: +networks: + challenges: + name: cortex-challenges diff --git a/deploy/env/master.env.example b/deploy/env/master.env.example index 99e30c656..f35aee39b 100644 --- a/deploy/env/master.env.example +++ b/deploy/env/master.env.example @@ -3,6 +3,11 @@ # Compose variables (CORTEX_IMAGE and bind address) are supplied with --env-file. CORTEX_IMAGE= BASE_GATEWAY_BIND_ADDRESS= +# Absolute host directory holding /internal.token (and admin.token, ...). +# The supervisor bind-mounts / read-only into each challenge container. +BASE_CHALLENGE_SECRETS_HOST_DIR= +# Host group owning /var/run/docker.sock (`stat -c %g /var/run/docker.sock`). +BASE_DOCKER_GID= BASE_NETUID=541 BASE_CHAIN_ENDPOINT=test BASE_CHAIN_FALLBACK_ENDPOINTS=[] @@ -11,8 +16,6 @@ BASE_EPOCH_REFRESH_SECS=12 BASE_EPOCH_STALE_SECS=60 BASE_CHALLENGES_MIN_VERSION=1 BASE_MEASUREMENTS_MIN_VERSION=1 -# HTTPS public feed; empty remains unavailable and emits no payable Bounty score. -BOUNTY_BACKEND_PUBLIC_URL= # Dedicated KVM host on the VPC. Empty URL keeps Proof unwired. PROOF_VM_ORCHESTRATOR_URL= PROOF_VM_ORCHESTRATOR_TOKEN_FILE=/run/secrets/proof-vm.token diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index b0e639ec8..45f3750bf 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -11,9 +11,8 @@ regular files, owned/readable by that identity, with mode 0400 or 0600. | File | Purpose | | --- | --- | | `gateway.key` | 32-byte sr25519 seed for bundle seals | -| `bounty.key` | Bounty leaf seed matching the trust root | +| `.key` | leaf seed of each trusted container challenge, e.g. `bounty.key` | | `proof.key` | Proof topic/leaf seed matching the trust root | -| `bounty-session.key` | at least 32 random bytes for opaque pairing sessions | | `operator.token` | bearer for master administrative routes | | `proof-vm.token` | bearer shared with the dedicated VM host | | `proof-vm-ca.pem` | CA used to verify the VM-host TLS certificate | @@ -22,12 +21,25 @@ The final two files are required only when Proof VM orchestration is configured. The CA certificate is public material but remains operator-managed because it controls the authenticated host boundary. +## Challenge secrets + +`$BASE_CHALLENGE_SECRETS_HOST_DIR//` is outside this repository. The gateway +mounts the whole directory read-only at `/run/challenge-secrets` and reads only +`/internal.token`. The supervisor bind-mounts `/` read-only at +`/run/secrets` inside that challenge container, without reading it: + +| File | Purpose | +| --- | --- | +| `internal.token` | master bearer for `get_weights` | +| `admin.token` | optional operator bearer for the challenge's admin routes | +| challenge-specific | e.g. Bounty `session.key`; see the challenge's operator guide | + ## Validator mounts The wallet tree is mounted read-only at `/run/wallets`. A separate validator identity directory is mounted at `/run/validator` and contains `consensus.key`, `tls.crt` and `tls.key`. The validator never receives gateway, -Bounty, Proof, operator or provider credentials. +challenge, Proof, operator or provider credentials. ## VM-host files diff --git a/docs/AGENTS.md b/docs/AGENTS.md index b655a2471..0b15ec915 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -10,7 +10,7 @@ not live product instructions. | --- | --- | | system topology and source map | `ARCHITECTURE.md` | | Proof control plane and RLM | `PROOF.md` | -| Bounty intake and score | `BOUNTY.md` | +| challenge container contract and updates | `CHALLENGES.md` | | compatibility names and domains | `NAMING.md` | | trust assumptions and residual risk | `THREAT_MODEL.md` | | operator release checklist | `OPERATOR_SECURITY.md` | @@ -30,7 +30,7 @@ short historical pointers so old URLs do not disappear. ## API changes -When Bounty or Proof changes a public route, payload, authentication rule, quota, +When Proof or a challenge contract changes a public route, payload, authentication rule, quota, timeout, scoring rule or failure response, update its miner guide in the same change. Examples must run against the Python CLI or current HTTP surface. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d00646704..5fcb3bd96 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,16 +1,25 @@ # Architecture -Cortex has two scoring products. Algorithm 2 assigns Bounty 3,000 basis points -and Proof 7,000. The owner-signed legacy 2,000/8,000 profile retains algorithm 1; -see [activation](how-to/trust-root.md#activate-proportional-bounty). -The signed trust root fixes their shares. All challenge execution belongs to -the master; validators independently verify sealed bundles and submit weights. +The owner-signed trust root lists the scoring challenges and their emission +shares: + +- algorithm 1: legacy bounty/proof 2,000/8,000; +- algorithm 2: bounty/proof 3,000/7,000; +- algorithm 3: any 1..64 challenges. + +Proof runs inside the master. Every other challenge is a Docker container +([contract](CHALLENGES.md)) started and auto-updated by `challenge-supervisor`. +All challenge execution belongs to the master; validators independently verify +sealed bundles and submit weights. ```mermaid flowchart LR Owner --> Master Miner --> Master Master --> Gateway[Durable gateway seals] + Supervisor[challenge-supervisor] -->|Docker API| Containers[Challenge containers] + Master -->|get_weights, /challenge/id proxy| Containers + Registry[GHCR + provenance] --> Supervisor Master --> Host[HTTPS KVM orchestrator] Host --> Topic[Persistent topic RLM guest] Topic --> Broker[Inference and memory broker] @@ -27,7 +36,7 @@ flowchart LR | `src/cortex/protocol` | SCALE wire format, signatures, Merkle roots, aggregation, trust roots | | `src/cortex/gateway` | Durable raw leaves and immutable sealed bundles | | `src/cortex/validator` | Independent historical chain reads and exact weight submission | -| `src/cortex/bounty` | Pairing, reports, adjudication and external-feed scoring | +| `src/cortex/challenges` | Registry, weights client, public proxy and auto-updating supervisor | | `src/cortex/proof` | Signed topics, setup, intake, evidence, quotas and payouts | | `src/cortex/rlm` | OpenRouter protocol, recursion, budgets, checkpoints and memory | | `src/cortex/vm` | Firecracker lifecycle, guest tools, setup export and host callbacks | @@ -35,6 +44,29 @@ flowchart LR | `src/cortex/cli.py` | Operator, miner, master and validator commands | | `tests` | Domain, service, protocol and lifecycle regressions | +## Container challenge epoch + +1. At the end of each epoch, the master calls + `GET /internal/v1/get_weights?epoch=` on each trusted, registered container, + with its private bearer. +2. It converts the weights of the sealed participant set into signed leaves: + `floor(10^12 * w / max(W, full_share_mass))` under algorithm 3. +3. A missing, failing or invalid answer becomes `NoScore(ChallengeInternal)`, + and that challenge's share burns. +4. The gateway seals all leaves. +5. Validators recompute `share * min(sum(leaves), 10^12) / 10^12` per challenge + from the signed leaves alone. + +The supervisor does the following: + +- pulls `stable`, `edge` or a pinned digest; +- verifies the labels and GitHub build provenance; +- runs a secretless canary; +- replaces the container and rolls back on failure. + +It holds the Docker socket and nothing else. Public challenge routes are +proxied under `/challenge//`; `internal/` paths never are. + ## Proof lifecycle An authenticated owner submits an objective and optional metric constraints. diff --git a/docs/BOUNTY.md b/docs/BOUNTY.md deleted file mode 100644 index 870ac6e73..000000000 --- a/docs/BOUNTY.md +++ /dev/null @@ -1,151 +0,0 @@ -# Bounty operator reference - -Bounty rewards useful vulnerability reports about the CortexLM backend. It is -up to 30% of subnet emission after -[algorithm 2 activation](how-to/trust-root.md#activate-proportional-bounty). -The legacy owner-signed profile retains its 20% allocation and algorithm 1. -Initial production pairing, intake and adjudication live -in CortexLM/backend. The Python subnet retains the compatibility intake below -and emits signed leaves, but does not export those local rows; the external -CortexLM/backend public feed is the sole scoring source. - -## Pairing and intake - -Before pairing, an operator verifies control of the named Cortex Chat account -out of band and creates a one-use authorization with authenticated -`POST /v1/admin/pair-grants`: - -```json -{ - "account_id": "dedicated-mining-account", - "hotkey": "5F...", - "expires_at": 1800000300 -} -``` - -The expiry must be in the future and no more than 300 seconds from the -operator's current time. The grant binds that exact account and hotkey. It is -stored in SQLite and consumed atomically only when pairing succeeds; an -expired or absent grant returns 403 without consuming the miner nonce. - -The miner then signs the exact UTF-8 payload -`cortex-bounty-v1|||` with its Bittensor hotkey. The -nonce is 16 to 64 hexadecimal characters and is single-use across pairings. -`POST /v1/pair` also requires explicit terms acceptance. A successful response -returns an opaque session token; only its hash is stored. - -The store consumes the nonce, grant and session change in one transaction. -Reusing an accepted nonce returns `409 nonce reused`, including an identical -signed request; the refusal returns no session data and does not consume a -new grant. This protection survives restarts and concurrent requests. -If a successful HTTP response or session token is lost, issue a fresh grant -and pair with a new nonce. Successful replacement revokes the previous session. - -Pairing a CortexLM account again revokes the previous session for that account. -An operator grant may replace its hotkey; the old session cannot follow that -replacement. `POST /v1/reports` accepts the session, title, report body and -reproduction steps. The optional hotkey field, when present, must match the -currently paired hotkey. - -Intake is transactional SQLite. Before storing a report it enforces: - -- a readable and internally consistent external scoring feed; -- at most five pending reports per hotkey; -- at least 60 seconds since that hotkey's previous report; -- at most one in-flight feed validation per hotkey; -- a nonempty title distinct from the body after normalization; -- at least 80 body characters, 20 reproduction characters and four distinct - evidence tokens; -- one canonical title/body fingerprint, so a duplicate never consumes a new - triage slot. - -Pairing and operator-adjudication JSON bodies are limited to 4096 bytes. Report -request bodies are limited to 262144 bytes. The service stops reading at the -limit and returns `413`, before signature, session or feed processing. - -An unavailable or malformed feed returns 503 and creates no row. There is no -local or simulated production scorer. - -## Adjudication - -Operator-authenticated routes list reports and apply one of four verdicts: -`valid`, `already_fixed_not_prod`, `invalid_malicious`, or `duplicate`. -`valid` requires a severity (`trivial`, `minor`, `major`, or `critical`) before -it can earn credit. `duplicate` requires the original report ID. Operator -bearer values are compared through stored SHA-256 hashes and never returned. - -These local rows support the triage workflow. They do not become weights until -the CortexLM/backend publishes the corresponding public reports and leaderboard. - -## Public-feed consistency - -For each scoring read, Cortex fetches: - -- `/v1/bounty/public/status` to pin one immutable publication revision and its - aggregate counters; -- `/v1/bounty/public/leaderboard?revision=`; -- every `/v1/bounty/public/reports` page for that same revision, following the - opaque cursor until `has_more` is false. - -Each response is bounded to 8 MiB and the complete report snapshot to 64 MiB. -Cortex requires API version 1, available adjudication, no unpriced valid report, -one revision across every response, complete pagination, unique report IDs and -leaderboard hotkeys, nonempty evidence, duplicate chains ending at a -non-duplicate report, and exact -agreement between status, leaderboard `valid_count` and the published reports. -A truncated leaderboard is accepted only when it is an exact ranked prefix; -Cortex reconstructs the complete ranking from the fully paginated reports. -Moving revisions, truncated report pagination or any inconsistency fail closed. -Transient transport, HTTP, JSON or revision errors receive at most three -read-only attempts inside one 30-second deadline; stable adjudication, pricing -and backlog gates fail immediately. A nonempty adjudication backlog with no -published report is treated as an unavailable scorer, not as a zero score. - -## Score - -Algorithm 2 signs each expected hotkey's exact count of `valid` reports as its -raw score. One valid report is one point regardless of severity. There is no -champion, precision gate, minimum author count or triage-noise gate. Invalid, -duplicate and already-fixed reports contribute zero points. Severity remains -required evidence for a valid publication, with no effect on its point value. - -Let `n_i` be author i's valid count and `N = sum(n_i)` over the epoch's expected -participants, selected by the owner-signed policy and sealed metagraph: - -```text -Bounty payout = 0.30 * min(N / 10, 1) -author i payout = 0.30 * n_i / max(10, N) -``` - -Five valid reports distribute 15% of subnet emission; ten or more distribute -30%, proportionally across authors. The remaining Bounty mass burns to UID0; -it never increases Proof's 70%. UID0 and unmapped author allocations also burn -without increasing other authors' allocations. Existing owner/permit submission -constraints are unchanged. - -Counts use the complete cumulative report history in one pinned external -publication, with no epoch reset or new rolling window. Only expected hotkeys -enter `N`; historical authors outside the metagraph/policy are excluded. -Validated report IDs and rooted duplicate chains prevent duplicate credit. -An author with zero valid reports gets `NotAttempted`. Feed failure produces -`ChallengeInternal` for every expected participant and burns the Bounty share. - -The legacy 2,000/8,000 owner profile retains algorithm 1, including its champion, -precision/severity scoring and signed encodings. A 3,000/7,000 owner profile -requires challenge-document version >=2 and algorithm 2. An algorithm 1 body -under that profile is rejected, even with a valid gateway signature. - -## Operational checks - -`GET /v1/status` reports `can_score` from a bounded feed probe. Successful -probes are shared for 15 seconds, failures for 5 seconds, and concurrent refresh -requests fail fast instead of multiplying full snapshot reads. Report intake -still performs an uncached feed read before storage. The response also includes -the bounded reason, operator-grant requirement, scoring constants, quotas and terms. -Before opening intake, verify that `can_score` is true, grant and pair a test -hotkey, submit a substantive report, adjudicate it through the operator route, -and confirm the public backend publishes a stable matching snapshot. The full -payment path still requires raw leaves, an immutable gateway seal, independent -validator recomputation and an on-chain submission. - -Miner-facing request examples live in [the Bounty guide](external-miner/bounty.md). diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md new file mode 100644 index 000000000..ed2b22ff7 --- /dev/null +++ b/docs/CHALLENGES.md @@ -0,0 +1,161 @@ + + +# Challenge containers + +A Cortex challenge is a Docker image that scores miners. The master runs it, +reads its weights once per completed epoch, signs one leaf per expected hotkey +and seals the epoch. Validators only verify the sealed bundle and submit it. + +```text +owner (offline) challenges.toml v>=3: id, leaf key, emission bps, policy +master operator challenge-registry.toml: image, channel, resources, env +challenge-supervisor pulls, verifies, canaries, runs and updates containers +challenge container public routes + GET /internal/v1/get_weights +master polls weights, signs leaves, seals, proxies /challenge// +validator GET /v1/weights/latest + /v1/bundle/, recompute, submit +``` + +The owner-signed trust root decides which challenges earn emission and how +much. The unsigned registry only decides what runs. A registered challenge that +is absent from the trust root runs without emission (burn-in). A trusted +challenge that is not registered or cannot score emits +`NoScore(ChallengeInternal)` for every expected hotkey and its share burns. + +## Container contract, version 1 + +The container serves plain HTTP on port `8000` of the private +`cortex-challenges` network. It never publishes a host port. It runs as UID +65532 with a read-only root filesystem, no capabilities, a `/tmp` tmpfs and one +writable named volume at `/data`. The image must create `/data` owned by +`65532:65532`, because Docker copies that ownership into a new named volume. + +### Environment + +| Variable | Value | +| --- | --- | +| `CHALLENGE_SLUG` | the challenge id, for example `bounty` | +| `CHALLENGE_STATE_DIR` | `/data` | +| `CHALLENGE_INTERNAL_TOKEN_FILE` | `/run/secrets/internal.token`, the master bearer | +| `CHALLENGE_ADMIN_TOKEN_FILE` | `/run/secrets/admin.token`, the operator bearer, optional | +| `CHALLENGE_MASTER_URL` | `http://cortex-master:8080`, for `/v1/metagraph/latest` | +| registry `env` | challenge-specific settings | + +Every file under `/run/secrets/` comes from the host directory +`//`, mounted read-only. A container never +receives a leaf-signing seed. + +### Routes + +| Route | Contract | +| --- | --- | +| `GET /health` | `200 {"ok": true}` when the container can score; `503` otherwise. Readiness only: an outage of an external dependency must not restart the container | +| `GET /version` | `200 {"slug", "version", "contract": 1, "capabilities": [...]}` | +| `GET /internal/v1/get_weights?epoch=` | weights for a completed epoch, described below | +| any other path | public, proxied when the capabilities include `proxy_routes` | + +`capabilities` is a subset of `get_weights` and `proxy_routes`. + +`get_weights` requires `Authorization: Bearer ` (`401` +otherwise) and `X-Platform-Challenge-Slug: ` (`403` on mismatch). It +returns: + +```json +{ + "challenge_slug": "bounty", + "epoch": 25316, + "weights": {"5F...": 3.0, "5G...": 1.0}, + "full_share_mass": 10.0, + "metadata": {}, + "computed_at": "2026-09-24T12:00:00Z" +} +``` + +- `weights` maps an SS58 or 64-hex hotkey to a finite non-negative number. It + holds at most 65,536 entries. +- `full_share_mass` is optional. When it is present, the challenge pays its full + share only once the total weight of expected hotkeys reaches it, and the + shortfall burns. When it is omitted or `null`, the expected weights are + normalized to the full share. +- The first successful answer for an epoch is final. The container persists it + and returns the same body for every later call with that epoch. +- `503` means the container cannot score. The master then burns the share for + that epoch. +- The call must answer within 60 seconds. + +### Leaves the master signs + +Let `E` be the owner-policy participant set of the sealed metagraph, `w_i` the +returned weight (0 when absent), `W = sum(w_i for i in E)` and +`D = max(W, full_share_mass or 0)`. A hotkey outside `E` is ignored and never +changes `D`. + +| Algorithm | Leaf score for `i` in `E` with `w_i > 0` | Challenge payout | +| --- | --- | --- | +| 3 (trust root version >= 3) | `floor(10^12 * w_i / D)` | `share * sum(leaves) / 10^12` | +| 2 (bounty/proof 3000/7000) | `w_i`, which must be an integer | Bounty: `share * min(N, 10) / 10`; Proof: `share` | +| 1 (legacy bounty/proof 2000/8000) | `w_i`, which must be an integer | `share` when any leaf is positive | + +Every other hotkey in `E` receives `NoScore(NotAttempted)`. A failed or invalid +call gives `NoScore(ChallengeInternal)` to every hotkey in `E`. Unpaid mass +always burns to UID0 and never moves to another challenge. Under algorithm 3, +Bounty with `full_share_mass = 10` pays exactly what algorithm 2 pays: +`share * min(N, 10) / 10` in total and `share * n_i / max(10, N)` per author. + +### Public proxy + +The master forwards `ANY /challenge//?` to +`http://cortex-challenge-:8000/?`. It refuses the following +with `404`: + +- a path under `internal/`; +- an empty, `.` or `..` segment; +- a backslash or a percent sign in the path. + +It forwards the request body, up to the registry `proxy_body_limit` (default 1 +MiB, `413` beyond it). Only the `content-type`, `accept` and `authorization` +request headers pass through. The master sets `X-Forwarded-For` itself, +overwriting any client value. It returns the status, +the `content-type` and at most 8 MiB of the response body. The request times out +after the registry `proxy_timeout_seconds` (default 30). A failure returns +`502`, and an unknown challenge returns `404`. + +`GET /v1/metagraph/latest` on the master returns +`{"epoch", "block", "netuid", "hotkeys": {"": uid}}` from the latest +sealed bundle, or `503` when no seal exists. Challenges use it to admit only +registered hotkeys. + +## Images and updates + +A challenge repository publishes `ghcr.io//`: + +| Tag | Moves | Built by | +| --- | --- | --- | +| `sha-` | never | every push to `main`, after tests | +| `edge` | every push to `main` | alias of the tested `sha-` digest | +| `vX.Y.Z` | never | annotated tag; aliases the existing `sha-` digest, no rebuild | +| `stable` | every release | alias of the release digest | + +Every image carries these labels: `org.opencontainers.image.source`, +`org.opencontainers.image.version`, `org.opencontainers.image.revision`, +`io.cortex.challenge.slug=` and `io.cortex.challenge.contract=1`. + +For each registry entry, the supervisor runs this loop every `poll_seconds`: + +1. Pull `image:channel`, or `image@pin` when a pin is set, and read the digest. + An unchanged digest only ensures the container is running. +2. Refuse the image when the slug label differs from the id, the contract label + is not `1`, or the source label differs from the registry `source`. The + current container keeps running. +3. Unless `attestation = false`, require a GitHub build-provenance attestation + for the digest in the `source` repository. +4. Start a canary with the same image, no secrets and a tmpfs `/data`. It must + answer `/version` with the expected slug and contract within 60 seconds. +5. Replace the container: stop the old one and start the new one on the same + volume. Then wait until `/version` returns the expected slug and contract. + `/health` is readiness and is only reported, because an external outage must + not trigger a rollback. +6. If the new container fails, recreate the previous digest and log the refusal. + The refused digest is not retried until the channel moves. + +A managed container whose id is no longer in the registry is stopped and +removed. Its volume is kept. diff --git a/docs/NAMING.md b/docs/NAMING.md index 6a4f7a292..521571835 100644 --- a/docs/NAMING.md +++ b/docs/NAMING.md @@ -30,8 +30,8 @@ The preserved signing domains are: | `base-dissent-v1` | validator dissent evidence | | `base-proof-topic-v1` | published Proof topics | | `base-proof-submit-v1` | miner Proof submissions | -| `base-bounty-report-v1` | Bounty report fingerprints | -| `base-bounty-session-v1` | Bounty session tokens | +| `base-bounty-report-v1` | Bounty report fingerprints, now in CortexLM/bounty | +| `base-bounty-session-v1` | Bounty session tokens, now in CortexLM/bounty | Inference and executor offer domains use the newer `cortex-*` prefix because they were introduced by the Python implementation and have no deployed legacy @@ -39,17 +39,14 @@ preimage. ## Live products -Only two challenge IDs are live: - -| ID | Emission share | -| --- | ---: | -| `bounty` | 2,000 bps | -| `proof` | 8,000 bps | - -The signed trust root must contain exactly those rows and total 10,000 basis -points. Design, Prism and Relearn names may appear in frozen or historical -documentation only. They are not services, routes, trust-root rows or emission -recipients. +The signed trust root lists the live challenge ids and their shares, which total +10,000 basis points: legacy `bounty` 2,000 / `proof` 8,000 (algorithm 1), +3,000 / 7,000 (algorithm 2), or any 1..64 ids matching `[a-z0-9][a-z0-9-]{0,62}` +(algorithm 3). `proof` is built in; every other id is a +[challenge container](CHALLENGES.md) named `cortex-challenge-`, with state in +the `cortex-challenge--data` volume and routes under `/challenge//`. +Design, Prism and Relearn names may appear in frozen or historical documentation +only. They are not services, routes, trust-root rows or emission recipients. ## File and command names diff --git a/docs/OPERATOR_SECURITY.md b/docs/OPERATOR_SECURITY.md index 3965cbd40..69a023428 100644 --- a/docs/OPERATOR_SECURITY.md +++ b/docs/OPERATOR_SECURITY.md @@ -9,8 +9,8 @@ recovery. It complements the [threat model](THREAT_MODEL.md). untracked regular file with mode 0400 or 0600 inside a 0700 directory. - [ ] No credential is present in an environment example, image layer, compose build argument, Terraform state, cloud-init payload, log or shell history. -- [ ] Master, Bounty, Proof and VM-host tokens are distinct and rotated - independently. +- [ ] Master operator, per-challenge internal/admin, Proof and VM-host tokens are + distinct and rotated independently. - [ ] Miner BYOK vault storage is durable only as long as queued work requires, and terminal jobs have no remaining secret files. - [ ] The OpenRouter owner key stays on the VM host and is not exposed to miner @@ -20,12 +20,16 @@ recovery. It complements the [threat model](THREAT_MODEL.md). - [ ] `cortex trust-verify` accepts the installed challenge and measurement documents at the deployment epoch. -- [ ] Bounty and Proof public keys match their mounted signing seeds, and the - gateway public key is different from both. -- [ ] The trust root contains only Bounty and Proof: legacy 2000/8000 with - algorithm 1, or 3000/7000 with algorithm 2 and challenge-document version >=2. - Complete [activation](how-to/trust-root.md#activate-proportional-bounty) before - switching profiles; preserve old journals and sealed bytes. +- [ ] Every challenge public key matches its mounted `.key` (or + `PROOF_SK_FILE`) seed, and the gateway public key differs from all of them. +- [ ] The trust root is legacy bounty/proof 2000/8000 (algorithm 1), 3000/7000 + (algorithm 2, version >=2), or unique ids summing to 10000 (algorithm 3, + version >=3). Complete the matching + [activation](how-to/trust-root.md#activate-container-challenges-algorithm-3) + before switching profiles; preserve old journals and sealed bytes. +- [ ] Every registry entry names the challenge's own `ghcr.io` repository and + GitHub `source`, keeps `attestation = true`, and runs on `stable` or a pin in + production. - [ ] Runtime, kernel, rootfs, evaluator and experiment-pack references use verified SHA-256 digests. No production image uses a floating tag. - [ ] Empty or unknown pins remain fail-closed; no digest was copied from an @@ -35,7 +39,9 @@ recovery. It complements the [threat model](THREAT_MODEL.md). ## Network and isolation -- [ ] The gateway and both challenge APIs run only on the master role. +- [ ] The gateway, Proof and challenge containers run only on the master role. +- [ ] Only `challenge-supervisor` mounts the Docker socket; challenge containers + publish no host port and join only `cortex-challenges`. - [ ] The validator role exposes no challenge execution route and reaches the master only through the configured VPC/TLS endpoint. - [ ] The VM orchestrator runs on a dedicated KVM-capable host with mutual @@ -50,11 +56,13 @@ recovery. It complements the [threat model](THREAT_MODEL.md). ## Service readiness -- [ ] Bounty `/v1/status` reports `can_score: true` after a real stable feed - probe, and a report outage test returns 503 without a row. -- [ ] In Bounty-only mode, `PROOF_VM_ORCHESTRATOR_URL` is empty, no Proof topic - is open, and a completed epoch contains signed `ChallengeInternal` Proof - leaves whose 7000 bps (8000 under algorithm 1) burn to UID 0 without blocking Bounty. +- [ ] Every registered challenge answers `GET /challenge//version`, and its + own readiness probe passes (for Bounty, `/challenge/bounty/v1/status` reports + `can_score: true` after a real stable feed probe). +- [ ] Stopping a challenge container makes the next epoch contain signed + `ChallengeInternal` leaves for it and burn its share without blocking others. +- [ ] While Proof is unwired, `PROOF_VM_ORCHESTRATOR_URL` is empty, no Proof topic + is open, and its share burns to UID 0 through signed `ChallengeInternal` leaves. - [ ] When Proof is enabled, `/v1/status` reports a valid topic, sealed baseline, registered runner, pinned image, open inference offer and compatible executor offer; its failure matrix returns 503 without a scored row. @@ -68,7 +76,8 @@ recovery. It complements the [threat model](THREAT_MODEL.md). - [ ] Master SQLite files and WAL state reside on a durable private volume and are backed up with the service quiesced or through SQLite's backup API. - [ ] Restore testing covers gateway seals, epoch journal, Proof jobs, setup - jobs, topic evidence, Bounty sessions and validator dispatch state. + jobs, topic evidence, each `cortex-challenge--data` volume and validator + dispatch state. - [ ] Pending external jobs are reconciled by stable job ID after restart; an uncertain paid operation is not blindly repeated. - [ ] Failed experiment VMs and bounded console tails are retained in the diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index cd8ec4c53..42b5fb394 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -22,9 +22,11 @@ challenge keys or generated topic rules that validators will faithfully accept. | Boundary | Trusted property | Residual risk | | --- | --- | --- | -| owner trust root | challenge keys, 20/80 shares, measurement digest | owner can sign a malicious replacement | +| owner trust root | challenge ids, keys, shares, measurement digest | owner can sign a malicious replacement | | master gateway | durable intake and immutable seals | availability and censorship remain operator risks | | validator | independent chain snapshot, recomputation and dispatch journal | chain RPC eclipse or colluding validators | +| challenge container | weights it returns for its own share | can misallocate or zero its own share; never holds a leaf seed, cannot move another challenge's mass | +| challenge-supervisor | Docker control, image provenance and label checks | holds the Docker socket (host-root equivalent); a compromised GitHub repository with valid provenance ships code | | CortexLM/backend feed | Bounty scoring publication | backend controls the underlying adjudication truth | | topic RLM guest | topic-scoped setup state | model output is untrusted until checked and signed | | experiment guest | measured run with no network and confirmed teardown | a compromised KVM host can forge its own evidence | @@ -58,6 +60,26 @@ of replayed. Compaction stores removed exchanges by digest but does not turn a model assertion into evidence. Shared knowledge is private and untrusted until an owner signature approves exact content and visibility. +## Challenge container controls + +- **The container.** Each container runs as UID 65532 with a read-only root, + no capabilities, `no-new-privileges`, memory, CPU and PID limits, and a single + `/data` volume. It is reachable only on the private `cortex-challenges` + network, never through a host port. +- **What the master exposes.** The master proxies public paths only, refuses + `internal/`, dot segments, `%` and `\`, forwards three request headers and caps + both bodies. +- **Weights.** Weights come only from the authenticated internal route and are + validated before signing: they must be finite, non-negative, at most 65,536 + entries and 8 MiB, and slug and epoch must match. +- **Image updates.** A new digest must carry the slug, contract and source + labels, and GitHub build provenance from the registered repository. It must + also pass a secretless canary. A failed rollout restores the previous digest. +- **Residual risk.** The supervisor checks that a provenance attestation + exists, but does not verify its Sigstore signature (documented upgrade path). + The Docker socket is host-root equivalent, so only that one supervisor service + mounts it. + ## Reward controls Challenge leaves are signed under keys in the owner root. The expected set comes @@ -67,9 +89,11 @@ its share rather than blocking every other challenge. The gateway seals only complete exact-epoch data. Individual raw-leaf intake cannot downgrade a positive score, while the master emitter atomically replaces -the complete participant set for one challenge and epoch. A feed outage therefore -replaces every Bounty participant with `NoScore(ChallengeInternal)` and burns the -full Bounty share without retaining stale positives. `GET /v1/weights/latest` +the complete participant set for one challenge and epoch. A challenge-container +outage or invalid answer therefore replaces every participant of that challenge +with `NoScore(ChallengeInternal)` and burns its full share without retaining +stale positives. A container's first answer per epoch is final, and the master +signs it once, so a later change cannot rewrite a sealed epoch. `GET /v1/weights/latest` returns an unsealed UID0 fallback when no valid seal exists; validators refuse that fallback. A sealed UID0 burn is valid and must still be submitted after independent verification. diff --git a/docs/external-miner/README.md b/docs/external-miner/README.md index c8aa16c81..7ecac5cb7 100644 --- a/docs/external-miner/README.md +++ b/docs/external-miner/README.md @@ -13,11 +13,13 @@ reproduction or on-chain payment. | `bounty` | up to 3000 bps (30%), algorithm 2 | [Pair an account and report bugs](bounty.md) | | `proof` | 7000 bps (70%), algorithm 2 | [Discover topics and submit research](proof.md) | -The legacy owner-signed profile remains 2000/8000 until -[algorithm 2 activation](../how-to/trust-root.md#activate-proportional-bounty). -These are the only live challenge ids. Proof topics are operator-published, -signed documents discovered through the API, never a built-in catalog. No -particular benchmark, runner, model or topic is promised by this repository. +The legacy owner-signed profile remains 2000/8000 (algorithm 1) until +[activation](../how-to/trust-root.md#activate-proportional-bounty). +A version-3 owner document may add further [challenge containers](../CHALLENGES.md), +such as [OpentypeAI/challenge](https://github.com/OpentypeAI/challenge), with +their own shares; each documents its miner API in its repository and is served +under `/challenge//`. Proof topics are operator-published, signed documents +discovered through the API, never a built-in catalog. ## Installation @@ -38,8 +40,8 @@ for installation and development commands. Get the gateway URL and independently pinned Proof public key from the subnet operator. `--gateway` is required; the CLI does not select a deployment for you. -The public challenge route prefixes are `/challenge/bounty` and -`/challenge/proof`. +Public challenge routes are prefixed with `/challenge/`, for example +`/challenge/bounty` and `/challenge/proof`. ```bash uv run cortex miner --gateway "$GATEWAY" \ @@ -69,7 +71,7 @@ not miner secrets. ## Support - [Proof guide](proof.md): signed topics, artifacts, BYOK and submission outcomes. -- [Bounty guide](bounty.md): terms, pairing, reports and the external scoring feed. +- [Bounty guide](bounty.md): pairing and reports; the challenge lives in CortexLM/bounty. - [Validator guide](validators.md): independently verify seals and peer roots. - [Troubleshooting](troubleshoot.md): refusal codes and safe retry behavior. diff --git a/docs/external-miner/bounty.md b/docs/external-miner/bounty.md index 14612e994..7fa90a62e 100644 --- a/docs/external-miner/bounty.md +++ b/docs/external-miner/bounty.md @@ -2,185 +2,32 @@ # Bounty miner guide -Bounty (`bounty`, up to 3000 bps under algorithm 2) accepts reproducible Cortex -product and backend bug reports associated with your Bittensor hotkey. Install the [Python CLI](README.md) -and obtain the gateway URL from the subnet operator. +Bounty rewards reproducible Cortex product and backend bug reports tied to a +Bittensor hotkey. Its code lives in [CortexLM/bounty](https://github.com/CortexLM/bounty). +It is a challenge container that the master runs, updates automatically and +serves through the gateway under `/challenge/bounty/`. The full miner guide, +API reference and scoring rules are in that repository: -The initial production miner flow pairs and files reports in CortexLM/backend. -The Python gateway routes documented here remain compatibility/operator-test -surfaces; they do not publish a local report into the backend and cannot make it -creditable by themselves. Only a report published by the backend public feed can -earn weight. +- [Miner guide](https://github.com/CortexLM/bounty/blob/main/docs/miner.md) +- [API reference](https://github.com/CortexLM/bounty/blob/main/docs/api.md) +- [Scoring](https://github.com/CortexLM/bounty/blob/main/docs/scoring.md) -## Check scoring availability +This CLI still signs and sends the two miner requests: ```bash -curl --fail-with-body "$GATEWAY/challenge/bounty/v1/status" -``` - -The response publishes research terms, quotas and `scoring_backend`. -`can_score` reflects a recent validated external-feed snapshot: successful -status probes are shared for up to 15 seconds and failures for up to 5 seconds. -Concurrent refreshes fail closed instead of multiplying full snapshot reads. -Every report POST performs its own uncached feed check, so an outage or -inconsistent publication after a successful status check can still return `503`. - -## Pair a dedicated account - -Use a dedicated Cortex Chat mining account. Read these terms before passing -`--accept-terms`: - -> By pairing a Bittensor hotkey to a Cortex Chat account for Bounty Challenge, -> you accept that this dedicated mining account, its logs, and its conversations -> may be used for research, to fix product and backend bugs, and to remunerate -> (or penalize) the bound miner hotkey. Do not pair a private personal account. - -Ask the subnet operator to verify that you control this account and authorize -the exact account/hotkey pair. That one-use authorization lasts at most five -minutes. Pairing without it, or after it expires, returns `403 pairing not -authorized by account operator`; request a fresh authorization and reuse your -unspent nonce. - -```bash -uv run cortex miner --gateway "$GATEWAY" \ - --wallet-name research --wallet-hotkey miner \ - bounty-pair --account-id "$CORTEX_ACCOUNT_ID" \ - --accept-terms --session-file ./bounty-session -``` - -For an encrypted hotkey add `--wallet-password-file /private/hotkey-password` -before `bounty-pair`. The wallet coldkey secret and a Proof public key are not -required. Wallet options and the development-only `--dev-seed-file` path are -described in the [mining index](README.md#usage). - -The CLI signs locally and posts to `/challenge/bounty/v1/pair`. It writes the -returned session to the specified new file with mode `0600`, refuses to -overwrite an existing file, and prints only `paired` and the public hotkey. -There is no automatic session cache or CLI-driven Chat activation flow in -this Python implementation. A session is a secret bearer credential. - -The pair API accepts `account_id`, `hotkey` (SS58 or public-key hex), `nonce`, -`exp`, `signature` and `terms_accepted: true`. The exact UTF-8 signing payload -is: - -```text -cortex-bounty-v1|{account_id}|{nonce}|{exp} -``` - -Use sr25519's **Substrate** signing context. This is different from the Cortex -context used by Proof and bundle signatures. The CLI supplies a random -32-character hexadecimal nonce and an expiry five minutes in the future. -Pairing requires explicit terms acceptance (`403` otherwise), a valid signature -and an unexpired signing window, plus the operator authorization above. -Successful pairing consumes the authorization and nonce and returns `201` with -session metadata. A nonce is single-use: retrying an accepted signed request -returns `409 nonce reused` with no session data, including after a restart. -This refusal does not consume a fresh operator authorization. If the response -or session file is lost, request a new authorization and pair with a new nonce. -Pairing the same account again revokes its previous session, which then returns -`401 invalid_session`, including an operator-authorized hotkey replacement. -Sessions do not currently expire automatically by age. -The complete pairing JSON body is limited to 4096 bytes; larger requests return -`413 pair request too large` before signature processing. - -## Submit a report - -```bash -uv run cortex miner --gateway "$GATEWAY" \ - --wallet-name research --wallet-hotkey miner \ - bounty-report --session-file ./bounty-session \ - --title "Reproducible failure in the research upload path" \ +uv run cortex miner --gateway "$GATEWAY" --wallet-name research --wallet-hotkey miner \ + bounty-pair --account-id "$CORTEX_ACCOUNT_ID" --accept-terms --session-file ./bounty-session +uv run cortex miner --gateway "$GATEWAY" --wallet-name research --wallet-hotkey miner \ + bounty-report --session-file ./bounty-session --title "..." \ --body-file report.md --repro-file reproduction.md ``` -The request goes to `/challenge/bounty/v1/reports`. Include enough distinct -evidence to reproduce the problem and explain its impact. The API accepts -`session`, optional `hotkey`, `title`, `body` and `repro_steps`; the CLI includes -its selected hotkey. A supplied hotkey must match the session (`403` on -mismatch). The API validates substance before inserting a report. - -| Limit | Value | -|-------|-------| -| Reports awaiting adjudication per hotkey | 5 | -| Minimum interval between reports | 60 seconds | -| Concurrent feed validations per hotkey | 1 | -| Minimum body length | 80 characters | -| Minimum reproduction length | 20 characters | -| Maximum complete report request body | 262144 bytes | - -An empty or repeated title/body and low-substance repeated-token content -return `400`. A quota violation or concurrent validation returns `429`. Framework schema failures may -return `422`; an oversized request returns `413`. Successful intake returns -`201` with `id`, `miner_hotkey`, -`state` and `fingerprint`; acceptance into triage is not a reward. - -Report reads (`GET /v1/reports` and `/v1/reports/{id}`) and -`POST /v1/admin/adjudicate` require operator bearer authentication. The public -gateway denies report reads. There is no public `bounty show` command. - -## Scoring and adjudication - -Cortex **reads** the CortexLM/backend public API configured by -`BOUNTY_BACKEND_PUBLIC_URL`. The required backend routes are -`/v1/bounty/public/status`, `/v1/bounty/public/leaderboard` and -`/v1/bounty/public/reports`. Status pins an immutable revision; Cortex requests -the leaderboard and every cursor-paginated report page for exactly that -revision. This subnet does not serve `/v1/public/*` or a substitute public -leaderboard. - -The external publication is the only scoring source. A local operator -adjudication alone does not place a report in that publication; the backend -must publish consistent, justified records. The Python subnet does not export -local reports or adjudications into the external backend automatically. -Leaderboard counts alone are not creditable evidence. Mixed revisions, -truncated report pagination, unavailable adjudication, unpriced valid reports, -any duplicate chain without a non-duplicate root, or disagreement between -status counters, reports and leaderboard fails closed. A leaderboard capped by the backend is -accepted only as an exact ranked prefix; Cortex rebuilds the complete ranking -from the report pages. Transient transport, HTTP, JSON or revision errors receive -at most three read-only attempts, all within the same 30-second snapshot deadline. -Stable adjudication, pricing and backlog gates are not retried. A waiting -adjudication backlog with no published report also fails closed instead of -scoring every miner as `NotAttempted`. - -| Adjudication | Algorithm 2 points | -|--------------|--------------------| -| `valid` with severity | 1, regardless of severity | -| `valid` without severity | Invalid publication; scoring fails closed | -| `already_fixed_not_prod` | 0 | -| `invalid_malicious` | 0 | -| `duplicate` | 0; the original valid report is counted once | - -Every author with valid evidence participates proportionally. There is no -champion, precision gate, minimum of three reports, severity weighting or -triage-noise gate. Severity (`trivial`, `minor`, `major`, `critical`) remains -required publication evidence and does not affect point value. - -For `N` valid reports across the epoch's expected participants, the total Bounty -payout is `0.30 * min(N / 10, 1)` of subnet emission. An author with `n` valid -reports gets `0.30 * n / max(10, N)`. Thus five valid reports distribute 15%; -ten or more distribute 30%. Unused mass burns to UID0, never to Proof or other -authors. Proof retains its separate 70% share. An allocation to UID0 or an -unmapped author burns without increasing another author's allocation. - -Counts include cumulative published history at one immutable revision, without -an epoch reset or rolling window. The population is the sealed metagraph's -hotkeys selected by the owner-signed participant policy. Historical authors -outside that population do not enter the total. Duplicate and rejected reports -never add points. Chain weights retain the protocol's independent u16 rounding. - -`GET /v1/status` exposes the active `scoring_version`, `points_per_valid_report`, -`full_share_reports`, population and window. Algorithm 2 requires the owner-signed -3000/7000 profile and document version >=2. Until the gateway and validators -complete [activation](../how-to/trust-root.md#activate-proportional-bounty), -legacy 2000/8000 deployments retain algorithm 1 and its champion/precision rules. - -If the backend is unreadable, unconfigured or inconsistent, report intake -returns `503` without storing a report. Emission pays nobody from Bounty and -covers the expected participant set with `NoScore(ChallengeInternal)` leaves. -The configured Bounty share then burns to uid 0 through normal sealing. There -is no offline scorer or forced simulation path. - -Validators independently verify the [sealed bundle](validators.md); they do -not rerun reports or fetch the Bounty feed. See [troubleshooting](troubleshoot.md) -for refusal and retry guidance. +`bounty-pair` posts to `/challenge/bounty/v1/pair` with `terms_accepted: true`. +It signs `cortex-bounty-v1|{account_id}|{nonce}|{exp}` with the hotkey's +Substrate signing context, not the Cortex one. `bounty-report` posts to +`/challenge/bounty/v1/reports`. Only a report that CortexLM/backend publishes as +`valid` earns weight: one point per valid report. Bounty pays +`share * min(N / 10, 1)`, split in proportion to each author's count, and the +rest burns to UID0. The master turns the container's weights into signed leaves +under the [challenge contract](../CHALLENGES.md). Validators never contact the +container. diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 4a9499520..38eb1d4b3 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -240,8 +240,9 @@ mass to the best eligible result, sharing exact ties. `discovery` divides a pass-floor pool among eligible miners and a novelty pool according to new improvement; duplicates receive no novelty pool. A miner's Proof score is the **sum of per-topic masses**, not a mean of binary passes. The fixed subnet -split is Bounty 3000 / Proof 7000 bps under algorithm 2. Legacy owner-signed -2000/8000 deployments retain algorithm 1 until +split is Bounty 3000 / Proof 7000 bps under algorithm 2; algorithm 3 takes any +owner-signed split. Legacy owner-signed 2000/8000 deployments retain algorithm 1 +until [activation](../how-to/trust-root.md#activate-proportional-bounty). Actual payment additionally needs signed leaves, a valid gateway seal and diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index be7df0edc..0d4040fc4 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -25,6 +25,9 @@ consensus-seed requirement; see the [validator guide](validators.md). ## Bounty +Bounty runs as the [CortexLM/bounty](https://github.com/CortexLM/bounty) container; +its routes are under `/challenge/bounty/`. + | Symptom | Meaning and next action | |---------|-------------------------| | `403 terms_required` | Read the terms and explicitly pass `--accept-terms` when pairing | @@ -44,8 +47,8 @@ consensus-seed requirement; see the [validator guide](validators.md). | Public report GET is denied | Reports require operator access; the gateway does not expose private report reads | A Bounty feed outage stores no new report and emits no positive bounty score. -It covers participants with `NoScore(ChallengeInternal)` so the share can burn -through a valid seal. Do not enable a substitute local scorer. +Any challenge-container outage covers participants with +`NoScore(ChallengeInternal)` so the share can burn through a valid seal. Do not enable a substitute local scorer. ## Proof intake diff --git a/docs/external-miner/validators.md b/docs/external-miner/validators.md index eb2c3d8e5..7688d0177 100644 --- a/docs/external-miner/validators.md +++ b/docs/external-miner/validators.md @@ -10,16 +10,18 @@ chain, and submits the verified vector on Bittensor. Cross-checking authenticated peer roots against other independently-operated validators is an opt-in hardening (`--peer-consensus`), off by default. -Validators never run Bounty or Proof -evaluation, rent GPUs or receive miner provider credentials. The only live -challenge shares are `bounty` 3000 bps and `proof` 7000 bps under algorithm 2. -The legacy 2000/8000 owner profile retains algorithm 1 until -[activation](../how-to/trust-root.md#activate-proportional-bounty). +Validators never run challenge containers or Proof evaluation, rent GPUs or +receive miner provider credentials; they need no registry, Docker or challenge +token. Challenge shares come from the owner-signed trust root: legacy +bounty/proof 2000/8000 (algorithm 1), `bounty` 3000 bps and `proof` 7000 bps +(algorithm 2), or any unique challenge set summing to 10000 bps (algorithm 3, +see [activation](../how-to/trust-root.md#activate-container-challenges-algorithm-3)). Sealed `GET /v1/weights/latest` responses expose `algorithm_version` from the signed bundle. `emission_shares` contains the configured ceilings; each `source_challenges[].emission_percent` reflects its allocated share, including -the ten-report Bounty ramp under algorithm 2. Author allocations to UID0 or +the ten-report Bounty ramp under algorithm 2 and +`min(sum(leaves), 10^12) / 10^12` of the share under algorithm 3. Author allocations to UID0 or unmapped hotkeys still burn. Validators always recompute from signed leaves; they never trust these display fields as consensus inputs. @@ -158,7 +160,9 @@ a reorg or a changed/unsealed latest response prevents submission. Under algorithm 2, quarantining Bounty retains Proof's absolute 7000 bps and burns Bounty's 3000 bps. Quarantining Proof leaves only 3000 bps and -cannot be submitted. No quarantined share is reassigned. Legacy algorithm 1 +cannot be submitted. Algorithm 3 applies the same rule to every challenge: the +quarantined share burns and the rest must keep at least 5000 bps. No +quarantined share is reassigned. Legacy algorithm 1 keeps its original 5000-bps gate and survivor renormalization. Valid `NoScore(ChallengeInternal)` leaves from an unavailable scorer still cover the expected participants and are not themselves a consensus fault. diff --git a/docs/how-to/trust-root.md b/docs/how-to/trust-root.md index cd9a99d93..2ac677ea3 100644 --- a/docs/how-to/trust-root.md +++ b/docs/how-to/trust-root.md @@ -1,7 +1,7 @@ # Sign and verify the trust root -The owner signs two immutable TOML documents: the two-challenge allocation and -the measurement allowlist. Cortex verifies both before serving or submitting a +The owner signs two immutable TOML documents: the challenge allocation and the +measurement allowlist. Cortex verifies both before serving or submitting a bundle. The committed keys are development fixtures; production ceremonies run offline with private files that never enter Git. @@ -18,6 +18,9 @@ cortex keygen \ cortex keygen \ --seed-out /private/cortex-ceremony/bounty.seed \ --public-out /private/cortex-ceremony/bounty.pubkey +cortex keygen \ + --seed-out /private/cortex-ceremony/opentype.seed \ + --public-out /private/cortex-ceremony/opentype.pubkey cortex keygen \ --seed-out /private/cortex-ceremony/proof.seed \ --public-out /private/cortex-ceremony/proof.pubkey @@ -25,13 +28,17 @@ cortex keygen \ `keygen` creates files exclusively and refuses to overwrite an existing path. Seeds are raw 32-byte sr25519 seeds with mode 0600. Copy only the public owner -key into `config/owner.pubkey`. Put the Bounty and Proof public keys in the -matching rows of the selected challenges document. The preserved development +key into `config/owner.pubkey`. Put each challenge public key in its row of the +selected challenges document. The master reads the seed of container challenge +`` from `$BASE_CHALLENGE_KEYS_DIR/.key` and the Proof seed from +`PROOF_SK_FILE`. The preserved development `config/challenges.toml` is the signed legacy 2000/8000 profile (algorithm 1). The unsigned `config/challenges-v2.example.toml` is the 3000/7000 activation template (algorithm 2, challenge-document version >=2). Replace its `"CHOOSE_ACTIVATION_EPOCH"` placeholder with the coordinated integer epoch; -the unchanged template cannot be signed. The gateway public key +the unchanged template cannot be signed. The unsigned +`config/challenges-v3.example.toml` is the algorithm 3 template (version >=3): +any 1..64 unique ids summing to 10,000 bps. The gateway public key is supplied to verification and is never a challenge row. ## Sign both documents @@ -120,7 +127,33 @@ requires bundle algorithm 2; changing only raw scores cannot implement it. proof of live chain submission. If Proof is unavailable, its 70% burns. Bounty pays at most 30%, with unused mass -burned according to [the report-count formula](../BOUNTY.md#score). Rollback must +burned according to the report-count formula in +[the challenge contract](../CHALLENGES.md#leaves-the-master-signs). Rollback must respect persisted version watermarks: restoring an older trust file is rejected. Stop submission and prepare an owner-authorized higher-version recovery profile through the same ceremony rather than deleting watermarks or replaying epochs. + +## Activate container challenges (algorithm 3) + +Algorithm 3 adds challenges without code changes: each trusted id pays +`share * min(sum(leaves), 10^12) / 10^12` and the rest burns to UID0. Bounty +keeps its algorithm 2 payout when its container returns `full_share_mass = 10`. + +1. Upgrade the master and every submitting validator to a release that accepts + algorithm 3. Validators only recompute signed leaves; they need no registry, + container or challenge credential. +2. On the master, add each container to `deploy/challenges/registry.toml`, put + its `internal.token` under `$BASE_CHALLENGE_SECRETS_HOST_DIR//` and its + leaf seed at `.key` in the master secrets directory. A registered id that + is not yet trusted runs without emission, which is the burn-in period: check + `GET /challenge//version` and its logs before you sign. +3. Drain and pause as for algorithm 2, then sign a `version >= 3` document from + `config/challenges-v3.example.toml` with the agreed `introduced_epoch`. +4. Install the signed documents and minimum version pins on the master and + validators, resume, and check a new `sealed: true` latest response with + algorithm 3 leaves and an independent validator recomputation. + +A trusted id that is missing from the registry, unhealthy or returning invalid +weights emits `NoScore(ChallengeInternal)`, so its share burns and never moves +to another challenge. To retire a challenge, sign a higher version without its +row first, then remove its registry entry. diff --git a/docs/index.md b/docs/index.md index a10706334..04f34694f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,14 +2,14 @@ - [Architecture](ARCHITECTURE.md): processes, source map and trust boundaries. - [Proof](PROOF.md): topic setup, recursive agent, VM evaluation and rewards. -- [Bounty](BOUNTY.md): pairing, intake, external-feed scoring and emission. +- [Challenge containers](CHALLENGES.md): contract, leaves, proxy, registry and auto-updates. - [Threat model](THREAT_MODEL.md): security claims, trust boundaries and limits. - [Operator security](OPERATOR_SECURITY.md): deployment and release checklist. - [Configuration](reference/configuration.md): Python services and credential files. - [Trust-root ceremony](how-to/trust-root.md): generate keys, sign and verify operator documents. - [Build a Proof guest rootfs](how-to/build-guest-rootfs.md): convert the pinned Python guest stage into measured ext4 bytes. - [Proof miner guide](external-miner/proof.md): discover topics and submit signed artifacts. -- [Bounty miner guide](external-miner/bounty.md): pair an account and report a vulnerability. +- [Bounty miner guide](external-miner/bounty.md): pointer to the CortexLM/bounty container. - [Validator guide](external-miner/validators.md): verify sealed weights and submit on-chain. - [Bundle specification](BUNDLE_SPEC.md): frozen consensus wire contract. - [Naming](NAMING.md): preserved `BASE_*` variables and signature domains. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index ad0a6b93b..0b1912a63 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -16,11 +16,11 @@ conflicting values are an error. Existing names and crypto domains are preserved | `BASE_CHALLENGES_FILE` | Signed challenge configuration; adjacent `.sig` required | | `BASE_MEASUREMENTS_FILE` | Signed measurement configuration; adjacent `.sig` required | | `BASE_GATEWAY_SK_FILE` | Gateway seal seed file | -| `BOUNTY_SK_FILE` | Bounty leaf seed file | | `PROOF_SK_FILE` | Proof topic and leaf seed file | -| `BOUNTY_SESSION_SECRET_FILE` | Separate pairing session secret | | `BASE_GATEWAY_ADMIN_TOKEN_FILE` | Required operator bearer file | -| `BOUNTY_BACKEND_PUBLIC_URL` | CortexLM/backend HTTPS public feed | +| `BASE_CHALLENGE_KEYS_DIR` | Directory of container challenge leaf seeds, `.key`; default `/run/secrets` | +| `BASE_CHALLENGE_REGISTRY_FILE` | Optional [challenge registry](../CHALLENGES.md); unset runs no container challenge | +| `BASE_CHALLENGE_SECRETS_DIR` | Per-challenge `/internal.token` bearers; default `/run/challenge-secrets` | | `PROOF_VM_ORCHESTRATOR_URL` | Dedicated VM host HTTPS origin | | `PROOF_VM_ORCHESTRATOR_TOKEN_FILE` | Rotating bearer file for that host | | `PROOF_VM_ORCHESTRATOR_CA_FILE` | CA file for the host's TLS certificate | @@ -37,14 +37,13 @@ Keys and tokens are files with mode 0400 or 0600, never image build arguments. Signing seeds are 32 bytes or 64 hexadecimal characters. SQLite state and miner credential vaults must be backed by durable private storage. -For the Bounty-only launch, set `BOUNTY_BACKEND_PUBLIC_URL` to the HTTPS -`CortexLM/backend` origin and leave `PROOF_VM_ORCHESTRATOR_URL` empty. The Proof -seed remains required because every completed epoch still needs signed Proof -absence leaves. The legacy owner-signed trust root is `bounty = 2000` and -`proof = 8000`; algorithm 2 requires a signed `bounty = 3000`, `proof = 7000` -profile with challenge-document version >=2 (see -[activation](../how-to/trust-root.md#activate-proportional-bounty)). The unavailable -Proof share burns and is never reassigned to Bounty. +Container challenges such as Bounty take their settings from the registry +`env` table, never from master variables; see [the challenge contract](../CHALLENGES.md). +A trusted challenge that is not registered, unhealthy or returns invalid weights +burns its share. The Proof seed remains required because every completed epoch +still needs signed Proof absence leaves. The unavailable Proof share burns and is +never reassigned. Trust-root profiles and their activation are described in +[the trust-root ceremony](../how-to/trust-root.md). One resource shape applies to the persistent topic VM and each fresh experiment VM. The examples explicitly request 1 vCPU, 1024 MiB RAM and 16384 MiB disk; @@ -56,6 +55,22 @@ topic VM also requires its image and resource shape to match exactly: changing these settings does not resize a running topic. See the [small-host sizing guide](../../deploy/README.md#small-host-sizing). +## Challenge supervisor + +`cortex challenge-supervisor` takes only arguments: + +| Option | Meaning | +| --- | --- | +| `--registry` | registry TOML, re-read every tick | +| `--secrets-host-dir` | absolute HOST path of `/` secret directories, bind-mounted read-only | +| `--docker-socket` | default `/var/run/docker.sock` | +| `--network` | private challenge network; default `cortex-challenges` | +| `--master-url` | URL challenges use for `/v1/metagraph/latest`; default `http://cortex-master:8080` | +| `--once` | reconcile every entry once and exit | + +The Compose role also needs `BASE_CHALLENGE_SECRETS_HOST_DIR` and +`BASE_DOCKER_GID` (the group owning the Docker socket). + ## Validator The Compose validator maps these host settings to required CLI arguments: diff --git a/scripts/check_deploy.py b/scripts/check_deploy.py index 6a252c69c..8d0587175 100644 --- a/scripts/check_deploy.py +++ b/scripts/check_deploy.py @@ -52,11 +52,14 @@ "BASE_CHALLENGES_FILE": "/etc/base/config/challenges.toml", "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway.key", - "BOUNTY_SK_FILE": "/run/secrets/bounty.key", "PROOF_SK_FILE": "/run/secrets/proof.key", - "BOUNTY_SESSION_SECRET_FILE": "/run/secrets/bounty-session.key", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator.token", + "BASE_CHALLENGE_KEYS_DIR": "/run/secrets", + "BASE_CHALLENGE_REGISTRY_FILE": "/etc/base/challenges/registry.toml", + "BASE_CHALLENGE_SECRETS_DIR": "/run/challenge-secrets", } +CHALLENGE_NETWORK = "cortex-challenges" +DOCKER_SOCKET = "/var/run/docker.sock" VALIDATOR_OPTIONS = { "--gateway", @@ -249,6 +252,66 @@ def _mounts(service: dict) -> dict[str, dict]: return by_target +def _network_names(service: dict) -> set[str]: + networks = service.get("networks") or {} + return set(networks if isinstance(networks, dict | list) else ()) + + +def validate_supervisor(config: dict, service: dict) -> None: + """The one master service allowed to hold the Docker socket; it holds nothing else.""" + validate_pin(service.get("image", "")) + if ( + service.get("init") is not True + or service.get("read_only") is not True + or service.get("restart") != "unless-stopped" + or service.get("privileged", False) + or service.get("user") != "65532:65532" + or set(service.get("cap_drop", [])) != {"ALL"} + or "no-new-privileges:true" not in service.get("security_opt", []) + or service.get("profiles") != ["master"] + or service.get("entrypoint") not in (None, [], ()) + ): + raise ValueError("challenge supervisor requires the hardened read-only master runtime") + if any(service.get(field) for field in ("cap_add", "devices", "ports", "environment")): + raise ValueError("challenge supervisor may not publish ports, add privileges or read env") + if any(service.get(field) == "host" for field in ("network_mode", "pid", "ipc")): + raise ValueError("challenge supervisor may not join host namespaces") + volumes = service.get("volumes", []) + mounts = {item.get("target"): item for item in volumes if isinstance(item, dict)} + if len(mounts) != len(volumes) or set(mounts) != {DOCKER_SOCKET, "/etc/base/challenges"}: + raise ValueError("challenge supervisor mounts only the Docker socket and the registry") + if any(item.get("type") != "bind" or item.get("read_only") is not True for item in volumes): + raise ValueError("challenge supervisor mounts must be read-only binds") + if mounts[DOCKER_SOCKET].get("source") != DOCKER_SOCKET: + raise ValueError("challenge supervisor must use the host Docker socket") + command = service.get("command", []) + if not isinstance(command, list) or command[:1] != ["challenge-supervisor"]: + raise ValueError("challenge supervisor command differs from its entrypoint") + options = dict(zip(command[1::2], command[2::2], strict=False)) + if len(command) % 2 == 0 or set(options) != { + "--registry", + "--secrets-host-dir", + "--network", + "--master-url", + }: + raise ValueError("challenge supervisor options differ from the runtime contract") + if ( + options["--registry"] != "/etc/base/challenges/registry.toml" + or options["--network"] != CHALLENGE_NETWORK + or options["--master-url"] != "http://cortex-master:8080" + or not options["--secrets-host-dir"].startswith(("/", "${")) + ): + raise ValueError("challenge supervisor paths differ from the runtime contract") + tmpfs = service.get("tmpfs", []) + if not isinstance(tmpfs, list) or not any(str(item).startswith("/tmp:") for item in tmpfs): + raise ValueError("challenge supervisor requires a bounded /tmp tmpfs") + networks = config.get("networks") or {} + if _network_names(service) != {"challenges"} or ( + (networks.get("challenges") or {}).get("name") != CHALLENGE_NETWORK + ): + raise ValueError("challenge supervisor must join only the cortex-challenges network") + + def _validate_common_service(service: dict) -> dict[str, dict]: validate_pin(service.get("image", "")) if ( @@ -279,8 +342,11 @@ def _validate_common_service(service: dict) -> dict[str, dict]: def validate_compose(config: dict, role: str) -> None: expected = "gateway" if role == "master" else "validator" services = _mapping(config.get("services"), "services") - if set(services) != {expected}: + allowed = {expected, "challenge-supervisor"} if role == "master" else {expected} + if expected not in services or set(services) - allowed: raise ValueError(f"{role} role must contain only its {expected} service") + if "challenge-supervisor" in services: + validate_supervisor(config, _mapping(services["challenge-supervisor"], "supervisor")) service = _mapping(services[expected], expected) mounts = _validate_common_service(service) command = service.get("command", []) @@ -297,10 +363,8 @@ def validate_compose(config: dict, role: str) -> None: forbidden_values = { "OPENROUTER_API_KEY", "BASE_GATEWAY_SK", - "BOUNTY_SK", "PROOF_SK", "BASE_GATEWAY_ADMIN_TOKEN", - "BOUNTY_SESSION_SECRET", "PROOF_VM_ORCHESTRATOR_TOKEN", } if forbidden_values.intersection(environment): @@ -313,6 +377,14 @@ def validate_compose(config: dict, role: str) -> None: raise ValueError("master credentials must be an operator bind mount") if "/run/wallets" in mounts: raise ValueError("master must not hold the validator wallet") + for target in ("/etc/base/challenges", "/run/challenge-secrets"): + if target in mounts and mounts[target].get("type") != "bind": + raise ValueError("master challenge registry and secrets must be operator binds") + if "challenge-supervisor" in services and not ( + {"/etc/base/challenges", "/run/challenge-secrets"} <= set(mounts) + and "challenges" in _network_names(service) + ): + raise ValueError("master must mount the challenge registry and join its network") health = service.get("healthcheck", {}).get("test", []) if health != MASTER_HEALTHCHECK: raise ValueError("master healthcheck must use its Python readiness endpoint") @@ -775,6 +847,8 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: master_required = { "CORTEX_IMAGE", "BASE_GATEWAY_BIND_ADDRESS", + "BASE_CHALLENGE_SECRETS_HOST_DIR", + "BASE_DOCKER_GID", "BASE_NETUID", "BASE_CHAIN_ENDPOINT", "BASE_CHAIN_FALLBACK_ENDPOINTS", @@ -783,7 +857,6 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: "BASE_EPOCH_STALE_SECS", "BASE_CHALLENGES_MIN_VERSION", "BASE_MEASUREMENTS_MIN_VERSION", - "BOUNTY_BACKEND_PUBLIC_URL", "PROOF_VM_ORCHESTRATOR_URL", "PROOF_VM_ORCHESTRATOR_TOKEN_FILE", "PROOF_VM_ORCHESTRATOR_CA_FILE", @@ -797,7 +870,8 @@ def validate_env_examples(master_source: str, validator_source: str) -> None: intentionally_unset = { "CORTEX_IMAGE", "BASE_GATEWAY_BIND_ADDRESS", - "BOUNTY_BACKEND_PUBLIC_URL", + "BASE_CHALLENGE_SECRETS_HOST_DIR", + "BASE_DOCKER_GID", "PROOF_VM_ORCHESTRATOR_URL", "PROOF_RLM_VM_IMAGE_DIGEST", "PROOF_INFERENCE_OFFER_COMMITMENT", @@ -898,6 +972,7 @@ def _default_sources(config: dict, role: str) -> None: expected = {"/etc/base/config": ROOT / "config"} if role == "master": expected["/run/secrets"] = ROOT / "deploy/secrets/master" + expected["/etc/base/challenges"] = ROOT / "deploy/challenges" # Some Compose releases fold `env_file` into `environment` in the JSON # render, so the committed YAML is the source of truth for the default. text = (ROOT / "deploy/compose/role-master.yml").read_text() @@ -934,12 +1009,15 @@ def check_examples() -> None: "BASE_MASTER_SECRETS_DIR", "BASE_TRUST_ROOT_DIR", "BASE_VALIDATOR_WALLETS_DIR", + "BASE_CHALLENGE_REGISTRY_DIR", } env = {key: value for key, value in os.environ.items() if key not in removed} env.update( { "CORTEX_IMAGE": "fixture.invalid/cortex@sha256:" + "f" * 64, "BASE_GATEWAY_BIND_ADDRESS": "127.0.0.1", + "BASE_CHALLENGE_SECRETS_HOST_DIR": "/fixture/challenge-secrets", + "BASE_DOCKER_GID": "999", "BASE_GATEWAY_ENDPOINT": "https://master.fixture.invalid", "BASE_NETUID": "541", "BASE_CHAIN_ENDPOINT": "test", diff --git a/scripts/check_repo.py b/scripts/check_repo.py index bb00e38fb..44bf80d98 100644 --- a/scripts/check_repo.py +++ b/scripts/check_repo.py @@ -25,14 +25,22 @@ } SHARES = {"bounty": 2000, "proof": 8000} PROPORTIONAL_SHARES = {"bounty": 3000, "proof": 7000} +CHALLENGE_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") DOC_CONTRACTS = { + "docs/CHALLENGES.md": ( + "/internal/v1/get_weights", + "X-Platform-Challenge-Slug", + "full_share_mass", + "io.cortex.challenge.contract", + "10^12", + "/v1/metagraph/latest", + ), "docs/external-miner/README.md": ("bounty", "proof", "3000", "7000"), "docs/external-miner/bounty.md": ( - "/v1/pair", - "/v1/reports", + "/challenge/bounty/v1/pair", + "/challenge/bounty/v1/reports", "terms_accepted", - "severity", - "503", + "CortexLM/bounty", "CortexLM/backend", ), "docs/external-miner/proof.md": ( @@ -60,18 +68,16 @@ ), } PUBLIC_ROUTES = { - "src/cortex/bounty/api.py": { - ("post", "/v1/pair"), - ("post", "/v1/reports"), - ("get", "/v1/status"), - }, "src/cortex/proof/api.py": { ("get", "/v1/proof/topics"), ("post", "/v1/submissions"), ("post", "/v1/submissions/lookup"), ("get", "/v1/status"), }, - "src/cortex/gateway/api.py": {("get", "/v1/weights/latest")}, + "src/cortex/gateway/api.py": { + ("get", "/v1/weights/latest"), + ("get", "/v1/metagraph/latest"), + }, } ARTIFACT_DIRS = { "__pycache__", @@ -155,24 +161,30 @@ def check_trust_roots(root: Path) -> list[str]: raise ValueError("symlinked trust root") document = tomllib.loads(path.read_text()) rows = document["challenges"] - if not isinstance(rows, list) or len(rows) != 2: - raise ValueError("exactly two challenges required") + version = document["version"] + if type(version) is not int or version < 1: + raise ValueError("invalid trust document version") + if not isinstance(rows, list) or not 1 <= len(rows) <= (64 if version >= 3 else 2): + raise ValueError("unsupported challenge count") actual = {} for row in rows: identifier, share = row["id"], row["emission_share_bps"] - if identifier in actual or type(share) is not int: + if identifier in actual or type(share) is not int or share < 0: raise ValueError("duplicate id or invalid share") + if not CHALLENGE_ID.fullmatch(identifier): + raise ValueError("invalid challenge id") if not re.fullmatch(r"[0-9a-f]{64}", row["public_key"]): raise ValueError("invalid challenge public key") actual[identifier] = share - version = document["version"] - if type(version) is not int or version < 1: - raise ValueError("invalid trust document version") - if actual != SHARES and not (actual == PROPORTIONAL_SHARES and version >= 2): + if version >= 3: + if sum(actual.values()) != 10000: + raise ValueError("version 3 shares must sum to 10000") + elif actual != SHARES and not (actual == PROPORTIONAL_SHARES and version >= 2): raise ValueError("unsupported shares or activation version") except (OSError, UnicodeError, ValueError, KeyError, TypeError): failures.append( - f"{name}: expected bounty/proof=2000/8000 or version >=2 with 3000/7000" + f"{name}: expected bounty/proof=2000/8000, version 2 with 3000/7000, " + "or version >=3 with unique ids summing to 10000" ) return failures @@ -241,8 +253,6 @@ def check_public_contracts(root: Path) -> list[str]: routes = declared_routes(path.read_text()) for method, route in sorted(required - routes): failures.append(f"{name}: public API removed: {method.upper()} {route}") - if "bounty" in name and any("/public/" in route for _, route in routes): - failures.append(f"{name}: Bounty public feed must remain an external backend API") except (OSError, UnicodeError, ValueError, SyntaxError): failures.append(f"{name}: public API source missing or invalid") return failures diff --git a/src/cortex/bounty/__init__.py b/src/cortex/bounty/__init__.py deleted file mode 100644 index dd8ee8a4f..000000000 --- a/src/cortex/bounty/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Durable Bounty ingest and external-feed-only scoring.""" - -from .api import create_router -from .backend import BackendUnavailable, PublicBackend, PublicSnapshot -from .service import BountyService, pair_payload -from .store import BountyStore - -__all__ = [ - "BackendUnavailable", - "BountyService", - "BountyStore", - "PublicBackend", - "PublicSnapshot", - "create_router", - "pair_payload", -] diff --git a/src/cortex/bounty/api.py b/src/cortex/bounty/api.py deleted file mode 100644 index 5a92cbaa3..000000000 --- a/src/cortex/bounty/api.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Internal Bounty HTTP routes; no public leaderboard or report API.""" - -from typing import Annotated - -from fastapi import APIRouter, Header, Request -from fastapi.responses import JSONResponse -from pydantic import BaseModel, ConfigDict, Field, ValidationError - -from cortex.protocol.models import BOUNTY_FULL_SHARE_REPORTS - -from .backend import BackendUnavailable, Severity, Verdict -from .scoring import MAX_TRIAGE_NOISE_BPS, MIN_PRECISION_BPS, SCORE_MAX, SEVERITY_BPS -from .service import PAIR_GRANT_MAX_TTL_SECONDS, TERMS_TEXT, BountyService -from .store import StoreError - -SMALL_WRITE_MAX_BODY_BYTES = 4096 -REPORT_MAX_BODY_BYTES = 256 * 1024 - - -class RequestBody(BaseModel): - model_config = ConfigDict(extra="forbid", strict=True) - - -class PairBody(RequestBody): - account_id: str = Field(max_length=128) - hotkey: str = Field(max_length=128) - nonce: str = Field(max_length=64) - exp: int - signature: str = Field(max_length=130) - terms_accepted: bool - - -class PairGrantBody(RequestBody): - account_id: str = Field(max_length=128) - hotkey: str = Field(max_length=128) - expires_at: int - - -class ReportBody(RequestBody): - session: str = Field(max_length=128) - hotkey: str | None = Field(default=None, max_length=128) - title: str = Field(max_length=512) - body: str = Field(max_length=100_000) - repro_steps: str | None = Field(default=None, max_length=100_000) - - -class AdjudicateBody(RequestBody): - report_id: str = Field(max_length=128) - verdict: Verdict - severity: Severity | None = None - duplicate_of: str | None = Field(default=None, max_length=128) - - -def _error(exc: StoreError) -> JSONResponse: - return JSONResponse({"error": str(exc)}, status_code=exc.status) - - -def _request_openapi(model: type[RequestBody]) -> dict[str, object]: - return { - "requestBody": { - "required": True, - "content": {"application/json": {"schema": model.model_json_schema()}}, - } - } - - -async def _read_request[RequestModel: RequestBody]( - request: Request, - model: type[RequestModel], - *, - limit: int, - label: str, -) -> RequestModel: - encoded = bytearray() - async for chunk in request.stream(): - if len(encoded) + len(chunk) > limit: - raise StoreError(413, f"{label} request too large") - encoded.extend(chunk) - try: - return model.model_validate_json(bytes(encoded)) - except ValidationError: - raise StoreError(422, f"invalid {label} request") from None - - -def create_router(service: BountyService) -> APIRouter: - router = APIRouter(tags=["bounty"]) - - @router.get("/health") - async def health(): - return {"ok": True, "challenge_id": "bounty", "scoring_version": service.scoring_version()} - - @router.get("/v1/status") - async def status(): - version = service.scoring_version() - reason = None - try: - await service.backend.probe() - can_score = True - except BackendUnavailable as error: - can_score = False - reason = str(error) - return { - "challenge_id": "bounty", - "scoring_version": version, - "score_max": SCORE_MAX if version == 1 else 2**64 - 1, - "champion_hotkey": None, - "scoring_backend": "backend_public" if service.backend.configured else "unconfigured", - "can_score": can_score, - "reason": reason, - "backend_public_configured": service.backend.configured, - "pairing": { - "requires_operator_grant": True, - "grant_max_ttl_secs": PAIR_GRANT_MAX_TTL_SECONDS, - }, - "scoring": { - "paid_on": ["valid_report_count"], - "points_per_valid_report": 1, - "full_share_reports": BOUNTY_FULL_SHARE_REPORTS, - "emission_share_bps": 3000, - "population": "expected_metagraph_hotkeys", - "window": "cumulative_published_history", - "off_score_gates": [], - "severities": list(SEVERITY_BPS), - } - if version == 2 - else { - "paid_on": ["precision", "severity_impact"], - "off_score_gates": ["triage_noise"], - "min_precision_bps": MIN_PRECISION_BPS, - "max_triage_noise_bps": MAX_TRIAGE_NOISE_BPS, - "severities": list(SEVERITY_BPS), - }, - "quotas": { - "max_pending_reports_per_hotkey": 5, - "max_concurrent_feed_validations_per_hotkey": 1, - "min_report_interval_secs": 60, - "min_report_body_chars": 80, - "min_repro_chars": 20, - "max_report_request_bytes": REPORT_MAX_BODY_BYTES, - }, - "terms": TERMS_TEXT, - } - - @router.post("/v1/pair", status_code=201, openapi_extra=_request_openapi(PairBody)) - async def pair(request: Request): - try: - body = await _read_request( - request, - PairBody, - limit=SMALL_WRITE_MAX_BODY_BYTES, - label="pair", - ) - return service.pair(body) - except StoreError as exc: - return _error(exc) - - @router.post( - "/v1/admin/pair-grants", - status_code=201, - openapi_extra=_request_openapi(PairGrantBody), - ) - async def grant_pair(request: Request, authorization: Annotated[str | None, Header()] = None): - try: - service.require_operator(authorization) - body = await _read_request( - request, - PairGrantBody, - limit=SMALL_WRITE_MAX_BODY_BYTES, - label="pair grant", - ) - return service.grant_pair(body) - except StoreError as exc: - return _error(exc) - - @router.post("/v1/reports", status_code=201, openapi_extra=_request_openapi(ReportBody)) - async def submit(request: Request): - try: - body = await _read_request( - request, - ReportBody, - limit=REPORT_MAX_BODY_BYTES, - label="report", - ) - row = await service.submit(body) - return {key: row[key] for key in ("id", "miner_hotkey", "state", "fingerprint")} - except StoreError as exc: - return _error(exc) - - @router.get("/v1/reports") - async def list_reports(authorization: Annotated[str | None, Header()] = None): - try: - service.require_operator(authorization) - return {"items": service.store.list_reports()} - except StoreError as exc: - return _error(exc) - - @router.get("/v1/reports/{report_id}") - async def get_report(report_id: str, authorization: Annotated[str | None, Header()] = None): - try: - service.require_operator(authorization) - return service.store.get_report(report_id) - except StoreError as exc: - return _error(exc) - - @router.post("/v1/admin/adjudicate", openapi_extra=_request_openapi(AdjudicateBody)) - async def adjudicate( - request: Request, - authorization: Annotated[str | None, Header()] = None, - ): - try: - service.require_operator(authorization) - body = await _read_request( - request, - AdjudicateBody, - limit=SMALL_WRITE_MAX_BODY_BYTES, - label="adjudication", - ) - return service.store.adjudicate( - body.report_id, body.verdict, body.severity, body.duplicate_of - ) - except StoreError as exc: - return _error(exc) - - return router diff --git a/src/cortex/bounty/backend.py b/src/cortex/bounty/backend.py deleted file mode 100644 index 121d8ef9a..000000000 --- a/src/cortex/bounty/backend.py +++ /dev/null @@ -1,470 +0,0 @@ -"""Read the external public feed; never substitute local adjudications.""" - -import asyncio -from collections import Counter -from time import monotonic -from typing import Annotated, Literal -from urllib.parse import urlsplit - -import httpx -from pydantic import AfterValidator, BaseModel, ConfigDict, Field, ValidationError, model_validator - -from cortex.protocol.crypto import decode_hotkey, encode_hotkey - -from .scoring import BountyScore, Holdout, judge_challenger - -Severity = Literal["trivial", "minor", "major", "critical"] -Verdict = Literal["valid", "invalid_malicious", "duplicate", "already_fixed_not_prod"] - - -def _canonical_revision(value: str) -> str: - if ( - not value - or any(character not in "0123456789" for character in value) - or (len(value) > 1 and value.startswith("0")) - or len(value) > 19 - or int(value) > 2**63 - 1 - ): - raise ValueError("revision must be a canonical non-negative i64") - return value - - -Revision = Annotated[str, AfterValidator(_canonical_revision)] - - -class BackendUnavailable(Exception): - """A public scoring snapshot cannot be trusted. Contains no upstream body.""" - - -class _RetryableBackendUnavailable(BackendUnavailable): - """A read-only snapshot attempt may succeed against the next rollout replica.""" - - -class FeedModel(BaseModel): - model_config = ConfigDict(extra="ignore", frozen=True, strict=True) - - -class LeaderboardRow(FeedModel): - hotkey: str - valid_count: int = Field(ge=0, le=2**64 - 1) - weight: int | None = Field(default=None, ge=0, le=2**64 - 1) - - @model_validator(mode="before") - @classmethod - def normalize_valid_count(cls, value: object) -> object: - if not isinstance(value, dict): - return value - if "valid_count" in value and "valid" in value and value["valid_count"] != value["valid"]: - raise ValueError("leaderboard valid counters disagree") - if "valid_count" not in value and "valid" in value: - normalized = dict(value) - normalized["valid_count"] = normalized["valid"] - return normalized - return value - - -class PublicReport(FeedModel): - id: str = Field(min_length=1) - hotkey: str - status: Verdict - problem_found: str - adjudicator: str - justification: str - severity: Severity | None = None - adjudicated_at: str - created_at: str - related_report_id: str | None = None - - -class PublicationStatus(FeedModel): - api_version: Literal[1] - revision: Revision - adjudication_available: bool - published: int = Field(ge=0, le=2**63 - 1) - valid: int = Field(ge=0, le=2**63 - 1) - duplicate: int = Field(ge=0, le=2**63 - 1) - already_fixed_not_prod: int = Field(ge=0, le=2**63 - 1) - invalid_malicious: int = Field(ge=0, le=2**63 - 1) - hotkeys: int = Field(ge=0, le=2**63 - 1) - awaiting_adjudication: int = Field(ge=0, le=2**63 - 1) - unpriced_valid: int = Field(ge=0, le=2**63 - 1) - - -class LeaderboardPage(FeedModel): - api_version: Literal[1] - revision: Revision - items: tuple[LeaderboardRow, ...] - has_more: bool - - -class ReportPage(FeedModel): - api_version: Literal[1] - revision: Revision - items: tuple[PublicReport, ...] - count: int = Field(ge=0, le=100) - has_more: bool - next_cursor: str | None = Field(default=None, min_length=1, max_length=1024) - - -class PublicSnapshot(FeedModel): - leaderboard: tuple[LeaderboardRow, ...] = () - reports: tuple[PublicReport, ...] = () - - @staticmethod - def leaderboard_from_reports( - reports: tuple[PublicReport, ...], - ) -> tuple[LeaderboardRow, ...]: - valid_counts: Counter[str] = Counter() - canonical: dict[str, str] = {} - try: - for report in reports: - public = decode_hotkey(report.hotkey) - key = public.hex() - canonical[key] = encode_hotkey(public) - if report.status == "valid": - valid_counts[key] += 1 - except ValueError: - raise BackendUnavailable("backend public invalid hotkey") from None - return tuple( - LeaderboardRow(hotkey=canonical[key], valid_count=valid_counts[key]) - for key in sorted(canonical, key=lambda item: (-valid_counts[item], canonical[item])) - ) - - @staticmethod - def validate_leaderboard_page( - page: LeaderboardPage, complete: tuple[LeaderboardRow, ...] - ) -> None: - try: - published = tuple( - (encode_hotkey(decode_hotkey(row.hotkey)), row.valid_count) for row in page.items - ) - except ValueError: - raise BackendUnavailable("backend public invalid hotkey") from None - if len({hotkey for hotkey, _ in published}) != len(published): - raise BackendUnavailable("backend public duplicate leaderboard hotkey") - expected = tuple((row.hotkey, row.valid_count) for row in complete) - if page.has_more: - if ( - not published - or len(published) >= len(expected) - or published != expected[: len(published)] - ): - raise BackendUnavailable("backend public truncated leaderboard is inconsistent") - return - published_hotkeys = {hotkey for hotkey, _ in published} - if any( - valid_count > 0 and hotkey not in published_hotkeys for hotkey, valid_count in expected - ) or published != tuple(row for row in expected if row[0] in published_hotkeys): - raise BackendUnavailable("backend public leaderboard and reports do not agree") - - def validate_publication(self) -> None: - """A stable pair can still be two permanently different revisions.""" - try: - valid_counts = Counter( - decode_hotkey(r.hotkey).hex() for r in self.reports if r.status == "valid" - ) - leaderboard = {decode_hotkey(r.hotkey).hex(): r.valid_count for r in self.leaderboard} - for report in self.reports: - decode_hotkey(report.hotkey) - except ValueError: - raise BackendUnavailable("backend public invalid hotkey") from None - if len(leaderboard) != len(self.leaderboard): - raise BackendUnavailable("backend public duplicate leaderboard hotkey") - report_ids = {row.id for row in self.reports} - if len(report_ids) != len(self.reports): - raise BackendUnavailable("backend public duplicate report id") - if any( - not report.problem_found.strip() - or not report.justification.strip() - or not report.adjudicator.strip() - for report in self.reports - ): - raise BackendUnavailable("backend public report evidence is incomplete") - if any( - (report.status == "valid") != (report.severity is not None) for report in self.reports - ): - raise BackendUnavailable("backend public report severity is inconsistent") - if any( - (report.status == "duplicate") != (report.related_report_id is not None) - or ( - report.related_report_id is not None - and ( - report.related_report_id == report.id - or report.related_report_id not in report_ids - ) - ) - for report in self.reports - ): - raise BackendUnavailable("backend public duplicate reference is invalid") - reports_by_id = {report.id: report for report in self.reports} - rooted = {report.id for report in self.reports if report.status != "duplicate"} - for report in self.reports: - if report.status != "duplicate" or report.id in rooted: - continue - path = [] - traversed = set() - current = report - while current.id not in rooted: - if current.id in traversed: - raise BackendUnavailable( - "backend public duplicate chain has no non-duplicate root" - ) - traversed.add(current.id) - path.append(current.id) - target = reports_by_id.get(current.related_report_id or "") - if target is None: - raise BackendUnavailable("backend public duplicate reference is invalid") - current = target - rooted.update(path) - if any(leaderboard.get(k) != count for k, count in valid_counts.items()) or any( - valid_counts.get(k, 0) != count for k, count in leaderboard.items() - ): - raise BackendUnavailable("backend public leaderboard and reports do not agree") - - def score(self, expected: list[str], *, scoring_version: int = 1) -> dict[str, BountyScore]: - self.validate_publication() - expected_keys = {decode_hotkey(raw).hex() for raw in expected} - if scoring_version == 2: - counts = Counter( - decode_hotkey(report.hotkey).hex() - for report in self.reports - if report.status == "valid" - ) - return { - key: BountyScore(value=counts[key]) - if counts[key] - else BountyScore(reason="NotAttempted") - for key in sorted(expected_keys) - } - if scoring_version != 1: - raise BackendUnavailable("unsupported Bounty scoring version") - holdouts: dict[str, Holdout] = {} - for report in self.reports: - hotkey = decode_hotkey(report.hotkey).hex() - holdouts.setdefault(hotkey, Holdout()).record(report.status, report.severity) - ranked = sorted(self.leaderboard, key=lambda row: (-row.valid_count, row.hotkey)) - order = [ - hotkey for row in ranked if (hotkey := decode_hotkey(row.hotkey).hex()) in expected_keys - ] - already_ranked = set(order) - order.extend( - key for key in sorted(holdouts) if key in expected_keys and key not in already_ranked - ) - champion, champion_holdout, lattice = None, Holdout(), 0 - for hotkey in order: - if hotkey not in holdouts: - continue - verdict = judge_challenger(champion_holdout, holdouts[hotkey]) - if verdict.eligible: - champion, champion_holdout, lattice = hotkey, holdouts[hotkey], verdict.lattice - scores = {} - for key in sorted(expected_keys): - if key == champion: - scores[key] = BountyScore(value=lattice) - else: - reason = ( - "InvalidResponse" - if key in holdouts and holdouts[key].net_credit < 0 - else "NotAttempted" - ) - scores[key] = BountyScore(reason=reason) - return scores - - def validate_status(self, status: PublicationStatus) -> None: - counts: Counter[Verdict] = Counter(report.status for report in self.reports) - expected: dict[Verdict, int] = { - "valid": status.valid, - "duplicate": status.duplicate, - "already_fixed_not_prod": status.already_fixed_not_prod, - "invalid_malicious": status.invalid_malicious, - } - if status.published != len(self.reports) or any( - counts.get(verdict, 0) != count for verdict, count in expected.items() - ): - raise BackendUnavailable("backend public status and reports do not agree") - try: - hotkeys = {decode_hotkey(report.hotkey).hex() for report in self.reports} - except ValueError: - raise BackendUnavailable("backend public invalid hotkey") from None - if status.hotkeys != len(hotkeys): - raise BackendUnavailable("backend public status hotkey count does not agree") - - -class PublicBackend: - """Read one immutable, fully paginated CortexLM/backend publication.""" - - _MAX_RESPONSE_BYTES = 8 * 1024 * 1024 - _MAX_REPORT_BYTES = 64 * 1024 * 1024 - _MAX_REPORT_PAGES = 10_000 - _SNAPSHOT_TIMEOUT_SECONDS = 30.0 - _SNAPSHOT_ATTEMPTS = 3 - _SNAPSHOT_RETRY_DELAY_SECONDS = 0.25 - _PROBE_SUCCESS_TTL_SECONDS = 15.0 - _PROBE_FAILURE_TTL_SECONDS = 5.0 - - def __init__(self, base_url: str | None, *, transport: httpx.AsyncBaseTransport | None = None): - self.base_url = (base_url or "").strip().rstrip("/") - if self.base_url: - parsed = urlsplit(self.base_url) - if ( - parsed.scheme != "https" - or not parsed.hostname - or parsed.username - or parsed.password - or parsed.query - or parsed.fragment - ): - raise ValueError("backend public URL must be HTTPS without credentials or query") - self.transport = transport - self._probe_lock = asyncio.Lock() - self._probe_snapshot: PublicSnapshot | None = None - self._probe_success_until = 0.0 - self._probe_error: str | None = None - self._probe_failure_until = 0.0 - - @property - def configured(self) -> bool: - return bool(self.base_url) - - async def probe(self) -> PublicSnapshot: - """Bound anonymous health probes without weakening intake freshness.""" - now = monotonic() - if self._probe_snapshot is not None and now < self._probe_success_until: - return self._probe_snapshot - if self._probe_error is not None and now < self._probe_failure_until: - raise BackendUnavailable(self._probe_error) - if self._probe_lock.locked(): - raise BackendUnavailable("backend public probe refresh already in progress") - async with self._probe_lock: - now = monotonic() - if self._probe_snapshot is not None and now < self._probe_success_until: - return self._probe_snapshot - if self._probe_error is not None and now < self._probe_failure_until: - raise BackendUnavailable(self._probe_error) - try: - snapshot = await self.fetch() - except BackendUnavailable as error: - self._probe_snapshot = None - self._probe_error = str(error) - self._probe_failure_until = monotonic() + self._PROBE_FAILURE_TTL_SECONDS - raise - self._probe_snapshot = snapshot - self._probe_success_until = monotonic() + self._PROBE_SUCCESS_TTL_SECONDS - self._probe_error = None - self._probe_failure_until = 0.0 - return snapshot - - async def fetch(self) -> PublicSnapshot: - if not self.configured: - raise BackendUnavailable("scoring unconfigured: set BOUNTY_BACKEND_PUBLIC_URL") - try: - async with asyncio.timeout(self._SNAPSHOT_TIMEOUT_SECONDS): - last_error: BackendUnavailable | None = None - for attempt in range(self._SNAPSHOT_ATTEMPTS): - if attempt: - await asyncio.sleep(self._SNAPSHOT_RETRY_DELAY_SECONDS * attempt) - try: - async with httpx.AsyncClient( - timeout=20, - transport=self.transport, - follow_redirects=False, - trust_env=False, - headers={"User-Agent": "cortex-bounty-challenge/python"}, - ) as client: - return await self._fetch_once(client) - except _RetryableBackendUnavailable as error: - last_error = error - except (httpx.HTTPError, ValueError, ValidationError): - last_error = _RetryableBackendUnavailable( - "backend public fetch or JSON validation failed" - ) - if last_error is None: - raise BackendUnavailable("backend public snapshot attempt unavailable") - raise last_error - except TimeoutError: - raise BackendUnavailable("backend public snapshot deadline exceeded") from None - - async def _fetch_once(self, client: httpx.AsyncClient) -> PublicSnapshot: - status, _ = await self._read_model(client, "status", PublicationStatus) - if not status.adjudication_available: - raise BackendUnavailable("backend public adjudication is unavailable") - if status.unpriced_valid: - raise BackendUnavailable("backend public feed has unpriced valid reports") - if status.awaiting_adjudication and not status.published: - raise BackendUnavailable("backend public adjudication backlog has no published reports") - leaderboard, _ = await self._read_model( - client, - "leaderboard", - LeaderboardPage, - params={"revision": status.revision}, - ) - if leaderboard.revision != status.revision: - raise _RetryableBackendUnavailable("backend public leaderboard revision changed") - reports = await self._read_reports(client, status.revision) - complete_leaderboard = PublicSnapshot.leaderboard_from_reports(reports) - snapshot = PublicSnapshot( - leaderboard=complete_leaderboard, - reports=reports, - ) - snapshot.validate_publication() - PublicSnapshot.validate_leaderboard_page(leaderboard, complete_leaderboard) - snapshot.validate_status(status) - return snapshot - - async def _read_reports( - self, client: httpx.AsyncClient, revision: str - ) -> tuple[PublicReport, ...]: - reports: list[PublicReport] = [] - cursor = None - seen_cursors = set() - total_bytes = 0 - for _ in range(self._MAX_REPORT_PAGES): - params = {"limit": "100", "revision": revision} - if cursor is not None: - params["cursor"] = cursor - page, size = await self._read_model( - client, - "reports", - ReportPage, - params=params, - ) - total_bytes += size - if total_bytes > self._MAX_REPORT_BYTES: - raise BackendUnavailable("backend public report snapshot is too large") - if page.revision != revision: - raise _RetryableBackendUnavailable("backend public report revision changed") - if page.count != len(page.items): - raise BackendUnavailable("backend public report page count is invalid") - reports.extend(page.items) - if not page.has_more: - if page.next_cursor is not None: - raise BackendUnavailable("backend public terminal page has a cursor") - return tuple(reports) - if not page.items or page.next_cursor is None or page.next_cursor in seen_cursors: - raise BackendUnavailable("backend public report pagination is invalid") - seen_cursors.add(page.next_cursor) - cursor = page.next_cursor - raise BackendUnavailable("backend public report pagination exceeded the page limit") - - async def _read_model[Model: FeedModel]( - self, - client: httpx.AsyncClient, - route: str, - model: type[Model], - *, - params: dict[str, str] | None = None, - ) -> tuple[Model, int]: - async with client.stream( - "GET", f"{self.base_url}/v1/bounty/public/{route}", params=params - ) as response: - if not 200 <= response.status_code < 300: - error = f"backend public fetch failed: HTTP {response.status_code}" - if response.status_code in {404, 408, 409, 425, 429} or response.status_code >= 500: - raise _RetryableBackendUnavailable(error) - raise BackendUnavailable(error) - content = bytearray() - async for part in response.aiter_bytes(): - if len(content) + len(part) > self._MAX_RESPONSE_BYTES: - raise BackendUnavailable("backend public response too large") - content.extend(part) - return model.model_validate_json(content), len(content) diff --git a/src/cortex/bounty/scoring.py b/src/cortex/bounty/scoring.py deleted file mode 100644 index 0fa3f6b48..000000000 --- a/src/cortex/bounty/scoring.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Bounty's integer-only precision, severity and displacement rules.""" - -from dataclasses import dataclass, field - -SCORE_MAX = 1_000_000 -MIN_PRECISION_BPS = 6_000 -MAX_TRIAGE_NOISE_BPS = 5_000 -SEVERITY_BPS = {"trivial": 625, "minor": 2_500, "major": 5_000, "critical": 10_000} - - -@dataclass -class Holdout: - valid_by_severity: dict[str, int] = field(default_factory=dict) - valid_unpriced: int = 0 - malicious: int = 0 - duplicate: int = 0 - already_fixed: int = 0 - - def record(self, verdict: str, severity: str | None = None) -> None: - if verdict == "valid": - if severity is None: - self.valid_unpriced += 1 - elif severity not in SEVERITY_BPS: - raise ValueError("unknown severity") - else: - self.valid_by_severity[severity] = self.valid_by_severity.get(severity, 0) + 1 - elif verdict == "invalid_malicious": - self.malicious += 1 - elif verdict == "duplicate": - self.duplicate += 1 - elif verdict == "already_fixed_not_prod": - self.already_fixed += 1 - else: - raise ValueError("unknown adjudication") - - @property - def valid(self) -> int: - return sum(self.valid_by_severity.values()) - - @property - def decided(self) -> int: - return self.valid + self.malicious - - @property - def precision_bps(self) -> int | None: - return self.valid * 10_000 // self.decided if self.decided else None - - @property - def impact_bps(self) -> int | None: - total = sum( - SEVERITY_BPS[severity] * count for severity, count in self.valid_by_severity.items() - ) - return total // self.valid if self.valid else None - - @property - def noise_bps(self) -> int | None: - noise = self.duplicate + self.already_fixed - total = self.decided + self.valid_unpriced + noise - return noise * 10_000 // total if total else None - - @property - def net_credit(self) -> int: - # Preserve the contract's per-report integer truncation (trivial = 6). - return ( - sum((100 * SEVERITY_BPS[s] // 10_000) * n for s, n in self.valid_by_severity.items()) - - 100 * self.malicious - ) - - -@dataclass(frozen=True) -class ChampionVerdict: - eligible: bool - lattice: int - failed: tuple[str, ...] - challenger_precision_bps: int | None - challenger_impact_bps: int | None - challenger_noise_bps: int | None - - -def judge_challenger(champion: Holdout, challenger: Holdout) -> ChampionVerdict: - failed = [] - precision, impact, noise = challenger.precision_bps, challenger.impact_bps, challenger.noise_bps - if challenger.decided < 3: - failed.append("thin_holdout") - if challenger.net_credit < 0: - failed.append("penalty") - if challenger.valid_unpriced: - failed.append("severity_evidence_missing") - if precision is not None and precision < MIN_PRECISION_BPS: - failed.append("precision_floor") - if noise is not None and noise > MAX_TRIAGE_NOISE_BPS: - failed.append("triage_noise") - if precision is None or ( - champion.precision_bps is not None and precision <= champion.precision_bps - ): - failed.append("no_precision_win") - lattice = ( - SCORE_MAX * precision * impact // 100_000_000 - if not failed and precision is not None and impact is not None - else 0 - ) - return ChampionVerdict(not failed, lattice, tuple(failed), precision, impact, noise) - - -@dataclass(frozen=True) -class BountyScore: - """Exactly one outcome per expected hotkey; absence always has zero value.""" - - value: int = 0 - reason: str | None = None diff --git a/src/cortex/bounty/service.py b/src/cortex/bounty/service.py deleted file mode 100644 index d823f90d9..000000000 --- a/src/cortex/bounty/service.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Bounty intake and scoring service; backend availability gates every report.""" - -import asyncio -import hashlib -import hmac -import re -import time -from collections.abc import Callable, Sequence - -from cortex.protocol.crypto import decode_hotkey, verify_substrate - -from .backend import BackendUnavailable, PublicBackend -from .scoring import BountyScore -from .store import BountyStore, StoreError, normalize_text - -TERMS_TEXT = ( - "By pairing a Bittensor hotkey to a Cortex Chat account for Bounty Challenge, you accept " - "that this dedicated mining account, its logs, and its conversations may be used for research, " - "to fix product and backend bugs, and to remunerate (or penalize) the bound miner hotkey. " - "Do not pair a private personal account." -) -PAIR_GRANT_MAX_TTL_SECONDS = 300 -_ACCOUNT_ID_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}") - - -def validate_account_id(account: str) -> None: - if not _ACCOUNT_ID_PATTERN.fullmatch(account): - raise StoreError(400, "invalid account_id") - - -def pair_payload(account: str, nonce: str, expiry: int) -> bytes: - validate_account_id(account) - if not re.fullmatch(r"[a-fA-F0-9]{16,64}", nonce): - raise StoreError(400, "invalid nonce") - if not 0 < expiry <= 2**64 - 1: - raise StoreError(400, "invalid or expired pairing window") - return f"cortex-bounty-v1|{account}|{nonce}|{expiry}".encode() - - -class BountyService: - def __init__( - self, - store: BountyStore, - backend: PublicBackend, - *, - session_secret: bytes, - admin_tokens: Sequence[str] = (), - admin_hashes: Sequence[str] = (), - clock: Callable[[], float] = time.time, - scoring_version: Callable[[], int] = lambda: 1, - ): - if len(session_secret) < 32: - raise ValueError("Bounty session secret must contain at least 32 bytes") - self.store, self.backend, self.clock = store, backend, clock - self.scoring_version = scoring_version - self._secret = session_secret - self._admin_hashes = tuple(admin_hashes) + tuple( - hashlib.sha256(t.encode()).hexdigest() for t in admin_tokens if t - ) - self._submission_locks: dict[str, asyncio.Lock] = {} - - def require_operator(self, authorization: str | None) -> None: - if not self._admin_hashes: - raise StoreError(503, "auth_unconfigured") - if not authorization or not authorization.startswith("Bearer "): - raise StoreError(401, "unauthorized") - token = authorization[7:].strip() - digest = hashlib.sha256(token.encode()).hexdigest() - matches = [hmac.compare_digest(digest, item) for item in self._admin_hashes] - if not token or not any(matches): - raise StoreError(401, "unauthorized") - - def pair(self, body) -> dict: - if not body.terms_accepted: - raise StoreError(403, "terms_required") - payload = pair_payload(body.account_id, body.nonce, body.exp) - now = int(self.clock()) - if now >= body.exp: - raise StoreError(400, "invalid or expired pairing window") - try: - hotkey = decode_hotkey(body.hotkey) - signature = bytes.fromhex(body.signature.removeprefix("0x")) - except ValueError: - raise StoreError(400, "invalid hotkey or signature") from None - if len(signature) != 64: - raise StoreError(400, "invalid signature") - if not verify_substrate(hotkey, payload, signature): - raise StoreError(401, "signature verification failed") - return self.store.bind_pair(body.account_id, hotkey.hex(), body.nonce, now, self._secret) - - def grant_pair(self, body) -> dict: - validate_account_id(body.account_id) - now = int(self.clock()) - if body.expires_at <= now: - raise StoreError(400, "pair grant must expire in the future") - if body.expires_at > now + PAIR_GRANT_MAX_TTL_SECONDS: - raise StoreError( - 400, - f"pair grant must expire within {PAIR_GRANT_MAX_TTL_SECONDS} seconds", - ) - try: - hotkey = decode_hotkey(body.hotkey).hex() - except ValueError: - raise StoreError(400, "invalid hotkey") from None - return self.store.grant_pair( - body.account_id, - hotkey, - expires_at=body.expires_at, - now=now, - ) - - async def submit(self, body) -> dict: - # The Rust intake only checked configuration. Actually reading the feed - # closes the documented 503/no-row contract during upstream outages. - if not self.backend.configured: - raise StoreError(503, "scoring unconfigured: set BOUNTY_BACKEND_PUBLIC_URL") - pairing = self.store.lookup_session(body.session, self._secret) - if body.hotkey: - try: - hotkey = decode_hotkey(body.hotkey).hex() - except ValueError: - raise StoreError(400, "invalid hotkey") from None - if hotkey != pairing["miner_hotkey"]: - raise StoreError(403, "hotkey_mismatch") - repro = body.repro_steps or "" - validate_substance(body.title, body.body, repro) - hotkey = pairing["miner_hotkey"] - lock = self._submission_locks.get(hotkey) - if lock is None: - lock = asyncio.Lock() - self._submission_locks[hotkey] = lock - if lock.locked(): - raise StoreError(429, "report validation already in progress for this hotkey") - async with lock: - pairing = self.store.lookup_session(body.session, self._secret) - self.store.check_report_admission(pairing, int(self.clock())) - try: - await self.backend.fetch() - except BackendUnavailable as exc: - raise StoreError(503, str(exc)) from None - pairing = self.store.lookup_session(body.session, self._secret) - return self.store.insert_report( - pairing, body.title, body.body, repro, int(self.clock()) - ) - - async def score(self, expected: list[str]) -> dict[str, BountyScore]: - """Produce exact-E outcomes, including ChallengeInternal on feed failure.""" - keys = sorted({decode_hotkey(raw).hex() for raw in expected}) - version = self.scoring_version() - try: - snapshot = await self.backend.fetch() - return snapshot.score(keys, scoring_version=version) - except BackendUnavailable: - return {key: BountyScore(reason="ChallengeInternal") for key in keys} - - -def validate_substance(title: str, body: str, repro: str) -> None: - if not title.strip() or not body.strip(): - raise StoreError(400, "title_and_body_required") - if normalize_text(title) == normalize_text(body): - raise StoreError(400, "title_and_body_must_differ") - if len(body.strip()) < 80: - raise StoreError(400, "body must be at least 80 characters") - if len(repro.strip()) < 20: - raise StoreError(400, "repro_steps must be at least 20 characters") - if len({token for token in normalize_text(body).split() if len(token) >= 3}) < 4: - raise StoreError(400, "body_lacks_distinct_evidence") diff --git a/src/cortex/bounty/store.py b/src/cortex/bounty/store.py deleted file mode 100644 index 2acd858d1..000000000 --- a/src/cortex/bounty/store.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Transactional SQLite persistence for Bounty intake, replay protection and quotas.""" - -import hashlib -import hmac -import sqlite3 -import threading -from contextlib import contextmanager -from pathlib import Path - -from cortex.state import secure_sqlite_path - - -class StoreError(Exception): - def __init__(self, status: int, message: str): - super().__init__(message) - self.status = status - - -def normalize_text(text: str) -> str: - # ASCII lowering preserves the existing Rust fingerprint contract. - return " ".join(text.split()).translate( - str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") - ) - - -def report_fingerprint(title: str, body: str) -> str: - return hashlib.sha256( - b"base-bounty-report-v1" - + normalize_text(title).encode() - + b"\xff" - + normalize_text(body).encode() - ).hexdigest() - - -def session_token(secret: bytes, session_id: str, account: str, hotkey: str) -> str: - payload = b"base-bounty-session-v1\x00" + b"\x00".join( - value.encode() for value in (session_id, account, hotkey) - ) - return hmac.new(secret, payload, hashlib.sha256).hexdigest() - - -class BountyStore: - """Writes use BEGIN IMMEDIATE, including quota and duplicate checks.""" - - def __init__(self, path: str | Path): - self.path = secure_sqlite_path(path) - self._lock = threading.RLock() - self._db = sqlite3.connect( - str(self.path), check_same_thread=False, isolation_level=None, timeout=10 - ) - self._db.row_factory = sqlite3.Row - self._db.execute("PRAGMA journal_mode=WAL") - self._db.execute("PRAGMA synchronous=FULL") - self._db.execute("PRAGMA foreign_keys=ON") - self._db.executescript(""" - CREATE TABLE IF NOT EXISTS bounty_nonces (nonce TEXT PRIMARY KEY); - CREATE TABLE IF NOT EXISTS bounty_pair_grants ( - account_id TEXT NOT NULL, - miner_hotkey TEXT NOT NULL, - expires_at INTEGER NOT NULL, - granted_at INTEGER NOT NULL, - PRIMARY KEY (account_id, miner_hotkey) - ); - CREATE TABLE IF NOT EXISTS bounty_ids (id INTEGER PRIMARY KEY AUTOINCREMENT); - CREATE TABLE IF NOT EXISTS bounty_sessions ( - session_id TEXT PRIMARY KEY, token_hash TEXT UNIQUE NOT NULL, - account_id TEXT NOT NULL, miner_hotkey TEXT NOT NULL, bound_at INTEGER NOT NULL - ); - CREATE TABLE IF NOT EXISTS bounty_pairings ( - account_id TEXT PRIMARY KEY, - session_id TEXT NOT NULL REFERENCES bounty_sessions(session_id) - ); - CREATE TABLE IF NOT EXISTS bounty_reports ( - id TEXT PRIMARY KEY, miner_hotkey TEXT NOT NULL, account_id TEXT NOT NULL, - title TEXT NOT NULL, body TEXT NOT NULL, repro_steps TEXT NOT NULL, - fingerprint TEXT NOT NULL, state TEXT NOT NULL, adjudication TEXT, - severity TEXT, duplicate_of TEXT REFERENCES bounty_reports(id), - created_at INTEGER NOT NULL - ); - CREATE INDEX IF NOT EXISTS bounty_reports_miner - ON bounty_reports(miner_hotkey, state, created_at); - CREATE INDEX IF NOT EXISTS bounty_reports_fingerprint - ON bounty_reports(fingerprint, id); - """) - - @contextmanager - def _transaction(self): - with self._lock: - self._db.execute("BEGIN IMMEDIATE") - try: - yield self._db - self._db.execute("COMMIT") - except BaseException: - self._db.execute("ROLLBACK") - raise - - def _next_id(self, prefix: str) -> str: - cursor = self._db.execute("INSERT INTO bounty_ids DEFAULT VALUES") - return f"{prefix}_{cursor.lastrowid:016x}" - - def grant_pair(self, account: str, hotkey: str, *, expires_at: int, now: int) -> dict: - if expires_at <= now: - raise StoreError(400, "pair grant must expire in the future") - with self._transaction() as db: - db.execute("DELETE FROM bounty_pair_grants WHERE expires_at<=?", (now,)) - db.execute( - "INSERT INTO bounty_pair_grants " - "(account_id, miner_hotkey, expires_at, granted_at) VALUES (?,?,?,?) " - "ON CONFLICT(account_id, miner_hotkey) DO UPDATE SET " - "expires_at=excluded.expires_at, granted_at=excluded.granted_at", - (account, hotkey, expires_at, now), - ) - return { - "account_id": account, - "miner_hotkey": hotkey, - "expires_at": expires_at, - } - - def bind_pair(self, account: str, hotkey: str, nonce: str, now: int, secret: bytes) -> dict: - with self._transaction() as db: - if db.execute("SELECT 1 FROM bounty_nonces WHERE nonce=?", (nonce,)).fetchone(): - raise StoreError(409, "nonce reused") - consumed = db.execute( - "DELETE FROM bounty_pair_grants " - "WHERE account_id=? AND miner_hotkey=? AND expires_at>?", - (account, hotkey, now), - ) - if consumed.rowcount != 1: - raise StoreError(403, "pairing not authorized by account operator") - db.execute("INSERT INTO bounty_nonces VALUES (?)", (nonce,)) - session_id = self._next_id("bs") - token = session_token(secret, session_id, account, hotkey) - db.execute( - "INSERT INTO bounty_sessions VALUES (?,?,?,?,?)", - (session_id, hashlib.sha256(token.encode()).hexdigest(), account, hotkey, now), - ) - db.execute( - "INSERT INTO bounty_pairings VALUES (?,?) ON CONFLICT(account_id) " - "DO UPDATE SET session_id=excluded.session_id", - (account, session_id), - ) - return { - "session": token, - "session_id": session_id, - "account_id": account, - "miner_hotkey": hotkey, - } - - def lookup_session(self, token: str, secret: bytes) -> dict: - with self._lock: - row = self._db.execute( - "SELECT s.* FROM bounty_sessions s " - "JOIN bounty_pairings p ON p.account_id=s.account_id AND p.session_id=s.session_id " - "WHERE s.token_hash=?", - (hashlib.sha256(token.encode()).hexdigest(),), - ).fetchone() - if row is None or not hmac.compare_digest( - token, - session_token(secret, row["session_id"], row["account_id"], row["miner_hotkey"]), - ): - raise StoreError(401, "invalid_session") - return { - key: row[key] for key in ("account_id", "miner_hotkey", "session_id", "bound_at") - } - - @staticmethod - def _check_report_admission(db: sqlite3.Connection, hotkey: str, now: int) -> None: - pending = db.execute( - "SELECT count(*) FROM bounty_reports WHERE miner_hotkey=? AND state='pending'", - (hotkey,), - ).fetchone()[0] - if pending >= 5: - raise StoreError(429, "5 reports already awaiting adjudication for this hotkey (max 5)") - last = db.execute( - "SELECT max(created_at) FROM bounty_reports WHERE miner_hotkey=?", (hotkey,) - ).fetchone()[0] - if last is not None and now - last < 60: - raise StoreError(429, "one report per 60s per hotkey") - - def check_report_admission(self, pairing: dict, now: int) -> None: - with self._lock: - self._check_report_admission(self._db, pairing["miner_hotkey"], now) - - def insert_report(self, pairing: dict, title: str, body: str, repro: str, now: int) -> dict: - fingerprint = report_fingerprint(title, body) - hotkey = pairing["miner_hotkey"] - with self._transaction() as db: - self._check_report_admission(db, hotkey, now) - original = db.execute( - "SELECT id FROM bounty_reports WHERE fingerprint=? ORDER BY id LIMIT 1", - (fingerprint,), - ).fetchone() - report_id = self._next_id("by") - state = "duplicate" if original else "pending" - db.execute( - "INSERT INTO bounty_reports VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", - ( - report_id, - hotkey, - pairing["account_id"], - title, - body, - repro, - fingerprint, - state, - "duplicate" if original else None, - None, - original["id"] if original else None, - now, - ), - ) - return self.get_report(report_id) - - def get_report(self, report_id: str) -> dict: - with self._lock: - row = self._db.execute( - "SELECT * FROM bounty_reports WHERE id=?", (report_id,) - ).fetchone() - if row is None: - raise StoreError(404, "not_found") - return {**dict(row), "champion_verdict": None} - - def list_reports(self) -> list[dict]: - with self._lock: - return [ - {**dict(row), "champion_verdict": None} - for row in self._db.execute("SELECT * FROM bounty_reports ORDER BY id DESC") - ] - - def adjudicate( - self, report_id: str, verdict: str, severity: str | None, duplicate_of: str | None - ) -> dict: - if verdict == "valid" and severity is None: - raise StoreError(409, "severity required for valid verdict") - if verdict != "valid" and severity is not None: - raise StoreError(409, "severity is only valid for a valid verdict") - with self._transaction() as db: - row = self.get_report(report_id) - if row["state"] != "pending" and row["adjudication"] != "duplicate": - raise StoreError(409, "already adjudicated") - if verdict == "duplicate": - if not duplicate_of: - raise StoreError(409, "duplicate_of required") - if duplicate_of == report_id: - raise StoreError(409, "report cannot duplicate itself") - self.get_report(duplicate_of) - elif duplicate_of is not None: - raise StoreError(409, "duplicate_of is only valid for duplicate verdicts") - db.execute( - "UPDATE bounty_reports SET state=?, adjudication=?, severity=?, duplicate_of=? " - "WHERE id=?", - ( - verdict, - verdict, - severity if verdict == "valid" else None, - duplicate_of, - report_id, - ), - ) - return self.get_report(report_id) - - def close(self) -> None: - with self._lock: - self._db.close() diff --git a/src/cortex/challenges/__init__.py b/src/cortex/challenges/__init__.py new file mode 100644 index 000000000..f05819c01 --- /dev/null +++ b/src/cortex/challenges/__init__.py @@ -0,0 +1 @@ +"""Challenge containers: operator registry, weights client, public proxy and auto-updater.""" diff --git a/src/cortex/challenges/__main__.py b/src/cortex/challenges/__main__.py new file mode 100644 index 000000000..7557e8592 --- /dev/null +++ b/src/cortex/challenges/__main__.py @@ -0,0 +1,59 @@ +"""`cortex challenge-supervisor`: the only Cortex process that controls Docker.""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +from pathlib import Path + +import httpx + +from .supervisor import Supervisor, SupervisorConfig + + +def parser() -> argparse.ArgumentParser: + arguments = argparse.ArgumentParser(description="Run and auto-update challenge containers") + arguments.add_argument("--registry", type=Path, required=True) + arguments.add_argument( + "--secrets-host-dir", + type=Path, + required=True, + help="absolute HOST path holding /{internal.token,admin.token,...}", + ) + arguments.add_argument("--docker-socket", default="/var/run/docker.sock") + arguments.add_argument("--network", default="cortex-challenges") + arguments.add_argument("--master-url", default="http://cortex-master:8080") + arguments.add_argument("--once", action="store_true", help="reconcile every entry once") + return arguments + + +async def run(arguments: argparse.Namespace) -> None: + config = SupervisorConfig( + registry_file=arguments.registry, + secrets_host_dir=arguments.secrets_host_dir, + network=arguments.network, + master_url=arguments.master_url, + ) + transport = httpx.AsyncHTTPTransport(uds=arguments.docker_socket) + async with ( + httpx.AsyncClient(transport=transport, base_url="http://docker/v1.44") as docker, + httpx.AsyncClient(trust_env=False, follow_redirects=False) as http, + ): + supervisor = Supervisor(config, docker, http) + if arguments.once: + await supervisor.tick(float("inf")) + return + await supervisor.run() + + +def main(argv: list[str] | None = None) -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + try: + asyncio.run(run(parser().parse_args(argv))) + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/src/cortex/challenges/client.py b/src/cortex/challenges/client.py new file mode 100644 index 000000000..4a6777519 --- /dev/null +++ b/src/cortex/challenges/client.py @@ -0,0 +1,113 @@ +"""Read container weights once per completed epoch and turn them into leaf scores.""" + +from __future__ import annotations + +import asyncio +import json +import math +from dataclasses import dataclass +from fractions import Fraction +from pathlib import Path + +import httpx + +from cortex.errors import ServiceError +from cortex.http import read_private_file +from cortex.protocol.crypto import decode_hotkey +from cortex.protocol.models import FULL_SHARE_SCORE, NoScore, NoScoreReason, Score + +from .registry import RegistryEntry + +MAX_RESPONSE_BYTES = 8 * 1024 * 1024 +MAX_WEIGHTS = 65536 +ATTEMPTS = 3 + + +@dataclass(frozen=True) +class ChallengeWeights: + weights: dict[bytes, Fraction] + full_share_mass: Fraction | None = None + + +def parse_weights(body: bytes, *, slug: str, epoch: int) -> ChallengeWeights: + def finite(value: object) -> Fraction: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("weight must be a number") + if not math.isfinite(value) or value < 0: + raise ValueError("weight must be finite and non-negative") + return Fraction(value) + + document = json.loads(body, parse_constant=lambda _: math.nan) + if ( + not isinstance(document, dict) + or document.get("challenge_slug") != slug + or document.get("epoch") != epoch + or not isinstance(document.get("weights"), dict) + or len(document["weights"]) > MAX_WEIGHTS + ): + raise ValueError("weights response does not match the requested challenge epoch") + weights: dict[bytes, Fraction] = {} + for key, value in document["weights"].items(): + hotkey = decode_hotkey(key) + if hotkey in weights: + raise ValueError("duplicate hotkey in weights response") + weights[hotkey] = finite(value) + mass = document.get("full_share_mass") + return ChallengeWeights(weights, None if mass is None else finite(mass)) + + +def leaf_scores( + answer: ChallengeWeights, expected: set[bytes], *, algorithm_version: int +) -> dict[bytes, Score | NoScore]: + """Exact-E scores. Hotkeys outside E are ignored and never change the denominator.""" + kept = {key: value for key, value in answer.weights.items() if key in expected and value > 0} + if algorithm_version == 3: + denominator = max(sum(kept.values(), Fraction(0)), answer.full_share_mass or Fraction(0)) + raw = { + key: math.floor(FULL_SHARE_SCORE * value / denominator) for key, value in kept.items() + } + else: + # Algorithms 1 and 2 sign raw integer counts; the protocol applies the Bounty cap. + if any(value.denominator != 1 for value in kept.values()): + raise ValueError("algorithm 1 and 2 weights must be integers") + raw = {key: int(value) for key, value in kept.items()} + return { + key: Score(raw[key]) if raw.get(key, 0) > 0 else NoScore(NoScoreReason.NOT_ATTEMPTED) + for key in expected + } + + +class ChallengeClient: + """The master's only call into a container; internal tokens never leave this process.""" + + def __init__(self, http: httpx.AsyncClient, secrets_dir: Path, *, retry_seconds: float = 5): + self.http, self.secrets_dir, self.retry_seconds = http, secrets_dir, retry_seconds + + async def weights(self, entry: RegistryEntry, epoch: int) -> ChallengeWeights: + token = read_private_file(self.secrets_dir / entry.id / "internal.token") + for attempt in range(ATTEMPTS): + try: + async with self.http.stream( + "GET", + f"{entry.url}/internal/v1/get_weights", + params={"epoch": str(epoch)}, + headers={ + "authorization": f"Bearer {token}", + "x-platform-challenge-slug": entry.id, + }, + timeout=60, + ) as response: + if response.status_code != 200: + raise ServiceError(503, f"challenge answered HTTP {response.status_code}") + body = bytearray() + async for chunk in response.aiter_bytes(): + body.extend(chunk) + if len(body) > MAX_RESPONSE_BYTES: + raise ServiceError(503, "challenge weights response too large") + return parse_weights(bytes(body), slug=entry.id, epoch=epoch) + except httpx.TransportError: + # A container restarting for an update must not burn a whole epoch. + if attempt + 1 == ATTEMPTS: + raise + await asyncio.sleep(self.retry_seconds * (attempt + 1)) + raise AssertionError("unreachable") diff --git a/src/cortex/challenges/proxy.py b/src/cortex/challenges/proxy.py new file mode 100644 index 000000000..fdd9c7633 --- /dev/null +++ b/src/cortex/challenges/proxy.py @@ -0,0 +1,74 @@ +"""Public reverse proxy from /challenge// to a registered container.""" + +from __future__ import annotations + +from collections.abc import Callable + +import httpx +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse, Response + +from .registry import RegistryEntry + +MAX_RESPONSE_BYTES = 8 * 1024 * 1024 +FORWARDED_HEADERS = ("content-type", "accept", "authorization") +METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] + + +def _error(status: int, reason: str) -> JSONResponse: + return JSONResponse({"error": reason}, status_code=status) + + +def _safe(path: str, raw_path: bytes) -> bool: + segments = path.split("/") + return ( + b"%" not in raw_path + and "\\" not in path + and segments[0] != "internal" + and all(segment not in {"", ".", ".."} for segment in segments) + ) + + +def create_router( + registry: Callable[[], dict[str, RegistryEntry]], http: httpx.AsyncClient +) -> APIRouter: + router = APIRouter(tags=["challenges"]) + + @router.api_route("/challenge/{challenge_id}/{path:path}", methods=METHODS) + async def proxy(challenge_id: str, path: str, request: Request): + entry = registry().get(challenge_id) + if entry is None or not _safe(path, request.scope.get("raw_path", b"")): + return _error(404, "not found") + body = bytearray() + async for chunk in request.stream(): + body.extend(chunk) + if len(body) > entry.proxy_body_limit: + return _error(413, "request body too large") + headers = { + name: request.headers[name] for name in FORWARDED_HEADERS if name in request.headers + } + if request.client is not None: + headers["x-forwarded-for"] = request.client.host + try: + async with http.stream( + request.method, + f"{entry.url}/{path}", + params=request.url.query or None, + content=bytes(body) if body else None, + headers=headers, + timeout=entry.proxy_timeout_seconds, + ) as upstream: + content = bytearray() + async for chunk in upstream.aiter_bytes(): + content.extend(chunk) + if len(content) > MAX_RESPONSE_BYTES: + return _error(502, "challenge response too large") + return Response( + bytes(content), + status_code=upstream.status_code, + media_type=upstream.headers.get("content-type"), + ) + except httpx.HTTPError: + return _error(502, "challenge unavailable") + + return router diff --git a/src/cortex/challenges/registry.py b/src/cortex/challenges/registry.py new file mode 100644 index 000000000..b8447d189 --- /dev/null +++ b/src/cortex/challenges/registry.py @@ -0,0 +1,139 @@ +"""Unsigned operator registry of challenge containers. + +The registry decides what runs; only the owner-signed trust root decides emission. +""" + +from __future__ import annotations + +import re +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import urlsplit + +from cortex.protocol.models import CHALLENGE_ID + +CHANNELS = frozenset({"stable", "edge"}) +IMAGE = re.compile(r"ghcr\.io/[a-z0-9][a-z0-9._-]*(?:/[a-z0-9][a-z0-9._-]*)+") +PIN = re.compile(r"sha256:[0-9a-f]{64}") +ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]{0,63}") +RESERVED_ENV = frozenset( + { + "CHALLENGE_SLUG", + "CHALLENGE_STATE_DIR", + "CHALLENGE_INTERNAL_TOKEN_FILE", + "CHALLENGE_ADMIN_TOKEN_FILE", + "CHALLENGE_MASTER_URL", + } +) +FIELDS = frozenset( + { + "id", + "image", + "channel", + "pin", + "source", + "attestation", + "poll_seconds", + "cpus", + "memory_mib", + "pids", + "proxy_body_limit", + "proxy_timeout_seconds", + "env", + } +) + + +def container_name(challenge_id: str) -> str: + return f"cortex-challenge-{challenge_id}" + + +@dataclass(frozen=True) +class RegistryEntry: + id: str + image: str + source: str + channel: str = "stable" + pin: str | None = None + attestation: bool = True + poll_seconds: int = 300 + cpus: float = 1.0 + memory_mib: int = 1024 + pids: int = 256 + proxy_body_limit: int = 1024 * 1024 + proxy_timeout_seconds: float = 30.0 + env: dict[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not CHALLENGE_ID.fullmatch(self.id) or self.id == "proof": + raise ValueError("challenge id must be lowercase [a-z0-9-] and not proof") + if not IMAGE.fullmatch(self.image): + raise ValueError(f"{self.id}: image must be a ghcr.io repository without tag") + source = urlsplit(self.source) + if ( + source.scheme != "https" + or source.hostname != "github.com" + or source.query + or source.fragment + or len(source.path.strip("/").split("/")) != 2 + ): + raise ValueError(f"{self.id}: source must be https://github.com//") + if self.channel not in CHANNELS: + raise ValueError(f"{self.id}: channel must be stable or edge") + if self.pin is not None and not PIN.fullmatch(self.pin): + raise ValueError(f"{self.id}: pin must be sha256:<64 hex>") + if type(self.attestation) is not bool: + raise ValueError(f"{self.id}: attestation must be a boolean") + if ( + not 30 <= self.poll_seconds <= 86400 + or not 0.1 <= self.cpus <= 64 + or not 64 <= self.memory_mib <= 262144 + or not 16 <= self.pids <= 32768 + or not 1024 <= self.proxy_body_limit <= 64 * 1024 * 1024 + or not 1 <= self.proxy_timeout_seconds <= 300 + ): + raise ValueError(f"{self.id}: resource or proxy limit out of range") + for name, value in self.env.items(): + if not ENV_NAME.fullmatch(name) or name in RESERVED_ENV or not isinstance(value, str): + raise ValueError(f"{self.id}: invalid or reserved env {name!r}") + + @property + def owner_repo(self) -> str: + return urlsplit(self.source).path.strip("/") + + @property + def reference(self) -> str: + """What the supervisor resolves: an explicit digest pin wins over the channel.""" + return f"{self.image}@{self.pin}" if self.pin else f"{self.image}:{self.channel}" + + @property + def url(self) -> str: + return f"http://{container_name(self.id)}:8000" + + +def parse_registry(document: dict) -> dict[str, RegistryEntry]: + if document.get("version") != 1 or set(document) - {"version", "challenge"}: + raise ValueError("challenge registry must be version 1 with [[challenge]] rows") + rows = document.get("challenge", []) + if not isinstance(rows, list) or len(rows) > 64: + raise ValueError("challenge registry holds at most 64 rows") + entries: dict[str, RegistryEntry] = {} + for row in rows: + if not isinstance(row, dict) or set(row) - FIELDS: + raise ValueError("unknown challenge registry field") + entry = RegistryEntry(**{**row, "env": dict(row.get("env", {}))}) + if entry.id in entries: + raise ValueError(f"duplicate challenge id {entry.id}") + entries[entry.id] = entry + return entries + + +def load_registry(path: Path | None) -> dict[str, RegistryEntry]: + """A missing optional registry means no container challenges.""" + if path is None: + return {} + try: + return parse_registry(tomllib.loads(path.read_text())) + except (OSError, tomllib.TOMLDecodeError, TypeError) as error: + raise ValueError(f"challenge registry unavailable: {type(error).__name__}") from None diff --git a/src/cortex/challenges/supervisor.py b/src/cortex/challenges/supervisor.py new file mode 100644 index 000000000..5c81ef7f5 --- /dev/null +++ b/src/cortex/challenges/supervisor.py @@ -0,0 +1,255 @@ +"""Pull, verify, canary, run and auto-update challenge containers through the Docker API. + +This is the only Cortex process with Docker control. It never reads a secret: +it passes host secret directories to Docker as read-only binds. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from pathlib import Path + +import httpx + +from .registry import RegistryEntry, container_name, load_registry + +CONTRACT = "1" +MANAGED = "io.cortex.managed" +LOG = logging.getLogger("cortex.challenges") + + +class SupervisorError(Exception): + """A refused image or failed rollout; the running container is left as it was.""" + + +@dataclass(frozen=True) +class SupervisorConfig: + registry_file: Path + secrets_host_dir: Path + network: str = "cortex-challenges" + master_url: str = "http://cortex-master:8080" + ready_seconds: float = 60 + + def __post_init__(self) -> None: + if not self.secrets_host_dir.is_absolute(): + raise ValueError("challenge secrets host directory must be absolute") + + +@dataclass +class Supervisor: + config: SupervisorConfig + docker: httpx.AsyncClient + http: httpx.AsyncClient + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep + refused: dict[str, str] = field(default_factory=dict) + checked: dict[str, float] = field(default_factory=dict) + + async def _docker(self, method: str, path: str, **kwargs) -> httpx.Response: + response = await self.docker.request(method, path, **kwargs) + if response.status_code >= 400 and response.status_code != 404: + raise SupervisorError(f"docker {method} {path.split('?')[0]}: {response.status_code}") + return response + + async def resolve(self, entry: RegistryEntry) -> tuple[str, dict[str, str]]: + """Pull the channel or pin and return (repository digest, labels).""" + tag = entry.pin or entry.channel + async with self.docker.stream( + "POST", "/images/create", params={"fromImage": entry.image, "tag": tag}, timeout=900 + ) as response: + if response.status_code != 200: + raise SupervisorError(f"{entry.id}: pull failed with {response.status_code}") + async for line in response.aiter_lines(): + if line.strip() and "error" in json.loads(line): + raise SupervisorError(f"{entry.id}: pull reported an error") + reference = f"{entry.image}@{entry.pin}" if entry.pin else f"{entry.image}:{tag}" + image = (await self._docker("GET", f"/images/{reference}/json")).json() + digests = [ + value.split("@", 1)[1] + for value in image.get("RepoDigests") or [] + if value.split("@", 1)[0] == entry.image + ] + if len(set(digests)) != 1: + raise SupervisorError(f"{entry.id}: image has no single repository digest") + return digests[0], (image.get("Config") or {}).get("Labels") or {} + + def verify_labels(self, entry: RegistryEntry, labels: dict[str, str]) -> None: + if ( + labels.get("io.cortex.challenge.slug") != entry.id + or labels.get("io.cortex.challenge.contract") != CONTRACT + or labels.get("org.opencontainers.image.source", "").rstrip("/") != entry.source + ): + raise SupervisorError(f"{entry.id}: image labels do not match the registry entry") + + async def verify_attestation(self, entry: RegistryEntry, digest: str) -> None: + # ponytail: checks that GitHub holds build provenance for this digest in the source + # repository, not the Sigstore bundle signature. Upgrade: verify the returned bundle + # with sigstore-python (or `gh attestation verify`) before trusting a new digest. + response = await self.http.get( + f"https://api.github.com/repos/{entry.owner_repo}/attestations/{digest}", + headers={"accept": "application/vnd.github+json"}, + timeout=30, + ) + if response.status_code != 200 or not response.json().get("attestations"): + raise SupervisorError(f"{entry.id}: no build provenance for {digest}") + + def _spec(self, entry: RegistryEntry, digest: str, *, canary: bool) -> dict: + env = { + **entry.env, + "CHALLENGE_SLUG": entry.id, + "CHALLENGE_STATE_DIR": "/data", + "CHALLENGE_INTERNAL_TOKEN_FILE": "/run/secrets/internal.token", + "CHALLENGE_ADMIN_TOKEN_FILE": "/run/secrets/admin.token", + "CHALLENGE_MASTER_URL": self.config.master_url, + } + mounts = [] + tmpfs = {"/tmp": "rw,noexec,nosuid,size=64m"} + if canary: + tmpfs["/data"] = "rw,noexec,nosuid,size=64m,uid=65532,gid=65532,mode=0700" + else: + mounts = [ + {"Type": "volume", "Source": f"{container_name(entry.id)}-data", "Target": "/data"}, + { + "Type": "bind", + "Source": str(self.config.secrets_host_dir / entry.id), + "Target": "/run/secrets", + "ReadOnly": True, + }, + ] + return { + "Image": f"{entry.image}@{digest}", + "Env": [f"{name}={value}" for name, value in sorted(env.items())], + "User": "65532:65532", + "Labels": { + MANAGED: "true", + "io.cortex.challenge.id": entry.id, + "io.cortex.challenge.digest": digest, + }, + "HostConfig": { + "ReadonlyRootfs": True, + "CapDrop": ["ALL"], + "SecurityOpt": ["no-new-privileges:true"], + "Init": True, + "Memory": entry.memory_mib * 1024 * 1024, + "NanoCpus": int(entry.cpus * 1e9), + "PidsLimit": entry.pids, + "Mounts": mounts, + "Tmpfs": tmpfs, + "NetworkMode": self.config.network, + "RestartPolicy": {"Name": "no" if canary else "unless-stopped"}, + "LogConfig": {"Type": "json-file", "Config": {"max-size": "20m", "max-file": "3"}}, + }, + } + + async def _remove(self, name: str) -> None: + await self._docker("DELETE", f"/containers/{name}", params={"force": "true"}) + + async def _start(self, name: str, spec: dict) -> None: + await self._docker("POST", "/containers/create", params={"name": name}, json=spec) + await self._docker("POST", f"/containers/{name}/start") + + async def _answers_version(self, name: str, entry: RegistryEntry) -> bool: + loop = asyncio.get_running_loop() + deadline = loop.time() + self.config.ready_seconds + while loop.time() < deadline: + try: + response = await self.http.get(f"http://{name}:8000/version", timeout=5) + body = response.json() if response.status_code == 200 else {} + if body.get("slug") == entry.id and str(body.get("contract")) == CONTRACT: + return True + except (httpx.HTTPError, ValueError): + pass + await self.sleep(1) + return False + + async def running_digest(self, entry: RegistryEntry) -> str | None: + response = await self._docker("GET", f"/containers/{container_name(entry.id)}/json") + if response.status_code == 404: + return None + state = response.json() + if not (state.get("State") or {}).get("Running"): + return None + return ((state.get("Config") or {}).get("Labels") or {}).get("io.cortex.challenge.digest") + + async def deployed_digest(self, entry: RegistryEntry) -> str | None: + response = await self._docker("GET", f"/containers/{container_name(entry.id)}/json") + if response.status_code == 404: + return None + return ((response.json().get("Config") or {}).get("Labels") or {}).get( + "io.cortex.challenge.digest" + ) + + async def reconcile(self, entry: RegistryEntry) -> str: + """Converge one challenge; returns the digest that is running afterwards.""" + digest, labels = await self.resolve(entry) + previous = await self.deployed_digest(entry) + if digest == previous: + if await self.running_digest(entry) != digest: + await self._docker("POST", f"/containers/{container_name(entry.id)}/start") + return digest + if self.refused.get(entry.id) == digest: + raise SupervisorError(f"{entry.id}: {digest} was refused; waiting for a new digest") + try: + self.verify_labels(entry, labels) + if entry.attestation: + await self.verify_attestation(entry, digest) + canary = container_name(entry.id) + "-canary" + await self._remove(canary) + await self._start(canary, self._spec(entry, digest, canary=True)) + try: + if not await self._answers_version(canary, entry): + raise SupervisorError(f"{entry.id}: canary did not answer /version") + finally: + await self._remove(canary) + except SupervisorError: + self.refused[entry.id] = digest + raise + name = container_name(entry.id) + await self._remove(name) + await self._start(name, self._spec(entry, digest, canary=False)) + if await self._answers_version(name, entry): + self.refused.pop(entry.id, None) + LOG.info("challenge %s now runs %s", entry.id, digest) + return digest + self.refused[entry.id] = digest + await self._remove(name) + if previous is not None: + await self._start(name, self._spec(entry, previous, canary=False)) + LOG.warning("challenge %s rolled back to %s", entry.id, previous) + raise SupervisorError(f"{entry.id}: {digest} failed after rollout") + + async def prune(self, registry: dict[str, RegistryEntry]) -> None: + filters = json.dumps({"label": [f"{MANAGED}=true"]}) + listed = await self._docker( + "GET", "/containers/json", params={"all": "true", "filters": filters} + ) + for container in listed.json(): + identifier = (container.get("Labels") or {}).get("io.cortex.challenge.id") + if identifier not in registry: + # The data volume is kept so a re-registered challenge resumes its state. + await self._remove(container["Id"]) + LOG.info("removed unregistered challenge container %s", identifier) + + async def tick(self, now: float) -> None: + registry = load_registry(self.config.registry_file) + await self.prune(registry) + for entry in registry.values(): + if now - self.checked.get(entry.id, float("-inf")) < entry.poll_seconds: + continue + self.checked[entry.id] = now + try: + await self.reconcile(entry) + except (SupervisorError, httpx.HTTPError) as error: + LOG.warning("challenge %s not updated: %s", entry.id, error) + + async def run(self) -> None: + loop = asyncio.get_running_loop() + while True: + try: + await self.tick(loop.time()) + except (ValueError, SupervisorError, httpx.HTTPError) as error: + LOG.warning("challenge supervisor tick failed: %s", error) + await self.sleep(15) diff --git a/src/cortex/cli.py b/src/cortex/cli.py index d1cc61e72..dc139e049 100644 --- a/src/cortex/cli.py +++ b/src/cortex/cli.py @@ -23,10 +23,13 @@ def parser() -> argparse.ArgumentParser: root = argparse.ArgumentParser(prog="cortex", description="Cortex research network") commands = root.add_subparsers(dest="command", required=True) - master = commands.add_parser("master", help="serve gateway, Bounty and Proof") + master = commands.add_parser("master", help="serve the gateway, Proof and challenge proxy") master.add_argument("--bind", default="127.0.0.1") master.add_argument("--port", type=int, default=8080) commands.add_parser("validator", help="verify seals and submit Bittensor weights") + commands.add_parser( + "challenge-supervisor", help="run and auto-update registered challenge containers" + ) reconcile = commands.add_parser( "validator-reconcile", help="resolve one ambiguous validator dispatch" ) @@ -300,6 +303,11 @@ def main(argv: list[str] | None = None) -> None: validator(values[1:]) return + if values and values[0] == "challenge-supervisor": + from cortex.challenges.__main__ import main as supervisor + + supervisor(values[1:]) + return args = parser().parse_args(values) logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") try: diff --git a/src/cortex/config.py b/src/cortex/config.py index 6e292f06e..ba475c4d7 100644 --- a/src/cortex/config.py +++ b/src/cortex/config.py @@ -104,13 +104,13 @@ class MasterConfig: challenges_file: Path measurements_file: Path gateway_seed_file: Path - bounty_seed_file: Path proof_seed_file: Path - bounty_session_secret_file: Path operator_token_file: Path + challenge_keys_dir: Path = Path("/run/secrets") + challenge_registry_file: Path | None = None + challenge_secrets_dir: Path = Path("/run/challenge-secrets") chain_endpoint: str = "finney" chain_fallback_endpoints: tuple[str, ...] = () - bounty_backend_url: str | None = None emit_poll_seconds: float = 120 epoch_refresh_seconds: float = 12 epoch_stale_seconds: float = 60 @@ -141,17 +141,16 @@ def __post_init__(self): raise ValueError("trust versions must be positive") validate_chain_endpoint(self.chain_endpoint) _chain_fallback_endpoints(self.chain_fallback_endpoints) - for url in (self.bounty_backend_url, self.proof_orchestrator_url): - if url: - parsed = urlsplit(url) - if ( - parsed.scheme != "https" - or not parsed.hostname - or parsed.username - or parsed.password - or parsed.fragment - ): - raise ValueError("backend URLs must be HTTPS without credentials") + if self.proof_orchestrator_url: + parsed = urlsplit(self.proof_orchestrator_url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.fragment + ): + raise ValueError("backend URLs must be HTTPS without credentials") if self.proof_orchestrator_url and self.proof_orchestrator_token_file is None: raise ValueError("Proof orchestrator token file required") if self.proof_orchestrator_url and self.proof_orchestrator_ca_file is None: @@ -189,13 +188,15 @@ def optional_path(name: str) -> Path | None: challenges_file=Path(value("BASE_CHALLENGES_FILE", "config/challenges.toml")), measurements_file=Path(value("BASE_MEASUREMENTS_FILE", "config/measurements.toml")), gateway_seed_file=required_path("BASE_GATEWAY_SK_FILE"), - bounty_seed_file=required_path("BOUNTY_SK_FILE"), proof_seed_file=required_path("PROOF_SK_FILE"), - bounty_session_secret_file=required_path("BOUNTY_SESSION_SECRET_FILE"), operator_token_file=required_path("BASE_GATEWAY_ADMIN_TOKEN_FILE"), + challenge_keys_dir=Path(value("BASE_CHALLENGE_KEYS_DIR", "/run/secrets")), + challenge_registry_file=optional_path("BASE_CHALLENGE_REGISTRY_FILE"), + challenge_secrets_dir=Path( + value("BASE_CHALLENGE_SECRETS_DIR", "/run/challenge-secrets") + ), chain_endpoint=value("BASE_CHAIN_ENDPOINT", "finney"), chain_fallback_endpoints=chain_fallback_endpoints, - bounty_backend_url=value("BOUNTY_BACKEND_PUBLIC_URL") or None, emit_poll_seconds=float( value("BASE_EMIT_POLL_SECS", value("PROOF_EMIT_POLL_SECS", "120")) ), @@ -213,6 +214,12 @@ def optional_path(name: str) -> Path | None: ), ) + def challenge_seed_file(self, challenge: bytes) -> Path: + """Proof keeps its topic key; every other challenge id uses /.key.""" + if challenge == b"proof": + return self.proof_seed_file + return self.challenge_keys_dir / f"{challenge.decode()}.key" + def trust_root(self, epoch: int) -> TrustRoot: try: trust = load_trust_root( @@ -226,10 +233,8 @@ def trust_root(self, epoch: int) -> TrustRoot: minimum_challenges_version=self.minimum_challenges_version, minimum_measurements_version=self.minimum_measurements_version, ) - for entry, path in zip( - trust.challenges, (self.bounty_seed_file, self.proof_seed_file), strict=True - ): - if public_key(read_seed(path)) != entry.public_key: + for entry in trust.challenges: + if public_key(read_seed(self.challenge_seed_file(entry.id))) != entry.public_key: raise ServiceError(503, "challenge signing key does not match owner trust") return trust except (OSError, ValueError): diff --git a/src/cortex/gateway/api.py b/src/cortex/gateway/api.py index 3a760a8d1..4b02027ba 100644 --- a/src/cortex/gateway/api.py +++ b/src/cortex/gateway/api.py @@ -54,6 +54,13 @@ async def seal(request: Request): async def latest(): return service.latest() + @router.get("/v1/metagraph/latest") + async def metagraph(): + try: + return service.metagraph() + except ServiceError as error: + return _error(error) + @router.get("/v1/bundle/{epoch}") async def bundle(epoch: str): try: diff --git a/src/cortex/gateway/projection.py b/src/cortex/gateway/projection.py index 899dd17e2..093e039cd 100644 --- a/src/cortex/gateway/projection.py +++ b/src/cortex/gateway/projection.py @@ -6,8 +6,8 @@ from uuid import UUID from cortex.protocol import Bundle, Score, aggregate_leaves +from cortex.protocol.aggregate import challenge_emission_percent from cortex.protocol.crypto import encode_hotkey -from cortex.protocol.models import BOUNTY_FULL_SHARE_REPORTS def _identity(digest: str) -> str: @@ -99,12 +99,14 @@ def project(bundle: Bundle | None, *, netuid: int, now: datetime, chain_endpoint hotkey = encode_hotkey(leaf.miner_hotkey) source_weights[hotkey] = source_weights.get(hotkey, 0.0) + leaf.score.value view["emission_shares"][slug] = bps / 10000 - emission_percent = bps / 100 - if body.algorithm_version == 2 and challenge == b"bounty": - emission_percent *= ( - min(sum(source_weights.values()), BOUNTY_FULL_SHARE_REPORTS) - / BOUNTY_FULL_SHARE_REPORTS - ) + raw_total = sum( + leaf.score.value + for leaf in leaves + if isinstance(leaf.score, Score) and leaf.score.value > 0 + ) + emission_percent = challenge_emission_percent( + challenge, bps, raw_total, algorithm_version=body.algorithm_version + ) view["source_challenges"].append( dict( slug=slug, diff --git a/src/cortex/gateway/service.py b/src/cortex/gateway/service.py index d3c69d2cc..a8161886e 100644 --- a/src/cortex/gateway/service.py +++ b/src/cortex/gateway/service.py @@ -7,7 +7,7 @@ from cortex.errors import ServiceError from cortex.protocol import Bundle, Leaf, ProtocolError, TrustRoot, aggregate_leaves, build_bundle -from cortex.protocol.crypto import BUNDLE_DOMAIN, RAW_WEIGHT_DOMAIN, verify_raw +from cortex.protocol.crypto import BUNDLE_DOMAIN, RAW_WEIGHT_DOMAIN, encode_hotkey, verify_raw from cortex.protocol.merkle import merkle_root from cortex.protocol.scale import uint from cortex.validator import ChainSnapshot @@ -212,6 +212,23 @@ def latest(self) -> dict: None, netuid=self.netuid, now=self.clock(), chain_endpoint=self._chain_endpoint() ) + def metagraph(self) -> dict: + """Hotkeys of the latest verified seal, for challenge intake filters.""" + try: + stored = self.store.latest() + if stored is None: + raise ServiceError(503, "no sealed metagraph") + self.refresh_trust(stored.epoch) + body = self._decode_stored(stored).body + except (sqlite3.Error, ProtocolError, ValueError): + raise ServiceError(503, "no sealed metagraph") from None + return { + "epoch": body.epoch, + "block": body.block_b, + "netuid": body.netuid, + "hotkeys": {encode_hotkey(key): uid for key, uid in body.uid_map}, + } + async def seal( self, epoch: int, *, netuid: int | None = None, block_b: int | None = None ) -> Bundle: diff --git a/src/cortex/master.py b/src/cortex/master.py index aabd4ae10..cfc1ae1a0 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -13,12 +13,13 @@ from typing import Any, Protocol, cast from urllib.parse import urlsplit -from fastapi import APIRouter, FastAPI, Request +import httpx +from fastapi import FastAPI from fastapi.responses import JSONResponse -from cortex.bounty import BountyService, BountyStore, PublicBackend -from cortex.bounty import create_router as bounty_router -from cortex.bounty.store import StoreError +from cortex.challenges.client import ChallengeClient, leaf_scores +from cortex.challenges.proxy import create_router as challenge_router +from cortex.challenges.registry import RegistryEntry, load_registry from cortex.config import MasterConfig, read_seed from cortex.errors import ServiceError from cortex.gateway import GatewayService, GatewayStore @@ -27,11 +28,13 @@ from cortex.proof.api import create_router as proof_router from cortex.proof.artifacts import FileVault from cortex.proof.models import Submission +from cortex.proof.scoring import SCORE_MAX from cortex.proof.service import EvaluationBackend, ProofService, UnwiredBackend from cortex.proof.store import ProofStore from cortex.protocol import NoScore, NoScoreReason, Score, TrustRoot, sign_leaf from cortex.protocol.crypto import decode_hotkey, public_key from cortex.protocol.merkle import canonical_rows +from cortex.protocol.models import FULL_SHARE_SCORE from cortex.protocol.scale import uint from cortex.state import prepare_master_state from cortex.validator import ChainSnapshot @@ -216,11 +219,25 @@ def close(self) -> None: self.connection.close() -_REASONS = { - "NotAttempted": NoScoreReason.NOT_ATTEMPTED, - "InvalidResponse": NoScoreReason.INVALID_RESPONSE, - "ChallengeInternal": NoScoreReason.CHALLENGE_INTERNAL, -} +class ChallengeRegistry: + """Re-read the operator registry when it changes; an invalid edit burns, never guesses.""" + + def __init__(self, path: Path | None): + self.path = path + self._stamp: int | None = None + self._entries: dict[str, RegistryEntry] = {} + + def __call__(self) -> dict[str, RegistryEntry]: + if self.path is None: + return {} + try: + stamp = self.path.stat().st_mtime_ns + if stamp != self._stamp: + self._entries, self._stamp = load_registry(self.path), stamp + except (OSError, ValueError) as error: + logging.warning("challenge registry unavailable (%s)", error) + self._entries, self._stamp = {}, None + return self._entries class _EpochTrackedProof(ProofService): @@ -249,28 +266,26 @@ def __init__( self, *, gateway: GatewayService, - bounty: BountyService, proof: _EpochTrackedProof, clock: EpochClock, journal: EmissionJournal, - challenge_seeds: dict[bytes, Callable[[], bytes]], + challenge_seed: Callable[[bytes], bytes], + registry: Callable[[], dict[str, RegistryEntry]], + challenges: ChallengeClient, ): - self.gateway, self.bounty, self.proof = gateway, bounty, proof - self.clock, self.journal, self.challenge_seeds = clock, journal, challenge_seeds + self.gateway, self.proof, self.clock, self.journal = gateway, proof, clock, journal + self.challenge_seed, self.registry, self.challenges = challenge_seed, registry, challenges self._lock = asyncio.Lock() async def _scores(self, challenge: bytes, epoch: int, expected: set[bytes]): + algorithm = self.gateway.trust.algorithm_version try: - if challenge == b"bounty": - outcomes = await self.bounty.score([key.hex() for key in sorted(expected)]) - return { - decode_hotkey(key): ( - Score(value.value) - if value.reason is None - else NoScore(_REASONS.get(value.reason, NoScoreReason.CHALLENGE_INTERNAL)) - ) - for key, value in outcomes.items() - } + if challenge != b"proof": + entry = self.registry().get(challenge.decode()) + if entry is None: + raise ServiceError(503, "challenge container not registered") + answer = await self.challenges.weights(entry, epoch) + return leaf_scores(answer, expected, algorithm_version=algorithm) # Backend readiness is still required before old rows can be emitted. readiness = await self.proof.backend.readiness() active = [topic for topic in self.proof.store.topics_at(epoch) if topic.active(epoch)] @@ -289,7 +304,10 @@ async def _scores(self, challenge: bytes, epoch: int, expected: set[bytes]): ): raise ServiceError(503, "Proof topic cannot score") scores = self.proof.scores(epoch) - return {decode_hotkey(key): Score(value) for key, value in scores.items()} + # Algorithm 3 pays a challenge sum(leaves) / 10^12 of its share, so topic mass + # without a winner burns instead of moving to other topics. + scale = FULL_SHARE_SCORE // SCORE_MAX if algorithm == 3 else 1 + return {decode_hotkey(key): Score(value * scale) for key, value in scores.items()} except Exception: logging.warning( "challenge emission unavailable challenge=%s epoch=%d", challenge.decode(), epoch @@ -299,12 +317,13 @@ async def _scores(self, challenge: bytes, epoch: int, expected: set[bytes]): async def _emit(self, epoch: int, snapshot: ChainSnapshot) -> None: self.gateway.refresh_trust(epoch) self.proof.topic_public_key = next( - entry.public_key for entry in self.gateway.trust.challenges if entry.id == b"proof" + (entry.public_key for entry in self.gateway.trust.challenges if entry.id == b"proof"), + self.proof.topic_public_key, ) rows = canonical_rows(snapshot.rows) for challenge in self.gateway.trust.challenges: expected = challenge.policy.expected(rows) - seed = self.challenge_seeds[challenge.id]() + seed = self.challenge_seed(challenge.id) if public_key(seed) != challenge.public_key: raise ServiceError(503, "challenge signing key does not match owner trust") outcomes = await self._scores(challenge.id, epoch, expected) @@ -361,33 +380,18 @@ async def tick(self) -> list[int]: return completed -class _FileAuthenticatedBounty(BountyService): - def __init__(self, *args, token_file: Path, **kwargs): - super().__init__(*args, **kwargs) - self._operator = OperatorAuth(token_file) - - def require_operator(self, authorization: str | None) -> None: - request = Request( - {"type": "http", "headers": [(b"authorization", (authorization or "").encode())]} - ) - try: - self._operator.require(request) - except ServiceError as error: - raise StoreError(error.status, error.reason) from None - - class MasterRuntime: def __init__( self, config: MasterConfig, gateway: GatewayService, - bounty: BountyService, proof: ProofService, clock: EpochClock, emitter: EpochEmitter, + http: httpx.AsyncClient, ): - self.config, self.gateway, self.bounty, self.proof = config, gateway, bounty, proof - self.clock, self.emitter = clock, emitter + self.config, self.gateway, self.proof = config, gateway, proof + self.clock, self.emitter, self.http = clock, emitter, http self._stop = asyncio.Event() self._tasks: list[asyncio.Task] = [] self.topic_setup = None @@ -432,9 +436,9 @@ async def close(self) -> None: if self.topic_setup is not None: await self.topic_setup.drain() self.gateway.store.close() - self.bounty.store.close() self.proof.store.close() self.emitter.journal.close() + await self.http.aclose() def app(self) -> FastAPI: @asynccontextmanager @@ -456,18 +460,7 @@ async def service_error(request, error: ServiceError): proof = proof_router(self.proof, operator, self.topic_setup) app.include_router(proof) app.include_router(proof, prefix="/challenge/proof", include_in_schema=False) - bounty = bounty_router(self.bounty) - app.include_router(bounty, prefix="/bounty") - app.include_router(bounty, prefix="/challenge/bounty", include_in_schema=False) - # Historical direct pair/report routes stay available; status is namespaced. - direct_bounty = APIRouter( - routes=[ - route - for route in bounty.routes - if getattr(route, "path", "") not in {"/v1/status", "/health"} - ] - ) - app.include_router(direct_bounty) + app.include_router(challenge_router(self.emitter.registry, self.http)) @app.get("/livez") async def health(): @@ -493,19 +486,18 @@ async def build_master( chain, epochs: EpochProvider, proof_backend: EvaluationBackend | None = None, - bounty_backend: PublicBackend | None = None, trust: TrustRoot | None = None, + challenge_http: httpx.AsyncClient | None = None, ) -> MasterRuntime: prepare_master_state(config.state_dir) clock = EpochClock(epochs, config.netuid, stale_seconds=config.epoch_stale_seconds) state = await clock.refresh() local_trust = trust or config.trust_root(state.epoch) read_private_file(config.operator_token_file) + load_registry(config.challenge_registry_file) # fail fast on an invalid operator registry with ExitStack() as cleanup: gateway_store = GatewayStore(config.state_dir / "gateway.sqlite3") cleanup.callback(gateway_store.close) - bounty_store = BountyStore(config.state_dir / "bounty.sqlite3") - cleanup.callback(bounty_store.close) proof_store = ProofStore(config.state_dir / "proof.sqlite3") cleanup.callback(proof_store.close) journal = EmissionJournal(config.state_dir / "emission.sqlite3") @@ -531,32 +523,27 @@ def chain_endpoint() -> str: chain_endpoint=chain_endpoint, trust_loader=None if trust is not None else config.trust_root, ) - bounty = _FileAuthenticatedBounty( - bounty_store, - bounty_backend or PublicBackend(config.bounty_backend_url), - session_secret=read_seed(config.bounty_session_secret_file), - token_file=config.operator_token_file, - scoring_version=lambda: gateway.trust.algorithm_version, - ) proof = _EpochTrackedProof( store=proof_store, - topic_public_key=next(c.public_key for c in local_trust.challenges if c.id == b"proof"), + topic_public_key=next( + (c.public_key for c in local_trust.challenges if c.id == b"proof"), + public_key(read_seed(config.proof_seed_file)), + ), vault=FileVault(config.state_dir / "miner-byok"), artifact_dir=config.state_dir / "artifacts", backend=proof_backend or UnwiredBackend(), epoch=clock, ) + http = challenge_http or httpx.AsyncClient(trust_env=False, follow_redirects=False) emitter = EpochEmitter( gateway=gateway, - bounty=bounty, proof=proof, clock=clock, journal=journal, - challenge_seeds={ - b"bounty": lambda: read_seed(config.bounty_seed_file), - b"proof": lambda: read_seed(config.proof_seed_file), - }, + challenge_seed=lambda challenge: read_seed(config.challenge_seed_file(challenge)), + registry=ChallengeRegistry(config.challenge_registry_file), + challenges=ChallengeClient(http, config.challenge_secrets_dir), ) - runtime = MasterRuntime(config, gateway, bounty, proof, clock, emitter) + runtime = MasterRuntime(config, gateway, proof, clock, emitter, http) cleanup.pop_all() return runtime diff --git a/src/cortex/miner.py b/src/cortex/miner.py index bc733e5d8..47f08f18b 100644 --- a/src/cortex/miner.py +++ b/src/cortex/miner.py @@ -5,13 +5,13 @@ import hashlib import json import os +import re import secrets import time from pathlib import Path import httpx -from cortex.bounty.service import pair_payload from cortex.errors import ServiceError from cortex.http import read_private_file from cortex.proof.artifacts import verify_artifact @@ -21,6 +21,15 @@ from cortex.wallet import HotkeySigner +def pair_payload(account: str, nonce: str, expiry: int) -> bytes: + """Exact Substrate-context preimage verified by the CortexLM/bounty challenge.""" + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", account): + raise ValueError("invalid account_id") + if not re.fullmatch(r"[a-fA-F0-9]{16,64}", nonce) or not 0 < expiry <= 2**64 - 1: + raise ValueError("invalid pairing nonce or expiry") + return f"cortex-bounty-v1|{account}|{nonce}|{expiry}".encode() + + def load_seed(path: Path) -> bytes: value = read_private_file(path, 128) try: diff --git a/src/cortex/protocol/aggregate.py b/src/cortex/protocol/aggregate.py index fd5138dbd..142a5ae11 100644 --- a/src/cortex/protocol/aggregate.py +++ b/src/cortex/protocol/aggregate.py @@ -9,7 +9,13 @@ from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass -from .models import BOUNTY_FULL_SHARE_REPORTS, PROPORTIONAL_SHARES, Leaf, Score +from .models import ( + BOUNTY_FULL_SHARE_REPORTS, + FULL_SHARE_SCORE, + PROPORTIONAL_SHARES, + Leaf, + Score, +) from .scale import ProtocolError, fixed, uint @@ -107,6 +113,18 @@ def aggregate_challenge_weights( return FinalWeights(tuple(uid for uid, _ in ordered), tuple(w for _, w in ordered), kept) +def challenge_emission_percent( + challenge: bytes, bps: int, raw_total: int, *, algorithm_version: int +) -> float: + """Share a challenge pays; the unpaid remainder burns and never moves elsewhere.""" + emission_percent = bps / 100.0 + if algorithm_version == 2 and challenge == b"bounty": + emission_percent *= min(raw_total, BOUNTY_FULL_SHARE_REPORTS) / BOUNTY_FULL_SHARE_REPORTS + elif algorithm_version == 3: + emission_percent *= min(raw_total, FULL_SHARE_SCORE) / FULL_SHARE_SCORE + return emission_percent + + def aggregate_leaves( leaves: Sequence[Leaf], shares: tuple[tuple[bytes, int], ...], @@ -114,7 +132,7 @@ def aggregate_leaves( *, algorithm_version: int = 1, ) -> FinalWeights: - if algorithm_version not in (1, 2): + if algorithm_version not in (1, 2, 3): raise ProtocolError("unsupported algorithm version") if algorithm_version == 2 and shares != PROPORTIONAL_SHARES: raise ProtocolError("algorithm 2 requires proportional shares") @@ -139,10 +157,8 @@ def aggregate_leaves( uint(bps, 2) miners = scores.get(challenge, {}) weights = {key.hex(): float(value) for key, value in sorted(miners.items()) if value > 0} - emission_percent = bps / 100.0 - if algorithm_version == 2 and challenge == b"bounty": - emission_percent *= ( - min(sum(miners.values()), BOUNTY_FULL_SHARE_REPORTS) / BOUNTY_FULL_SHARE_REPORTS - ) + emission_percent = challenge_emission_percent( + challenge, bps, sum(miners.values()), algorithm_version=algorithm_version + ) results.append(ChallengeWeights(challenge.hex(), emission_percent, weights)) return aggregate_challenge_weights(results, {key.hex(): uid for key, uid in sorted(uid_map)}) diff --git a/src/cortex/protocol/bundle.py b/src/cortex/protocol/bundle.py index 17c1841d1..306f4cba2 100644 --- a/src/cortex/protocol/bundle.py +++ b/src/cortex/protocol/bundle.py @@ -169,14 +169,14 @@ def verify_inputs( def recompute(body: BundleBody, quarantined: set[bytes], minimum_share_mass: int = 5000): - if body.emission_shares == PROPORTIONAL_SHARES and body.algorithm_version != 2: + if body.emission_shares == PROPORTIONAL_SHARES and body.algorithm_version == 1: raise ProtocolError("proportional shares require algorithm version 2") shares = tuple(pair for pair in body.emission_shares if pair[0] not in quarantined) mass = sum(value for _, value in shares) if mass < minimum_share_mass or not mass: raise ProtocolError("surviving share mass below threshold") - if body.algorithm_version == 2: - # Quarantined mass burns; the surviving challenge cannot inherit its share. + if body.algorithm_version >= 2: + # Quarantined mass burns; the surviving challenges cannot inherit its share. shares = body.emission_shares elif quarantined: apportioned = {name: value * 10000 // mass for name, value in shares} diff --git a/src/cortex/protocol/models.py b/src/cortex/protocol/models.py index 461b074e9..f0e6026a1 100644 --- a/src/cortex/protocol/models.py +++ b/src/cortex/protocol/models.py @@ -1,5 +1,6 @@ """Immutable SCALE v1 types. Field order mirrors the frozen Rust contract.""" +import re from dataclasses import dataclass from enum import IntEnum @@ -8,6 +9,9 @@ LIVE_SHARES = ((b"bounty", 2000), (b"proof", 8000)) PROPORTIONAL_SHARES = ((b"bounty", 3000), (b"proof", 7000)) BOUNTY_FULL_SHARE_REPORTS = 10 +# Algorithm 3: a challenge's leaves sum to at most this; the shortfall burns to UID0. +FULL_SHARE_SCORE = 10**12 +CHALLENGE_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") class NoScoreReason(IntEnum): @@ -160,6 +164,8 @@ def shares(self) -> tuple[tuple[bytes, int], ...]: @property def algorithm_version(self) -> int: self.validate() + if self.challenges_version >= 3: + return 3 return 2 if self.shares == PROPORTIONAL_SHARES else 1 def challenges_body(self) -> bytes: @@ -167,7 +173,18 @@ def challenges_body(self) -> bytes: return vector(self.challenges, ChallengeEntry.encode) def validate(self) -> None: - if self.shares not in (LIVE_SHARES, PROPORTIONAL_SHARES): + if self.challenges_version >= 3: + ids = [entry.id for entry in self.challenges] + if ( + not 1 <= len(ids) <= 64 + or len(set(ids)) != len(ids) + or sum(entry.emission_share_bps for entry in self.challenges) != 10000 + or not all(CHALLENGE_ID.fullmatch(key.decode("ascii", "replace")) for key in ids) + ): + raise ProtocolError( + "version 3 challenges need 1..64 unique lowercase ids summing to 10000 bps" + ) + elif self.shares not in (LIVE_SHARES, PROPORTIONAL_SHARES): raise ProtocolError("shares must be bounty/proof=2000/8000 or 3000/7000") if self.shares == PROPORTIONAL_SHARES and self.challenges_version < 2: raise ProtocolError("proportional shares require challenges version >= 2") diff --git a/src/cortex/state.py b/src/cortex/state.py index 23b97ec8c..70a7da5f9 100644 --- a/src/cortex/state.py +++ b/src/cortex/state.py @@ -121,6 +121,6 @@ def prepare_master_state(path: str | Path) -> Path: """Prepare every durable database before a master service opens SQLite.""" directory = secure_state_directory(path) - for name in ("gateway.sqlite3", "bounty.sqlite3", "proof.sqlite3", "emission.sqlite3"): + for name in ("gateway.sqlite3", "proof.sqlite3", "emission.sqlite3"): secure_sqlite_path(directory / name) return directory diff --git a/tests/bounty/test_api.py b/tests/bounty/test_api.py deleted file mode 100644 index a4e22b431..000000000 --- a/tests/bounty/test_api.py +++ /dev/null @@ -1,684 +0,0 @@ -"""Real sr25519 signatures, SQLite and ASGI; only upstream I/O and clock vary.""" - -import asyncio -from dataclasses import dataclass -from types import SimpleNamespace - -import httpx -import pytest -import sr25519 -from fastapi import FastAPI - -from cortex.bounty import BountyService, BountyStore, PublicBackend, create_router - - -@dataclass -class Clock: - now: int = 1_800_000_000 - - def __call__(self): - return self.now - - -def signed_pair(account="account-a", nonce="12" * 16, *, seed_byte=7): - public, secret = sr25519.pair_from_seed(bytes([seed_byte]) * 32) - expiry = 1_800_000_900 - challenge = f"cortex-bounty-v1|{account}|{nonce}|{expiry}".encode() - return { - "account_id": account, - "hotkey": public.hex(), - "nonce": nonce, - "exp": expiry, - "signature": sr25519.sign((public, secret), challenge).hex(), - "terms_accepted": True, - } - - -def report_body(session, number=0): - return { - "session": session, - "title": f"Gateway accepts unauthorized request {number}", - "body": f"Request {number} to the operator endpoint succeeds without credentials. " - "An anonymous caller can change the backend configuration " - "and invalidate the current bundle.", - "repro_steps": "Call the operator endpoint without an Authorization header.", - } - - -@pytest.fixture -def service(tmp_path): - clock = Clock() - upstream = {"status": 200, "leaderboard": [], "reports": []} - - def transport(request): - if upstream["status"] != 200: - return httpx.Response(upstream["status"]) - route = request.url.path.rsplit("/", 1)[-1] - reports = upstream["reports"] - if route == "status": - body = { - "api_version": 1, - "revision": "1", - "adjudication_available": True, - "published": len(reports), - "valid": sum(row["status"] == "valid" for row in reports), - "duplicate": sum(row["status"] == "duplicate" for row in reports), - "already_fixed_not_prod": sum( - row["status"] == "already_fixed_not_prod" for row in reports - ), - "invalid_malicious": sum(row["status"] == "invalid_malicious" for row in reports), - "hotkeys": len({row["hotkey"] for row in reports}), - "awaiting_adjudication": 0, - "unpriced_valid": 0, - } - elif route == "leaderboard": - body = { - "api_version": 1, - "revision": "1", - "items": upstream["leaderboard"], - "has_more": False, - } - else: - body = { - "api_version": 1, - "revision": "1", - "items": reports, - "count": len(reports), - "has_more": False, - "next_cursor": None, - } - return httpx.Response(200, json=body) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(transport)) - store = BountyStore(tmp_path / "bounty.sqlite3") - svc = BountyService( - store, - backend, - session_secret=b"session-secret-for-tests" * 2, - admin_tokens=["operator-token"], - clock=clock, - ) - yield svc, clock, upstream - store.close() - - -@pytest.fixture -async def client(service): - app = FastAPI() - app.include_router(create_router(service[0])) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://test" - ) as client: - yield client - - -async def pair(client): - payload = signed_pair() - await grant_pair(client, payload) - response = await client.post("/v1/pair", json=payload) - assert response.status_code == 201, response.text - return response.json()["session"] - - -async def grant_pair(client, payload): - response = await client.post( - "/v1/admin/pair-grants", - headers={"Authorization": "Bearer operator-token"}, - json={ - "account_id": payload["account_id"], - "hotkey": payload["hotkey"], - "expires_at": 1_800_000_300, - }, - ) - assert response.status_code == 201, response.text - return response - - -def test_bounded_write_routes_keep_their_openapi_request_schemas(service): - app = FastAPI() - app.include_router(create_router(service[0])) - schema = app.openapi() - - expected = { - "/v1/pair": "PairBody", - "/v1/admin/pair-grants": "PairGrantBody", - "/v1/reports": "ReportBody", - "/v1/admin/adjudicate": "AdjudicateBody", - } - for path, title in expected.items(): - body = schema["paths"][path]["post"]["requestBody"] - assert body["required"] is True - assert body["content"]["application/json"]["schema"]["title"] == title - - -async def test_adjudication_is_durable_but_only_published_backend_rows_are_paid(service, client): - svc, clock, upstream = service - session = await pair(client) - reports = [] - for number in range(3): - response = await client.post("/v1/reports", json=report_body(session, number)) - assert response.status_code == 201, response.text - report_id = response.json()["id"] - verdict = await client.post( - "/v1/admin/adjudicate", - headers={"Authorization": "Bearer operator-token"}, - json={"report_id": report_id, "verdict": "valid", "severity": "major"}, - ) - assert verdict.status_code == 200, verdict.text - reports.append(verdict.json()) - clock.now += 60 - - assert svc.store.get_report(reports[0]["id"])["severity"] == "major" - hotkey = signed_pair()["hotkey"] - assert (await svc.score([hotkey]))[hotkey].value == 0 - upstream["leaderboard"] = [{"hotkey": hotkey, "valid_count": 3}] - upstream["reports"] = [ - { - "id": row["id"], - "hotkey": hotkey, - "status": "valid", - "severity": "major", - "problem_found": row["title"], - "justification": "Reproduced with an unauthorized request", - "adjudicator": "operator", - "adjudicated_at": "2026-09-16T00:00:00Z", - "created_at": "2026-09-16T00:00:00Z", - } - for row in reports - ] - - scores = await svc.score([hotkey, "ab" * 32]) - - assert scores[hotkey].value == 500_000 - assert scores["ab" * 32].reason == "NotAttempted" - - -@pytest.mark.parametrize( - "payload,error", - [ - ({"verdict": "valid"}, "severity required for valid verdict"), - ( - {"verdict": "invalid_malicious", "severity": "critical"}, - "severity is only valid for a valid verdict", - ), - ], -) -async def test_adjudication_requires_severity_exactly_for_valid_reports( - service, client, payload, error -): - svc, _, _ = service - session = await pair(client) - report = (await client.post("/v1/reports", json=report_body(session))).json() - - response = await client.post( - "/v1/admin/adjudicate", - headers={"Authorization": "Bearer operator-token"}, - json={"report_id": report["id"], **payload}, - ) - - assert response.status_code == 409 - assert response.json() == {"error": error} - assert svc.store.get_report(report["id"])["state"] == "pending" - - -@pytest.mark.parametrize("configured,status", [(False, 200), (True, 503)]) -async def test_unavailable_scoring_refuses_report_before_any_row( - service, client, configured, status -): - svc, _, upstream = service - session = await pair(client) - upstream["status"] = status - if not configured: - svc.backend = PublicBackend(None) - - response = await client.post("/v1/reports", json=report_body(session)) - - assert response.status_code == 503 - assert svc.store.list_reports() == [] - - -@pytest.mark.parametrize("mutation,status", [("signature", 401), ("terms", 403), ("expiry", 400)]) -async def test_pairing_rejects_forgery_missing_terms_and_expired_challenge( - service, client, mutation, status -): - payload = signed_pair() - await grant_pair(client, payload) - if mutation == "signature": - payload["signature"] = "00" * 64 - elif mutation == "terms": - payload["terms_accepted"] = False - else: - payload["exp"] = 1 - - response = await client.post("/v1/pair", json=payload) - - assert response.status_code == status - - -async def test_first_pairing_requires_an_operator_grant_without_burning_the_nonce(client): - payload = signed_pair() - - refused = await client.post("/v1/pair", json=payload) - await grant_pair(client, payload) - accepted = await client.post("/v1/pair", json=payload) - - assert refused.status_code == 403 - assert refused.json() == {"error": "pairing not authorized by account operator"} - assert accepted.status_code == 201 - - -async def test_pair_grant_requires_operator_authentication(client): - payload = signed_pair() - - response = await client.post( - "/v1/admin/pair-grants", - headers={"Authorization": "Bearer wrong"}, - json={ - "account_id": payload["account_id"], - "hotkey": payload["hotkey"], - "expires_at": 1_800_000_300, - }, - ) - - assert response.status_code == 401 - - -async def test_pair_grant_authentication_precedes_body_parsing(client): - response = await client.post( - "/v1/admin/pair-grants", - headers={"Authorization": "Bearer wrong", "Content-Type": "application/json"}, - content=b"{not-json", - ) - - assert response.status_code == 401 - assert response.json() == {"error": "unauthorized"} - - -async def test_pair_grant_body_has_a_hard_size_limit(client): - response = await client.post( - "/v1/admin/pair-grants", - headers={"Authorization": "Bearer operator-token", "Content-Type": "application/json"}, - content=b" " * 4097, - ) - - assert response.status_code == 413 - assert response.json() == {"error": "pair grant request too large"} - - -@pytest.mark.parametrize( - "route,size,error", - [ - ("/v1/pair", 4097, "pair request too large"), - ("/v1/reports", 256 * 1024 + 1, "report request too large"), - ], -) -async def test_public_bounty_writes_have_hard_body_limits(client, route, size, error): - response = await client.post( - route, - headers={"Content-Type": "application/json"}, - content=b" " * size, - ) - - assert response.status_code == 413 - assert response.json() == {"error": error} - - -async def test_adjudication_authentication_precedes_bounded_body_parsing(client): - unauthorized = await client.post( - "/v1/admin/adjudicate", - headers={"Authorization": "Bearer wrong", "Content-Type": "application/json"}, - content=b"{not-json", - ) - oversized = await client.post( - "/v1/admin/adjudicate", - headers={"Authorization": "Bearer operator-token", "Content-Type": "application/json"}, - content=b" " * 4097, - ) - - assert unauthorized.status_code == 401 - assert unauthorized.json() == {"error": "unauthorized"} - assert oversized.status_code == 413 - assert oversized.json() == {"error": "adjudication request too large"} - - -async def test_expired_pair_grant_does_not_authorize_or_burn_nonce(service, client): - _, clock, _ = service - payload = signed_pair() - grant = await client.post( - "/v1/admin/pair-grants", - headers={"Authorization": "Bearer operator-token"}, - json={ - "account_id": payload["account_id"], - "hotkey": payload["hotkey"], - "expires_at": clock.now + 1, - }, - ) - assert grant.status_code == 201 - clock.now += 1 - - refused = await client.post("/v1/pair", json=payload) - renewed = await client.post( - "/v1/admin/pair-grants", - headers={"Authorization": "Bearer operator-token"}, - json={ - "account_id": payload["account_id"], - "hotkey": payload["hotkey"], - "expires_at": clock.now + 300, - }, - ) - accepted = await client.post("/v1/pair", json=payload) - - assert refused.status_code == 403 - assert renewed.status_code == 201 - assert accepted.status_code == 201 - - -async def test_successful_pairing_consumes_its_operator_grant(client): - payload = signed_pair() - await grant_pair(client, payload) - assert (await client.post("/v1/pair", json=payload)).status_code == 201 - payload = signed_pair(nonce="34" * 16) - - response = await client.post("/v1/pair", json=payload) - - assert response.status_code == 403 - assert response.json() == {"error": "pairing not authorized by account operator"} - - -async def test_pair_retry_refuses_used_nonce_without_returning_session_data(client): - payload = signed_pair() - await grant_pair(client, payload) - - first = await client.post("/v1/pair", json=payload) - retry = await client.post("/v1/pair", json=payload) - - assert first.status_code == 201 - assert retry.status_code == 409 - assert retry.json() == {"error": "nonce reused"} - - -@pytest.mark.parametrize( - "account,seed_byte", [("account-b", 7), ("account-b", 8), ("account-a", 8)] -) -async def test_used_nonce_refuses_different_identity_without_consuming_its_grant( - client, account, seed_byte -): - original_session = await pair(client) - payload = signed_pair(account, seed_byte=seed_byte) - await grant_pair(client, payload) - - reused = await client.post("/v1/pair", json=payload) - fresh = signed_pair(account, nonce="34" * 16, seed_byte=seed_byte) - retry = await client.post("/v1/pair", json=fresh) - - assert reused.status_code == 409 - assert reused.json() == {"error": "nonce reused"} - assert retry.status_code == 201 - retained = await client.post("/v1/reports", json=report_body(original_session)) - assert retained.status_code == (401 if account == "account-a" else 201) - - -async def test_pair_grant_expiry_is_bounded_to_five_minutes(service, client): - _, clock, _ = service - payload = signed_pair() - - response = await client.post( - "/v1/admin/pair-grants", - headers={"Authorization": "Bearer operator-token"}, - json={ - "account_id": payload["account_id"], - "hotkey": payload["hotkey"], - "expires_at": clock.now + 301, - }, - ) - - assert response.status_code == 400 - assert response.json() == {"error": "pair grant must expire within 300 seconds"} - - -async def test_restart_preserves_sessions_nonces_reports_and_quotas(service, client): - svc, clock, _ = service - session = await pair(client) - created = (await client.post("/v1/reports", json=report_body(session))).json() - path = svc.store.path - svc.store.close() - svc.store = BountyStore(path) - - repeated_pair = await client.post("/v1/pair", json=signed_pair()) - repeated_report = await client.post("/v1/reports", json=report_body(session, 1)) - - assert repeated_pair.status_code == 409 - assert repeated_pair.json() == {"error": "nonce reused"} - assert repeated_report.status_code == 429 - assert svc.store.get_report(created["id"])["state"] == "pending" - clock.now += 60 - assert (await client.post("/v1/reports", json=report_body(session, 1))).status_code == 201 - svc.store.close() - - -async def test_repairing_an_account_revokes_its_previous_session(service, client): - first = await pair(client) - payload = signed_pair(nonce="34" * 16) - await grant_pair(client, payload) - reused = await client.post("/v1/pair", json=signed_pair()) - replacement = await client.post("/v1/pair", json=payload) - - old_report = await client.post("/v1/reports", json=report_body(first)) - new_report = await client.post( - "/v1/reports", json=report_body(replacement.json()["session"], 1) - ) - - assert reused.status_code == 409 - assert reused.json() == {"error": "nonce reused"} - assert replacement.status_code == 201 - assert old_report.status_code == 401 - assert new_report.status_code == 201 - - -async def test_repairing_during_feed_validation_revokes_the_inflight_session(service, client): - svc, clock, _ = service - old_session = await pair(client) - original_fetch = svc.backend.fetch - - async def fetch_after_repair(): - replacement = signed_pair(nonce="34" * 16) - svc.grant_pair( - SimpleNamespace( - account_id=replacement["account_id"], - hotkey=replacement["hotkey"], - expires_at=clock.now + 300, - ) - ) - svc.pair(SimpleNamespace(**replacement)) - return await original_fetch() - - svc.backend.fetch = fetch_after_repair - - response = await client.post("/v1/reports", json=report_body(old_session)) - - assert response.status_code == 401 - assert response.json() == {"error": "invalid_session"} - assert svc.store.list_reports() == [] - - -async def test_operator_granted_hotkey_replacement_revokes_the_existing_session(service, client): - original = await pair(client) - - payload = signed_pair(nonce="34" * 16, seed_byte=8) - await grant_pair(client, payload) - replacement = await client.post("/v1/pair", json=payload) - revoked = await client.post("/v1/reports", json=report_body(original)) - accepted = await client.post("/v1/reports", json=report_body(replacement.json()["session"], 1)) - - assert replacement.status_code == 201 - assert replacement.json()["miner_hotkey"] == payload["hotkey"] - assert revoked.status_code == 401 - assert accepted.status_code == 201 - - -async def test_status_probes_the_external_feed_instead_of_claiming_configured_is_ready( - service, client -): - _, _, upstream = service - upstream["status"] = 503 - - status = await client.get("/v1/status") - - assert status.status_code == 200 - assert status.json()["backend_public_configured"] is True - assert status.json()["can_score"] is False - assert "fetch failed" in status.json()["reason"] - assert status.json()["pairing"] == { - "requires_operator_grant": True, - "grant_max_ttl_secs": 300, - } - assert status.json()["quotas"] == { - "max_pending_reports_per_hotkey": 5, - "max_concurrent_feed_validations_per_hotkey": 1, - "min_report_interval_secs": 60, - "min_report_body_chars": 80, - "min_repro_chars": 20, - "max_report_request_bytes": 256 * 1024, - } - - -async def test_successful_status_cache_never_masks_an_intake_outage(service, client): - svc, _, upstream = service - session = await pair(client) - assert (await client.get("/v1/status")).json()["can_score"] is True - upstream["status"] = 503 - - response = await client.post("/v1/reports", json=report_body(session)) - - assert response.status_code == 503 - assert svc.store.list_reports() == [] - - -async def test_concurrent_reports_from_one_hotkey_start_only_one_feed_snapshot(service, client): - svc, _, _ = service - session = await pair(client) - entered = asyncio.Event() - release = asyncio.Event() - - class BlockingBackend: - configured = True - reads = 0 - - async def fetch(self): - self.reads += 1 - entered.set() - await release.wait() - - backend = BlockingBackend() - svc.backend = backend - first = asyncio.create_task(client.post("/v1/reports", json=report_body(session))) - await entered.wait() - try: - second = await asyncio.wait_for( - client.post("/v1/reports", json=report_body(session, 1)), - timeout=0.1, - ) - except TimeoutError: - second = None - finally: - release.set() - first_response = await first - - assert first_response.status_code == 201 - assert second is not None and second.status_code == 429 - assert backend.reads == 1 - - -async def test_duplicate_of_closed_report_never_reopens_triage(service, client): - svc, clock, _ = service - session = await pair(client) - original = (await client.post("/v1/reports", json=report_body(session))).json() - await client.post( - "/v1/admin/adjudicate", - headers={"Authorization": "Bearer operator-token"}, - json={"report_id": original["id"], "verdict": "invalid_malicious"}, - ) - clock.now += 60 - payload = report_body(session) - payload["title"] = " " + payload["title"].upper() + " " - - duplicate = await client.post("/v1/reports", json=payload) - - assert duplicate.status_code == 201 - assert duplicate.json()["state"] == "duplicate" - assert svc.store.get_report(duplicate.json()["id"])["duplicate_of"] == original["id"] - - -async def test_report_reads_require_operator_and_public_routes_do_not_exist(service, client): - session = await pair(client) - report = (await client.post("/v1/reports", json=report_body(session))).json() - - responses = [ - await client.get("/v1/reports"), - await client.get(f"/v1/reports/{report['id']}"), - await client.get("/v1/public/reports"), - ] - - assert [response.status_code for response in responses] == [401, 401, 404] - assert all("repro_steps" not in response.text for response in responses) - - -async def test_pending_quota_and_substance_rejections_do_not_create_rows(service, client): - svc, clock, _ = service - session = await pair(client) - thin = report_body(session) - thin["body"] = "fabricated " * 10 - assert (await client.post("/v1/reports", json=thin)).status_code == 400 - for index in range(5): - assert ( - await client.post("/v1/reports", json=report_body(session, index)) - ).status_code == 201 - clock.now += 60 - - response = await client.post("/v1/reports", json=report_body(session, 6)) - - assert response.status_code == 429 - assert len(svc.store.list_reports()) == 5 - - -@pytest.mark.parametrize("mutation,status", [("session", 401), ("hotkey", 403)]) -async def test_report_cannot_impersonate_another_hotkey(service, client, mutation, status): - session = await pair(client) - body = report_body(session) - if mutation == "session": - body["session"] = "00" * 32 - else: - body["hotkey"] = "ab" * 32 - - response = await client.post("/v1/reports", json=body) - - assert response.status_code == status - assert service[0].store.list_reports() == [] - - -async def test_empty_admin_configuration_never_exposes_private_reports(service): - svc, _, _ = service - locked = BountyService(svc.store, svc.backend, session_secret=b"s" * 32) - app = FastAPI() - app.include_router(create_router(locked)) - - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://test" - ) as client: - response = await client.get("/v1/reports") - - assert response.status_code == 503 - assert response.json() == {"error": "auth_unconfigured"} - - -async def test_scoring_outage_covers_every_participant_without_payment(service): - svc, _, upstream = service - upstream["status"] = 503 - hotkeys = ["ab" * 32, "cd" * 32] - - scores = await svc.score(hotkeys) - - assert set(scores) == set(hotkeys) - assert all( - score.value == 0 and score.reason == "ChallengeInternal" for score in scores.values() - ) diff --git a/tests/bounty/test_backend.py b/tests/bounty/test_backend.py deleted file mode 100644 index 75ced9dc3..000000000 --- a/tests/bounty/test_backend.py +++ /dev/null @@ -1,674 +0,0 @@ -"""Reject inconsistent publications before they can produce any paid leaf.""" - -import asyncio -import hashlib - -import httpx -import pytest - -from cortex.bounty import BackendUnavailable, PublicBackend -from cortex.protocol.crypto import decode_hotkey, public_key - -HOTKEY = "ab" * 32 -CURRENT_HOTKEY = public_key(bytes([7]) * 32).hex() -HISTORICAL_HOTKEY = public_key(bytes([8]) * 32).hex() - - -def published_report(report_id="r1", **changes): - return { - "id": report_id, - "hotkey": HOTKEY, - "status": "valid", - "severity": "critical", - "problem_found": "Unauthorized configuration change", - "adjudicator": "operator", - "justification": "Reproduced from an unauthenticated client", - "created_at": "2026-09-16T00:00:00Z", - "adjudicated_at": "2026-09-16T01:00:00Z", - **changes, - } - - -def feed_body(route, leaderboard, reports, *, revision="1"): - if route == "status": - return { - "api_version": 1, - "revision": revision, - "adjudication_available": True, - "published": len(reports), - "valid": sum(row["status"] == "valid" for row in reports), - "duplicate": sum(row["status"] == "duplicate" for row in reports), - "already_fixed_not_prod": sum( - row["status"] == "already_fixed_not_prod" for row in reports - ), - "invalid_malicious": sum(row["status"] == "invalid_malicious" for row in reports), - "hotkeys": len({row["hotkey"] for row in reports}), - "awaiting_adjudication": 0, - "unpriced_valid": 0, - } - if route == "leaderboard": - return { - "api_version": 1, - "revision": revision, - "items": leaderboard, - "has_more": False, - } - return { - "api_version": 1, - "revision": revision, - "items": reports, - "count": len(reports), - "has_more": False, - "next_cursor": None, - } - - -def backend_for(leaderboard, reports, /, **tokens): - def handle(request): - route = request.url.path.rsplit("/", 1)[-1] - body = feed_body( - route, - leaderboard, - reports, - revision=tokens.get(route, tokens.get("status", "1")), - ) - return httpx.Response(200, json=body) - - return PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - -@pytest.mark.parametrize( - "leaderboard,reports,tokens", - [ - ([{"hotkey": HOTKEY, "valid_count": 2}], [published_report()], {}), - ([], [published_report()], {}), - ([{"hotkey": HOTKEY, "valid_count": 1}] * 2, [published_report()], {}), - ([{"hotkey": HOTKEY, "valid_count": 2}], [published_report()] * 2, {}), - ( - [{"hotkey": HOTKEY, "valid_count": 1}], - [published_report()], - {"leaderboard": "2", "reports": "3"}, - ), - ([{"hotkey": "invalid", "valid_count": 1}], [published_report(hotkey="invalid")], {}), - ], -) -async def test_stable_but_incoherent_feed_is_refused(leaderboard, reports, tokens): - backend = backend_for(leaderboard, reports, **tokens) - - with pytest.raises(BackendUnavailable): - await backend.fetch() - - -async def test_moving_feed_cannot_be_mistaken_for_stable_scores(): - def handle(request): - route = request.url.path.rsplit("/", 1)[-1] - revision = "1" if route == "status" else "2" - return httpx.Response(200, json=feed_body(route, [], [], revision=revision)) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - with pytest.raises(BackendUnavailable, match="revision changed"): - await backend.fetch() - - -async def test_transient_json_and_revision_rollout_errors_are_retried(): - status_reads = 0 - - def handle(request): - nonlocal status_reads - route = request.url.path.rsplit("/", 1)[-1] - if route == "status": - status_reads += 1 - if status_reads == 1: - return httpx.Response(200, content=b"{") - revision = "2" if route == "leaderboard" and status_reads == 2 else "1" - return httpx.Response(200, json=feed_body(route, [], [], revision=revision)) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - backend._SNAPSHOT_RETRY_DELAY_SECONDS = 0 - - snapshot = await backend.fetch() - - assert snapshot.reports == () - assert status_reads == 3 - - -async def test_transient_failures_stop_after_the_bounded_attempt_count(): - requests = 0 - - def unavailable(request): - nonlocal requests - requests += 1 - return httpx.Response(503) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(unavailable)) - backend._SNAPSHOT_RETRY_DELAY_SECONDS = 0 - - with pytest.raises(BackendUnavailable, match="HTTP 503"): - await backend.fetch() - - assert requests == backend._SNAPSHOT_ATTEMPTS - - -async def test_ignored_metadata_does_not_make_a_stable_feed_unreadable(): - request_number = 0 - - def handle(request): - nonlocal request_number - request_number += 1 - route = request.url.path.rsplit("/", 1)[-1] - body = feed_body(route, [], []) - body["generated_at"] = request_number - return httpx.Response(200, json=body) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - snapshot = await backend.fetch() - - assert snapshot.score([HOTKEY])[HOTKEY].reason == "NotAttempted" - - -async def test_backend_public_leaderboard_valid_field_is_supported(): - backend = backend_for([{"hotkey": HOTKEY, "valid": 1}], [published_report()]) - - snapshot = await backend.fetch() - - assert snapshot.leaderboard[0].valid_count == 1 - - -async def test_v2_counts_every_valid_report_without_precision_severity_or_champion_gates(): - other = "02" * 32 - historical = "03" * 32 - reports = [ - published_report(id="a", severity="trivial"), - published_report(id="b", hotkey=other, severity="critical"), - published_report(id="c", hotkey=other, severity="minor"), - published_report(id="old", hotkey=historical), - *[ - published_report(id=f"invalid-{index}", status="invalid_malicious", severity=None) - for index in range(5) - ], - published_report(id="duplicate", status="duplicate", severity=None, related_report_id="a"), - ] - backend = backend_for( - [ - {"hotkey": other, "valid": 2}, - {"hotkey": historical, "valid": 1}, - {"hotkey": HOTKEY, "valid": 1}, - ], - reports, - ) - snapshot = await backend.fetch() - scores = snapshot.score([HOTKEY, other, "04" * 32], scoring_version=2) - assert set(scores) == {HOTKEY, other, "04" * 32} - assert scores[HOTKEY].value == 1 and scores[HOTKEY].reason is None - assert scores[other].value == 2 and scores[other].reason is None - assert scores["04" * 32].value == 0 and scores["04" * 32].reason == "NotAttempted" - - -async def test_backend_rejects_conflicting_leaderboard_count_aliases(): - backend = backend_for( - [{"hotkey": HOTKEY, "valid": 1, "valid_count": 2}], - [published_report()], - ) - - with pytest.raises(BackendUnavailable): - await backend.fetch() - - -@pytest.mark.parametrize( - "revision", - ["-1", "01", "x1", "1x", str(2**63)], -) -async def test_backend_rejects_noncanonical_or_out_of_range_revisions(revision): - backend = backend_for([], [], status=revision) - - with pytest.raises(BackendUnavailable): - await backend.fetch() - - -async def test_backend_report_read_uses_the_largest_supported_page(): - seen = [] - - def handle(request): - seen.append(request.url) - route = request.url.path.rsplit("/", 1)[-1] - return httpx.Response(200, json=feed_body(route, [], [])) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - await backend.fetch() - - report_reads = [url for url in seen if url.path.endswith("/reports")] - assert report_reads and all(url.params.get("limit") == "100" for url in report_reads) - - -async def test_backend_reads_every_report_page_at_the_status_revision(): - seen = [] - - def handle(request): - seen.append(request.url) - route = request.url.path.rsplit("/", 1)[-1] - if route == "status": - return httpx.Response( - 200, - json={ - "api_version": 1, - "revision": "7", - "adjudication_available": True, - "published": 2, - "valid": 2, - "duplicate": 0, - "already_fixed_not_prod": 0, - "invalid_malicious": 0, - "hotkeys": 1, - "awaiting_adjudication": 0, - "unpriced_valid": 0, - }, - ) - if route == "leaderboard": - return httpx.Response( - 200, - json={ - "api_version": 1, - "revision": "7", - "items": [{"hotkey": HOTKEY, "valid": 2}], - "has_more": False, - }, - ) - cursor = request.url.params.get("cursor") - item = published_report("r2" if cursor else "r1") - return httpx.Response( - 200, - json={ - "api_version": 1, - "revision": "7", - "items": [item], - "count": 1, - "has_more": cursor is None, - "next_cursor": "next" if cursor is None else None, - }, - ) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - snapshot = await backend.fetch() - - assert [report.id for report in snapshot.reports] == ["r1", "r2"] - report_reads = [url for url in seen if url.path.endswith("/reports")] - assert [url.params.get("revision") for url in report_reads] == ["7", "7"] - assert [url.params.get("cursor") for url in report_reads] == [None, "next"] - - -@pytest.mark.parametrize( - "page", - [ - {"items": [], "count": 1, "has_more": False, "next_cursor": None}, - {"items": [], "count": 0, "has_more": False, "next_cursor": "unexpected"}, - {"items": [], "count": 0, "has_more": True, "next_cursor": "next"}, - { - "items": [published_report()], - "count": 1, - "has_more": True, - "next_cursor": None, - }, - ], -) -async def test_backend_rejects_malformed_report_pagination(page): - def handle(request): - route = request.url.path.rsplit("/", 1)[-1] - body = feed_body(route, [], []) - if route == "reports": - body.update(page) - return httpx.Response(200, json=body) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - with pytest.raises(BackendUnavailable, match="page count|pagination|terminal"): - await backend.fetch() - - -async def test_backend_rejects_a_repeated_report_cursor(): - def handle(request): - route = request.url.path.rsplit("/", 1)[-1] - if route != "reports": - return httpx.Response(200, json=feed_body(route, [], [])) - report_id = "r2" if request.url.params.get("cursor") else "r1" - return httpx.Response( - 200, - json={ - "api_version": 1, - "revision": "1", - "items": [published_report(report_id)], - "count": 1, - "has_more": True, - "next_cursor": "repeated", - }, - ) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - with pytest.raises(BackendUnavailable, match="pagination"): - await backend.fetch() - - -async def test_backend_reconstructs_truncated_leaderboard_from_complete_reports(): - hotkeys = [hashlib.sha256(str(index).encode()).hexdigest() for index in range(1001)] - reports = [published_report(f"r{index}", hotkey=hotkey) for index, hotkey in enumerate(hotkeys)] - ordered = sorted(hotkeys) - - def handle(request): - route = request.url.path.rsplit("/", 1)[-1] - body = feed_body(route, [], reports) - if route == "leaderboard": - body["items"] = [{"hotkey": hotkey, "valid": 1} for hotkey in ordered[:1000]] - body["has_more"] = True - elif route == "reports": - cursor = request.url.params.get("cursor") - page = int(cursor.removeprefix("page-")) if cursor else 0 - items = reports[page * 100 : (page + 1) * 100] - has_more = (page + 1) * 100 < len(reports) - body.update( - items=items, - count=len(items), - has_more=has_more, - next_cursor=f"page-{page + 1}" if has_more else None, - ) - return httpx.Response(200, json=body) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - snapshot = await backend.fetch() - - assert len(snapshot.leaderboard) == 1001 - assert {decode_hotkey(row.hotkey).hex() for row in snapshot.leaderboard} == set(hotkeys) - assert all(row.valid_count == 1 for row in snapshot.leaderboard) - - -async def test_backend_rejects_an_inconsistent_truncated_leaderboard_prefix(): - reports = [published_report("r1"), published_report("r2", hotkey=CURRENT_HOTKEY)] - - def handle(request): - route = request.url.path.rsplit("/", 1)[-1] - body = feed_body(route, [], reports) - if route == "leaderboard": - body["items"] = [{"hotkey": HOTKEY, "valid": 2}] - body["has_more"] = True - return httpx.Response(200, json=body) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - with pytest.raises(BackendUnavailable, match="truncated leaderboard"): - await backend.fetch() - - -async def test_backend_enforces_per_response_and_complete_report_size_limits(): - backend = backend_for([], []) - backend._MAX_RESPONSE_BYTES = 32 - with pytest.raises(BackendUnavailable, match="response too large"): - await backend.fetch() - - backend = backend_for([], []) - backend._MAX_REPORT_BYTES = 1 - with pytest.raises(BackendUnavailable, match="snapshot is too large"): - await backend.fetch() - - -@pytest.mark.parametrize( - "change,reason", - [ - ({"adjudication_available": False}, "adjudication"), - ({"unpriced_valid": 1}, "unpriced"), - ], -) -async def test_backend_status_must_be_ready_and_fully_priced(change, reason): - requests = 0 - - def handle(request): - nonlocal requests - requests += 1 - route = request.url.path.rsplit("/", 1)[-1] - if route == "status": - body = { - "api_version": 1, - "revision": "0", - "adjudication_available": True, - "published": 0, - "valid": 0, - "duplicate": 0, - "already_fixed_not_prod": 0, - "invalid_malicious": 0, - "hotkeys": 0, - "awaiting_adjudication": 0, - "unpriced_valid": 0, - **change, - } - return httpx.Response(200, json=body) - return httpx.Response( - 200, - json={ - "api_version": 1, - "revision": "0", - "items": [], - "has_more": False, - **({"count": 0, "next_cursor": None} if route == "reports" else {}), - }, - ) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - with pytest.raises(BackendUnavailable, match=reason): - await backend.fetch() - - assert requests == 1, "a stable gate is not retried" - - -async def test_backend_refuses_an_empty_publication_with_a_waiting_backlog(): - def handle(request): - route = request.url.path.rsplit("/", 1)[-1] - body = feed_body(route, [], []) - if route == "status": - body["awaiting_adjudication"] = 1 - return httpx.Response(200, json=body) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - with pytest.raises(BackendUnavailable, match="backlog"): - await backend.fetch() - - -async def test_backend_snapshot_has_one_global_deadline(): - entered = asyncio.Event() - - async def stalled(request): - entered.set() - await asyncio.Event().wait() - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(stalled)) - backend._SNAPSHOT_TIMEOUT_SECONDS = 0.01 - - with pytest.raises(BackendUnavailable, match="deadline"): - await backend.fetch() - assert entered.is_set() - - -async def test_public_probe_reuses_a_short_cache_but_intake_fetch_does_not(): - requests = 0 - available = True - - def handle(request): - nonlocal requests - requests += 1 - if not available: - return httpx.Response(503) - route = request.url.path.rsplit("/", 1)[-1] - return httpx.Response(200, json=feed_body(route, [], [])) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - - await backend.probe() - await backend.probe() - assert requests == 3 - - available = False - with pytest.raises(BackendUnavailable, match="HTTP 503"): - await backend.fetch() - - -async def test_public_probe_refuses_parallel_refresh_without_duplicate_upstream_reads(): - entered = asyncio.Event() - release = asyncio.Event() - requests = 0 - - async def handle(request): - nonlocal requests - requests += 1 - route = request.url.path.rsplit("/", 1)[-1] - if route == "status": - entered.set() - await release.wait() - return httpx.Response(200, json=feed_body(route, [], [])) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(handle)) - first = asyncio.create_task(backend.probe()) - await entered.wait() - - with pytest.raises(BackendUnavailable, match="refresh already in progress"): - await backend.probe() - - assert requests == 1 - release.set() - await first - - -async def test_public_probe_caches_failures_briefly(): - requests = 0 - - def unavailable(request): - nonlocal requests - requests += 1 - return httpx.Response(503) - - backend = PublicBackend("https://backend.invalid", transport=httpx.MockTransport(unavailable)) - - for _ in range(2): - with pytest.raises(BackendUnavailable, match="HTTP 503"): - await backend.probe() - - assert requests == backend._SNAPSHOT_ATTEMPTS, "a failure is cached after bounded retries" - - -async def test_unpriced_valid_is_a_gate_even_after_three_priced_reports(): - reports = [published_report(f"r{index}") for index in range(3)] - reports.append(published_report("unpriced", severity=None)) - backend = backend_for([{"hotkey": HOTKEY, "valid_count": 4}], reports) - - with pytest.raises(BackendUnavailable, match="severity"): - await backend.fetch() - - -async def test_nonvalid_public_report_cannot_publish_a_severity(): - backend = backend_for( - [], - [published_report(status="duplicate", severity="critical")], - ) - - with pytest.raises(BackendUnavailable, match="severity"): - await backend.fetch() - - -@pytest.mark.parametrize( - "report", - [ - published_report(status="duplicate", severity=None), - published_report(status="duplicate", severity=None, related_report_id="missing"), - published_report(status="duplicate", severity=None, related_report_id="r1"), - published_report(related_report_id="other"), - ], -) -async def test_backend_rejects_invalid_duplicate_references(report): - backend = backend_for([], [report]) - - with pytest.raises(BackendUnavailable, match="duplicate reference"): - await backend.fetch() - - -async def test_backend_rejects_a_duplicate_cycle_without_a_root_report(): - reports = [ - published_report("r1", status="duplicate", severity=None, related_report_id="r2"), - published_report("r2", status="duplicate", severity=None, related_report_id="r1"), - ] - backend = backend_for([], reports) - - with pytest.raises(BackendUnavailable, match="non-duplicate root"): - await backend.fetch() - - -async def test_backend_accepts_a_duplicate_chain_with_a_root_report(): - reports = [ - published_report("r1", status="duplicate", severity=None, related_report_id="r2"), - published_report("r2", status="duplicate", severity=None, related_report_id="r3"), - published_report("r3"), - ] - backend = backend_for([{"hotkey": HOTKEY, "valid": 1}], reports) - - snapshot = await backend.fetch() - - assert [report.id for report in snapshot.reports] == ["r1", "r2", "r3"] - - -async def test_malicious_published_row_without_severity_burns_the_hotkey(): - backend = backend_for([], [published_report(status="invalid_malicious", severity=None)]) - - scores = (await backend.fetch()).score([HOTKEY]) - - assert scores[HOTKEY].value == 0 - assert scores[HOTKEY].reason == "InvalidResponse" - - -async def test_leaderboard_weight_cannot_invent_evidence(): - reports = [published_report(f"r{index}", justification="") for index in range(3)] - backend = backend_for([{"hotkey": HOTKEY, "valid_count": 3, "weight": 1_000_000}], reports) - - with pytest.raises(BackendUnavailable, match="evidence"): - await backend.fetch() - - -@pytest.mark.parametrize("field", ["problem_found", "justification", "adjudicator"]) -async def test_blank_public_evidence_makes_the_feed_unavailable(field): - backend = backend_for( - [{"hotkey": HOTKEY, "valid": 1}], - [published_report(**{field: ""})], - ) - - with pytest.raises(BackendUnavailable, match="evidence"): - await backend.fetch() - - -async def test_historical_hotkey_cannot_block_the_current_champion(): - reports = [ - *[published_report(f"old-{index}", hotkey=HISTORICAL_HOTKEY) for index in range(4)], - *[published_report(f"current-{index}", hotkey=CURRENT_HOTKEY) for index in range(3)], - ] - backend = backend_for( - [ - {"hotkey": HISTORICAL_HOTKEY, "valid": 4}, - {"hotkey": CURRENT_HOTKEY, "valid": 3}, - ], - reports, - ) - - scores = (await backend.fetch()).score([CURRENT_HOTKEY]) - - assert scores[CURRENT_HOTKEY].value == 1_000_000 - - -@pytest.mark.parametrize("body", [b"not json", b'{"items":{}}', b'{"items":[{}]}']) -async def test_unparseable_feed_is_a_scoring_outage(body): - backend = PublicBackend( - "https://backend.invalid", - transport=httpx.MockTransport(lambda request: httpx.Response(200, content=body)), - ) - - with pytest.raises(BackendUnavailable): - await backend.fetch() diff --git a/tests/bounty/test_scoring.py b/tests/bounty/test_scoring.py deleted file mode 100644 index cf0c45918..000000000 --- a/tests/bounty/test_scoring.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Money-path contracts ported from the live Bounty scorer.""" - -import pytest - -from cortex.bounty.scoring import Holdout, judge_challenger - - -@pytest.mark.parametrize( - ("severity", "expected"), - [("trivial", 62_500), ("minor", 250_000), ("major", 500_000), ("critical", 1_000_000)], -) -def test_reward_is_precision_times_mean_severity(severity, expected): - challenger = Holdout() - for _ in range(3): - challenger.record("valid", severity) - - verdict = judge_challenger(Holdout(), challenger) - - assert verdict.eligible - assert verdict.lattice == expected - - -@pytest.mark.parametrize( - ("verdict", "severity", "count", "gate"), - [ - ("valid", None, 1, "severity_evidence_missing"), - ("duplicate", None, 4, "triage_noise"), - ("already_fixed_not_prod", None, 4, "triage_noise"), - ("invalid_malicious", None, 4, "penalty"), - ], -) -def test_unpriced_noise_and_penalty_gates_cannot_be_paid(verdict, severity, count, gate): - challenger = Holdout() - for _ in range(3): - challenger.record("valid", "critical") - for _ in range(count): - challenger.record(verdict, severity) - - result = judge_challenger(Holdout(), challenger) - - assert not result.eligible - assert result.lattice == 0 - assert gate in result.failed - - -def test_impact_does_not_override_strict_precision_displacement(): - incumbent = Holdout() - challenger = Holdout() - for _ in range(3): - incumbent.record("valid", "trivial") - challenger.record("valid", "critical") - - verdict = judge_challenger(incumbent, challenger) - - assert not verdict.eligible - assert "no_precision_win" in verdict.failed - - -def test_malicious_rows_without_evidence_still_count_against_precision(): - challenger = Holdout() - for _ in range(4): - challenger.record("valid", "critical") - challenger.record("invalid_malicious") - - verdict = judge_challenger(Holdout(), challenger) - - assert verdict.eligible - assert verdict.lattice == 800_000 diff --git a/tests/bounty/test_store.py b/tests/bounty/test_store.py deleted file mode 100644 index f04d260b2..000000000 --- a/tests/bounty/test_store.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Persistence guarantees that span independent service processes.""" - -import sqlite3 -from concurrent.futures import ThreadPoolExecutor -from threading import Barrier - -import pytest - -from cortex.bounty import BountyStore -from cortex.bounty.store import StoreError - - -def test_two_connections_cannot_race_the_same_hotkey_rate_limit(tmp_path): - path = tmp_path / "bounty.sqlite3" - stores = [BountyStore(path), BountyStore(path)] - barrier = Barrier(2) - pairing = {"miner_hotkey": "ab" * 32, "account_id": "account"} - - def submit(index): - barrier.wait(timeout=5) - try: - stores[index].insert_report(pairing, f"title {index}", "body", "reproduce", 100) - return 201 - except StoreError as exc: - return exc.status - - try: - with ThreadPoolExecutor(max_workers=2) as pool: - outcomes = list(pool.map(submit, range(2))) - - assert sorted(outcomes) == [201, 429] - assert len(stores[0].list_reports()) == 1 - finally: - for store in stores: - store.close() - - -def test_two_connections_cannot_consume_the_same_pairing_nonce(tmp_path): - path = tmp_path / "bounty.sqlite3" - stores = [BountyStore(path), BountyStore(path)] - barrier = Barrier(2) - stores[0].grant_pair("account", "ab" * 32, expires_at=200, now=100) - - def bind(index): - barrier.wait(timeout=5) - try: - result = stores[index].bind_pair("account", "ab" * 32, "12" * 16, 100, b"s" * 32) - return 201, result - except StoreError as exc: - return exc.status, {"error": str(exc)} - - try: - with ThreadPoolExecutor(max_workers=2) as pool: - outcomes = list(pool.map(bind, range(2))) - - assert sorted(status for status, _ in outcomes) == [201, 409] - refusal = next(body for status, body in outcomes if status == 409) - assert refusal == {"error": "nonce reused"} - accepted = next(body for status, body in outcomes if status == 201) - assert stores[0].lookup_session(accepted["session"], b"s" * 32)["account_id"] == "account" - finally: - for store in stores: - store.close() - - -def test_failed_session_insert_rolls_back_the_grant_and_nonce_across_restart(tmp_path): - path = tmp_path / "bounty.sqlite3" - store = BountyStore(path) - store.grant_pair("account", "ab" * 32, expires_at=200, now=100) - with sqlite3.connect(path) as connection: - connection.execute( - "CREATE TRIGGER fail_session BEFORE INSERT ON bounty_sessions " - "BEGIN SELECT RAISE(ABORT, 'session write failed'); END" - ) - try: - with pytest.raises(sqlite3.IntegrityError, match="session write failed"): - store.bind_pair("account", "ab" * 32, "12" * 16, 100, b"s" * 32) - finally: - store.close() - with sqlite3.connect(path) as connection: - connection.execute("DROP TRIGGER fail_session") - - restarted = BountyStore(path) - try: - result = restarted.bind_pair("account", "ab" * 32, "12" * 16, 100, b"s" * 32) - - assert restarted.lookup_session(result["session"], b"s" * 32)["account_id"] == "account" - with pytest.raises(StoreError, match="nonce reused") as repeated: - restarted.bind_pair("account", "ab" * 32, "12" * 16, 100, b"s" * 32) - assert repeated.value.status == 409 - with pytest.raises(StoreError, match="pairing not authorized") as spent_grant: - restarted.bind_pair("account", "ab" * 32, "34" * 16, 100, b"s" * 32) - assert spent_grant.value.status == 403 - finally: - restarted.close() - - -def test_two_connections_cannot_consume_one_pair_grant_with_different_nonces(tmp_path): - path = tmp_path / "bounty.sqlite3" - stores = [BountyStore(path), BountyStore(path)] - barrier = Barrier(2) - stores[0].grant_pair("account", "ab" * 32, expires_at=200, now=100) - - def bind(index): - barrier.wait(timeout=5) - try: - stores[index].bind_pair("account", "ab" * 32, f"{index + 1:02x}" * 16, 100, b"s" * 32) - return 201 - except StoreError as exc: - return exc.status - - try: - with ThreadPoolExecutor(max_workers=2) as pool: - outcomes = list(pool.map(bind, range(2))) - - assert sorted(outcomes) == [201, 403] - finally: - for store in stores: - store.close() diff --git a/tests/challenges/test_contract.py b/tests/challenges/test_contract.py new file mode 100644 index 000000000..d3ffe0028 --- /dev/null +++ b/tests/challenges/test_contract.py @@ -0,0 +1,123 @@ +"""Registry validation, exact leaf conversion and public proxy refusals.""" + +from fractions import Fraction + +import httpx +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from cortex.challenges.client import ChallengeWeights, leaf_scores, parse_weights +from cortex.challenges.proxy import create_router +from cortex.challenges.registry import parse_registry +from cortex.protocol.models import FULL_SHARE_SCORE, NoScore, NoScoreReason, Score + +ROW = { + "id": "opentype", + "image": "ghcr.io/opentypeai/challenge", + "source": "https://github.com/OpentypeAI/challenge", +} + + +@pytest.mark.parametrize( + "change", + [ + {"id": "proof"}, + {"id": "Bad"}, + {"image": "docker.io/opentypeai/challenge"}, + {"image": "ghcr.io/opentypeai/challenge:latest"}, + {"source": "http://github.com/OpentypeAI/challenge"}, + {"channel": "latest"}, + {"pin": "sha256:abc"}, + {"memory_mib": 1}, + {"env": {"CHALLENGE_SLUG": "bounty"}}, + {"unknown": 1}, + ], +) +def test_registry_refuses_unsafe_rows(change): + with pytest.raises((ValueError, TypeError)): + parse_registry({"version": 1, "challenge": [{**ROW, **change}]}) + + +def test_registry_refuses_duplicate_ids(): + with pytest.raises(ValueError, match="duplicate"): + parse_registry({"version": 1, "challenge": [ROW, ROW]}) + + +def test_algorithm_three_scores_are_exact_and_ignore_hotkeys_outside_the_expected_set(): + a, b, outsider, idle = (bytes([n]) * 32 for n in (1, 2, 3, 4)) + answer = ChallengeWeights( + {a: Fraction(3), b: Fraction(1), outsider: Fraction(1000)}, Fraction(10) + ) + + scores = leaf_scores(answer, {a, b, idle}, algorithm_version=3) + + assert scores == { + a: Score(3 * FULL_SHARE_SCORE // 10), + b: Score(FULL_SHARE_SCORE // 10), + idle: NoScore(NoScoreReason.NOT_ATTEMPTED), + } + normalized = leaf_scores( + ChallengeWeights({a: Fraction(1), b: Fraction(2)}), {a, b}, algorithm_version=3 + ) + assert sum(score.value for score in normalized.values()) == FULL_SHARE_SCORE - 1 + + +def test_legacy_algorithms_refuse_fractional_weights(): + with pytest.raises(ValueError, match="integers"): + leaf_scores( + ChallengeWeights({b"\1" * 32: Fraction(1, 2)}), {b"\1" * 32}, algorithm_version=2 + ) + + +@pytest.mark.parametrize( + "body", + [ + b'{"challenge_slug":"bounty","epoch":7,"weights":{}}', + b'{"challenge_slug":"opentype","epoch":8,"weights":{}}', + b'{"challenge_slug":"opentype","epoch":7,"weights":{"' + b"a" * 64 + b'":-1}}', + b'{"challenge_slug":"opentype","epoch":7,"weights":{"' + b"a" * 64 + b'":NaN}}', + b'{"challenge_slug":"opentype","epoch":7,"weights":{"not-a-key":1}}', + ], +) +def test_weights_for_another_challenge_epoch_or_with_invalid_values_are_rejected(body): + with pytest.raises(ValueError): + parse_weights(body, slug="opentype", epoch=7) + + +def test_proxy_forwards_public_routes_only(): + seen = [] + + def upstream(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json={"ok": True}) + + entry = parse_registry({"version": 1, "challenge": [ROW]}) + app = FastAPI() + http = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + app.include_router(create_router(lambda: entry, http)) + client = TestClient(app) + + response = client.get( + "/challenge/opentype/v1/status?x=1", + headers={"authorization": "Bearer miner", "cookie": "secret", "x-forwarded-for": "1.2.3.4"}, + ) + assert response.status_code == 200 + forwarded = seen[-1] + assert str(forwarded.url) == "http://cortex-challenge-opentype:8000/v1/status?x=1" + assert forwarded.headers["authorization"] == "Bearer miner" + assert "cookie" not in forwarded.headers + assert forwarded.headers["x-forwarded-for"] == "testclient" + + count = len(seen) + for path in ( + "/challenge/opentype/internal/v1/get_weights?epoch=1", + "/challenge/opentype/v1/%2e%2e/internal/v1/get_weights", + "/challenge/opentype/v1//status", + "/challenge/unknown/v1/status", + ): + assert client.get(path).status_code == 404 + assert ( + client.post("/challenge/opentype/v1/x", content=b"x" * (1024 * 1024 + 1)).status_code == 413 + ) + assert len(seen) == count diff --git a/tests/challenges/test_supervisor.py b/tests/challenges/test_supervisor.py new file mode 100644 index 000000000..d69b51dbc --- /dev/null +++ b/tests/challenges/test_supervisor.py @@ -0,0 +1,211 @@ +"""Auto-updater behaviour against a fake Docker Engine API and a fake GitHub.""" + +import json +from pathlib import Path +from urllib.parse import unquote + +import httpx +import pytest + +from cortex.challenges.registry import parse_registry +from cortex.challenges.supervisor import Supervisor, SupervisorConfig, SupervisorError + +IMAGE = "ghcr.io/cortexlm/bounty" +GOOD, NEXT, BROKEN = ("sha256:" + c * 64 for c in "abc") +LABELS = { + "io.cortex.challenge.slug": "bounty", + "io.cortex.challenge.contract": "1", + "org.opencontainers.image.source": "https://github.com/CortexLM/bounty", +} + + +class FakeEngine: + """Enough of Docker Engine v1.44 for the supervisor: images, containers, labels.""" + + def __init__(self): + self.tags = {"stable": GOOD} + self.labels = {GOOD: LABELS, NEXT: LABELS, BROKEN: LABELS} + self.containers: dict[str, dict] = {} + self.created: list[tuple[str, dict]] = [] + + def handle(self, request: httpx.Request) -> httpx.Response: + path = unquote(request.url.path).removeprefix("/v1.44") + params = request.url.params + if path == "/images/create": + return httpx.Response(200, text=json.dumps({"status": "pulled"}) + "\n") + if path.startswith("/images/"): + reference = path.removeprefix("/images/").removesuffix("/json") + digest = ( + reference.split("@", 1)[1] + if "@" in reference + else self.tags[reference.rsplit(":", 1)[1]] + ) + return httpx.Response( + 200, + json={ + "RepoDigests": [f"{IMAGE}@{digest}"], + "Config": {"Labels": self.labels[digest]}, + }, + ) + if path == "/containers/json": + return httpx.Response( + 200, + json=[{"Id": name, "Labels": c["Labels"]} for name, c in self.containers.items()], + ) + if path == "/containers/create": + spec = json.loads(request.content) + self.containers[params["name"]] = {"Labels": spec["Labels"], "Running": False} + self.created.append((params["name"], spec)) + return httpx.Response(201, json={"Id": params["name"]}) + name = path.split("/")[2] + if request.method == "DELETE": + return httpx.Response(204 if self.containers.pop(name, None) else 404) + if name not in self.containers: + return httpx.Response(404) + if path.endswith("/start"): + self.containers[name]["Running"] = True + return httpx.Response(204) + container = self.containers[name] + return httpx.Response( + 200, + json={ + "Config": {"Labels": container["Labels"]}, + "State": {"Running": container["Running"]}, + }, + ) + + def digest(self, name: str) -> str | None: + container = self.containers.get(name) + return container and container["Labels"]["io.cortex.challenge.digest"] + + +def network(engine: FakeEngine, attested: set[str]): + def handle(request: httpx.Request) -> httpx.Response: + if request.url.host == "api.github.com": + digest = request.url.path.rsplit("/", 1)[1] + return httpx.Response(200, json={"attestations": [{}] if digest in attested else []}) + container = engine.containers.get(request.url.host) + if container is None or not container["Running"]: + raise httpx.ConnectError("no route", request=request) + if container["Labels"]["io.cortex.challenge.digest"] == BROKEN: + return httpx.Response(500) + return httpx.Response(200, json={"slug": "bounty", "contract": 1, "version": "1.0.0"}) + + return handle + + +def registry(**overrides): + row = { + "id": "bounty", + "image": IMAGE, + "source": "https://github.com/CortexLM/bounty", + "env": {"BOUNTY_BACKEND_PUBLIC_URL": "https://backend.invalid"}, + } + return parse_registry({"version": 1, "challenge": [{**row, **overrides}]})["bounty"] + + +@pytest.fixture +def world(tmp_path): + engine = FakeEngine() + attested = {GOOD, NEXT, BROKEN} + + async def no_wait(_seconds): + return None + + docker = httpx.AsyncClient( + transport=httpx.MockTransport(engine.handle), base_url="http://docker/v1.44" + ) + http = httpx.AsyncClient(transport=httpx.MockTransport(network(engine, attested))) + config = SupervisorConfig( + registry_file=tmp_path / "registry.toml", + secrets_host_dir=Path("/srv/challenge-secrets"), + ready_seconds=0.05, + ) + return engine, attested, Supervisor(config, docker, http, sleep=no_wait) + + +async def test_rollout_runs_hardened_container_with_secrets_only_after_a_secretless_canary(world): + engine, _, supervisor = world + + assert await supervisor.reconcile(registry()) == GOOD + + (canary_name, canary), (name, spec) = engine.created + assert canary_name == "cortex-challenge-bounty-canary" and name == "cortex-challenge-bounty" + assert canary["HostConfig"]["Mounts"] == [] and "/data" in canary["HostConfig"]["Tmpfs"] + host = spec["HostConfig"] + assert spec["Image"] == f"{IMAGE}@{GOOD}" and spec["User"] == "65532:65532" + assert host["ReadonlyRootfs"] and host["CapDrop"] == ["ALL"] + assert host["NetworkMode"] == "cortex-challenges" and "PortBindings" not in host + assert { + "Type": "bind", + "Source": "/srv/challenge-secrets/bounty", + "Target": "/run/secrets", + "ReadOnly": True, + } in host["Mounts"] + assert "CHALLENGE_SLUG=bounty" in spec["Env"] + assert "BOUNTY_BACKEND_PUBLIC_URL=https://backend.invalid" in spec["Env"] + assert set(engine.containers) == {"cortex-challenge-bounty"} + + +async def test_channel_move_updates_and_a_failed_rollout_rolls_back_and_is_not_retried(world): + engine, _, supervisor = world + await supervisor.reconcile(registry()) + engine.tags["stable"] = NEXT + assert await supervisor.reconcile(registry()) == NEXT + + engine.tags["stable"] = BROKEN + with pytest.raises(SupervisorError, match="canary"): + await supervisor.reconcile(registry()) + assert engine.digest("cortex-challenge-bounty") == NEXT + created = len(engine.created) + with pytest.raises(SupervisorError, match="refused"): + await supervisor.reconcile(registry()) + assert len(engine.created) == created # the refused digest is not pulled into a canary again + + +@pytest.mark.parametrize("fault", ["slug", "source", "contract", "attestation"]) +async def test_untrusted_image_is_refused_and_the_running_digest_keeps_serving(world, fault): + engine, attested, supervisor = world + await supervisor.reconcile(registry()) + engine.tags["stable"] = NEXT + if fault == "attestation": + attested.discard(NEXT) + else: + key = { + "slug": "io.cortex.challenge.slug", + "source": "org.opencontainers.image.source", + "contract": "io.cortex.challenge.contract", + }[fault] + engine.labels[NEXT] = { + **LABELS, + key: "https://github.com/evil/fork" if fault == "source" else "2", + } + + with pytest.raises(SupervisorError): + await supervisor.reconcile(registry()) + assert engine.digest("cortex-challenge-bounty") == GOOD + assert not any(name.endswith("-canary") for name in engine.containers) + + +async def test_pin_freezes_the_digest_even_when_the_channel_moves(world): + engine, _, supervisor = world + engine.tags["stable"] = NEXT + assert await supervisor.reconcile(registry(pin=GOOD)) == GOOD + + +async def test_stopped_container_is_restarted_without_a_new_rollout(world): + engine, _, supervisor = world + await supervisor.reconcile(registry()) + engine.containers["cortex-challenge-bounty"]["Running"] = False + created = len(engine.created) + await supervisor.reconcile(registry()) + assert engine.containers["cortex-challenge-bounty"]["Running"] + assert len(engine.created) == created + + +async def test_unregistered_container_is_pruned_on_tick(world): + engine, _, supervisor = world + await supervisor.reconcile(registry()) + supervisor.config.registry_file.write_text("version = 1\n") + await supervisor.tick(0) + assert engine.containers == {} diff --git a/tests/test_deploy_contract.py b/tests/test_deploy_contract.py index 112904523..055bad0b8 100644 --- a/tests/test_deploy_contract.py +++ b/tests/test_deploy_contract.py @@ -147,10 +147,11 @@ def master_config(): "BASE_CHALLENGES_FILE": "/etc/base/config/challenges.toml", "BASE_MEASUREMENTS_FILE": "/etc/base/config/measurements.toml", "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway.key", - "BOUNTY_SK_FILE": "/run/secrets/bounty.key", "PROOF_SK_FILE": "/run/secrets/proof.key", - "BOUNTY_SESSION_SECRET_FILE": "/run/secrets/bounty-session.key", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator.token", + "BASE_CHALLENGE_KEYS_DIR": "/run/secrets", + "BASE_CHALLENGE_REGISTRY_FILE": "/etc/base/challenges/registry.toml", + "BASE_CHALLENGE_SECRETS_DIR": "/run/challenge-secrets", }, "healthcheck": { "test": [ @@ -230,6 +231,115 @@ def test_compose_accepts_isolated_python_roles_with_durable_state(role): CHECK["validate_compose"](config, role) +def supervised_master_config(): + config = master_config() + gateway = config["services"]["gateway"] + gateway["networks"] = {"default": None, "challenges": {"aliases": ["cortex-master"]}} + gateway["volumes"] += [ + { + "type": "bind", + "source": "/fixture/challenges", + "target": "/etc/base/challenges", + "read_only": True, + }, + { + "type": "bind", + "source": "/fixture/challenge-secrets", + "target": "/run/challenge-secrets", + "read_only": True, + }, + ] + config["networks"] = {"challenges": {"name": "cortex-challenges"}} + config["services"]["challenge-supervisor"] = { + "profiles": ["master"], + "image": PINNED_IMAGE, + "init": True, + "read_only": True, + "restart": "unless-stopped", + "user": "65532:65532", + "group_add": ["999"], + "cap_drop": ["ALL"], + "security_opt": ["no-new-privileges:true"], + "command": [ + "challenge-supervisor", + "--registry", + "/etc/base/challenges/registry.toml", + "--secrets-host-dir", + "/fixture/challenge-secrets", + "--network", + "cortex-challenges", + "--master-url", + "http://cortex-master:8080", + ], + "networks": {"challenges": None}, + "tmpfs": ["/tmp:size=16m,mode=1777"], + "volumes": [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + }, + { + "type": "bind", + "source": "/fixture/challenges", + "target": "/etc/base/challenges", + "read_only": True, + }, + ], + } + return config + + +def test_master_accepts_the_one_docker_controlling_challenge_supervisor(): + CHECK["validate_compose"](supervised_master_config(), "master") + + +@pytest.mark.parametrize( + "mutation", + ["gateway-socket", "secrets", "ports", "env", "privileged", "network", "registry", "command"], +) +def test_docker_control_stays_confined_to_the_secretless_supervisor(mutation): + config = supervised_master_config() + gateway = config["services"]["gateway"] + supervisor = config["services"]["challenge-supervisor"] + if mutation == "gateway-socket": + gateway["volumes"].append( + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + } + ) + elif mutation == "secrets": + supervisor["volumes"].append( + { + "type": "bind", + "source": "/fixture/secrets", + "target": "/run/secrets", + "read_only": True, + } + ) + elif mutation == "ports": + supervisor["ports"] = [{"target": 2375, "host_ip": "10.0.0.1"}] + elif mutation == "env": + supervisor["environment"] = {"PROOF_SK_FILE": "/run/secrets/proof.key"} + elif mutation == "privileged": + supervisor["privileged"] = True + elif mutation == "network": + supervisor["networks"] = {"challenges": None, "default": None} + elif mutation == "registry": + gateway["volumes"] = [ + item for item in gateway["volumes"] if item["target"] != "/etc/base/challenges" + ] + else: + supervisor["command"] += ["--once"] + + with pytest.raises(ValueError): + CHECK["validate_compose"](config, "master") + + @pytest.mark.parametrize( ("flag", "value"), [ diff --git a/tests/test_master.py b/tests/test_master.py index 4e2195f9f..2dde1217b 100644 --- a/tests/test_master.py +++ b/tests/test_master.py @@ -8,7 +8,6 @@ import httpx import pytest -from cortex.bounty import PublicBackend from cortex.config import MasterConfig, read_seed from cortex.errors import ServiceError from cortex.master import BittensorEpochProvider, EpochClock, EpochState, build_master @@ -76,6 +75,7 @@ def secret(name, value): path.chmod(0o600) return path + secret("bounty.key", bytes([1]) * 32) return MasterConfig( netuid=541, state_dir=tmp_path / "state", @@ -83,10 +83,10 @@ def secret(name, value): challenges_file=tmp_path / "challenges.toml", measurements_file=tmp_path / "measurements.toml", gateway_seed_file=secret("gateway.key", bytes([7]) * 32), - bounty_seed_file=secret("bounty.key", bytes([1]) * 32), proof_seed_file=secret("proof.key", bytes([2]) * 32), - bounty_session_secret_file=secret("session.key", bytes([8]) * 32), operator_token_file=secret("operator.token", b"fixture-operator-token"), + challenge_keys_dir=tmp_path, + challenge_secrets_dir=tmp_path / "challenge-secrets", ) @@ -387,71 +387,13 @@ async def test_private_routes_require_rotating_file_and_master_routes_are_compos assert readiness.status_code == 200 assert readiness.json() == {"ready": True, "epoch": 12, "role": "master"} assert (await client.get("/v1/proof/topics")).json() == {"topics": []} - assert (await client.get("/bounty/v1/status")).json()["challenge_id"] == "bounty" - assert (await client.get("/v1/reports")).status_code == 401 + assert (await client.get("/challenge/bounty/v1/status")).status_code == 404 + drain = "/v1/admin/proof/drain" + assert (await client.post(drain)).status_code == 401 master.config.operator_token_file.write_text("rotated-token") assert ( - await client.get("/v1/reports", headers={"authorization": "Bearer rotated-token"}) - ).status_code == 200 - - -async def test_master_ready_is_independent_of_a_readable_bounty_feed(master): - def empty_feed(request): - route = request.url.path.rsplit("/", 1)[-1] - if route == "status": - body = { - "api_version": 1, - "revision": "1", - "adjudication_available": True, - "published": 0, - "valid": 0, - "duplicate": 0, - "already_fixed_not_prod": 0, - "invalid_malicious": 0, - "hotkeys": 0, - "awaiting_adjudication": 0, - "unpriced_valid": 0, - } - else: - body = { - "api_version": 1, - "revision": "1", - "items": [], - "has_more": False, - **({"count": 0, "next_cursor": None} if route == "reports" else {}), - } - return httpx.Response(200, json=body) - - master.runtime.bounty.backend = PublicBackend( - "https://backend.invalid", - transport=httpx.MockTransport(empty_feed), - ) - - async with httpx.AsyncClient( - transport=httpx.ASGITransport(master.runtime.app()), base_url="https://master" - ) as client: - readiness = await client.get("/readyz") - - assert readiness.status_code == 200 - assert readiness.json() == {"ready": True, "epoch": 12, "role": "master"} - - -async def test_master_readiness_does_not_probe_the_bounty_feed(master): - def unexpected_probe(request): - raise AssertionError("master readiness must not call the external scorer") - - master.runtime.bounty.backend = PublicBackend( - "https://backend.invalid", - transport=httpx.MockTransport(unexpected_probe), - ) - - async with httpx.AsyncClient( - transport=httpx.ASGITransport(master.runtime.app()), base_url="https://master" - ) as client: - readiness = await client.get("/readyz") - - assert readiness.status_code == 200 - assert readiness.json() == {"ready": True, "epoch": 12, "role": "master"} + await client.post(drain, headers={"authorization": "Bearer rotated-token"}) + ).status_code != 401 def test_private_seed_rejects_symlinks_permissions_and_accepts_raw_or_hex(tmp_path): @@ -472,10 +414,9 @@ def test_config_base_aliases_fail_closed_and_do_not_accept_inline_secret_values( env = { "BASE_NETUID": "541", "BASE_GATEWAY_SK_FILE": "/private/gateway", - "BOUNTY_SK_FILE": "/private/bounty", "PROOF_SK_FILE": "/private/proof", - "BOUNTY_SESSION_SECRET_FILE": "/private/session", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/private/token", + "BASE_CHALLENGE_KEYS_DIR": "/private", } alias = {key.replace("BASE_", "CORTEX_"): value for key, value in env.items()} assert MasterConfig.from_env(env) == MasterConfig.from_env(alias) @@ -489,7 +430,12 @@ def test_config_base_aliases_fail_closed_and_do_not_accept_inline_secret_values( def test_backend_config_rejects_credentials_in_url(tmp_path): with pytest.raises(ValueError, match="HTTPS without credentials"): - replace(master_config(tmp_path), bounty_backend_url="https://key@example.org") + replace( + master_config(tmp_path), + proof_orchestrator_url="https://key@example.org", + proof_orchestrator_token_file=tmp_path / "token", + proof_orchestrator_ca_file=tmp_path / "ca", + ) @pytest.mark.parametrize("value", [float("inf"), float("nan"), 0, -1]) diff --git a/tests/test_master_config.py b/tests/test_master_config.py index d7bfe2dcb..5ddeea55a 100644 --- a/tests/test_master_config.py +++ b/tests/test_master_config.py @@ -12,9 +12,7 @@ def configured_environment(): return { "BASE_NETUID": "541", "BASE_GATEWAY_SK_FILE": "/run/secrets/gateway", - "BOUNTY_SK_FILE": "/run/secrets/bounty", "PROOF_SK_FILE": "/run/secrets/proof", - "BOUNTY_SESSION_SECRET_FILE": "/run/secrets/session", "BASE_GATEWAY_ADMIN_TOKEN_FILE": "/run/secrets/operator", "PROOF_VM_ORCHESTRATOR_URL": "https://vm.example", "PROOF_VM_ORCHESTRATOR_TOKEN_FILE": "/run/secrets/vm-token", diff --git a/tests/test_miner_wallet.py b/tests/test_miner_wallet.py index 7a19f1f5b..aa25dedd7 100644 --- a/tests/test_miner_wallet.py +++ b/tests/test_miner_wallet.py @@ -13,10 +13,9 @@ import pytest from bittensor_wallet import Keypair, Wallet -from cortex.bounty.service import pair_payload from cortex.cli import parser from cortex.errors import ServiceError -from cortex.miner import MinerClient +from cortex.miner import MinerClient, pair_payload from cortex.proof.models import Baseline, Metric, Submission, Topic, digest from cortex.proof.service import SUBMIT_DOMAIN, sign_topic from cortex.protocol.crypto import public_key, verify_raw, verify_substrate diff --git a/tests/test_network_e2e.py b/tests/test_network_e2e.py index 707efe5e2..6c444e58a 100644 --- a/tests/test_network_e2e.py +++ b/tests/test_network_e2e.py @@ -5,18 +5,20 @@ import json import os import tarfile -import time +from dataclasses import replace from types import SimpleNamespace import httpx import pytest +import sr25519 +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse from test_master import FakeEpochChain, master_config from vm.test_research import MemoryCallback from vm.test_setup_agent import ExecutingHypervisor, SetupProvider -from cortex.bounty import PublicBackend from cortex.master import EpochState, build_master -from cortex.miner import MinerClient +from cortex.miner import MinerClient, pair_payload from cortex.proof.backend import VmBackend from cortex.protocol import ChallengeEntry, MetagraphRow, NoScore, NoScoreReason, Score, TrustRoot from cortex.protocol.crypto import encode_hotkey, public_key @@ -119,52 +121,70 @@ async def exchange(self, socket_path, message, budget_seconds): return await guest.handle(message) -class BountyFeed: +class FakeChallenges: + """Contract-v1 challenge containers, routed by the container host name.""" + def __init__(self): - self.available = True - self.leaderboard = [] - self.reports = [] - self.revision = "1" + self.weights: dict[str, dict[str, float]] = {} + self.full_share_mass: dict[str, float | None] = {} + self.failing: set[str] = set() + self.calls: list[tuple[str, int]] = [] + self.pairs: list[dict] = [] + app = FastAPI() - def handle(self, request): - if not self.available: - return httpx.Response(503) - route = request.url.path.rsplit("/", 1)[-1] - if route == "status": - body = { - "api_version": 1, - "revision": self.revision, - "adjudication_available": True, - "published": len(self.reports), - "valid": sum(row["status"] == "valid" for row in self.reports), - "duplicate": sum(row["status"] == "duplicate" for row in self.reports), - "already_fixed_not_prod": sum( - row["status"] == "already_fixed_not_prod" for row in self.reports - ), - "invalid_malicious": sum( - row["status"] == "invalid_malicious" for row in self.reports - ), - "hotkeys": len({row["hotkey"] for row in self.reports}), - "awaiting_adjudication": 0, - "unpriced_valid": 0, - } - elif route == "leaderboard": - body = { - "api_version": 1, - "revision": self.revision, - "items": self.leaderboard, - "has_more": False, - } - else: - body = { - "api_version": 1, - "revision": self.revision, - "items": self.reports, - "count": len(self.reports), - "has_more": False, - "next_cursor": None, + @app.get("/internal/v1/get_weights") + async def get_weights(request: Request, epoch: int): + slug = request.headers["host"].removeprefix("cortex-challenge-").split(":")[0] + if request.headers.get("authorization") != f"Bearer {slug}-internal": + return JSONResponse({"error": "unauthorized"}, status_code=401) + if request.headers.get("x-platform-challenge-slug") != slug: + return JSONResponse({"error": "forbidden"}, status_code=403) + self.calls.append((slug, epoch)) + if slug in self.failing: + return JSONResponse({"error": "feed unavailable"}, status_code=503) + return { + "challenge_slug": slug, + "epoch": epoch, + "weights": self.weights.get(slug, {}), + "full_share_mass": self.full_share_mass.get(slug), + "metadata": {}, + "computed_at": "2026-09-24T00:00:00Z", } - return httpx.Response(200, json=body) + + @app.post("/v1/pair", status_code=201) + async def pair(request: Request): + body = await request.json() + payload = pair_payload(body["account_id"], body["nonce"], body["exp"]) + public = bytes.fromhex(body["hotkey"]) if len(body["hotkey"]) == 64 else None + assert public is None or sr25519.verify( + bytes.fromhex(body["signature"]), payload, public + ) + self.pairs.append(body) + return {"session": "fixture-session"} + + self.transport = httpx.ASGITransport(app) + + +def challenge_secrets(config, *slugs): + for slug in slugs: + directory = config.challenge_secrets_dir / slug + directory.mkdir(parents=True, exist_ok=True) + token = directory / "internal.token" + token.write_text(f"{slug}-internal") + token.chmod(0o600) + + +def registry_file(tmp_path, *slugs): + path = tmp_path / "challenge-registry.toml" + path.write_text( + "version = 1\n" + + "".join( + f'[[challenge]]\nid = "{slug}"\nimage = "ghcr.io/fixture/{slug}"\n' + f'source = "https://github.com/fixture/{slug}"\n' + for slug in slugs + ) + ) + return path @pytest.mark.parametrize( @@ -177,12 +197,21 @@ def handle(self, request): (2, (4, 5), (0.12, 0.15)), (2, (4, 6), (0.12, 0.18)), (2, (8, 12), (0.12, 0.18)), + (3, (0, 0), (0, 0)), + (3, (1, 0), (0.03, 0)), + (3, (2, 3), (0.06, 0.09)), + (3, (4, 6), (0.12, 0.18)), + (3, (8, 12), (0.12, 0.18)), ], ) -async def test_bounty_intake_external_feed_seal_and_validator_dispatch( +async def test_bounty_container_weights_seal_and_validator_dispatch( tmp_path, version, counts, payouts ): - config = master_config(tmp_path) + """Algorithm 3 with full_share_mass=10 pays exactly what algorithm 2 pays.""" + config = replace( + master_config(tmp_path), challenge_registry_file=registry_file(tmp_path, "bounty") + ) + challenge_secrets(config, "bounty") chain = FakeEpochChain() second_key = public_key(bytes([22]) * 32) chain.rows += (MetagraphRow(second_key, 2),) @@ -196,16 +225,21 @@ async def test_bounty_intake_external_feed_seal_and_validator_dispatch( public_key(bytes([7]) * 32), challenges_version=version, ) - feed = BountyFeed() + fake = FakeChallenges() + miner_key = public_key(bytes([21]) * 32) + fake.weights["bounty"] = { + encode_hotkey(key): count + for key, count in zip((miner_key, second_key), counts, strict=True) + if count + } + fake.weights["bounty"][encode_hotkey(public_key(bytes([99]) * 32))] = 50 # not registered + fake.full_share_mass["bounty"] = 10 runtime = await build_master( config, chain=chain, epochs=chain, trust=trust, - bounty_backend=PublicBackend( - "https://backend.fixture", - transport=httpx.MockTransport(feed.handle), - ), + challenge_http=httpx.AsyncClient(transport=fake.transport), ) journal = SubmissionJournal(tmp_path / "validator.sqlite3") try: @@ -214,95 +248,33 @@ async def test_bounty_intake_external_feed_seal_and_validator_dispatch( base_url="https://master.fixture", ) as http: miner = MinerClient( - base_url="https://master.fixture", - seed=bytes([21]) * 32, - client=http, - ) - now = int(time.time()) - runtime.bounty.clock = lambda: now - grant = await http.post( - "/v1/admin/pair-grants", - headers={"authorization": "Bearer fixture-operator-token"}, - json={ - "account_id": "bounty-fixture", - "hotkey": miner.hotkey, - "expires_at": now + 300, - }, + base_url="https://master.fixture", seed=bytes([21]) * 32, client=http ) - assert grant.status_code == 201, grant.text - pairing = await miner.pair_bounty(account_id="bounty-fixture", accept_terms=True) - - feed.available = False - refused = await http.post( - "/challenge/bounty/v1/reports", - json={ - "session": pairing["session"], - "hotkey": miner.hotkey, - "title": "External feed outage blocks durable intake", - "body": ( - "The external scoring publication is unavailable during intake, so " - "this otherwise substantive report must not create any durable local row." - ), - "repro_steps": "Disable the feed and submit the signed miner session report.", - }, - ) - assert refused.status_code == 503 - assert runtime.bounty.store.list_reports() == [] - - feed.available = True - local_report = await miner.report_bounty( - session=pairing["session"], - title="Unauthorized configuration mutation", - body=( - "An unauthenticated request changes the operator configuration and " - "invalidates the current sealed bundle for every validator observing " - "the gateway." - ), - repro_steps=( - "Call the operator endpoint without authorization and read the changed value." - ), - ) - assert local_report["state"] == "pending" - authors = (miner.hotkey, encode_hotkey(second_key)) - feed.leaderboard = sorted( - [ - {"hotkey": author, "valid": count} - for author, count in zip(authors, counts, strict=True) - if count - ], - key=lambda row: (-row["valid"], row["hotkey"]), - ) - feed.reports = [ - { - "id": f"backend-{author}-{index}", - "hotkey": author, - "status": "valid", - "severity": "critical" if version == 1 else "trivial", - "problem_found": "Unauthorized configuration mutation", - "adjudicator": "fixture-adjudicator", - "justification": "Reproduced from a clean unauthenticated client.", - "adjudicated_at": "2026-09-18T01:00:00Z", - "created_at": "2026-09-18T00:00:00Z", - } - for author, count in zip(authors, counts, strict=True) - for index in range(count) - ] - feed.revision = "2" + paired = await miner.pair_bounty(account_id="bounty-fixture", accept_terms=True) + assert paired == {"session": "fixture-session"} + assert fake.pairs[0]["hotkey"] == miner.hotkey + internal = await http.get("/challenge/bounty/internal/v1/get_weights?epoch=12") + assert internal.status_code == 404 await runtime.emitter.tick() chain.state = EpochState(13, 100, 105) assert await runtime.emitter.tick() == [12] + assert fake.calls == [("bounty", 12)] leaves = runtime.gateway.store.leaves(12) - miner_key = public_key(bytes([21]) * 32) - assert next( + assert {leaf.miner_hotkey for leaf in leaves if leaf.challenge_id == b"bounty"} == { + row.hotkey for row in chain.rows + } + mine = next( leaf for leaf in leaves if leaf.challenge_id == b"bounty" and leaf.miner_hotkey == miner_key - ).score == ( - Score(1_000_000 if version == 1 else counts[0]) - if counts[0] - else NoScore(NoScoreReason.NOT_ATTEMPTED) ) + if not counts[0]: + assert mine.score == NoScore(NoScoreReason.NOT_ATTEMPTED) + elif version == 3: + assert mine.score == Score(10**12 * counts[0] // max(10, sum(counts))) + else: + assert mine.score == Score(counts[0]) assert all( leaf.score == NoScore(NoScoreReason.CHALLENGE_INTERNAL) for leaf in leaves @@ -311,17 +283,12 @@ async def test_bounty_intake_external_feed_seal_and_validator_dispatch( latest = (await http.get("/v1/weights/latest")).json() assert latest["sealed"] is True assert latest["algorithm_version"] == version - assert latest["source_challenges"][0]["emission_percent"] == pytest.approx( - 20 if version == 1 else sum(payouts) * 100 - ) weights = dict(zip(latest["uids"], latest["weights"], strict=True)) assert weights[0] == pytest.approx(1 - sum(payouts)) assert weights.get(1, 0) == pytest.approx(payouts[0]) assert weights.get(2, 0) == pytest.approx(payouts[1]) - status = (await http.get("/challenge/bounty/v1/status")).json() - assert status["scoring_version"] == version - if version == 2: - assert status["scoring"]["paid_on"] == ["valid_report_count"] + metagraph = (await http.get("/v1/metagraph/latest")).json() + assert metagraph["hotkeys"][miner.hotkey] == 1 preflight = Validator( gateway_url="https://master.fixture", @@ -334,7 +301,6 @@ async def test_bounty_intake_external_feed_seal_and_validator_dispatch( ) assert (await preflight.run_once()).outcome == "verified" assert chain.submissions == [] - validator = Validator( gateway_url="https://master.fixture", netuid=541, @@ -350,6 +316,62 @@ async def test_bounty_intake_external_feed_seal_and_validator_dispatch( await runtime.close() +async def test_three_container_challenges_burn_failures_and_unpaid_mass(tmp_path): + config = replace( + master_config(tmp_path), + challenge_registry_file=registry_file(tmp_path, "bounty", "opentype"), + ) + challenge_secrets(config, "bounty", "opentype") + for slug, seed in (("opentype", 3), ("audit", 4)): + key = tmp_path / f"{slug}.key" + key.write_bytes(bytes([seed]) * 32) + key.chmod(0o600) + chain = FakeEpochChain() + trust = TrustRoot( + ( + ChallengeEntry(b"audit", public_key(bytes([4]) * 32), 1000), + ChallengeEntry(b"bounty", public_key(bytes([1]) * 32), 3000), + ChallengeEntry(b"opentype", public_key(bytes([3]) * 32), 4000), + ChallengeEntry(b"proof", public_key(bytes([2]) * 32), 2000), + ), + hashlib.sha256(b"\x00").digest(), + public_key(bytes([7]) * 32), + challenges_version=3, + ) + fake = FakeChallenges() + miner = encode_hotkey(chain.rows[1].hotkey) + fake.weights["opentype"] = {miner: 0.25} + fake.full_share_mass["opentype"] = 1.0 + fake.failing.add("bounty") + runtime = await build_master( + config, + chain=chain, + epochs=chain, + trust=trust, + challenge_http=httpx.AsyncClient(transport=fake.transport), + ) + try: + await runtime.emitter.tick() + chain.state = EpochState(13, 100, 105) + assert await runtime.emitter.tick() == [12] + leaves = { + (leaf.challenge_id, leaf.miner_hotkey): leaf.score + for leaf in runtime.gateway.store.leaves(12) + } + # Registered in trust but not in the registry, and a failing container: both burn. + for challenge in (b"audit", b"bounty"): + assert {score for (cid, _), score in leaves.items() if cid == challenge} == { + NoScore(NoScoreReason.CHALLENGE_INTERNAL) + } + assert leaves[(b"opentype", chain.rows[1].hotkey)] == Score(250_000_000_000) + latest = runtime.gateway.latest() + weights = dict(zip(latest["uids"], latest["weights"], strict=True)) + assert weights[1] == pytest.approx(0.4 * 0.25) + assert weights[0] == pytest.approx(1 - 0.4 * 0.25) + finally: + await runtime.close() + + async def test_owner_setup_miner_submission_seal_and_validator_dispatch(tmp_path, monkeypatch): config = master_config(tmp_path) chain = FakeEpochChain() diff --git a/tests/test_repo_contract.py b/tests/test_repo_contract.py index ca759cd1b..ebfa35386 100644 --- a/tests/test_repo_contract.py +++ b/tests/test_repo_contract.py @@ -59,6 +59,21 @@ def test_trust_root_rejects_emission_drift_or_extra_products(tmp_path, mutation) assert len(CHECK["check_trust_roots"](tmp_path)) == 1 +def test_version_three_accepts_any_unique_challenge_set_summing_to_full_emission(tmp_path): + rows = "".join( + f'[[challenges]]\nid = "{name}"\npublic_key = "{"c" * 64}"\nemission_share_bps = {share}\n' + for name, share in (("bounty", 3000), ("opentype", 5000), ("proof", 2000)) + ) + path = put(tmp_path, "config/challenges.toml", "version = 3\n" + rows) + assert CHECK["check_trust_roots"](tmp_path) == [] + path.write_text("version = 3\n" + rows.replace("5000", "4999")) + assert len(CHECK["check_trust_roots"](tmp_path)) == 1 + path.write_text("version = 3\n" + rows.replace('"opentype"', '"Open_Type"')) + assert len(CHECK["check_trust_roots"](tmp_path)) == 1 + path.write_text("version = 2\n" + rows) + assert len(CHECK["check_trust_roots"](tmp_path)) == 1 + + @pytest.mark.parametrize("version,accepted", [(1, False), (2, True), (3, True)]) def test_proportional_trust_template_requires_new_owner_version(tmp_path, version, accepted): put(tmp_path, "config/challenges.toml", trust_root()) diff --git a/tests/test_state_security.py b/tests/test_state_security.py index c14617935..c5c628380 100644 --- a/tests/test_state_security.py +++ b/tests/test_state_security.py @@ -5,7 +5,6 @@ import pytest -from cortex.bounty import BountyStore from cortex.gateway import GatewayStore from cortex.proof.store import ProofStore from cortex.state import ( @@ -26,7 +25,6 @@ def test_prepare_master_state_creates_private_directory_and_databases(tmp_path): assert mode(state) == 0o700 assert {path.name for path in state.iterdir()} == { "gateway.sqlite3", - "bounty.sqlite3", "proof.sqlite3", "emission.sqlite3", } @@ -76,7 +74,7 @@ def test_sqlite_path_refuses_a_group_writable_parent(tmp_path): secure_sqlite_path(state / "database.sqlite3") -@pytest.mark.parametrize("store_type", [BountyStore, ProofStore, GatewayStore]) +@pytest.mark.parametrize("store_type", [ProofStore, GatewayStore]) def test_stores_create_private_regular_databases(tmp_path, store_type): path = tmp_path / f"{store_type.__name__}.sqlite3" From fe08adefa671c96029e09b193b31c2d83b5c9198 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:28:05 +0000 Subject: [PATCH 2/4] fix(challenges): restore serving container and fail whole answers - Rollout now stops the running container and keeps it aside. It removes that container only after the replacement answers /version. A failed create, start or readiness check restarts the previous container unchanged. - Each container carries a fingerprint of its full spec. A registry change (env, resources, limits) now redeploys even when the digest stays the same. - An algorithm 2 answer with any weight that is not a u64 integer is rejected as a whole, so the challenge burns instead of partly paying. - Algorithm 1 signed Bounty's legacy champion lattice, which containers never compute, so container challenges burn under it. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/CHALLENGES.md | 23 ++++-- src/cortex/challenges/client.py | 13 +++- src/cortex/challenges/supervisor.py | 114 +++++++++++++++++----------- src/cortex/master.py | 2 + tests/challenges/test_contract.py | 17 +++-- tests/challenges/test_supervisor.py | 55 +++++++++++++- tests/test_network_e2e.py | 9 ++- 7 files changed, 168 insertions(+), 65 deletions(-) diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index ed2b22ff7..9566d7314 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -92,11 +92,14 @@ changes `D`. | Algorithm | Leaf score for `i` in `E` with `w_i > 0` | Challenge payout | | --- | --- | --- | | 3 (trust root version >= 3) | `floor(10^12 * w_i / D)` | `share * sum(leaves) / 10^12` | -| 2 (bounty/proof 3000/7000) | `w_i`, which must be an integer | Bounty: `share * min(N, 10) / 10`; Proof: `share` | -| 1 (legacy bounty/proof 2000/8000) | `w_i`, which must be an integer | `share` when any leaf is positive | +| 2 (bounty/proof 3000/7000) | `w_i`, which must be an integer below 2^64 | Bounty: `share * min(N, 10) / 10`; Proof: `share` | +| 1 (legacy bounty/proof 2000/8000) | not scored: every hotkey in `E` gets `NoScore(ChallengeInternal)` | nothing; the share burns | Every other hotkey in `E` receives `NoScore(NotAttempted)`. A failed or invalid -call gives `NoScore(ChallengeInternal)` to every hotkey in `E`. Unpaid mass +call gives `NoScore(ChallengeInternal)` to every hotkey in `E`: one invalid +weight invalidates the whole answer, so an answer is never partially paid. +Algorithm 1 signed Bounty's legacy champion score, which no container computes, +so containers are not scored under it; trust watermarks never return to it. Unpaid mass always burns to UID0 and never moves to another challenge. Under algorithm 3, Bounty with `full_share_mass = 10` pays exactly what algorithm 2 pays: `share * min(N, 10) / 10` in total and `share * n_i / max(10, N)` per author. @@ -150,12 +153,18 @@ For each registry entry, the supervisor runs this loop every `poll_seconds`: for the digest in the `source` repository. 4. Start a canary with the same image, no secrets and a tmpfs `/data`. It must answer `/version` with the expected slug and contract within 60 seconds. -5. Replace the container: stop the old one and start the new one on the same - volume. Then wait until `/version` returns the expected slug and contract. +5. Replace the container: stop the old one and keep it aside, then start the new + one on the same volume. Then wait until `/version` returns the expected slug + and contract. `/health` is readiness and is only reported, because an external outage must not trigger a rollback. -6. If the new container fails, recreate the previous digest and log the refusal. - The refused digest is not retried until the channel moves. +6. If the new container cannot be created, started or reached, remove it and + restart the previous container unchanged. The refused digest and + configuration are not retried until the channel or the registry entry moves. + +A registry change that keeps the same digest, such as `env`, resources or limits, +redeploys through steps 5 and 6 without a new canary. The container carries a +fingerprint of its full specification for this purpose. A managed container whose id is no longer in the registry is stopped and removed. Its volume is kept. diff --git a/src/cortex/challenges/client.py b/src/cortex/challenges/client.py index 4a6777519..449bf659d 100644 --- a/src/cortex/challenges/client.py +++ b/src/cortex/challenges/client.py @@ -20,6 +20,7 @@ MAX_RESPONSE_BYTES = 8 * 1024 * 1024 MAX_WEIGHTS = 65536 +MAX_SCORE = 2**64 - 1 ATTEMPTS = 3 @@ -66,11 +67,15 @@ def leaf_scores( raw = { key: math.floor(FULL_SHARE_SCORE * value / denominator) for key, value in kept.items() } - else: - # Algorithms 1 and 2 sign raw integer counts; the protocol applies the Bounty cap. - if any(value.denominator != 1 for value in kept.values()): - raise ValueError("algorithm 1 and 2 weights must be integers") + elif algorithm_version == 2: + # Algorithm 2 signs raw integer counts; the protocol applies the Bounty cap. + if any(value.denominator != 1 or value > MAX_SCORE for value in kept.values()): + raise ValueError("algorithm 2 weights must be u64 integers") raw = {key: int(value) for key, value in kept.items()} + else: + # Algorithm 1 signed Bounty's legacy champion lattice, which a container never + # computes; its share burns instead of paying a score with other semantics. + raise ValueError("container challenges are not scored under algorithm 1") return { key: Score(raw[key]) if raw.get(key, 0) > 0 else NoScore(NoScoreReason.NOT_ATTEMPTED) for key in expected diff --git a/src/cortex/challenges/supervisor.py b/src/cortex/challenges/supervisor.py index 5c81ef7f5..371584515 100644 --- a/src/cortex/challenges/supervisor.py +++ b/src/cortex/challenges/supervisor.py @@ -7,11 +7,13 @@ from __future__ import annotations import asyncio +import hashlib import json import logging from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from pathlib import Path +from typing import Any import httpx @@ -19,6 +21,7 @@ CONTRACT = "1" MANAGED = "io.cortex.managed" +CONFIG = "io.cortex.challenge.config" LOG = logging.getLogger("cortex.challenges") @@ -119,7 +122,7 @@ def _spec(self, entry: RegistryEntry, digest: str, *, canary: bool) -> dict: "ReadOnly": True, }, ] - return { + spec: dict[str, Any] = { "Image": f"{entry.image}@{digest}", "Env": [f"{name}={value}" for name, value in sorted(env.items())], "User": "65532:65532", @@ -143,6 +146,11 @@ def _spec(self, entry: RegistryEntry, digest: str, *, canary: bool) -> dict: "LogConfig": {"Type": "json-file", "Config": {"max-size": "20m", "max-file": "3"}}, }, } + # Any registry, secret-path or network change alters this, so it redeploys even + # when the image digest stays the same. + canonical = json.dumps(spec, sort_keys=True, separators=(",", ":")).encode() + spec["Labels"][CONFIG] = hashlib.sha256(canonical).hexdigest() + return spec async def _remove(self, name: str) -> None: await self._docker("DELETE", f"/containers/{name}", params={"force": "true"}) @@ -165,61 +173,77 @@ async def _answers_version(self, name: str, entry: RegistryEntry) -> bool: await self.sleep(1) return False - async def running_digest(self, entry: RegistryEntry) -> str | None: - response = await self._docker("GET", f"/containers/{container_name(entry.id)}/json") + async def _inspect(self, name: str) -> tuple[dict[str, str], bool] | None: + """(labels, running) of a container, or None when it does not exist.""" + response = await self._docker("GET", f"/containers/{name}/json") if response.status_code == 404: return None state = response.json() - if not (state.get("State") or {}).get("Running"): - return None - return ((state.get("Config") or {}).get("Labels") or {}).get("io.cortex.challenge.digest") + labels = (state.get("Config") or {}).get("Labels") or {} + return labels, bool((state.get("State") or {}).get("Running")) - async def deployed_digest(self, entry: RegistryEntry) -> str | None: - response = await self._docker("GET", f"/containers/{container_name(entry.id)}/json") - if response.status_code == 404: - return None - return ((response.json().get("Config") or {}).get("Labels") or {}).get( - "io.cortex.challenge.digest" - ) + async def _rollout(self, entry: RegistryEntry, spec: dict, *, replace: bool) -> bool: + """Swap in `spec`; any failure restores the exact previous container.""" + name = container_name(entry.id) + backup = f"{name}-previous" + await self._remove(backup) + if replace: + await self._docker("POST", f"/containers/{name}/stop", params={"t": "30"}) + await self._docker("POST", f"/containers/{name}/rename", params={"name": backup}) + try: + await self._start(name, spec) + ready = await self._answers_version(name, entry) + except (SupervisorError, httpx.HTTPError): + ready = False + if ready: + await self._remove(backup) + return True + await self._remove(name) + if replace: + await self._docker("POST", f"/containers/{backup}/rename", params={"name": name}) + await self._docker("POST", f"/containers/{name}/start") + if await self._answers_version(name, entry): + LOG.warning("challenge %s rolled back to its previous container", entry.id) + else: + LOG.error("challenge %s previous container did not come back", entry.id) + return False async def reconcile(self, entry: RegistryEntry) -> str: """Converge one challenge; returns the digest that is running afterwards.""" digest, labels = await self.resolve(entry) - previous = await self.deployed_digest(entry) - if digest == previous: - if await self.running_digest(entry) != digest: - await self._docker("POST", f"/containers/{container_name(entry.id)}/start") + name = container_name(entry.id) + spec = self._spec(entry, digest, canary=False) + fingerprint = spec["Labels"][CONFIG] + deployed = await self._inspect(name) + if deployed is not None and deployed[0].get(CONFIG) == fingerprint: + if not deployed[1]: + await self._docker("POST", f"/containers/{name}/start") return digest - if self.refused.get(entry.id) == digest: - raise SupervisorError(f"{entry.id}: {digest} was refused; waiting for a new digest") - try: - self.verify_labels(entry, labels) - if entry.attestation: - await self.verify_attestation(entry, digest) - canary = container_name(entry.id) + "-canary" - await self._remove(canary) - await self._start(canary, self._spec(entry, digest, canary=True)) + if self.refused.get(entry.id) == fingerprint: + raise SupervisorError(f"{entry.id}: {digest} was refused; waiting for a change") + # A configuration-only change reuses the running, already verified digest. + if deployed is None or deployed[0].get("io.cortex.challenge.digest") != digest: try: - if not await self._answers_version(canary, entry): - raise SupervisorError(f"{entry.id}: canary did not answer /version") - finally: + self.verify_labels(entry, labels) + if entry.attestation: + await self.verify_attestation(entry, digest) + canary = f"{name}-canary" await self._remove(canary) - except SupervisorError: - self.refused[entry.id] = digest - raise - name = container_name(entry.id) - await self._remove(name) - await self._start(name, self._spec(entry, digest, canary=False)) - if await self._answers_version(name, entry): - self.refused.pop(entry.id, None) - LOG.info("challenge %s now runs %s", entry.id, digest) - return digest - self.refused[entry.id] = digest - await self._remove(name) - if previous is not None: - await self._start(name, self._spec(entry, previous, canary=False)) - LOG.warning("challenge %s rolled back to %s", entry.id, previous) - raise SupervisorError(f"{entry.id}: {digest} failed after rollout") + try: + await self._start(canary, self._spec(entry, digest, canary=True)) + if not await self._answers_version(canary, entry): + raise SupervisorError(f"{entry.id}: canary did not answer /version") + finally: + await self._remove(canary) + except SupervisorError: + self.refused[entry.id] = fingerprint + raise + if not await self._rollout(entry, spec, replace=deployed is not None): + self.refused[entry.id] = fingerprint + raise SupervisorError(f"{entry.id}: {digest} failed after rollout") + self.refused.pop(entry.id, None) + LOG.info("challenge %s now runs %s", entry.id, digest) + return digest async def prune(self, registry: dict[str, RegistryEntry]) -> None: filters = json.dumps({"label": [f"{MANAGED}=true"]}) diff --git a/src/cortex/master.py b/src/cortex/master.py index cfc1ae1a0..f3b51d4b1 100644 --- a/src/cortex/master.py +++ b/src/cortex/master.py @@ -284,6 +284,8 @@ async def _scores(self, challenge: bytes, epoch: int, expected: set[bytes]): entry = self.registry().get(challenge.decode()) if entry is None: raise ServiceError(503, "challenge container not registered") + if algorithm == 1: + raise ServiceError(503, "container challenges need algorithm 2 or 3") answer = await self.challenges.weights(entry, epoch) return leaf_scores(answer, expected, algorithm_version=algorithm) # Backend readiness is still required before old rows can be emitted. diff --git a/tests/challenges/test_contract.py b/tests/challenges/test_contract.py index d3ffe0028..f18b935ff 100644 --- a/tests/challenges/test_contract.py +++ b/tests/challenges/test_contract.py @@ -63,11 +63,18 @@ def test_algorithm_three_scores_are_exact_and_ignore_hotkeys_outside_the_expecte assert sum(score.value for score in normalized.values()) == FULL_SHARE_SCORE - 1 -def test_legacy_algorithms_refuse_fractional_weights(): - with pytest.raises(ValueError, match="integers"): - leaf_scores( - ChallengeWeights({b"\1" * 32: Fraction(1, 2)}), {b"\1" * 32}, algorithm_version=2 - ) +@pytest.mark.parametrize("weight", [Fraction(1, 2), Fraction(2**64)]) +def test_algorithm_two_refuses_the_whole_answer_on_one_non_u64_weight(weight): + good, bad = b"\1" * 32, b"\2" * 32 + answer = ChallengeWeights({good: Fraction(3), bad: weight}) + with pytest.raises(ValueError, match="u64"): + leaf_scores(answer, {good, bad}, algorithm_version=2) + + +def test_algorithm_one_never_signs_a_container_weight(): + hotkey = b"\1" * 32 + with pytest.raises(ValueError, match="algorithm 1"): + leaf_scores(ChallengeWeights({hotkey: Fraction(3)}), {hotkey}, algorithm_version=1) @pytest.mark.parametrize( diff --git a/tests/challenges/test_supervisor.py b/tests/challenges/test_supervisor.py index d69b51dbc..5d41dcc5a 100644 --- a/tests/challenges/test_supervisor.py +++ b/tests/challenges/test_supervisor.py @@ -27,6 +27,7 @@ def __init__(self): self.labels = {GOOD: LABELS, NEXT: LABELS, BROKEN: LABELS} self.containers: dict[str, dict] = {} self.created: list[tuple[str, dict]] = [] + self.fail: set[str] = set() # "create" or "start" of the production container def handle(self, request: httpx.Request) -> httpx.Response: path = unquote(request.url.path).removeprefix("/v1.44") @@ -54,7 +55,13 @@ def handle(self, request: httpx.Request) -> httpx.Response: ) if path == "/containers/create": spec = json.loads(request.content) - self.containers[params["name"]] = {"Labels": spec["Labels"], "Running": False} + if "create" in self.fail and params["name"] == "cortex-challenge-bounty": + return httpx.Response(500) + self.containers[params["name"]] = { + "Labels": spec["Labels"], + "Running": False, + "Spec": spec, + } self.created.append((params["name"], spec)) return httpx.Response(201, json={"Id": params["name"]}) name = path.split("/")[2] @@ -63,8 +70,19 @@ def handle(self, request: httpx.Request) -> httpx.Response: if name not in self.containers: return httpx.Response(404) if path.endswith("/start"): + fresh = self.containers[name]["Spec"] is self.created[-1][1] + if "start" in self.fail and name == "cortex-challenge-bounty" and fresh: + return httpx.Response(500) self.containers[name]["Running"] = True return httpx.Response(204) + if path.endswith("/stop"): + self.containers[name]["Running"] = False + return httpx.Response(204) + if path.endswith("/rename"): + if params["name"] in self.containers: + return httpx.Response(409) + self.containers[params["name"]] = self.containers.pop(name) + return httpx.Response(204) container = self.containers[name] return httpx.Response( 200, @@ -78,6 +96,9 @@ def digest(self, name: str) -> str | None: container = self.containers.get(name) return container and container["Labels"]["io.cortex.challenge.digest"] + def env(self, name: str) -> list[str]: + return self.containers[name]["Spec"]["Env"] + def network(engine: FakeEngine, attested: set[str]): def handle(request: httpx.Request) -> httpx.Response: @@ -209,3 +230,35 @@ async def test_unregistered_container_is_pruned_on_tick(world): supervisor.config.registry_file.write_text("version = 1\n") await supervisor.tick(0) assert engine.containers == {} + + +@pytest.mark.parametrize("fault", ["create", "start"]) +async def test_production_create_or_start_failure_restores_the_serving_container(world, fault): + engine, _, supervisor = world + await supervisor.reconcile(registry()) + engine.tags["stable"] = NEXT + engine.fail.add(fault) + + with pytest.raises(SupervisorError, match="after rollout"): + await supervisor.reconcile(registry()) + + assert set(engine.containers) == {"cortex-challenge-bounty"} + assert engine.digest("cortex-challenge-bounty") == GOOD + assert engine.containers["cortex-challenge-bounty"]["Running"] + + +async def test_registry_change_with_the_same_digest_redeploys_without_a_canary(world): + engine, _, supervisor = world + await supervisor.reconcile(registry()) + created = len(engine.created) + + changed = registry(memory_mib=2048, env={"BOUNTY_BACKEND_PUBLIC_URL": "https://new.invalid"}) + assert await supervisor.reconcile(changed) == GOOD + + assert [name for name, _ in engine.created[created:]] == ["cortex-challenge-bounty"] + host = engine.containers["cortex-challenge-bounty"]["Spec"]["HostConfig"] + assert host["Memory"] == 2048 * 1024 * 1024 + assert "BOUNTY_BACKEND_PUBLIC_URL=https://new.invalid" in engine.env("cortex-challenge-bounty") + created = len(engine.created) + await supervisor.reconcile(changed) + assert len(engine.created) == created # converged: no redeploy loop diff --git a/tests/test_network_e2e.py b/tests/test_network_e2e.py index 6c444e58a..78f56b736 100644 --- a/tests/test_network_e2e.py +++ b/tests/test_network_e2e.py @@ -190,7 +190,7 @@ def registry_file(tmp_path, *slugs): @pytest.mark.parametrize( "version,counts,payouts", [ - (1, (3, 0), (0.2, 0)), + (1, (3, 0), (0, 0)), (2, (0, 0), (0, 0)), (2, (1, 0), (0.03, 0)), (2, (2, 3), (0.06, 0.09)), @@ -259,7 +259,8 @@ async def test_bounty_container_weights_seal_and_validator_dispatch( await runtime.emitter.tick() chain.state = EpochState(13, 100, 105) assert await runtime.emitter.tick() == [12] - assert fake.calls == [("bounty", 12)] + # Algorithm 1 never asks a container: its legacy lattice is not a weight. + assert fake.calls == ([] if version == 1 else [("bounty", 12)]) leaves = runtime.gateway.store.leaves(12) assert {leaf.miner_hotkey for leaf in leaves if leaf.challenge_id == b"bounty"} == { row.hotkey for row in chain.rows @@ -269,7 +270,9 @@ async def test_bounty_container_weights_seal_and_validator_dispatch( for leaf in leaves if leaf.challenge_id == b"bounty" and leaf.miner_hotkey == miner_key ) - if not counts[0]: + if version == 1: + assert mine.score == NoScore(NoScoreReason.CHALLENGE_INTERNAL) + elif not counts[0]: assert mine.score == NoScore(NoScoreReason.NOT_ATTEMPTED) elif version == 3: assert mine.score == Score(10**12 * counts[0] // max(10, sum(counts))) From 0f80ec96bca6acb8c72033e50d718ab90b56432e Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:40:23 +0000 Subject: [PATCH 3/4] fix(challenges): recover interrupted rollouts and reverify sources - A supervisor restart after it set the serving container aside no longer deletes that backup. The next reconcile first restores it. - Labels and build provenance are re-checked against the current registry entry for every new container spec. A source edit with the same digest can no longer deploy an image that was never verified against that source. Only the canary is skipped for configuration-only changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/CHALLENGES.md | 7 ++++-- src/cortex/challenges/supervisor.py | 22 +++++++++++++++---- tests/challenges/test_supervisor.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 9566d7314..0268faa67 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -162,8 +162,11 @@ For each registry entry, the supervisor runs this loop every `poll_seconds`: restart the previous container unchanged. The refused digest and configuration are not retried until the channel or the registry entry moves. -A registry change that keeps the same digest, such as `env`, resources or limits, -redeploys through steps 5 and 6 without a new canary. The container carries a +A registry change that keeps the same digest, such as `env`, `source`, resources +or limits, redeploys through steps 2, 3, 5 and 6: labels and provenance are always +re-checked against the current entry, and only the canary is skipped. A supervisor +restart in the middle of step 5 restores the set-aside container before anything +else. The container carries a fingerprint of its full specification for this purpose. A managed container whose id is no longer in the registry is stopped and diff --git a/src/cortex/challenges/supervisor.py b/src/cortex/challenges/supervisor.py index 371584515..1fa3477ae 100644 --- a/src/cortex/challenges/supervisor.py +++ b/src/cortex/challenges/supervisor.py @@ -215,18 +215,32 @@ async def reconcile(self, entry: RegistryEntry) -> str: spec = self._spec(entry, digest, canary=False) fingerprint = spec["Labels"][CONFIG] deployed = await self._inspect(name) + if deployed is None and await self._inspect(f"{name}-previous") is not None: + # A restart mid-rollout left only the set-aside container: it is the last + # known-good service, so restore it instead of letting _rollout delete it. + await self._docker("POST", f"/containers/{name}-previous/rename", params={"name": name}) + deployed = await self._inspect(name) + LOG.warning( + "challenge %s recovered its container from an interrupted rollout", entry.id + ) if deployed is not None and deployed[0].get(CONFIG) == fingerprint: if not deployed[1]: await self._docker("POST", f"/containers/{name}/start") return digest if self.refused.get(entry.id) == fingerprint: raise SupervisorError(f"{entry.id}: {digest} was refused; waiting for a change") - # A configuration-only change reuses the running, already verified digest. + try: + # Every new spec re-checks labels and provenance against the current entry, + # so a source edit is verified even when the digest stays the same. + self.verify_labels(entry, labels) + if entry.attestation: + await self.verify_attestation(entry, digest) + except SupervisorError: + self.refused[entry.id] = fingerprint + raise + # A configuration-only change skips the canary: that digest already booted here. if deployed is None or deployed[0].get("io.cortex.challenge.digest") != digest: try: - self.verify_labels(entry, labels) - if entry.attestation: - await self.verify_attestation(entry, digest) canary = f"{name}-canary" await self._remove(canary) try: diff --git a/tests/challenges/test_supervisor.py b/tests/challenges/test_supervisor.py index 5d41dcc5a..ac4fe95ad 100644 --- a/tests/challenges/test_supervisor.py +++ b/tests/challenges/test_supervisor.py @@ -262,3 +262,36 @@ async def test_registry_change_with_the_same_digest_redeploys_without_a_canary(w created = len(engine.created) await supervisor.reconcile(changed) assert len(engine.created) == created # converged: no redeploy loop + + +async def test_restart_mid_rollout_recovers_the_set_aside_container(world): + engine, _, supervisor = world + await supervisor.reconcile(registry()) + # The supervisor died after stopping and renaming the serving container. + engine.containers["cortex-challenge-bounty-previous"] = engine.containers.pop( + "cortex-challenge-bounty" + ) + engine.containers["cortex-challenge-bounty-previous"]["Running"] = False + engine.tags["stable"] = NEXT + engine.fail.add("start") + + with pytest.raises(SupervisorError, match="after rollout"): + await supervisor.reconcile(registry()) + + assert set(engine.containers) == {"cortex-challenge-bounty"} + assert engine.digest("cortex-challenge-bounty") == GOOD + assert engine.containers["cortex-challenge-bounty"]["Running"] + + +async def test_source_edit_with_the_same_digest_is_verified_before_rollout(world): + engine, attested, supervisor = world + await supervisor.reconcile(registry()) + created = len(engine.created) + + moved = registry(source="https://github.com/Other/bounty", memory_mib=2048) + with pytest.raises(SupervisorError, match="labels"): + await supervisor.reconcile(moved) + + assert len(engine.created) == created + assert engine.digest("cortex-challenge-bounty") == GOOD + assert engine.containers["cortex-challenge-bounty"]["Running"] From b59a14f42b0d8240bf59a2f6b542eb13e9f75ddd Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:49:49 +0000 Subject: [PATCH 4/4] fix(challenges): retry attestation outages instead of refusing A GitHub 403/429/5xx or network error on the attestation lookup is now a TransientError. The next poll retries it. Only a definitive answer (no provenance, bad labels, failed canary or rollout) is remembered as a refusal of that configuration. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/cortex/challenges/supervisor.py | 21 +++++++++++++++----- tests/challenges/test_supervisor.py | 30 ++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/cortex/challenges/supervisor.py b/src/cortex/challenges/supervisor.py index 1fa3477ae..aa085c9b7 100644 --- a/src/cortex/challenges/supervisor.py +++ b/src/cortex/challenges/supervisor.py @@ -29,6 +29,10 @@ class SupervisorError(Exception): """A refused image or failed rollout; the running container is left as it was.""" +class TransientError(SupervisorError): + """An outage (GitHub 429/5xx, network) that decides nothing; retried next poll.""" + + @dataclass(frozen=True) class SupervisorConfig: registry_file: Path @@ -91,11 +95,16 @@ async def verify_attestation(self, entry: RegistryEntry, digest: str) -> None: # ponytail: checks that GitHub holds build provenance for this digest in the source # repository, not the Sigstore bundle signature. Upgrade: verify the returned bundle # with sigstore-python (or `gh attestation verify`) before trusting a new digest. - response = await self.http.get( - f"https://api.github.com/repos/{entry.owner_repo}/attestations/{digest}", - headers={"accept": "application/vnd.github+json"}, - timeout=30, - ) + try: + response = await self.http.get( + f"https://api.github.com/repos/{entry.owner_repo}/attestations/{digest}", + headers={"accept": "application/vnd.github+json"}, + timeout=30, + ) + except httpx.HTTPError as error: + raise TransientError(f"{entry.id}: attestation lookup failed: {error}") from None + if response.status_code in {403, 429} or response.status_code >= 500: + raise TransientError(f"{entry.id}: attestation lookup HTTP {response.status_code}") if response.status_code != 200 or not response.json().get("attestations"): raise SupervisorError(f"{entry.id}: no build provenance for {digest}") @@ -235,6 +244,8 @@ async def reconcile(self, entry: RegistryEntry) -> str: self.verify_labels(entry, labels) if entry.attestation: await self.verify_attestation(entry, digest) + except TransientError: + raise except SupervisorError: self.refused[entry.id] = fingerprint raise diff --git a/tests/challenges/test_supervisor.py b/tests/challenges/test_supervisor.py index ac4fe95ad..4e66fcc67 100644 --- a/tests/challenges/test_supervisor.py +++ b/tests/challenges/test_supervisor.py @@ -8,7 +8,12 @@ import pytest from cortex.challenges.registry import parse_registry -from cortex.challenges.supervisor import Supervisor, SupervisorConfig, SupervisorError +from cortex.challenges.supervisor import ( + Supervisor, + SupervisorConfig, + SupervisorError, + TransientError, +) IMAGE = "ghcr.io/cortexlm/bounty" GOOD, NEXT, BROKEN = ("sha256:" + c * 64 for c in "abc") @@ -28,6 +33,7 @@ def __init__(self): self.containers: dict[str, dict] = {} self.created: list[tuple[str, dict]] = [] self.fail: set[str] = set() # "create" or "start" of the production container + self.github_outage = 0 # HTTP status the attestation API answers while down def handle(self, request: httpx.Request) -> httpx.Response: path = unquote(request.url.path).removeprefix("/v1.44") @@ -102,6 +108,8 @@ def env(self, name: str) -> list[str]: def network(engine: FakeEngine, attested: set[str]): def handle(request: httpx.Request) -> httpx.Response: + if request.url.host == "api.github.com" and engine.github_outage: + return httpx.Response(engine.github_outage) if request.url.host == "api.github.com": digest = request.url.path.rsplit("/", 1)[1] return httpx.Response(200, json={"attestations": [{}] if digest in attested else []}) @@ -295,3 +303,23 @@ async def test_source_edit_with_the_same_digest_is_verified_before_rollout(world assert len(engine.created) == created assert engine.digest("cortex-challenge-bounty") == GOOD assert engine.containers["cortex-challenge-bounty"]["Running"] + + +@pytest.mark.parametrize("status", [429, 503]) +async def test_github_outage_is_retried_not_remembered_as_a_refusal(world, status): + engine, _, supervisor = world + await supervisor.reconcile(registry()) + changed = registry(memory_mib=2048) + engine.github_outage = status + + with pytest.raises(TransientError): + await supervisor.reconcile(changed) + assert ( + engine.containers["cortex-challenge-bounty"]["Spec"]["HostConfig"]["Memory"] == 1024 * 2**20 + ) + + engine.github_outage = 0 + await supervisor.reconcile(changed) + assert ( + engine.containers["cortex-challenge-bounty"]["Spec"]["HostConfig"]["Memory"] == 2048 * 2**20 + )