diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3e4a727 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Check text out with LF on every platform. Pinned digests (schema, manifests, +# native sources) are computed over the committed bytes, so a CRLF checkout on +# Windows would not match them. +* text=auto eol=lf + +*.png binary +*.ico binary +*.icns binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c12bc13..5c73625 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,31 @@ -name: coinfly validation +# CI validates source contracts. It does not sign, publish, enable live orders, +# certify regional eligibility or substitute for native release evidence. +name: CI + on: + push: + branches: [main] + pull_request: workflow_dispatch: - # push: - # pull_request: + permissions: contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + UV_VERSION: "0.13.1" + PYTHON_VERSION: "3.14.8" + # Use uv-managed CPython on every runner. A preinstalled runner Python can link + # an older SQLite (the Windows python.org build ships 3.50.4) than the 3.51.3 + # minimum; uv's managed builds bundle a current one. + UV_MANAGED_PYTHON: "1" + jobs: contracts: + name: Contracts (${{ matrix.os }}) strategy: fail-fast: false matrix: @@ -14,21 +33,27 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 25 steps: - - uses: actions/checkout@v5 - - uses: astral-sh/setup-uv@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 (no floating major tags) with: - version: "0.12.15" - python-version: "3.14.7" - enable-cache: true - - run: uv sync --locked --extra desktop --extra research --extra training + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + - name: Install locked dependencies + run: uv sync --locked --extra desktop --extra research --extra training - name: Verify linked SQLite requirement run: uv run python -c "import sqlite3; assert sqlite3.sqlite_version_info >= (3,51,3), sqlite3.sqlite_version" - - run: uv run ruff check src tests scripts packaging dev.py - - run: uv run mypy src/coinfly - - run: uv run python -m pytest -q + - name: Lint + run: uv run ruff check src tests scripts packaging dev.py + - name: Type-check + run: uv run mypy src/coinfly + - name: Test + run: uv run python -m pytest -q - name: Verify fail-closed doctor diagnostics run: uv run python scripts/verify_doctor.py - - run: uv build --no-sources + - name: Build wheel and sdist + run: uv build --no-sources - name: Inspect wheel payload run: uv run python scripts/verify_artifact.py - name: Compile macOS ARM64 native sources with vault access disabled @@ -46,16 +71,20 @@ jobs: run: | cmake -S native -B build/native -A x64 cmake --build build/native --config Release + dependencies: + name: Dependency audit runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v5 - - uses: astral-sh/setup-uv@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - version: "0.12.15" - python-version: "3.14.7" - - run: uv sync --locked --extra desktop --extra research --extra training - - run: uv run pip-audit --local --skip-editable -# CI validates source contracts. It does not sign, publish, enable live orders, -# certify regional eligibility or substitute for native release evidence. + persist-credentials: false + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 (no floating major tags) + with: + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + - name: Install locked dependencies + run: uv sync --locked --extra desktop --extra research --extra training + - name: Audit installed packages + run: uv run pip-audit --local --skip-editable diff --git a/Makefile b/Makefile index f5dda52..9376015 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,6 @@ github-bootstrap: @$(MAKE) github-repo-create @$(MAKE) github-verify -# Trigger the manual-only validation workflow (.github/workflows/ci.yml) on GitHub. +# Trigger the validation workflow (.github/workflows/ci.yml) on main manually; it also runs on pushes and PRs. github-run-ci: github-check gh workflow run ci.yml --repo $(GITHUB_FULL) --ref main \ No newline at end of file diff --git a/README.md b/README.md index 9cd12d9..bce90a4 100644 --- a/README.md +++ b/README.md @@ -1,125 +1,133 @@ -# Coinfly +

+ Coinfly +

+ +

Coinfly

+ +

+ Paper-trading research desktop that drives the full MaleCNS fruit-fly connectome with crypto market data. +

+ +

+ CI + MIT License + Python 3.11+ + Platforms +

+ +> [!WARNING] +> **Experimental research software.** Coinfly is shared as-is for research and +> education. There is no guarantee that it works, is stable or will be updated, +> and no claim that it trades profitably. Paper execution is the default and live +> trading is closed. This is not financial advice. Use it at your own risk. + +## Overview + +Coinfly feeds crypto order-book data into the complete retained **MaleCNS v1.0** +male fruit-fly connectome (166,700 neurons, 25,582,938 edges, no pruning) and lets +identified descending neurons propose paper trades. A separate worker process owns +the autonomous loop, exact accounting, risk limits and isolated paper execution; +the desktop app only sends commands and shows status. + +## Highlights + +- **The whole graph, no shortcuts.** A native C++ spiking kernel runs every retained + neuron. A HOLD is never replaced by a scripted trade, and no LLM policy or hidden + profit rule selects actions. +- **Exact, durable accounting.** Decimal quantities, order intent persisted before + dispatch, and a fresh SQLite ledger pinned by schema digests. +- **Fail-closed execution.** Stale data, sequence gaps and silent feeds block + decisions. Unknown order outcomes halt execution until reconciliation and are + never blindly retried. +- **Risk can veto, never trade.** The risk guard may reject an order; it never + chooses a different one. +- **Labeled market data.** Coinbase public REST/WebSocket (no credentials), recorded + replays or synthetic fixtures, each labeled end to end. -**1.0.0** · **com.devsolux.coinfly** · Publisher: **devsolux** · [MIT License](LICENSE) +## How it works -> **Experimental.** Coinfly is shared as-is for research and education. There is -> no guarantee that it works, is stable or will be updated, and no claim that it -> trades profitably. It is not financial advice. Use it at your own risk. +```mermaid +flowchart LR + Observe --> Sense --> Propagate --> Decode --> Map --> Admit --> Execute + Execute -. research learning only .-> Reinforce +``` -Coinfly is desktop trading research software. It feeds crypto market data into -the full retained **MaleCNS v1.0** male fruit-fly connectome (166,700 neurons, -25,582,938 edges, no pruning) and lets identified output neurons propose paper -trades. An independent worker owns the autonomous loop, exact accounting, risk -limits and isolated paper execution; the desktop only sends commands and shows -status. Targets: **macOS ARM64, Windows AMD64 and Linux AMD64**. +| Stage | What happens | +| --- | --- | +| Observe | Read an order book from a labeled source. | +| Sense | Four fixed features become an 8-bit RGB frame delivered through a photoreceptor proxy (an engineered input, not a market model). | +| Propagate | The native kernel runs the whole graph. | +| Decode | A fixed decoder reads the DNp20 (left/right) and DNpe017 descending neurons as BUY, SELL or HOLD. | +| Map | A fixed mapping moves only the chosen instrument's target weight, by at most the configured step. | +| Admit | An exact planner, turnover guard and risk actor check capital, exposure, drawdown and loss limits. | +| Execute | The intent is stored, filled in the paper book and reconciled. | +| Reinforce | *Research learning only:* paper profit/loss stimulates two identified dopamine cells (PAM11, PPL101), and a bounded three-factor rule adjusts existing KC→MBON synapses. | -## How it works +Reinforcement is an engineered input. It does not model pain, pleasure or +consciousness, and profitable learning is unverified. See the +[autonomous lifecycle](docs/autonomous-lifecycle.md) for the full loop. -Each worker tick runs these separate stages for every strategy-owned slot: - -1. **Observe** – read an order book from a labeled source: Coinbase public - REST/WebSocket, a recorded replay file, or a synthetic test fixture. Stale - data, sequence gaps and silent feeds block decisions. -2. **Sense** – four fixed features (mid offset, spread, depth imbalance, one-step - return) become exact integer ticks, are scaled only with statistics fitted on - earlier data, and are rendered as an 8-bit RGB frame. A photoreceptor proxy - delivers that frame to the graph. This is an engineered input, not a market model. -3. **Propagate** – the native C++ spiking kernel runs the whole graph. -4. **Decode** – a fixed decoder reads the DNp20 (left/right) and DNpe017 - descending neurons as BUY, SELL or HOLD. A HOLD is never replaced by a - scripted trade. -5. **Map** – a fixed mapping moves only the chosen instrument's target weight, - by at most the configured step (default 0.05 of strategy capital, never above - the 0.05 turnover limit). Paper perpetuals use a separate long/short mapping. -6. **Admit** – an exact Decimal planner, turnover guard and risk actor check - capital, exposure, drawdown and loss limits. Risk may reject an order; it - never chooses a different one. -7. **Execute** – the order intent is stored before dispatch, filled in the paper - book and reconciled. An unknown outcome halts execution until reconciliation; - orders are never blindly retried. -8. **Reinforce** *(research learning mode only)* – paper-ledger profit/loss - becomes engineered stimulation of two identified dopamine cells (PAM11, - PPL101). A bounded three-factor rule changes only existing KC→MBON synapses. - This does not model pain, pleasure or consciousness. - -Baseline and frozen-checkpoint controllers do not learn. Profitable learning is -unverified. Details: [autonomous lifecycle](docs/autonomous-lifecycle.md), -[dataset bootstrap](docs/malecns-bootstrap.md), [training](docs/malecns-training.md) -and [plasticity kernel](docs/malecns-rstdp-kernel.md). - -## What is supported - -| Area | Supported | Not supported or not yet qualified | +## Project status + +| Area | Available | Not yet supported or qualified | | --- | --- | --- | -| Platforms | macOS ARM64, Windows AMD64, Linux AMD64 from source; Python ≥3.11 with SQLite ≥3.51.3 | Latest recorded run is macOS ARM64 lint, type checks, tests and package build ([validation evidence](docs/evidence-matrix.md)); no native desktop smoke test, full-graph run or Windows/Linux native run is recorded | -| Execution | Paper (SANDBOX) trading with an exact local ledger: spot and synthetic perpetuals | Real-money trading; live mode is closed by default | -| Venue | Coinbase public market data (no credentials); live Advanced Trade spot code path | Live spot qualification (enrollment, signing, account evidence); live derivatives; other exchanges | -| Market data | Coinbase public REST and WebSocket level2, JSON-lines replay, labeled synthetic fixtures | — | -| Neural controllers | Full preserved graph in baseline inference, frozen checkpoint, or single-slot paper research learning (`malecns-risk-adjusted-experiment-v2`/`-v3`) | Separate NeuralHost process; model promotion workflow; any validated trading result | -| Research tooling | Optuna studies with paired frozen controls; training and plasticity benchmarks | No development-fold or holdout result has been produced | -| Regions | GLOBAL and GERMANY policies through the capability resolver | Verified regional entitlements | +| Execution | Paper (SANDBOX) spot and synthetic perpetuals with an exact local ledger | Real-money trading; live mode is closed by default | +| Market data | Coinbase public REST and WebSocket level2, JSON-lines replay, synthetic fixtures | Other exchanges | +| Controllers | Full-graph baseline inference, frozen checkpoints, single-slot paper research learning | Any validated trading result | +| Platforms | macOS ARM64, Windows AMD64 and Linux AMD64 from source | Native desktop smoke tests and native runs on every target | | Credentials | macOS Keychain, Windows Credential Manager, Linux Secret Service; no plaintext fallback | Linux native execution-vault bridge | -| Desktop | Tk/CustomTkinter UI in English and German | Packaged smoke tests on every target | | Storage | Fresh 1.0.0 SQLite ledger pinned by schema digests | Migration or import of older or modified ledgers | -The full matrix is in [implementation status](docs/implementation-status.md) and -[venue capabilities](docs/venues/capability-matrix.md). +Details: [implementation status](docs/implementation-status.md), +[venue capabilities](docs/venues/capability-matrix.md) and the +[latest validation evidence](docs/evidence-matrix.md). -## Native development +## Getting started -Use native Python ≥3.11 linked to **SQLite ≥3.51.3**, with Tk, uv and the -locked dependencies. A native C++ toolchain and CMake are needed for the explicit -local research kernel build, not for ordinary application startup. +**Requirements:** Python ≥3.11 linked to SQLite ≥3.51.3 with Tk, +[uv](https://docs.astral.sh/uv/), and CMake with a C++ toolchain for the native kernel. ```sh -python dev.py setup -python dev.py prepare -python dev.py kernel -python dev.py run +python dev.py setup # create the locked environment +python dev.py prepare # download and verify the MaleCNS v1.0 dataset (about 1.1 GB on first use) +python dev.py kernel # build the native spiking kernel +python dev.py run # start the desktop app ``` -On Windows, `py dev.py …` or `./dev.ps1 …` uses the same launcher. On macOS, -`./dev.sh` remains a convenience entry point. The portable launcher selects the -native virtual-environment layout and rejects unsupported CPU/OS combinations. - -Preparation verifies all retained neurons and edges against pinned source/array -identities (about 1.1 GB of source downloads on first use). Application data -lives under the native account's `~/.devsolux/coinfly` root; development workers -have checkout-specific ledgers and authenticated local IPC. Paper execution never -acquires exchange-mutation authority. +On Windows use `py dev.py …` or `./dev.ps1 …`; on macOS `./dev.sh …` is +equivalent. In **Overview**, configure the `malecns-autonomous` strategy (capital, +slot limit, feed and controller) and enable automation. The +[paper startup runbook](docs/runbooks/paper-startup.md) walks through each step. +Paper mode needs no exchange credentials, and application data lives under +`~/.devsolux/coinfly`. -In Overview, configure `malecns-autonomous`, its capital allocation, slot limit, -feed and baseline/frozen-checkpoint controller, then enable automation. The worker -selects eligible instruments and manages owned slots. A native kernel and valid -observations are required. Fixture controllers are for isolated tests, not -ordinary strategy choices. +## Development ```sh -python dev.py test -python dev.py test-ui -python dev.py check +python dev.py check # lint, strict type checks and the full test suite +python dev.py test # tests only; pytest arguments pass through +python dev.py test-ui # native desktop smoke tests (needs a desktop session) +python dev.py doctor # environment and production-readiness diagnostics ``` -## Fresh 1.0.0 format +CI runs lint, type checks, tests, package verification and a native compile on +macOS, Windows and Linux for every pull request and push to `main`. Project rules +for contributors and agents are in [AGENTS.md](AGENTS.md). -This is a fresh-install project, not an upgrade distribution. `storage/schema.sql` -and final component DDL are pinned by format/digest contracts. Old or modified -ledgers are rejected without automatic migration, deletion, import or reseeding. -Application version 1.0.0 is distinct from preserved scientific v2/v3 profiles. +macOS release builds (`./dev.sh build-unsigned`, `./dev.sh build`, optionally with +`--preflight`) stay closed until genuine production prerequisites are supplied. See +[desktop packaging](packaging/README.md) and +[macOS production builds](docs/devsh-production-builds.md). -## Delivery boundaries +## Documentation -No live credentials, orders or transfers were used in testing. Production -enrollment/evidence providers, Linux native execution signing and production -target qualification remain incomplete, so the default release path stays closed -until genuine prerequisites are supplied; no test evidence satisfies them. +The [documentation index](docs/README.md) covers operation, runbooks, venues and +architecture. Start with: -macOS production entry points are `./dev.sh build-unsigned` and `./dev.sh build`, -with optional `--preflight`. Both read reviewed publisher metadata from -`~/.devsolux/coinfly` by default and require the same production gates; missing -inputs never select development packaging. See [desktop packaging](packaging/README.md), -[setup and storage](docs/desktop-initialization-and-settings.md) and the -[documentation index](docs/README.md). +- [Paper startup without credentials](docs/runbooks/paper-startup.md) +- [Full-graph provenance and dataset bootstrap](docs/malecns-bootstrap.md) +- [Experimental training and evaluation](docs/malecns-training.md) +- [Safety invariants and supporting tests](docs/requirements/invariants.md) ## License diff --git a/docs/evidence-matrix.md b/docs/evidence-matrix.md index f2a11ee..7c99931 100644 --- a/docs/evidence-matrix.md +++ b/docs/evidence-matrix.md @@ -32,7 +32,7 @@ The 14 skips are expected on this host: | --- | --- | | Native desktop smoke tests (`test-ui`) | Not run | | Full MaleCNS graph (`prepare`, native kernel, full-graph paper run) | Not run; unit tests use small synthetic graphs | -| Windows AMD64 and Linux AMD64 native runs | Not run; the CI workflow is manual-only | +| Windows AMD64 and Linux AMD64 native runs | Not run locally; CI only compiles the native sources on each target | | Signed or unsigned production builds | Not run; see [macOS production builds](devsh-production-builds.md) | | Live accounts, orders, transfers or enrollment | Never used in testing | diff --git a/docs/implementation-status.md b/docs/implementation-status.md index cb3d407..3c678b0 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -18,7 +18,7 @@ uses a fresh-install format. | Cancellation and account truth | A cancel acknowledgement does not release reservations; reconciliation establishes terminal facts; incomplete account evidence invalidates readiness; cash subtotal labeled as cash | Real venue qualification, scope/placement coverage verification and worst-case reconciliation testing | | Regional/derivative routing | Existing capability resolver and distinct product contracts; existing synthetic paper derivative model | Germany/Global entitlement evidence, route verification, live derivative gateway/engine and exact venue economics; enforcement of the declared `margin_mode` and `max_maintenance_utilization` strategy fields | | Secrets | Explicit Keychain, Credential Manager and Linux Secret Service setup backends; no plaintext fallback | Linux native execution-vault bridge; genuine signing/enrollment qualification on all targets | -| Packaging and CI | Existing macOS signing builder; common desktop/worker entry point; Windows/Linux paper development spec; three-target CI matrix (manual `workflow_dispatch` only) | Actual locked CI runs, signed Windows/Linux production installers and platform-specific native qualification | +| Packaging and CI | Existing macOS signing builder; common desktop/worker entry point; Windows/Linux paper development spec; three-target CI matrix on pushes to `main`, pull requests and manual dispatch | A passing CI history on all three targets, signed Windows/Linux production installers and platform-specific native qualification | `python dev.py run` is explicitly local paper development. A verified dataset, local kernel, stored credential or successful read-only connection test does not diff --git a/docs/runbooks/paper-startup.md b/docs/runbooks/paper-startup.md index 0fd58d9..b904b85 100644 --- a/docs/runbooks/paper-startup.md +++ b/docs/runbooks/paper-startup.md @@ -120,6 +120,26 @@ intent cause `VENUE_SETTLEMENT`, never as a strategy trade. Both appear in `automation.status.settlements` as synthetic. None of this is presented as venue behavior. +Funding and expiry are booked only at a fresh mark: a two-sided book observed +no more than the paper venue's book age limit (`book_max_age`, 5 s, the same +bound a resting order needs to fill) before the settlement check. If the +contract's book is older (for example, the venue stopped quoting it or the +feed or worker was down at the boundary or expiry) or missing (no book since a +restart), nothing is booked and the event stays due. It is retried on every +later tick until a fresh book arrives. A crossed funding boundary is then +charged for the quantity held when it was crossed, at that fresh mark. An +expiring position, ledger and risk books stay open until then. Each deferral +appears once in `automation.status.settlements` as `PAPER_FUNDING_DEFERRED` or +`PAPER_EXPIRY_DEFERRED`, with reason `..._MARK_STALE` or `..._MARK_MISSING` and +the mark's observation time. A due funding boundary whose position closed +before any fresh mark priced it is not charged; it is reported as +`PAPER_FUNDING_NOT_CHARGED` (`PAPER_FUNDING_POSITION_CLOSED`). Due funding +boundaries live in the worker's memory and, like the settlement clock, do not +survive a restart. A due expiry does, because it is checked against the +contract's expiry on every tick. No substitute mark is ever used, and new +targets are refused (`POSITION_MARK_UNVERIFIED`) while any open position lacks +a fresh mark. + ## 5. Pause, stop, restart The development worker runs in the background and keeps running after the diff --git a/scripts/build_release_archive.py b/scripts/build_release_archive.py index 0976fdd..1cbb26b 100644 --- a/scripts/build_release_archive.py +++ b/scripts/build_release_archive.py @@ -46,8 +46,9 @@ def main() -> None: entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) entry.compress_type = zipfile.ZIP_DEFLATED entry.create_system = 3 - # Keep launchers executable after extraction, with normalized modes. - permissions = 0o755 if path.stat().st_mode & 0o111 else 0o644 + # Keep launchers executable after extraction, with normalized modes. Windows + # file modes carry no execute bit, so shell launchers are named explicitly. + permissions = 0o755 if path.stat().st_mode & 0o111 or path.suffix == ".sh" else 0o644 entry.external_attr = (0o100000 | permissions) << 16 archive.writestr(entry, data) source_files += 1 diff --git a/src/coinfly/bootstrap/development.py b/src/coinfly/bootstrap/development.py index 4340f61..6fd64cb 100644 --- a/src/coinfly/bootstrap/development.py +++ b/src/coinfly/bootstrap/development.py @@ -166,7 +166,7 @@ def install(self, envelope: bytes | None, *, now: datetime, prepare_graph(staging, controls, progress) dataset = self._verified(staging, controls, progress) for name in ("graph.npz", "normalized/neurons.feather"): - with (staging / name).open("rb") as source: + with (staging / name).open("r+b") as source: # Windows flushes only writable handles os.fsync(source.fileno()) _fsync_directory(staging / "normalized") _fsync_directory(staging) diff --git a/src/coinfly/development.py b/src/coinfly/development.py index b84b4c8..1a22452 100644 --- a/src/coinfly/development.py +++ b/src/coinfly/development.py @@ -62,7 +62,8 @@ def development_socket(checkout: Path) -> Path: from coinfly.ipc.windows_native import pipe_path, process_sid return Path(pipe_path(process_sid(os.getpid()), checkout=identity)) - return default_socket_path().with_name(f"development-{identity}.sock") + else: + return default_socket_path().with_name(f"development-{identity}.sock") # How long a launcher waits for a stale worker to retire (automation stopped, diff --git a/src/coinfly/paths.py b/src/coinfly/paths.py index 3444637..875d161 100644 --- a/src/coinfly/paths.py +++ b/src/coinfly/paths.py @@ -50,9 +50,10 @@ def native_user_home() -> Path: if function(None, 0x0028, None, 0, buffer) != 0 or not buffer.value: raise DomainError("NATIVE_HOME_UNAVAILABLE") return Path(buffer.value) - import pwd + else: + import pwd - return Path(pwd.getpwuid(os.getuid()).pw_dir) + return Path(pwd.getpwuid(os.getuid()).pw_dir) def is_link(path: Path) -> bool: diff --git a/src/coinfly/research/optimization.py b/src/coinfly/research/optimization.py index a51f8ea..59d1a8a 100644 --- a/src/coinfly/research/optimization.py +++ b/src/coinfly/research/optimization.py @@ -544,13 +544,16 @@ def evaluate_fold(resources: StudyResources, fold: Fold, profile: TrainingProfil def _max_rss_bytes() -> int | None: """Process peak RSS in bytes (Linux reports KiB, macOS bytes); None where unmeasured.""" - try: - import resource # POSIX only; Windows hosts report None rather than a guess. - - peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - except (ImportError, AttributeError, OSError, ValueError): - return None - return int(peak) if sys.platform == "darwin" else int(peak) * 1024 + if sys.platform == "win32": + return None # no POSIX resource module: reported as unmeasured rather than a guess + else: + try: + import resource + + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + except (ImportError, AttributeError, OSError, ValueError): + return None + return int(peak) if sys.platform == "darwin" else int(peak) * 1024 def comparable_metrics(evaluation: dict[str, Any]) -> dict[str, float | None]: diff --git a/src/coinfly/worker/paper_engine.py b/src/coinfly/worker/paper_engine.py index 07c65ff..f8830d7 100644 --- a/src/coinfly/worker/paper_engine.py +++ b/src/coinfly/worker/paper_engine.py @@ -109,6 +109,10 @@ def __init__( self.config, self.context = config, context self.writer, self.reducer, self.interlock = writer, reducer, interlock self._settled_events: set[str] = set() + # Funding boundaries crossed but not yet charged: event id -> (instrument, interval start, + # quantity held at the crossing, contract multiplier); and the deferral reason last reported per event. + self._funding_due: dict[str, tuple[str, datetime, Decimal, Decimal]] = {} + self._deferrals: dict[str, str] = {} self._settlement_cursor: datetime | None = None self._ready, self._on_change = ready, on_change self._lock = threading.RLock() @@ -598,68 +602,104 @@ def settle(self, now: datetime) -> tuple[dict[str, Any], ...]: The paper perpetual book applies funding on a fixed schedule per declared subtype (``PERPETUAL_NO_EXPIRY``: every 8 h at 00/08/16 UTC; ``PERPETUAL_STYLE_FUTURE``: hourly) at the declared synthetic rate - (``PAPER_FUNDING_RATE_PER_INTERVAL``, long pays when positive) against the - last mark, once per interval (idempotent by event id), journaled as - ``PAPER_FUNDING``. A ``DATED_FUTURE`` position is cash-settled at its - catalog expiry (``PAPER_EXPIRY_SETTLEMENT``): the margin position and the - execution ledger are both closed at the mark by one synthetic, fee-free - fill, so lots, positions, journal and risk books agree. Every event is - labeled synthetic; none is venue evidence. + (``PAPER_FUNDING_RATE_PER_INTERVAL``, long pays when positive), once per + interval boundary the clock crosses (idempotent by event id), for the + quantity held when it was crossed, journaled as ``PAPER_FUNDING``. A + ``DATED_FUTURE`` position is cash-settled at its catalog expiry + (``PAPER_EXPIRY_SETTLEMENT``): the margin position and the execution + ledger are both closed at the mark by one synthetic, fee-free fill, so + lots, positions, journal and risk books agree. Every event is labeled + synthetic; none is venue evidence. + + Both are booked only at a fresh mark: the mid of a two-sided book observed + no more than the matcher's ``book_max_age`` (the age past which no paper + order fills) before ``now``. Without one the event stays due and is + retried on every later call, and a ``PAPER_FUNDING_DEFERRED`` or + ``PAPER_EXPIRY_DEFERRED`` record with reason ``..._MARK_STALE`` or + ``..._MARK_MISSING`` is returned once per reason. No other mark is ever + substituted. A due funding boundary whose position closed before a fresh + mark priced it cannot be charged (the margin book holds funding only + against an open position): it is dropped and reported as + ``PAPER_FUNDING_NOT_CHARGED`` (``PAPER_FUNDING_POSITION_CLOSED``). All of it commits together with the matcher snapshot. If that commit fails - the in-memory book, the settled-event ids and the settlement cursor are - rolled back, so the next call retries the same due events. + the in-memory book, the settled-event ids, the due and reported events and + the settlement cursor are rolled back, so the next call retries the same + due events. """ with self.interlock.lock, self._lock: if self._retired or self.matcher.margin is None: return () margin = self.matcher.margin events: list[dict[str, Any]] = [] + notices: list[dict[str, Any]] = [] previous = self._settlement_cursor self._settlement_cursor = now if previous is None or now <= previous: return () # no retroactive funding: boundaries are charged when the clock crosses them snapshot: dict[str, object] | None = None # matcher state before the first mutation added: list[str] = [] + due, reported = dict(self._funding_due), dict(self._deferrals) try: + held = {item.instrument_id: item for item in margin.positions} + contracts = {item.instrument_id: item for item in self._instruments} + for instrument, position in held.items(): + contract = contracts.get(instrument) + interval = PAPER_FUNDING_INTERVALS.get(contract.declared_subtype) if contract is not None else None + if contract is None or interval is None: + continue + boundary = _interval_start(now, interval) + while boundary > previous: + event_id = f"paper-funding:{instrument}:{boundary.strftime('%Y%m%dT%H%M%SZ')}" + if event_id not in self._settled_events: + self._funding_due.setdefault(event_id, (instrument, boundary, position.quantity, contract.contract_multiplier)) + boundary -= interval + for event_id, (instrument, boundary, quantity, multiplier) in sorted( + self._funding_due.items(), key=lambda item: item[1][1], reverse=True): + if instrument not in held: + del self._funding_due[event_id] + self._deferrals.pop(event_id, None) + notices.append({"kind": "PAPER_FUNDING_NOT_CHARGED", "event_id": event_id, "instrument": instrument, + "reason": "PAPER_FUNDING_POSITION_CLOSED", "interval_start": boundary.isoformat(), "synthetic": True}) + continue + mark, observed = self._settlement_mark(instrument, now) + if mark is None: + self._defer(notices, "PAPER_FUNDING", event_id, instrument, observed, interval_start=boundary.isoformat()) + continue + if snapshot is None: + snapshot = self.matcher.dump_state() + payment = -(PAPER_FUNDING_RATE_PER_INTERVAL * quantity * mark * multiplier) + applied = margin.apply_funding(event_id, instrument, payment) + self._settled_events.add(event_id) + added.append(event_id) + del self._funding_due[event_id] + self._deferrals.pop(event_id, None) + if applied: + events.append({"kind": "PAPER_FUNDING", "event_id": event_id, "instrument": instrument, "payment": str(payment), + "quantity": str(quantity), "mark": str(mark), "rate": str(PAPER_FUNDING_RATE_PER_INTERVAL), + "synthetic": True, "interval_start": boundary.isoformat()}) for position in margin.positions: instrument = position.instrument_id - contract = next((item for item in self._instruments if item.instrument_id == instrument), None) - depth = self._depth.get(instrument) - if contract is None or depth is None or not depth[0] or not depth[1]: + contract = contracts.get(instrument) + if contract is None or contract.declared_subtype is not ProductSubtype.DATED_FUTURE \ + or contract.expiry is None or now < contract.expiry: continue - mark = (depth[0][0].price + depth[1][0].price) / 2 - subtype = contract.declared_subtype - interval = PAPER_FUNDING_INTERVALS.get(subtype) - if interval is not None: - boundary = _interval_start(now, interval) - while boundary > previous: - stamp = boundary.strftime("%Y%m%dT%H%M%SZ") - event_id = f"paper-funding:{instrument}:{stamp}" - if event_id not in self._settled_events: - if snapshot is None: - snapshot = self.matcher.dump_state() - payment = -(PAPER_FUNDING_RATE_PER_INTERVAL * position.quantity * mark * contract.contract_multiplier) - applied = margin.apply_funding(event_id, instrument, payment) - self._settled_events.add(event_id) - added.append(event_id) - if applied: - events.append({"kind": "PAPER_FUNDING", "event_id": event_id, "instrument": instrument, "payment": str(payment), - "rate": str(PAPER_FUNDING_RATE_PER_INTERVAL), "synthetic": True, - "interval_start": boundary.isoformat()}) - boundary -= interval - expiry = getattr(contract, "expiry", None) - if subtype is ProductSubtype.DATED_FUTURE and expiry is not None and now >= expiry: - event_id = f"{PAPER_EXPIRY_PREFIX}{instrument}:{expiry.strftime('%Y%m%dT%H%M%SZ')}" - if event_id not in self._settled_events: - if snapshot is None: - snapshot = self.matcher.dump_state() - realized = margin.apply_fill(instrument, -position.quantity, mark, multiplier=contract.contract_multiplier, - fee=Decimal(0), reduce_only=True) - self._settled_events.add(event_id) - added.append(event_id) - events.append({"kind": "PAPER_EXPIRY_SETTLEMENT", "event_id": event_id, "instrument": instrument, - "quantity": str(position.quantity), "mark": str(mark), "realized": str(realized), "synthetic": True}) + event_id = f"{PAPER_EXPIRY_PREFIX}{instrument}:{contract.expiry.strftime('%Y%m%dT%H%M%SZ')}" + if event_id in self._settled_events: + continue + mark, observed = self._settlement_mark(instrument, now) + if mark is None: + self._defer(notices, "PAPER_EXPIRY", event_id, instrument, observed) + continue + if snapshot is None: + snapshot = self.matcher.dump_state() + realized = margin.apply_fill(instrument, -position.quantity, mark, multiplier=contract.contract_multiplier, + fee=Decimal(0), reduce_only=True) + self._settled_events.add(event_id) + self._deferrals.pop(event_id, None) + added.append(event_id) + events.append({"kind": "PAPER_EXPIRY_SETTLEMENT", "event_id": event_id, "instrument": instrument, + "quantity": str(position.quantity), "mark": str(mark), "realized": str(realized), "synthetic": True}) incidents: list[str] = [] if events: def save(connection: sqlite3.Connection) -> None: @@ -685,13 +725,39 @@ def save(connection: sqlite3.Connection) -> None: if snapshot is not None: self._matcher = PaperMatcher.restore(self.context, self._instruments, snapshot) self._settled_events.difference_update(added) + self._funding_due, self._deferrals = due, reported self._settlement_cursor = previous raise for incident in incidents: self.interlock.provisional_fence(self.context.authority_id, incident) - if events: + if events or notices: self._notify() - return tuple(events) + return (*events, *notices) + + def _settlement_mark(self, instrument: str, now: datetime) -> tuple[Decimal | None, datetime | None]: + """The mid a settlement may book at ``now``, and when its book was observed. + + No mid without a two-sided book (then no observation time either), or when + that book is older than the matcher's ``book_max_age``. + """ + depth = self._depth.get(instrument) + if depth is None or not depth[0] or not depth[1]: + return None, None + bids, asks, observed = depth + if now - observed > self.matcher.book_max_age: + return None, observed + return (bids[0].price + asks[0].price) / 2, observed + + def _defer(self, notices: list[dict[str, Any]], kind: str, event_id: str, instrument: str, + observed: datetime | None, **detail: str) -> None: + """Report a settlement event left due for want of a fresh mark, once per reason.""" + reason = f"{kind}_MARK_MISSING" if observed is None else f"{kind}_MARK_STALE" + if self._deferrals.get(event_id) == reason: + return + self._deferrals[event_id] = reason + notices.append({"kind": f"{kind}_DEFERRED", "event_id": event_id, "instrument": instrument, "reason": reason, + "mark_observed_at": observed.isoformat() if observed is not None else None, + "max_mark_age_seconds": str(self.matcher.book_max_age.total_seconds()), **detail, "synthetic": True}) @exact_decimal def _settle_expiry_tx(self, connection: sqlite3.Connection, event: dict[str, Any]) -> FillResult: diff --git a/tests/architecture/test_context_boundaries.py b/tests/architecture/test_context_boundaries.py index 675dce3..e041c6c 100644 --- a/tests/architecture/test_context_boundaries.py +++ b/tests/architecture/test_context_boundaries.py @@ -76,7 +76,7 @@ def test_active_imports_do_not_include_ambient_sdk_or_dotenv(self): root = Path(__file__).resolve().parents[2] / "src" / "coinfly" bad = {"dotenv", "pydantic_settings", "coinbase_agentkit", "coinbase.rest"} for path in root.rglob("*.py"): - for node in ast.walk(ast.parse(path.read_text())): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): modules = ([a.name for a in node.names] if isinstance(node, ast.Import) else [node.module or ""] if isinstance(node, ast.ImportFrom) else []) for module in modules: @@ -87,7 +87,7 @@ def test_ui_cannot_import_worker_database_or_broker(self): root = Path(__file__).resolve().parents[2] / "src" / "coinfly" / "ui" forbidden = ("sqlite3", "coinfly.storage", "coinfly.execution", "coinfly.security", "coinfly.venues") for path in root.rglob("*.py"): - for node in ast.walk(ast.parse(path.read_text())): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): modules = ([a.name for a in node.names] if isinstance(node, ast.Import) else [node.module or ""] if isinstance(node, ast.ImportFrom) else []) self.assertFalse(any(m == f or m.startswith(f + ".") for m in modules for f in forbidden), diff --git a/tests/bootstrap/test_development_bootstrap.py b/tests/bootstrap/test_development_bootstrap.py index c7a61c5..8852184 100644 --- a/tests/bootstrap/test_development_bootstrap.py +++ b/tests/bootstrap/test_development_bootstrap.py @@ -2,6 +2,7 @@ import asyncio import hashlib +import sys import threading from datetime import datetime, timezone from pathlib import Path @@ -149,7 +150,8 @@ def test_development_state_and_socket_are_isolated_and_live_is_rejected(tmp_path paths = WorkerPaths(root) assert paths.database.is_relative_to(application.storage / "development") assert not (tmp_path / "checkout").exists() - assert development_socket(tmp_path) != default_socket_path() + if sys.platform != "win32": # Windows has no Unix default socket; checkouts still differ below + assert development_socket(tmp_path) != default_socket_path() assert development_socket(tmp_path) != development_socket(tmp_path / "another-checkout") config = default_config().model_copy(update={"environment": Environment.LIVE, "run_profile": RunProfile.STANDARD}) with pytest.raises(DomainError, match="DEVELOPMENT_PAPER_ONLY"): diff --git a/tests/bootstrap/test_development_distribution.py b/tests/bootstrap/test_development_distribution.py index 6da259b..f13240c 100644 --- a/tests/bootstrap/test_development_distribution.py +++ b/tests/bootstrap/test_development_distribution.py @@ -83,7 +83,7 @@ def test_frozen_development_uses_bundled_worker_without_checkout(packaged, tmp_p desktop = tmp_path / "coinfly.app/Contents/MacOS/coinfly" desktop.parent.mkdir(parents=True) - worker = desktop.with_name("coinfly-worker") + worker = desktop.with_name("coinfly-worker.exe" if sys.platform == "win32" else "coinfly-worker") worker.touch() monkeypatch.setattr(sys, "executable", str(desktop)) assert development.checkout_root() == desktop.parent diff --git a/tests/conftest.py b/tests/conftest.py index 14a7d1f..7487587 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Test-process network boundary: local IPC only, no exchange connectivity.""" +import ipaddress import socket import pytest @@ -11,26 +12,36 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) def deny_internet_sockets(monkeypatch): - # Synthetic transport tests inject in-process responses. AF_UNIX remains - # available for the real worker IPC tests. Neither live nor sandbox HTTP is - # needed by this suite, so reject Internet sockets before DNS or connection. + # Synthetic transport tests inject in-process responses. AF_UNIX and loopback + # remain available for the real worker IPC tests (and for Windows asyncio, + # which wakes its event loop through a loopback socket pair). Neither live nor + # sandbox HTTP is needed by this suite, so reject Internet sockets before DNS + # or connection. def blocked(*_args, **_kwargs): raise AssertionError("Tests must use synthetic transports; Internet access is forbidden") + def internet(sock, address): + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + try: + return not ipaddress.ip_address(address[0]).is_loopback + except (TypeError, ValueError, IndexError): + return True + connect, connect_ex, sendto = socket.socket.connect, socket.socket.connect_ex, socket.socket.sendto def local_connect(sock, address): - if sock.family in (socket.AF_INET, socket.AF_INET6): + if internet(sock, address): return blocked() return connect(sock, address) def local_connect_ex(sock, address): - if sock.family in (socket.AF_INET, socket.AF_INET6): + if internet(sock, address): return blocked() return connect_ex(sock, address) def local_sendto(sock, *args): - if sock.family in (socket.AF_INET, socket.AF_INET6): + if internet(sock, args[-1]): return blocked() return sendto(sock, *args) diff --git a/tests/ipc/test_handshake.py b/tests/ipc/test_handshake.py index bcd0bca..42d1807 100644 --- a/tests/ipc/test_handshake.py +++ b/tests/ipc/test_handshake.py @@ -8,12 +8,13 @@ import errno import os +import sys import tempfile import threading import time from pathlib import Path from typing import Any -from unittest import TestCase +from unittest import TestCase, skipIf from coinfly import __application_id__, __version__ from coinfly.ipc.auth import OwnerOnlySandboxPolicy @@ -25,6 +26,7 @@ from tests.worker.test_automation_lifecycle import TestOnlyBootstrap +@skipIf(sys.platform == "win32", "Unix-domain socket IPC; the Windows transport uses named pipes") class HandshakeTests(TestCase): def setUp(self) -> None: self.directory = tempfile.TemporaryDirectory(prefix="cf-hello-") diff --git a/tests/network/test_websocket_client.py b/tests/network/test_websocket_client.py index 2072276..d914a09 100644 --- a/tests/network/test_websocket_client.py +++ b/tests/network/test_websocket_client.py @@ -1,6 +1,6 @@ """RFC 6455 client: framing, masking rules, handshake validation, ping/pong, silence. -The server side is an in-process thread on an AF_UNIX socketpair; the test never +The server side is an in-process thread on a local socketpair; the test never opens an Internet socket (the suite forbids it), so ``socket.create_connection`` is replaced for the duration of each connection. """ @@ -65,7 +65,7 @@ class FakeServer: """Minimal WebSocket server on one end of a socketpair.""" def __init__(self, *, accept_correctly: bool = True, status: bytes = b"101 Switching Protocols") -> None: - self.server, self.client_end = socket.socketpair(socket.AF_UNIX) + self.server, self.client_end = socket.socketpair() # AF_UNIX, or loopback on Windows self.accept_correctly = accept_correctly self.status = status self.received: list[bytes] = [] diff --git a/tests/support.py b/tests/support.py new file mode 100644 index 0000000..f2e1097 --- /dev/null +++ b/tests/support.py @@ -0,0 +1,15 @@ +"""Shared test helpers that hold on every supported platform.""" + +import os + +from coinfly.ipc.auth import PeerIdentity, current_uid + + +def local_peer() -> PeerIdentity: + """This process as an IPC peer, as the platform transport identifies it. + + POSIX transports report uid/gid; the Windows named-pipe transport reports + the user's principal id for both. + """ + uid = current_uid() + return PeerIdentity(uid=uid, gid=os.getgid() if hasattr(os, "getgid") else uid, pid=os.getpid()) diff --git a/tests/test_macos_release.py b/tests/test_macos_release.py index 469952b..2c7e66f 100644 --- a/tests/test_macos_release.py +++ b/tests/test_macos_release.py @@ -5,6 +5,7 @@ import plistlib import runpy import subprocess +import sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -147,6 +148,7 @@ def test_explicit_invalid_publisher_never_falls_back(builder, tmp_path, failure) assert not (stage / "development-distribution").exists() +@pytest.mark.skipif(sys.platform == "win32", reason="macOS release tooling relies on POSIX rename over an empty directory") def test_development_initialization_is_idempotent_atomic_and_preserves_existing(builder, tmp_path): directory = tmp_path / "local metadata" directory.mkdir() # dev.sh's mkdir -p is supported. diff --git a/tests/ui/test_startup_handoff.py b/tests/ui/test_startup_handoff.py index ff0d364..9939c30 100644 --- a/tests/ui/test_startup_handoff.py +++ b/tests/ui/test_startup_handoff.py @@ -1,3 +1,4 @@ +import sys import threading import pytest @@ -11,7 +12,7 @@ def test_packaged_desktop_launches_sibling_worker(tmp_path, monkeypatch): executable = tmp_path / "coinfly.app/Contents/MacOS/coinfly" executable.parent.mkdir(parents=True) - worker = executable.with_name("coinfly-worker") + worker = executable.with_name("coinfly-worker.exe" if sys.platform == "win32" else "coinfly-worker") monkeypatch.setattr("coinfly.cli.sys.executable", str(executable)) monkeypatch.setattr("coinfly.cli.sys.frozen", True, raising=False) with pytest.raises(DomainError, match="PACKAGED_WORKER_MISSING"): diff --git a/tests/worker/test_automation_lifecycle.py b/tests/worker/test_automation_lifecycle.py index 5edee5e..18485b0 100644 --- a/tests/worker/test_automation_lifecycle.py +++ b/tests/worker/test_automation_lifecycle.py @@ -12,7 +12,6 @@ from __future__ import annotations -import os import time from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -29,7 +28,6 @@ from coinfly.domain.context import DomainError, EngineState, ProductTrack, RegionPolicy from coinfly.domain.products import ProductSubtype from coinfly.execution.intent_store import IntentStore -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import RPCRequest from coinfly.market.feed import fixture_catalog from coinfly.market.observations import ( @@ -44,6 +42,7 @@ from coinfly.worker.automation import AutomationState from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer _SENTINEL = object() @@ -166,7 +165,7 @@ def rpc(self, method: str, params: dict[str, Any] | None = None, *, request_id: epoch = self.worker.coordinator.status(self.worker.authority_id).transition_epoch request = RPCRequest.parse({"jsonrpc": "2.0", "id": request_id or method, "method": method, "params": params or {}, "epoch": epoch, "deadline_ms": int(time.time() * 1000) + 10000}) - result = self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid())) + result = self.worker.handle(request, local_peer()) return cast(dict[str, Any], result) def initialize(self) -> None: diff --git a/tests/worker/test_automation_real_feed.py b/tests/worker/test_automation_real_feed.py index c659771..6c55ba2 100644 --- a/tests/worker/test_automation_real_feed.py +++ b/tests/worker/test_automation_real_feed.py @@ -13,7 +13,6 @@ from __future__ import annotations import json -import os import time from collections.abc import Mapping from datetime import datetime, timezone @@ -28,13 +27,13 @@ from coinfly.domain.context import DomainError, ProductTrack, RegionPolicy from coinfly.domain.products import ProductSubtype from coinfly.execution.intent_store import IntentStore -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import RPCRequest from coinfly.market.coinbase_public import CoinbasePublicRestFeed from coinfly.market.observations import DataSource from coinfly.worker.automation import AutomationConfig, AutomationState from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer from .test_automation_lifecycle import TestOnlyBootstrap @@ -110,7 +109,7 @@ def rpc(self, method: str, params: dict[str, Any] | None = None, *, request_id: epoch = self.worker.coordinator.status(self.worker.authority_id).transition_epoch request = RPCRequest.parse({"jsonrpc": "2.0", "id": request_id or method, "method": method, "params": params or {}, "epoch": epoch, "deadline_ms": int(time.time() * 1000) + 10000}) - return cast(dict[str, Any], self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid()))) + return cast(dict[str, Any], self.worker.handle(request, local_peer())) def initialize(self, config: EngineConfig | None = None) -> None: config = config or default_config() diff --git a/tests/worker/test_database_bootstrap.py b/tests/worker/test_database_bootstrap.py index 90bc58e..16fc3f7 100644 --- a/tests/worker/test_database_bootstrap.py +++ b/tests/worker/test_database_bootstrap.py @@ -1,6 +1,7 @@ """Real SQLite contention and a responsive diagnostic boundary; no operator data.""" import sqlite3 +import sys import threading import time from pathlib import Path @@ -151,6 +152,7 @@ def factory(**kwargs): startup.close() +@pytest.mark.skipif(sys.platform == "win32", reason="Unix-domain socket IPC; the Windows transport uses named pipes (sockets under /tmp)") def test_diagnostic_ipc_remains_available_until_database_lock_is_released(tmp_path): # Short AF_UNIX path, with the same private directory checks as production. import tempfile diff --git a/tests/worker/test_initialization_diagnostics.py b/tests/worker/test_initialization_diagnostics.py index 1295cdb..37323bb 100644 --- a/tests/worker/test_initialization_diagnostics.py +++ b/tests/worker/test_initialization_diagnostics.py @@ -22,12 +22,12 @@ from coinfly.bootstrap.manifest import BootstrapError from coinfly.config.models import default_config from coinfly.control.transition_state import TransitionPhase -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import RPCRequest from coinfly.observability.diagnostics import redacted_chain, source_fingerprint from coinfly.ui.state import DesktopState from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer _SENTINEL = object() @@ -69,7 +69,7 @@ def rpc(self, method: str, params: dict[str, Any] | None = None, *, request_id: epoch = self.worker.coordinator.status(self.worker.authority_id).transition_epoch request = RPCRequest.parse({"jsonrpc": "2.0", "id": request_id or method, "method": method, "params": params or {}, "epoch": epoch, "deadline_ms": int(time.time() * 1000) + 10000}) - result = self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid())) + result = self.worker.handle(request, local_peer()) return cast(dict[str, Any], result) def apply(self, request_id: str) -> dict[str, Any]: diff --git a/tests/worker/test_live_arming.py b/tests/worker/test_live_arming.py index 0f9a9d2..5c7f86f 100644 --- a/tests/worker/test_live_arming.py +++ b/tests/worker/test_live_arming.py @@ -8,7 +8,6 @@ from __future__ import annotations -import os import time from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -21,7 +20,6 @@ from coinfly.config.models import EngineConfig, default_config from coinfly.domain.context import DomainError, Environment, ProductTrack, RunProfile from coinfly.domain.products import ProductSubtype -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import OWNER_METHODS, READ_METHODS, RPCRequest from coinfly.operations.live_arming import ( MANDATE_STATEMENT, @@ -32,6 +30,7 @@ from coinfly.venues.capability import LocalQualifications from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer NOW = datetime(2026, 9, 21, 12, 0, tzinfo=timezone.utc) @@ -100,7 +99,7 @@ def setUp(self) -> None: def rpc(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]: request = RPCRequest.parse({"jsonrpc": "2.0", "id": method, "method": method, "params": params or {}, "epoch": 0, "deadline_ms": int(time.time() * 1000) + 10000}) - return cast(dict[str, Any], self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid()))) + return cast(dict[str, Any], self.worker.handle(request, local_peer())) def test_prerequisites_report_and_mandate_lifecycle(self) -> None: self.assertIn("live.prerequisites", READ_METHODS) diff --git a/tests/worker/test_paper_settlement.py b/tests/worker/test_paper_settlement.py index 6ea9099..2fe217b 100644 --- a/tests/worker/test_paper_settlement.py +++ b/tests/worker/test_paper_settlement.py @@ -45,6 +45,7 @@ def test_funding_is_charged_when_the_clock_crosses_an_interval_boundary_only(sel now = datetime.now(timezone.utc) self.assertEqual(engine.settle(now + timedelta(seconds=1)), ()) # inside the current 8 h interval: nothing boundary = _interval_start(now, timedelta(hours=8)) + timedelta(hours=8) + self.refresh_mark(engine, boundary) events = engine.settle(boundary + timedelta(seconds=1)) self.assertEqual(len(events), 1) [event] = events @@ -56,6 +57,7 @@ def test_funding_is_charged_when_the_clock_crosses_an_interval_boundary_only(sel # Idempotent per boundary; a later call inside the same interval charges nothing. self.assertEqual(engine.settle(boundary + timedelta(seconds=2)), ()) # Two boundaries crossed at once are two events. + self.refresh_mark(engine, boundary + timedelta(hours=16)) events = engine.settle(boundary + timedelta(hours=16, seconds=1)) self.assertEqual([item["interval_start"] for item in events], [(boundary + timedelta(hours=16)).isoformat(), (boundary + timedelta(hours=8)).isoformat()]) @@ -74,6 +76,12 @@ def redeclare_as_dated(engine: Any, expiry: datetime) -> None: matcher.instruments = tuple(dated if item.instrument_id == "BTC-USD" else item for item in matcher.instruments) matcher._instruments["BTC-USD"] = dated + @staticmethod + def refresh_mark(engine: Any, observed_at: datetime, bid: str = "99.80", ask: str = "99.90") -> None: + """Feed a BTC-USD book observed at ``observed_at`` (an expiry settles only at a fresh mark).""" + engine.update_market("BTC-USD", bids=(DepthLevel(price=Decimal(bid), quantity=Decimal("5")),), + asks=(DepthLevel(price=Decimal(ask), quantity=Decimal("5")),), now=observed_at) + def durable_collateral(self) -> Decimal: state = self.worker.writer.call(lambda connection: connection.execute("SELECT state_json FROM wp_paper_books").fetchone()[0]) return Decimal(json.loads(state)["margin"]["collateral"]) @@ -86,6 +94,7 @@ def test_dated_future_is_cash_settled_at_expiry(self) -> None: expiry = datetime.now(timezone.utc) + timedelta(days=1) self.redeclare_as_dated(engine, expiry) self.assertEqual(engine.settle(expiry - timedelta(seconds=1)), ()) + self.refresh_mark(engine, expiry) events = engine.settle(expiry + timedelta(seconds=1)) kinds = [item["kind"] for item in events] self.assertIn("PAPER_EXPIRY_SETTLEMENT", kinds) @@ -105,6 +114,7 @@ def settled_at_expiry(self) -> tuple[Any, datetime, dict[str, Any]]: before = engine.admission.drawdown.read(engine.context, engine.scope) assert before is not None self.before_daily_result = before.daily_net_result + self.refresh_mark(engine, expiry) [event] = engine.settle(expiry + timedelta(seconds=1)) self.assertEqual(event["kind"], "PAPER_EXPIRY_SETTLEMENT") return engine, expiry, event @@ -182,6 +192,7 @@ def test_failed_funding_commit_rolls_the_margin_book_and_event_ids_back(self) -> now = datetime.now(timezone.utc) boundary = _interval_start(now, timedelta(hours=8)) + timedelta(hours=8) self.assertEqual(engine.settle(now + timedelta(seconds=1)), ()) + self.refresh_mark(engine, boundary) with patch.object(engine, "_persist_tx", side_effect=DomainError("SIMULATED_COMMIT_FAILURE")): with self.assertRaisesRegex(DomainError, "SIMULATED_COMMIT_FAILURE"): engine.settle(boundary + timedelta(seconds=1)) @@ -197,11 +208,62 @@ def test_failed_funding_commit_rolls_the_margin_book_and_event_ids_back(self) -> self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 1) self.assertEqual(engine.settle(boundary + timedelta(seconds=3)), ()) + def crossed_at_a_stale_mark(self) -> tuple[Any, datetime, dict[str, Any]]: + engine = self.open_long() + now = datetime.now(timezone.utc) + boundary = _interval_start(now, timedelta(hours=8)) + timedelta(hours=8) + self.assertEqual(engine.settle(now + timedelta(seconds=1)), ()) + self.collateral = engine.matcher.margin.collateral + # The last BTC-USD book was observed when the long opened, hours before the boundary. + [deferred] = engine.settle(boundary + timedelta(seconds=1)) + self.assertEqual((deferred["kind"], deferred["reason"], deferred["instrument"], deferred["interval_start"], deferred["synthetic"]), + ("PAPER_FUNDING_DEFERRED", "PAPER_FUNDING_MARK_STALE", "BTC-USD", boundary.isoformat(), True)) + return engine, boundary, deferred + + def test_funding_at_a_stale_mark_is_deferred_and_charged_at_the_next_fresh_mark(self) -> None: + engine, boundary, deferred = self.crossed_at_a_stale_mark() + margin = engine.matcher.margin + # Nothing is charged, in memory or durably. + self.assertEqual((margin.collateral, self.durable_collateral()), (self.collateral, self.collateral)) + self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 0) + # Reported once per reason; the cursor moves on but the crossed boundary stays due. + self.assertEqual(engine.settle(boundary + timedelta(seconds=2)), ()) + engine._depth.pop("BTC-USD") + [missing] = engine.settle(boundary + timedelta(seconds=3)) + self.assertEqual((missing["kind"], missing["reason"]), ("PAPER_FUNDING_DEFERRED", "PAPER_FUNDING_MARK_MISSING")) + # A book no older than the venue's limit charges that boundary, for the quantity held when it was crossed. + observed = boundary + timedelta(seconds=4) + self.refresh_mark(engine, observed, "100.00", "100.20") + [event] = engine.settle(observed + engine.matcher.book_max_age) + self.assertEqual((event["kind"], event["event_id"], event["interval_start"]), + ("PAPER_FUNDING", deferred["event_id"], boundary.isoformat())) + expected = -(PAPER_FUNDING_RATE_PER_INTERVAL * Decimal("0.04") * Decimal("100.10")) + self.assertEqual(Decimal(event["payment"]), expected) + self.assertEqual((margin.collateral, self.durable_collateral()), (self.collateral + expected, self.collateral + expected)) + self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 1) + self.assertEqual(engine.settle(observed + timedelta(minutes=1)), ()) + Journal(self.worker.writer).verify() + + def test_funding_due_on_a_position_closed_before_a_fresh_mark_is_reported_not_charged(self) -> None: + engine, boundary, deferred = self.crossed_at_a_stale_mark() + margin = engine.matcher.margin + # The position closes before any fresh mark prices the boundary (as a resting exit filling at the returning book would). + margin.apply_fill("BTC-USD", Decimal("-0.04"), Decimal("99.85"), multiplier=Decimal(1), fee=Decimal(0), reduce_only=True) + collateral = margin.collateral + self.refresh_mark(engine, boundary + timedelta(seconds=2)) + [dropped] = engine.settle(boundary + timedelta(seconds=3)) + self.assertEqual((dropped["kind"], dropped["reason"], dropped["event_id"]), + ("PAPER_FUNDING_NOT_CHARGED", "PAPER_FUNDING_POSITION_CLOSED", deferred["event_id"])) + self.assertEqual(margin.collateral, collateral) + self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 0) + self.assertEqual(engine.settle(boundary + timedelta(seconds=4)), ()) + def test_failed_expiry_commit_leaves_margin_and_ledger_open_until_the_retry(self) -> None: engine = self.open_long() expiry = datetime.now(timezone.utc) + timedelta(days=1) self.redeclare_as_dated(engine, expiry) engine.settle(expiry - timedelta(seconds=1)) + self.refresh_mark(engine, expiry) with patch.object(engine, "_persist_tx", side_effect=DomainError("SIMULATED_COMMIT_FAILURE")): with self.assertRaisesRegex(DomainError, "SIMULATED_COMMIT_FAILURE"): engine.settle(expiry + timedelta(seconds=1)) @@ -214,6 +276,57 @@ def test_failed_expiry_commit_leaves_margin_and_ledger_open_until_the_retry(self self.assertEqual(self.position(), Decimal(0)) self.worker.writer.call(lambda connection: assert_context_closed_tx(connection, engine.context)) + def test_expiry_at_a_stale_mark_is_deferred_and_retried_never_booked(self) -> None: + engine = self.open_long() + margin = engine.matcher.margin + assert margin is not None + collateral = margin.collateral + expiry = datetime.now(timezone.utc) + timedelta(days=1) + self.redeclare_as_dated(engine, expiry) + engine.settle(expiry - timedelta(seconds=1)) + # The last BTC-USD book was observed when the long opened, a day before expiry: far past the venue's book age. + [deferred] = engine.settle(expiry + timedelta(seconds=1)) + self.assertEqual((deferred["kind"], deferred["reason"], deferred["instrument"], deferred["synthetic"]), + ("PAPER_EXPIRY_DEFERRED", "PAPER_EXPIRY_MARK_STALE", "BTC-USD", True)) + # Nothing is booked: margin book, durable book, ledger and events are untouched, and nothing is fenced. + self.assertEqual([item.quantity for item in margin.positions], [Decimal("0.04")]) + self.assertEqual((margin.collateral, self.durable_collateral()), (collateral, collateral)) + self.assertEqual(self.position(), Decimal("0.04")) + self.assertEqual(self.ledger_rows("SELECT 1 FROM ex_fills WHERE trade_id LIKE '%paper-expiry%'"), []) + self.assertEqual(self.ledger_rows("SELECT 1 FROM ex_events WHERE event_kind='PAPER_EXPIRY_SETTLEMENT'"), []) + self.assertFalse(self.worker.interlock.is_provisionally_fenced(engine.context.authority_id)) + # Reported once per reason; the event stays due and unsettled. + self.assertEqual(engine.settle(expiry + timedelta(seconds=2)), ()) + self.assertEqual(self.position(), Decimal("0.04")) + # With no book at all (as after a restart) it stays deferred under its own reason. + engine._depth.pop("BTC-USD") + [missing] = engine.settle(expiry + timedelta(seconds=3)) + self.assertEqual((missing["kind"], missing["reason"]), ("PAPER_EXPIRY_DEFERRED", "PAPER_EXPIRY_MARK_MISSING")) + # A book no older than the venue's limit settles the same event at that mark. + observed = expiry + timedelta(seconds=4) + self.refresh_mark(engine, observed, "100.00", "100.20") + [event] = engine.settle(observed + engine.matcher.book_max_age) + self.assertEqual((event["kind"], event["event_id"]), ("PAPER_EXPIRY_SETTLEMENT", deferred["event_id"])) + self.assertEqual(Decimal(event["mark"]), Decimal("100.10")) + self.assertEqual(margin.positions, ()) + self.assertEqual(self.position(), Decimal(0)) + self.assertEqual(engine.settle(observed + timedelta(minutes=1)), ()) + Journal(self.worker.writer).verify() + + def test_expiry_deferral_is_visible_in_automation_status(self) -> None: + engine = self.open_long() + self.worker.automation.pause_entries() + # BTC-USD drops out of the feed: the engine keeps its last book, which has aged past the venue's limit. + self.feed.subscribed = ("ETH-USD",) + bids, asks, observed = engine._depth["BTC-USD"] + engine._depth["BTC-USD"] = (bids, asks, observed - timedelta(minutes=1)) + self.redeclare_as_dated(engine, datetime.now(timezone.utc) - timedelta(seconds=1)) + self.tick() + settlements = [item for item in self.worker.automation.status()["settlements"] if item["instrument"] == "BTC-USD"] + self.assertEqual([(item["kind"], item["reason"]) for item in settlements], + [("PAPER_EXPIRY_DEFERRED", "PAPER_EXPIRY_MARK_STALE")]) + self.assertEqual(self.position(), Decimal("0.04")) + def test_drain_refused_by_the_matcher_fails_fast_once_and_does_not_fence(self) -> None: """A deterministic paper refusal is a proven non-creation: the drain reports it instead of re-planning 1000 times.""" engine = self.open_long() diff --git a/tests/worker/test_worker_replacement.py b/tests/worker/test_worker_replacement.py index c6fa651..3730a5b 100644 --- a/tests/worker/test_worker_replacement.py +++ b/tests/worker/test_worker_replacement.py @@ -12,16 +12,16 @@ from __future__ import annotations import os +import sys import tempfile import threading import time from pathlib import Path from typing import Any -from unittest import TestCase +from unittest import TestCase, skipIf from unittest.mock import patch from coinfly.config.models import default_config -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.local import LocalRPCClient, LocalRPCServer from coinfly.ipc.protocol import PROTOCOL_VERSION, RPCRequest from coinfly.observability import diagnostics @@ -31,6 +31,7 @@ from coinfly.worker.paths import WorkerPaths from coinfly.worker.startup import WorkerStartup from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer from .test_automation_lifecycle import TestOnlyBootstrap @@ -47,6 +48,7 @@ def hello_params(fingerprint: str | None = None) -> dict[str, Any]: return identity +@skipIf(sys.platform == "win32", "Unix-domain socket IPC; the Windows transport uses named pipes (sockets under /tmp)") class ReplacementFixture(TestCase): def setUp(self) -> None: # Short, private path: Unix socket paths are limited to ~104 bytes on macOS. @@ -78,7 +80,7 @@ def slow_factory(**kwargs: Any) -> WorkerSupervisor: startup = WorkerStartup(socket_path=self.socket, factory=slow_factory, timeout=30).start() self.addCleanup(startup.close) self.addCleanup(release.set) - peer = PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid()) + peer = local_peer() hello = startup.handle(request("worker.hello", hello_params()), peer) self.assertEqual((hello["protocol_version"], hello["build_match"], hello["initializing"]), (PROTOCOL_VERSION, True, True)) stale = startup.handle(request("worker.hello", hello_params("0" * 64)), peer)