From 8eb4bb6f7533613ee403b9735483d1afbe15172f Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:04:33 +0200 Subject: [PATCH 1/9] Defer paper expiry settlement at a stale or missing mark A DATED_FUTURE expiry close books fee-free realized P&L into lots, journal, drawdown and strategy capital at the last mid. That mid had no age check, so a book left behind by a contract that stopped being quoted (or a feed/worker outage across expiry) became permanent realized P&L. settle() now books the expiry only at a two-sided book observed no more than the matcher's book_max_age (the bound a resting paper order needs to fill) before now. Otherwise nothing is booked, the event stays unsettled and is retried every call, and a PAPER_EXPIRY_DEFERRED record with reason PAPER_EXPIRY_MARK_STALE or PAPER_EXPIRY_MARK_MISSING is returned once per reason, so it shows in automation.status.settlements. No mark is substituted; commit rollback also restores the reported-deferral state. The subscription keep list already carries every held instrument first and is not re-planned mid-run, so it is not the cause and is unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/runbooks/paper-startup.md | 12 ++++++ src/coinfly/worker/paper_engine.py | 37 ++++++++++++++--- tests/worker/test_paper_settlement.py | 60 +++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 6 deletions(-) diff --git a/docs/runbooks/paper-startup.md b/docs/runbooks/paper-startup.md index 0fd58d9..248cca1 100644 --- a/docs/runbooks/paper-startup.md +++ b/docs/runbooks/paper-startup.md @@ -120,6 +120,18 @@ intent cause `VENUE_SETTLEMENT`, never as a strategy trade. Both appear in `automation.status.settlements` as synthetic. None of this is presented as venue behavior. +Expiry settles only at a fresh mark: a two-sided book observed no more than the +paper venue's book age limit (`book_max_age`, 5 s, the same bound a resting +order needs to fill) before the settlement check. If the contract's book is +older (for example, the venue stopped quoting it or the feed or worker was +down across expiry) or missing (no book since a restart), nothing is booked. +The position, ledger and risk books stay open, and settlement is retried on +every later tick until a fresh book arrives. Each deferral appears once in +`automation.status.settlements` as `PAPER_EXPIRY_DEFERRED` with reason +`PAPER_EXPIRY_MARK_STALE` or `PAPER_EXPIRY_MARK_MISSING` and the mark's +observation time. No substitute mark is ever used, and new targets are refused +(`POSITION_MARK_UNVERIFIED`) while any open position lacks a fresh mark. + ## 5. Pause, stop, restart The development worker runs in the background and keeps running after the diff --git a/src/coinfly/worker/paper_engine.py b/src/coinfly/worker/paper_engine.py index 07c65ff..5c486a7 100644 --- a/src/coinfly/worker/paper_engine.py +++ b/src/coinfly/worker/paper_engine.py @@ -109,6 +109,7 @@ def __init__( self.config, self.context = config, context self.writer, self.reducer, self.interlock = writer, reducer, interlock self._settled_events: set[str] = set() + self._expiry_deferrals: dict[str, str] = {} # due expiry event id -> reason last reported self._settlement_cursor: datetime | None = None self._ready, self._on_change = ready, on_change self._lock = threading.RLock() @@ -606,6 +607,14 @@ def settle(self, now: datetime) -> tuple[dict[str, Any], ...]: fill, so lots, positions, journal and risk books agree. Every event is labeled synthetic; none is venue evidence. + The expiry close realizes P&L permanently, so it is booked only at a mark + the paper venue would still fill against: a two-sided book observed no + more than the matcher's ``book_max_age`` before ``now``. Otherwise the + event is deferred: nothing is booked, it stays unsettled and is retried on + every later call, and a ``PAPER_EXPIRY_DEFERRED`` record with reason + ``PAPER_EXPIRY_MARK_STALE`` or ``PAPER_EXPIRY_MARK_MISSING`` is returned + once per reason. No other mark is ever substituted. + All of it commits together with the matcher snapshot. If that commit fails the in-memory book, the settled-event ids and the settlement cursor are rolled back, so the next call retries the same due events. @@ -615,23 +624,28 @@ def settle(self, now: datetime) -> tuple[dict[str, Any], ...]: return () margin = self.matcher.margin events: list[dict[str, Any]] = [] + deferred: list[dict[str, Any]] = [] previous = self._settlement_cursor self._settlement_cursor = now if previous is None or now <= previous: return () # no retroactive funding: boundaries are charged when the clock crosses them snapshot: dict[str, object] | None = None # matcher state before the first mutation added: list[str] = [] + reported = dict(self._expiry_deferrals) try: for position in margin.positions: instrument = position.instrument_id contract = next((item for item in self._instruments if item.instrument_id == instrument), None) - depth = self._depth.get(instrument) - if contract is None or depth is None or not depth[0] or not depth[1]: + if contract is None: continue - mark = (depth[0][0].price + depth[1][0].price) / 2 + depth = self._depth.get(instrument) + mark: Decimal | None = None + observed: datetime | None = None + if depth is not None and depth[0] and depth[1]: + mark, observed = (depth[0][0].price + depth[1][0].price) / 2, depth[2] subtype = contract.declared_subtype interval = PAPER_FUNDING_INTERVALS.get(subtype) - if interval is not None: + if interval is not None and mark is not None: boundary = _interval_start(now, interval) while boundary > previous: stamp = boundary.strftime("%Y%m%dT%H%M%SZ") @@ -652,11 +666,21 @@ def settle(self, now: datetime) -> tuple[dict[str, Any], ...]: if subtype is ProductSubtype.DATED_FUTURE and expiry is not None and now >= expiry: event_id = f"{PAPER_EXPIRY_PREFIX}{instrument}:{expiry.strftime('%Y%m%dT%H%M%SZ')}" if event_id not in self._settled_events: + max_age = self.matcher.book_max_age + if mark is None or observed is None or now - observed > max_age: + reason = "PAPER_EXPIRY_MARK_MISSING" if observed is None else "PAPER_EXPIRY_MARK_STALE" + if self._expiry_deferrals.get(event_id) != reason: + self._expiry_deferrals[event_id] = reason + deferred.append({"kind": "PAPER_EXPIRY_DEFERRED", "event_id": event_id, "instrument": instrument, + "reason": reason, "mark_observed_at": observed.isoformat() if observed is not None else None, + "max_mark_age_seconds": str(max_age.total_seconds()), "synthetic": True}) + continue if snapshot is None: snapshot = self.matcher.dump_state() realized = margin.apply_fill(instrument, -position.quantity, mark, multiplier=contract.contract_multiplier, fee=Decimal(0), reduce_only=True) self._settled_events.add(event_id) + self._expiry_deferrals.pop(event_id, None) added.append(event_id) events.append({"kind": "PAPER_EXPIRY_SETTLEMENT", "event_id": event_id, "instrument": instrument, "quantity": str(position.quantity), "mark": str(mark), "realized": str(realized), "synthetic": True}) @@ -685,13 +709,14 @@ def save(connection: sqlite3.Connection) -> None: if snapshot is not None: self._matcher = PaperMatcher.restore(self.context, self._instruments, snapshot) self._settled_events.difference_update(added) + self._expiry_deferrals = reported self._settlement_cursor = previous raise for incident in incidents: self.interlock.provisional_fence(self.context.authority_id, incident) - if events: + if events or deferred: self._notify() - return tuple(events) + return (*events, *deferred) @exact_decimal def _settle_expiry_tx(self, connection: sqlite3.Connection, event: dict[str, Any]) -> FillResult: diff --git a/tests/worker/test_paper_settlement.py b/tests/worker/test_paper_settlement.py index 6ea9099..01ea5fc 100644 --- a/tests/worker/test_paper_settlement.py +++ b/tests/worker/test_paper_settlement.py @@ -74,6 +74,12 @@ def redeclare_as_dated(engine: Any, expiry: datetime) -> None: matcher.instruments = tuple(dated if item.instrument_id == "BTC-USD" else item for item in matcher.instruments) matcher._instruments["BTC-USD"] = dated + @staticmethod + def refresh_mark(engine: Any, observed_at: datetime, bid: str = "99.80", ask: str = "99.90") -> None: + """Feed a BTC-USD book observed at ``observed_at`` (an expiry settles only at a fresh mark).""" + engine.update_market("BTC-USD", bids=(DepthLevel(price=Decimal(bid), quantity=Decimal("5")),), + asks=(DepthLevel(price=Decimal(ask), quantity=Decimal("5")),), now=observed_at) + def durable_collateral(self) -> Decimal: state = self.worker.writer.call(lambda connection: connection.execute("SELECT state_json FROM wp_paper_books").fetchone()[0]) return Decimal(json.loads(state)["margin"]["collateral"]) @@ -86,6 +92,7 @@ def test_dated_future_is_cash_settled_at_expiry(self) -> None: expiry = datetime.now(timezone.utc) + timedelta(days=1) self.redeclare_as_dated(engine, expiry) self.assertEqual(engine.settle(expiry - timedelta(seconds=1)), ()) + self.refresh_mark(engine, expiry) events = engine.settle(expiry + timedelta(seconds=1)) kinds = [item["kind"] for item in events] self.assertIn("PAPER_EXPIRY_SETTLEMENT", kinds) @@ -105,6 +112,7 @@ def settled_at_expiry(self) -> tuple[Any, datetime, dict[str, Any]]: before = engine.admission.drawdown.read(engine.context, engine.scope) assert before is not None self.before_daily_result = before.daily_net_result + self.refresh_mark(engine, expiry) [event] = engine.settle(expiry + timedelta(seconds=1)) self.assertEqual(event["kind"], "PAPER_EXPIRY_SETTLEMENT") return engine, expiry, event @@ -202,6 +210,7 @@ def test_failed_expiry_commit_leaves_margin_and_ledger_open_until_the_retry(self expiry = datetime.now(timezone.utc) + timedelta(days=1) self.redeclare_as_dated(engine, expiry) engine.settle(expiry - timedelta(seconds=1)) + self.refresh_mark(engine, expiry) with patch.object(engine, "_persist_tx", side_effect=DomainError("SIMULATED_COMMIT_FAILURE")): with self.assertRaisesRegex(DomainError, "SIMULATED_COMMIT_FAILURE"): engine.settle(expiry + timedelta(seconds=1)) @@ -214,6 +223,57 @@ def test_failed_expiry_commit_leaves_margin_and_ledger_open_until_the_retry(self self.assertEqual(self.position(), Decimal(0)) self.worker.writer.call(lambda connection: assert_context_closed_tx(connection, engine.context)) + def test_expiry_at_a_stale_mark_is_deferred_and_retried_never_booked(self) -> None: + engine = self.open_long() + margin = engine.matcher.margin + assert margin is not None + collateral = margin.collateral + expiry = datetime.now(timezone.utc) + timedelta(days=1) + self.redeclare_as_dated(engine, expiry) + engine.settle(expiry - timedelta(seconds=1)) + # The last BTC-USD book was observed when the long opened, a day before expiry: far past the venue's book age. + [deferred] = engine.settle(expiry + timedelta(seconds=1)) + self.assertEqual((deferred["kind"], deferred["reason"], deferred["instrument"], deferred["synthetic"]), + ("PAPER_EXPIRY_DEFERRED", "PAPER_EXPIRY_MARK_STALE", "BTC-USD", True)) + # Nothing is booked: margin book, durable book, ledger and events are untouched, and nothing is fenced. + self.assertEqual([item.quantity for item in margin.positions], [Decimal("0.04")]) + self.assertEqual((margin.collateral, self.durable_collateral()), (collateral, collateral)) + self.assertEqual(self.position(), Decimal("0.04")) + self.assertEqual(self.ledger_rows("SELECT 1 FROM ex_fills WHERE trade_id LIKE '%paper-expiry%'"), []) + self.assertEqual(self.ledger_rows("SELECT 1 FROM ex_events WHERE event_kind='PAPER_EXPIRY_SETTLEMENT'"), []) + self.assertFalse(self.worker.interlock.is_provisionally_fenced(engine.context.authority_id)) + # Reported once per reason; the event stays due and unsettled. + self.assertEqual(engine.settle(expiry + timedelta(seconds=2)), ()) + self.assertEqual(self.position(), Decimal("0.04")) + # With no book at all (as after a restart) it stays deferred under its own reason. + engine._depth.pop("BTC-USD") + [missing] = engine.settle(expiry + timedelta(seconds=3)) + self.assertEqual((missing["kind"], missing["reason"]), ("PAPER_EXPIRY_DEFERRED", "PAPER_EXPIRY_MARK_MISSING")) + # A book no older than the venue's limit settles the same event at that mark. + observed = expiry + timedelta(seconds=4) + self.refresh_mark(engine, observed, "100.00", "100.20") + [event] = engine.settle(observed + engine.matcher.book_max_age) + self.assertEqual((event["kind"], event["event_id"]), ("PAPER_EXPIRY_SETTLEMENT", deferred["event_id"])) + self.assertEqual(Decimal(event["mark"]), Decimal("100.10")) + self.assertEqual(margin.positions, ()) + self.assertEqual(self.position(), Decimal(0)) + self.assertEqual(engine.settle(observed + timedelta(minutes=1)), ()) + Journal(self.worker.writer).verify() + + def test_expiry_deferral_is_visible_in_automation_status(self) -> None: + engine = self.open_long() + self.worker.automation.pause_entries() + # BTC-USD drops out of the feed: the engine keeps its last book, which has aged past the venue's limit. + self.feed.subscribed = ("ETH-USD",) + bids, asks, observed = engine._depth["BTC-USD"] + engine._depth["BTC-USD"] = (bids, asks, observed - timedelta(minutes=1)) + self.redeclare_as_dated(engine, datetime.now(timezone.utc) - timedelta(seconds=1)) + self.tick() + settlements = [item for item in self.worker.automation.status()["settlements"] if item["instrument"] == "BTC-USD"] + self.assertEqual([(item["kind"], item["reason"]) for item in settlements], + [("PAPER_EXPIRY_DEFERRED", "PAPER_EXPIRY_MARK_STALE")]) + self.assertEqual(self.position(), Decimal("0.04")) + def test_drain_refused_by_the_matcher_fails_fast_once_and_does_not_fence(self) -> None: """A deterministic paper refusal is a proven non-creation: the drain reports it instead of re-planning 1000 times.""" engine = self.open_long() From 2dffb712300a2b46bc76e015acc63b362014bd0e Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:12:56 +0200 Subject: [PATCH 2/9] Defer paper funding at a stale or missing mark Synthetic PAPER_FUNDING was charged against the last mid with no age check, so a book left behind by a dropped feed set a permanent collateral debit or credit. Each funding boundary the clock crosses is now recorded as due, with the quantity held at the crossing, and charged only at a fresh mark (a two-sided book no older than the matcher's book_max_age), the same rule as expiry settlement. Until then it stays due across calls and is reported once per reason as PAPER_FUNDING_DEFERRED (PAPER_FUNDING_MARK_STALE / _MISSING). A due boundary whose position closed before a fresh mark priced it cannot be charged by the margin book; it is dropped and reported as PAPER_FUNDING_NOT_CHARGED (PAPER_FUNDING_POSITION_CLOSED). Expiry and funding now share one mark helper and one deferral report. Funding events also carry the quantity and mark they were charged at. Co-Authored-By: Claude Opus 5.5 --- docs/runbooks/paper-startup.md | 30 +++-- src/coinfly/worker/paper_engine.py | 183 ++++++++++++++++---------- tests/worker/test_paper_settlement.py | 53 ++++++++ 3 files changed, 184 insertions(+), 82 deletions(-) diff --git a/docs/runbooks/paper-startup.md b/docs/runbooks/paper-startup.md index 248cca1..b904b85 100644 --- a/docs/runbooks/paper-startup.md +++ b/docs/runbooks/paper-startup.md @@ -120,17 +120,25 @@ intent cause `VENUE_SETTLEMENT`, never as a strategy trade. Both appear in `automation.status.settlements` as synthetic. None of this is presented as venue behavior. -Expiry settles only at a fresh mark: a two-sided book observed no more than the -paper venue's book age limit (`book_max_age`, 5 s, the same bound a resting -order needs to fill) before the settlement check. If the contract's book is -older (for example, the venue stopped quoting it or the feed or worker was -down across expiry) or missing (no book since a restart), nothing is booked. -The position, ledger and risk books stay open, and settlement is retried on -every later tick until a fresh book arrives. Each deferral appears once in -`automation.status.settlements` as `PAPER_EXPIRY_DEFERRED` with reason -`PAPER_EXPIRY_MARK_STALE` or `PAPER_EXPIRY_MARK_MISSING` and the mark's -observation time. No substitute mark is ever used, and new targets are refused -(`POSITION_MARK_UNVERIFIED`) while any open position lacks a fresh mark. +Funding and expiry are booked only at a fresh mark: a two-sided book observed +no more than the paper venue's book age limit (`book_max_age`, 5 s, the same +bound a resting order needs to fill) before the settlement check. If the +contract's book is older (for example, the venue stopped quoting it or the +feed or worker was down at the boundary or expiry) or missing (no book since a +restart), nothing is booked and the event stays due. It is retried on every +later tick until a fresh book arrives. A crossed funding boundary is then +charged for the quantity held when it was crossed, at that fresh mark. An +expiring position, ledger and risk books stay open until then. Each deferral +appears once in `automation.status.settlements` as `PAPER_FUNDING_DEFERRED` or +`PAPER_EXPIRY_DEFERRED`, with reason `..._MARK_STALE` or `..._MARK_MISSING` and +the mark's observation time. A due funding boundary whose position closed +before any fresh mark priced it is not charged; it is reported as +`PAPER_FUNDING_NOT_CHARGED` (`PAPER_FUNDING_POSITION_CLOSED`). Due funding +boundaries live in the worker's memory and, like the settlement clock, do not +survive a restart. A due expiry does, because it is checked against the +contract's expiry on every tick. No substitute mark is ever used, and new +targets are refused (`POSITION_MARK_UNVERIFIED`) while any open position lacks +a fresh mark. ## 5. Pause, stop, restart diff --git a/src/coinfly/worker/paper_engine.py b/src/coinfly/worker/paper_engine.py index 5c486a7..f8830d7 100644 --- a/src/coinfly/worker/paper_engine.py +++ b/src/coinfly/worker/paper_engine.py @@ -109,7 +109,10 @@ def __init__( self.config, self.context = config, context self.writer, self.reducer, self.interlock = writer, reducer, interlock self._settled_events: set[str] = set() - self._expiry_deferrals: dict[str, str] = {} # due expiry event id -> reason last reported + # Funding boundaries crossed but not yet charged: event id -> (instrument, interval start, + # quantity held at the crossing, contract multiplier); and the deferral reason last reported per event. + self._funding_due: dict[str, tuple[str, datetime, Decimal, Decimal]] = {} + self._deferrals: dict[str, str] = {} self._settlement_cursor: datetime | None = None self._ready, self._on_change = ready, on_change self._lock = threading.RLock() @@ -599,91 +602,104 @@ def settle(self, now: datetime) -> tuple[dict[str, Any], ...]: The paper perpetual book applies funding on a fixed schedule per declared subtype (``PERPETUAL_NO_EXPIRY``: every 8 h at 00/08/16 UTC; ``PERPETUAL_STYLE_FUTURE``: hourly) at the declared synthetic rate - (``PAPER_FUNDING_RATE_PER_INTERVAL``, long pays when positive) against the - last mark, once per interval (idempotent by event id), journaled as - ``PAPER_FUNDING``. A ``DATED_FUTURE`` position is cash-settled at its - catalog expiry (``PAPER_EXPIRY_SETTLEMENT``): the margin position and the - execution ledger are both closed at the mark by one synthetic, fee-free - fill, so lots, positions, journal and risk books agree. Every event is - labeled synthetic; none is venue evidence. - - The expiry close realizes P&L permanently, so it is booked only at a mark - the paper venue would still fill against: a two-sided book observed no - more than the matcher's ``book_max_age`` before ``now``. Otherwise the - event is deferred: nothing is booked, it stays unsettled and is retried on - every later call, and a ``PAPER_EXPIRY_DEFERRED`` record with reason - ``PAPER_EXPIRY_MARK_STALE`` or ``PAPER_EXPIRY_MARK_MISSING`` is returned - once per reason. No other mark is ever substituted. + (``PAPER_FUNDING_RATE_PER_INTERVAL``, long pays when positive), once per + interval boundary the clock crosses (idempotent by event id), for the + quantity held when it was crossed, journaled as ``PAPER_FUNDING``. A + ``DATED_FUTURE`` position is cash-settled at its catalog expiry + (``PAPER_EXPIRY_SETTLEMENT``): the margin position and the execution + ledger are both closed at the mark by one synthetic, fee-free fill, so + lots, positions, journal and risk books agree. Every event is labeled + synthetic; none is venue evidence. + + Both are booked only at a fresh mark: the mid of a two-sided book observed + no more than the matcher's ``book_max_age`` (the age past which no paper + order fills) before ``now``. Without one the event stays due and is + retried on every later call, and a ``PAPER_FUNDING_DEFERRED`` or + ``PAPER_EXPIRY_DEFERRED`` record with reason ``..._MARK_STALE`` or + ``..._MARK_MISSING`` is returned once per reason. No other mark is ever + substituted. A due funding boundary whose position closed before a fresh + mark priced it cannot be charged (the margin book holds funding only + against an open position): it is dropped and reported as + ``PAPER_FUNDING_NOT_CHARGED`` (``PAPER_FUNDING_POSITION_CLOSED``). All of it commits together with the matcher snapshot. If that commit fails - the in-memory book, the settled-event ids and the settlement cursor are - rolled back, so the next call retries the same due events. + the in-memory book, the settled-event ids, the due and reported events and + the settlement cursor are rolled back, so the next call retries the same + due events. """ with self.interlock.lock, self._lock: if self._retired or self.matcher.margin is None: return () margin = self.matcher.margin events: list[dict[str, Any]] = [] - deferred: list[dict[str, Any]] = [] + notices: list[dict[str, Any]] = [] previous = self._settlement_cursor self._settlement_cursor = now if previous is None or now <= previous: return () # no retroactive funding: boundaries are charged when the clock crosses them snapshot: dict[str, object] | None = None # matcher state before the first mutation added: list[str] = [] - reported = dict(self._expiry_deferrals) + due, reported = dict(self._funding_due), dict(self._deferrals) try: + held = {item.instrument_id: item for item in margin.positions} + contracts = {item.instrument_id: item for item in self._instruments} + for instrument, position in held.items(): + contract = contracts.get(instrument) + interval = PAPER_FUNDING_INTERVALS.get(contract.declared_subtype) if contract is not None else None + if contract is None or interval is None: + continue + boundary = _interval_start(now, interval) + while boundary > previous: + event_id = f"paper-funding:{instrument}:{boundary.strftime('%Y%m%dT%H%M%SZ')}" + if event_id not in self._settled_events: + self._funding_due.setdefault(event_id, (instrument, boundary, position.quantity, contract.contract_multiplier)) + boundary -= interval + for event_id, (instrument, boundary, quantity, multiplier) in sorted( + self._funding_due.items(), key=lambda item: item[1][1], reverse=True): + if instrument not in held: + del self._funding_due[event_id] + self._deferrals.pop(event_id, None) + notices.append({"kind": "PAPER_FUNDING_NOT_CHARGED", "event_id": event_id, "instrument": instrument, + "reason": "PAPER_FUNDING_POSITION_CLOSED", "interval_start": boundary.isoformat(), "synthetic": True}) + continue + mark, observed = self._settlement_mark(instrument, now) + if mark is None: + self._defer(notices, "PAPER_FUNDING", event_id, instrument, observed, interval_start=boundary.isoformat()) + continue + if snapshot is None: + snapshot = self.matcher.dump_state() + payment = -(PAPER_FUNDING_RATE_PER_INTERVAL * quantity * mark * multiplier) + applied = margin.apply_funding(event_id, instrument, payment) + self._settled_events.add(event_id) + added.append(event_id) + del self._funding_due[event_id] + self._deferrals.pop(event_id, None) + if applied: + events.append({"kind": "PAPER_FUNDING", "event_id": event_id, "instrument": instrument, "payment": str(payment), + "quantity": str(quantity), "mark": str(mark), "rate": str(PAPER_FUNDING_RATE_PER_INTERVAL), + "synthetic": True, "interval_start": boundary.isoformat()}) for position in margin.positions: instrument = position.instrument_id - contract = next((item for item in self._instruments if item.instrument_id == instrument), None) - if contract is None: + contract = contracts.get(instrument) + if contract is None or contract.declared_subtype is not ProductSubtype.DATED_FUTURE \ + or contract.expiry is None or now < contract.expiry: continue - depth = self._depth.get(instrument) - mark: Decimal | None = None - observed: datetime | None = None - if depth is not None and depth[0] and depth[1]: - mark, observed = (depth[0][0].price + depth[1][0].price) / 2, depth[2] - subtype = contract.declared_subtype - interval = PAPER_FUNDING_INTERVALS.get(subtype) - if interval is not None and mark is not None: - boundary = _interval_start(now, interval) - while boundary > previous: - stamp = boundary.strftime("%Y%m%dT%H%M%SZ") - event_id = f"paper-funding:{instrument}:{stamp}" - if event_id not in self._settled_events: - if snapshot is None: - snapshot = self.matcher.dump_state() - payment = -(PAPER_FUNDING_RATE_PER_INTERVAL * position.quantity * mark * contract.contract_multiplier) - applied = margin.apply_funding(event_id, instrument, payment) - self._settled_events.add(event_id) - added.append(event_id) - if applied: - events.append({"kind": "PAPER_FUNDING", "event_id": event_id, "instrument": instrument, "payment": str(payment), - "rate": str(PAPER_FUNDING_RATE_PER_INTERVAL), "synthetic": True, - "interval_start": boundary.isoformat()}) - boundary -= interval - expiry = getattr(contract, "expiry", None) - if subtype is ProductSubtype.DATED_FUTURE and expiry is not None and now >= expiry: - event_id = f"{PAPER_EXPIRY_PREFIX}{instrument}:{expiry.strftime('%Y%m%dT%H%M%SZ')}" - if event_id not in self._settled_events: - max_age = self.matcher.book_max_age - if mark is None or observed is None or now - observed > max_age: - reason = "PAPER_EXPIRY_MARK_MISSING" if observed is None else "PAPER_EXPIRY_MARK_STALE" - if self._expiry_deferrals.get(event_id) != reason: - self._expiry_deferrals[event_id] = reason - deferred.append({"kind": "PAPER_EXPIRY_DEFERRED", "event_id": event_id, "instrument": instrument, - "reason": reason, "mark_observed_at": observed.isoformat() if observed is not None else None, - "max_mark_age_seconds": str(max_age.total_seconds()), "synthetic": True}) - continue - if snapshot is None: - snapshot = self.matcher.dump_state() - realized = margin.apply_fill(instrument, -position.quantity, mark, multiplier=contract.contract_multiplier, - fee=Decimal(0), reduce_only=True) - self._settled_events.add(event_id) - self._expiry_deferrals.pop(event_id, None) - added.append(event_id) - events.append({"kind": "PAPER_EXPIRY_SETTLEMENT", "event_id": event_id, "instrument": instrument, - "quantity": str(position.quantity), "mark": str(mark), "realized": str(realized), "synthetic": True}) + event_id = f"{PAPER_EXPIRY_PREFIX}{instrument}:{contract.expiry.strftime('%Y%m%dT%H%M%SZ')}" + if event_id in self._settled_events: + continue + mark, observed = self._settlement_mark(instrument, now) + if mark is None: + self._defer(notices, "PAPER_EXPIRY", event_id, instrument, observed) + continue + if snapshot is None: + snapshot = self.matcher.dump_state() + realized = margin.apply_fill(instrument, -position.quantity, mark, multiplier=contract.contract_multiplier, + fee=Decimal(0), reduce_only=True) + self._settled_events.add(event_id) + self._deferrals.pop(event_id, None) + added.append(event_id) + events.append({"kind": "PAPER_EXPIRY_SETTLEMENT", "event_id": event_id, "instrument": instrument, + "quantity": str(position.quantity), "mark": str(mark), "realized": str(realized), "synthetic": True}) incidents: list[str] = [] if events: def save(connection: sqlite3.Connection) -> None: @@ -709,14 +725,39 @@ def save(connection: sqlite3.Connection) -> None: if snapshot is not None: self._matcher = PaperMatcher.restore(self.context, self._instruments, snapshot) self._settled_events.difference_update(added) - self._expiry_deferrals = reported + self._funding_due, self._deferrals = due, reported self._settlement_cursor = previous raise for incident in incidents: self.interlock.provisional_fence(self.context.authority_id, incident) - if events or deferred: + if events or notices: self._notify() - return (*events, *deferred) + return (*events, *notices) + + def _settlement_mark(self, instrument: str, now: datetime) -> tuple[Decimal | None, datetime | None]: + """The mid a settlement may book at ``now``, and when its book was observed. + + No mid without a two-sided book (then no observation time either), or when + that book is older than the matcher's ``book_max_age``. + """ + depth = self._depth.get(instrument) + if depth is None or not depth[0] or not depth[1]: + return None, None + bids, asks, observed = depth + if now - observed > self.matcher.book_max_age: + return None, observed + return (bids[0].price + asks[0].price) / 2, observed + + def _defer(self, notices: list[dict[str, Any]], kind: str, event_id: str, instrument: str, + observed: datetime | None, **detail: str) -> None: + """Report a settlement event left due for want of a fresh mark, once per reason.""" + reason = f"{kind}_MARK_MISSING" if observed is None else f"{kind}_MARK_STALE" + if self._deferrals.get(event_id) == reason: + return + self._deferrals[event_id] = reason + notices.append({"kind": f"{kind}_DEFERRED", "event_id": event_id, "instrument": instrument, "reason": reason, + "mark_observed_at": observed.isoformat() if observed is not None else None, + "max_mark_age_seconds": str(self.matcher.book_max_age.total_seconds()), **detail, "synthetic": True}) @exact_decimal def _settle_expiry_tx(self, connection: sqlite3.Connection, event: dict[str, Any]) -> FillResult: diff --git a/tests/worker/test_paper_settlement.py b/tests/worker/test_paper_settlement.py index 01ea5fc..2fe217b 100644 --- a/tests/worker/test_paper_settlement.py +++ b/tests/worker/test_paper_settlement.py @@ -45,6 +45,7 @@ def test_funding_is_charged_when_the_clock_crosses_an_interval_boundary_only(sel now = datetime.now(timezone.utc) self.assertEqual(engine.settle(now + timedelta(seconds=1)), ()) # inside the current 8 h interval: nothing boundary = _interval_start(now, timedelta(hours=8)) + timedelta(hours=8) + self.refresh_mark(engine, boundary) events = engine.settle(boundary + timedelta(seconds=1)) self.assertEqual(len(events), 1) [event] = events @@ -56,6 +57,7 @@ def test_funding_is_charged_when_the_clock_crosses_an_interval_boundary_only(sel # Idempotent per boundary; a later call inside the same interval charges nothing. self.assertEqual(engine.settle(boundary + timedelta(seconds=2)), ()) # Two boundaries crossed at once are two events. + self.refresh_mark(engine, boundary + timedelta(hours=16)) events = engine.settle(boundary + timedelta(hours=16, seconds=1)) self.assertEqual([item["interval_start"] for item in events], [(boundary + timedelta(hours=16)).isoformat(), (boundary + timedelta(hours=8)).isoformat()]) @@ -190,6 +192,7 @@ def test_failed_funding_commit_rolls_the_margin_book_and_event_ids_back(self) -> now = datetime.now(timezone.utc) boundary = _interval_start(now, timedelta(hours=8)) + timedelta(hours=8) self.assertEqual(engine.settle(now + timedelta(seconds=1)), ()) + self.refresh_mark(engine, boundary) with patch.object(engine, "_persist_tx", side_effect=DomainError("SIMULATED_COMMIT_FAILURE")): with self.assertRaisesRegex(DomainError, "SIMULATED_COMMIT_FAILURE"): engine.settle(boundary + timedelta(seconds=1)) @@ -205,6 +208,56 @@ def test_failed_funding_commit_rolls_the_margin_book_and_event_ids_back(self) -> self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 1) self.assertEqual(engine.settle(boundary + timedelta(seconds=3)), ()) + def crossed_at_a_stale_mark(self) -> tuple[Any, datetime, dict[str, Any]]: + engine = self.open_long() + now = datetime.now(timezone.utc) + boundary = _interval_start(now, timedelta(hours=8)) + timedelta(hours=8) + self.assertEqual(engine.settle(now + timedelta(seconds=1)), ()) + self.collateral = engine.matcher.margin.collateral + # The last BTC-USD book was observed when the long opened, hours before the boundary. + [deferred] = engine.settle(boundary + timedelta(seconds=1)) + self.assertEqual((deferred["kind"], deferred["reason"], deferred["instrument"], deferred["interval_start"], deferred["synthetic"]), + ("PAPER_FUNDING_DEFERRED", "PAPER_FUNDING_MARK_STALE", "BTC-USD", boundary.isoformat(), True)) + return engine, boundary, deferred + + def test_funding_at_a_stale_mark_is_deferred_and_charged_at_the_next_fresh_mark(self) -> None: + engine, boundary, deferred = self.crossed_at_a_stale_mark() + margin = engine.matcher.margin + # Nothing is charged, in memory or durably. + self.assertEqual((margin.collateral, self.durable_collateral()), (self.collateral, self.collateral)) + self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 0) + # Reported once per reason; the cursor moves on but the crossed boundary stays due. + self.assertEqual(engine.settle(boundary + timedelta(seconds=2)), ()) + engine._depth.pop("BTC-USD") + [missing] = engine.settle(boundary + timedelta(seconds=3)) + self.assertEqual((missing["kind"], missing["reason"]), ("PAPER_FUNDING_DEFERRED", "PAPER_FUNDING_MARK_MISSING")) + # A book no older than the venue's limit charges that boundary, for the quantity held when it was crossed. + observed = boundary + timedelta(seconds=4) + self.refresh_mark(engine, observed, "100.00", "100.20") + [event] = engine.settle(observed + engine.matcher.book_max_age) + self.assertEqual((event["kind"], event["event_id"], event["interval_start"]), + ("PAPER_FUNDING", deferred["event_id"], boundary.isoformat())) + expected = -(PAPER_FUNDING_RATE_PER_INTERVAL * Decimal("0.04") * Decimal("100.10")) + self.assertEqual(Decimal(event["payment"]), expected) + self.assertEqual((margin.collateral, self.durable_collateral()), (self.collateral + expected, self.collateral + expected)) + self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 1) + self.assertEqual(engine.settle(observed + timedelta(minutes=1)), ()) + Journal(self.worker.writer).verify() + + def test_funding_due_on_a_position_closed_before_a_fresh_mark_is_reported_not_charged(self) -> None: + engine, boundary, deferred = self.crossed_at_a_stale_mark() + margin = engine.matcher.margin + # The position closes before any fresh mark prices the boundary (as a resting exit filling at the returning book would). + margin.apply_fill("BTC-USD", Decimal("-0.04"), Decimal("99.85"), multiplier=Decimal(1), fee=Decimal(0), reduce_only=True) + collateral = margin.collateral + self.refresh_mark(engine, boundary + timedelta(seconds=2)) + [dropped] = engine.settle(boundary + timedelta(seconds=3)) + self.assertEqual((dropped["kind"], dropped["reason"], dropped["event_id"]), + ("PAPER_FUNDING_NOT_CHARGED", "PAPER_FUNDING_POSITION_CLOSED", deferred["event_id"])) + self.assertEqual(margin.collateral, collateral) + self.assertEqual(self.journal_kinds().count("PAPER_FUNDING"), 0) + self.assertEqual(engine.settle(boundary + timedelta(seconds=4)), ()) + def test_failed_expiry_commit_leaves_margin_and_ledger_open_until_the_retry(self) -> None: engine = self.open_long() expiry = datetime.now(timezone.utc) + timedelta(days=1) From 28c9677488e2affe2daadcb59f96e95da70c7a7a Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:24:20 +0200 Subject: [PATCH 3/9] Run CI on pushes and pull requests with current pins The validation workflow only ran on manual dispatch, so no change was ever checked remotely. It now runs on pushes to main, on pull requests and on demand, and superseded pull-request runs are cancelled. - actions/checkout v5 -> v7 without persisted credentials. - astral-sh/setup-uv v7 -> v10; caching follows the action's safer default instead of being forced on. - uv 0.12.15 -> 0.13.1 and Python 3.14.7 -> 3.14.8 (the local toolchain), pinned once at workflow level. - Named jobs and steps; the checks themselves are unchanged. Docs and the Makefile no longer describe the workflow as manual-only. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 67 ++++++++++++++++++++++++----------- Makefile | 2 +- docs/evidence-matrix.md | 2 +- docs/implementation-status.md | 2 +- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c12bc13..ffbb8cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,27 @@ -name: coinfly validation +# CI validates source contracts. It does not sign, publish, enable live orders, +# certify regional eligibility or substitute for native release evidence. +name: CI + on: + push: + branches: [main] + pull_request: workflow_dispatch: - # push: - # pull_request: + permissions: contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + UV_VERSION: "0.13.1" + PYTHON_VERSION: "3.14.8" + jobs: contracts: + name: Contracts (${{ matrix.os }}) strategy: fail-fast: false matrix: @@ -14,21 +29,27 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 25 steps: - - uses: actions/checkout@v5 - - uses: astral-sh/setup-uv@v7 + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v10 with: - version: "0.12.15" - python-version: "3.14.7" - enable-cache: true - - run: uv sync --locked --extra desktop --extra research --extra training + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + - name: Install locked dependencies + run: uv sync --locked --extra desktop --extra research --extra training - name: Verify linked SQLite requirement run: uv run python -c "import sqlite3; assert sqlite3.sqlite_version_info >= (3,51,3), sqlite3.sqlite_version" - - run: uv run ruff check src tests scripts packaging dev.py - - run: uv run mypy src/coinfly - - run: uv run python -m pytest -q + - name: Lint + run: uv run ruff check src tests scripts packaging dev.py + - name: Type-check + run: uv run mypy src/coinfly + - name: Test + run: uv run python -m pytest -q - name: Verify fail-closed doctor diagnostics run: uv run python scripts/verify_doctor.py - - run: uv build --no-sources + - name: Build wheel and sdist + run: uv build --no-sources - name: Inspect wheel payload run: uv run python scripts/verify_artifact.py - name: Compile macOS ARM64 native sources with vault access disabled @@ -46,16 +67,20 @@ jobs: run: | cmake -S native -B build/native -A x64 cmake --build build/native --config Release + dependencies: + name: Dependency audit runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v5 - - uses: astral-sh/setup-uv@v7 + - uses: actions/checkout@v7 with: - version: "0.12.15" - python-version: "3.14.7" - - run: uv sync --locked --extra desktop --extra research --extra training - - run: uv run pip-audit --local --skip-editable -# CI validates source contracts. It does not sign, publish, enable live orders, -# certify regional eligibility or substitute for native release evidence. + persist-credentials: false + - uses: astral-sh/setup-uv@v10 + with: + version: ${{ env.UV_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} + - name: Install locked dependencies + run: uv sync --locked --extra desktop --extra research --extra training + - name: Audit installed packages + run: uv run pip-audit --local --skip-editable diff --git a/Makefile b/Makefile index f5dda52..9376015 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,6 @@ github-bootstrap: @$(MAKE) github-repo-create @$(MAKE) github-verify -# Trigger the manual-only validation workflow (.github/workflows/ci.yml) on GitHub. +# Trigger the validation workflow (.github/workflows/ci.yml) on main manually; it also runs on pushes and PRs. github-run-ci: github-check gh workflow run ci.yml --repo $(GITHUB_FULL) --ref main \ No newline at end of file diff --git a/docs/evidence-matrix.md b/docs/evidence-matrix.md index f2a11ee..7c99931 100644 --- a/docs/evidence-matrix.md +++ b/docs/evidence-matrix.md @@ -32,7 +32,7 @@ The 14 skips are expected on this host: | --- | --- | | Native desktop smoke tests (`test-ui`) | Not run | | Full MaleCNS graph (`prepare`, native kernel, full-graph paper run) | Not run; unit tests use small synthetic graphs | -| Windows AMD64 and Linux AMD64 native runs | Not run; the CI workflow is manual-only | +| Windows AMD64 and Linux AMD64 native runs | Not run locally; CI only compiles the native sources on each target | | Signed or unsigned production builds | Not run; see [macOS production builds](devsh-production-builds.md) | | Live accounts, orders, transfers or enrollment | Never used in testing | diff --git a/docs/implementation-status.md b/docs/implementation-status.md index cb3d407..3c678b0 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -18,7 +18,7 @@ uses a fresh-install format. | Cancellation and account truth | A cancel acknowledgement does not release reservations; reconciliation establishes terminal facts; incomplete account evidence invalidates readiness; cash subtotal labeled as cash | Real venue qualification, scope/placement coverage verification and worst-case reconciliation testing | | Regional/derivative routing | Existing capability resolver and distinct product contracts; existing synthetic paper derivative model | Germany/Global entitlement evidence, route verification, live derivative gateway/engine and exact venue economics; enforcement of the declared `margin_mode` and `max_maintenance_utilization` strategy fields | | Secrets | Explicit Keychain, Credential Manager and Linux Secret Service setup backends; no plaintext fallback | Linux native execution-vault bridge; genuine signing/enrollment qualification on all targets | -| Packaging and CI | Existing macOS signing builder; common desktop/worker entry point; Windows/Linux paper development spec; three-target CI matrix (manual `workflow_dispatch` only) | Actual locked CI runs, signed Windows/Linux production installers and platform-specific native qualification | +| Packaging and CI | Existing macOS signing builder; common desktop/worker entry point; Windows/Linux paper development spec; three-target CI matrix on pushes to `main`, pull requests and manual dispatch | A passing CI history on all three targets, signed Windows/Linux production installers and platform-specific native qualification | `python dev.py run` is explicitly local paper development. A verified dataset, local kernel, stored credential or successful read-only connection test does not From 21f1ab2050321392fc20e0a76679d89548112a8c Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:24:20 +0200 Subject: [PATCH 4/9] Restructure README for a concise project landing page Header with icon and CI/license/Python/platform badges, a warning callout, highlights, a pipeline diagram with a stage table, a status table, getting started, development and CI, and documentation links. Claims are carried over from the previous README; detailed caveats stay in docs. Co-Authored-By: Claude Opus 5.5 --- README.md | 204 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 106 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 9cd12d9..bce90a4 100644 --- a/README.md +++ b/README.md @@ -1,125 +1,133 @@ -# Coinfly +

+ Coinfly +

+ +

Coinfly

+ +

+ Paper-trading research desktop that drives the full MaleCNS fruit-fly connectome with crypto market data. +

+ +

+ CI + MIT License + Python 3.11+ + Platforms +

+ +> [!WARNING] +> **Experimental research software.** Coinfly is shared as-is for research and +> education. There is no guarantee that it works, is stable or will be updated, +> and no claim that it trades profitably. Paper execution is the default and live +> trading is closed. This is not financial advice. Use it at your own risk. + +## Overview + +Coinfly feeds crypto order-book data into the complete retained **MaleCNS v1.0** +male fruit-fly connectome (166,700 neurons, 25,582,938 edges, no pruning) and lets +identified descending neurons propose paper trades. A separate worker process owns +the autonomous loop, exact accounting, risk limits and isolated paper execution; +the desktop app only sends commands and shows status. + +## Highlights + +- **The whole graph, no shortcuts.** A native C++ spiking kernel runs every retained + neuron. A HOLD is never replaced by a scripted trade, and no LLM policy or hidden + profit rule selects actions. +- **Exact, durable accounting.** Decimal quantities, order intent persisted before + dispatch, and a fresh SQLite ledger pinned by schema digests. +- **Fail-closed execution.** Stale data, sequence gaps and silent feeds block + decisions. Unknown order outcomes halt execution until reconciliation and are + never blindly retried. +- **Risk can veto, never trade.** The risk guard may reject an order; it never + chooses a different one. +- **Labeled market data.** Coinbase public REST/WebSocket (no credentials), recorded + replays or synthetic fixtures, each labeled end to end. -**1.0.0** · **com.devsolux.coinfly** · Publisher: **devsolux** · [MIT License](LICENSE) +## How it works -> **Experimental.** Coinfly is shared as-is for research and education. There is -> no guarantee that it works, is stable or will be updated, and no claim that it -> trades profitably. It is not financial advice. Use it at your own risk. +```mermaid +flowchart LR + Observe --> Sense --> Propagate --> Decode --> Map --> Admit --> Execute + Execute -. research learning only .-> Reinforce +``` -Coinfly is desktop trading research software. It feeds crypto market data into -the full retained **MaleCNS v1.0** male fruit-fly connectome (166,700 neurons, -25,582,938 edges, no pruning) and lets identified output neurons propose paper -trades. An independent worker owns the autonomous loop, exact accounting, risk -limits and isolated paper execution; the desktop only sends commands and shows -status. Targets: **macOS ARM64, Windows AMD64 and Linux AMD64**. +| Stage | What happens | +| --- | --- | +| Observe | Read an order book from a labeled source. | +| Sense | Four fixed features become an 8-bit RGB frame delivered through a photoreceptor proxy (an engineered input, not a market model). | +| Propagate | The native kernel runs the whole graph. | +| Decode | A fixed decoder reads the DNp20 (left/right) and DNpe017 descending neurons as BUY, SELL or HOLD. | +| Map | A fixed mapping moves only the chosen instrument's target weight, by at most the configured step. | +| Admit | An exact planner, turnover guard and risk actor check capital, exposure, drawdown and loss limits. | +| Execute | The intent is stored, filled in the paper book and reconciled. | +| Reinforce | *Research learning only:* paper profit/loss stimulates two identified dopamine cells (PAM11, PPL101), and a bounded three-factor rule adjusts existing KC→MBON synapses. | -## How it works +Reinforcement is an engineered input. It does not model pain, pleasure or +consciousness, and profitable learning is unverified. See the +[autonomous lifecycle](docs/autonomous-lifecycle.md) for the full loop. -Each worker tick runs these separate stages for every strategy-owned slot: - -1. **Observe** – read an order book from a labeled source: Coinbase public - REST/WebSocket, a recorded replay file, or a synthetic test fixture. Stale - data, sequence gaps and silent feeds block decisions. -2. **Sense** – four fixed features (mid offset, spread, depth imbalance, one-step - return) become exact integer ticks, are scaled only with statistics fitted on - earlier data, and are rendered as an 8-bit RGB frame. A photoreceptor proxy - delivers that frame to the graph. This is an engineered input, not a market model. -3. **Propagate** – the native C++ spiking kernel runs the whole graph. -4. **Decode** – a fixed decoder reads the DNp20 (left/right) and DNpe017 - descending neurons as BUY, SELL or HOLD. A HOLD is never replaced by a - scripted trade. -5. **Map** – a fixed mapping moves only the chosen instrument's target weight, - by at most the configured step (default 0.05 of strategy capital, never above - the 0.05 turnover limit). Paper perpetuals use a separate long/short mapping. -6. **Admit** – an exact Decimal planner, turnover guard and risk actor check - capital, exposure, drawdown and loss limits. Risk may reject an order; it - never chooses a different one. -7. **Execute** – the order intent is stored before dispatch, filled in the paper - book and reconciled. An unknown outcome halts execution until reconciliation; - orders are never blindly retried. -8. **Reinforce** *(research learning mode only)* – paper-ledger profit/loss - becomes engineered stimulation of two identified dopamine cells (PAM11, - PPL101). A bounded three-factor rule changes only existing KC→MBON synapses. - This does not model pain, pleasure or consciousness. - -Baseline and frozen-checkpoint controllers do not learn. Profitable learning is -unverified. Details: [autonomous lifecycle](docs/autonomous-lifecycle.md), -[dataset bootstrap](docs/malecns-bootstrap.md), [training](docs/malecns-training.md) -and [plasticity kernel](docs/malecns-rstdp-kernel.md). - -## What is supported - -| Area | Supported | Not supported or not yet qualified | +## Project status + +| Area | Available | Not yet supported or qualified | | --- | --- | --- | -| Platforms | macOS ARM64, Windows AMD64, Linux AMD64 from source; Python ≥3.11 with SQLite ≥3.51.3 | Latest recorded run is macOS ARM64 lint, type checks, tests and package build ([validation evidence](docs/evidence-matrix.md)); no native desktop smoke test, full-graph run or Windows/Linux native run is recorded | -| Execution | Paper (SANDBOX) trading with an exact local ledger: spot and synthetic perpetuals | Real-money trading; live mode is closed by default | -| Venue | Coinbase public market data (no credentials); live Advanced Trade spot code path | Live spot qualification (enrollment, signing, account evidence); live derivatives; other exchanges | -| Market data | Coinbase public REST and WebSocket level2, JSON-lines replay, labeled synthetic fixtures | — | -| Neural controllers | Full preserved graph in baseline inference, frozen checkpoint, or single-slot paper research learning (`malecns-risk-adjusted-experiment-v2`/`-v3`) | Separate NeuralHost process; model promotion workflow; any validated trading result | -| Research tooling | Optuna studies with paired frozen controls; training and plasticity benchmarks | No development-fold or holdout result has been produced | -| Regions | GLOBAL and GERMANY policies through the capability resolver | Verified regional entitlements | +| Execution | Paper (SANDBOX) spot and synthetic perpetuals with an exact local ledger | Real-money trading; live mode is closed by default | +| Market data | Coinbase public REST and WebSocket level2, JSON-lines replay, synthetic fixtures | Other exchanges | +| Controllers | Full-graph baseline inference, frozen checkpoints, single-slot paper research learning | Any validated trading result | +| Platforms | macOS ARM64, Windows AMD64 and Linux AMD64 from source | Native desktop smoke tests and native runs on every target | | Credentials | macOS Keychain, Windows Credential Manager, Linux Secret Service; no plaintext fallback | Linux native execution-vault bridge | -| Desktop | Tk/CustomTkinter UI in English and German | Packaged smoke tests on every target | | Storage | Fresh 1.0.0 SQLite ledger pinned by schema digests | Migration or import of older or modified ledgers | -The full matrix is in [implementation status](docs/implementation-status.md) and -[venue capabilities](docs/venues/capability-matrix.md). +Details: [implementation status](docs/implementation-status.md), +[venue capabilities](docs/venues/capability-matrix.md) and the +[latest validation evidence](docs/evidence-matrix.md). -## Native development +## Getting started -Use native Python ≥3.11 linked to **SQLite ≥3.51.3**, with Tk, uv and the -locked dependencies. A native C++ toolchain and CMake are needed for the explicit -local research kernel build, not for ordinary application startup. +**Requirements:** Python ≥3.11 linked to SQLite ≥3.51.3 with Tk, +[uv](https://docs.astral.sh/uv/), and CMake with a C++ toolchain for the native kernel. ```sh -python dev.py setup -python dev.py prepare -python dev.py kernel -python dev.py run +python dev.py setup # create the locked environment +python dev.py prepare # download and verify the MaleCNS v1.0 dataset (about 1.1 GB on first use) +python dev.py kernel # build the native spiking kernel +python dev.py run # start the desktop app ``` -On Windows, `py dev.py …` or `./dev.ps1 …` uses the same launcher. On macOS, -`./dev.sh` remains a convenience entry point. The portable launcher selects the -native virtual-environment layout and rejects unsupported CPU/OS combinations. - -Preparation verifies all retained neurons and edges against pinned source/array -identities (about 1.1 GB of source downloads on first use). Application data -lives under the native account's `~/.devsolux/coinfly` root; development workers -have checkout-specific ledgers and authenticated local IPC. Paper execution never -acquires exchange-mutation authority. +On Windows use `py dev.py …` or `./dev.ps1 …`; on macOS `./dev.sh …` is +equivalent. In **Overview**, configure the `malecns-autonomous` strategy (capital, +slot limit, feed and controller) and enable automation. The +[paper startup runbook](docs/runbooks/paper-startup.md) walks through each step. +Paper mode needs no exchange credentials, and application data lives under +`~/.devsolux/coinfly`. -In Overview, configure `malecns-autonomous`, its capital allocation, slot limit, -feed and baseline/frozen-checkpoint controller, then enable automation. The worker -selects eligible instruments and manages owned slots. A native kernel and valid -observations are required. Fixture controllers are for isolated tests, not -ordinary strategy choices. +## Development ```sh -python dev.py test -python dev.py test-ui -python dev.py check +python dev.py check # lint, strict type checks and the full test suite +python dev.py test # tests only; pytest arguments pass through +python dev.py test-ui # native desktop smoke tests (needs a desktop session) +python dev.py doctor # environment and production-readiness diagnostics ``` -## Fresh 1.0.0 format +CI runs lint, type checks, tests, package verification and a native compile on +macOS, Windows and Linux for every pull request and push to `main`. Project rules +for contributors and agents are in [AGENTS.md](AGENTS.md). -This is a fresh-install project, not an upgrade distribution. `storage/schema.sql` -and final component DDL are pinned by format/digest contracts. Old or modified -ledgers are rejected without automatic migration, deletion, import or reseeding. -Application version 1.0.0 is distinct from preserved scientific v2/v3 profiles. +macOS release builds (`./dev.sh build-unsigned`, `./dev.sh build`, optionally with +`--preflight`) stay closed until genuine production prerequisites are supplied. See +[desktop packaging](packaging/README.md) and +[macOS production builds](docs/devsh-production-builds.md). -## Delivery boundaries +## Documentation -No live credentials, orders or transfers were used in testing. Production -enrollment/evidence providers, Linux native execution signing and production -target qualification remain incomplete, so the default release path stays closed -until genuine prerequisites are supplied; no test evidence satisfies them. +The [documentation index](docs/README.md) covers operation, runbooks, venues and +architecture. Start with: -macOS production entry points are `./dev.sh build-unsigned` and `./dev.sh build`, -with optional `--preflight`. Both read reviewed publisher metadata from -`~/.devsolux/coinfly` by default and require the same production gates; missing -inputs never select development packaging. See [desktop packaging](packaging/README.md), -[setup and storage](docs/desktop-initialization-and-settings.md) and the -[documentation index](docs/README.md). +- [Paper startup without credentials](docs/runbooks/paper-startup.md) +- [Full-graph provenance and dataset bootstrap](docs/malecns-bootstrap.md) +- [Experimental training and evaluation](docs/malecns-training.md) +- [Safety invariants and supporting tests](docs/requirements/invariants.md) ## License From e03bd28c25a5be4aa940e9bd6236725181dad19e Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:33:05 +0200 Subject: [PATCH 5/9] Pin CI actions to release commits setup-uv stopped publishing floating major tags in v8, so `setup-uv@v10` could not be resolved and every job failed at set-up. Pin it and checkout to the commit SHAs of their exact releases (v10.3.0, v7.0.1). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffbb8cc..2d6213a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,10 +29,10 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 25 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: astral-sh/setup-uv@v10 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 (no floating major tags) with: version: ${{ env.UV_VERSION }} python-version: ${{ env.PYTHON_VERSION }} @@ -73,10 +73,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: astral-sh/setup-uv@v10 + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 (no floating major tags) with: version: ${{ env.UV_VERSION }} python-version: ${{ env.PYTHON_VERSION }} From ac7f7340bb33f556efdebb9f8f2bff537a957004 Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:37:15 +0200 Subject: [PATCH 6/9] Require uv-managed Python in CI The Windows job resolved Python 3.14.8 to the runner's preinstalled python.org build, which links SQLite 3.50.4, below the 3.51.3 minimum, so the SQLite gate failed. uv prefers an existing system interpreter over a download; UV_MANAGED_PYTHON makes every runner use uv's managed CPython, which bundles a current SQLite. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d6213a..5c73625 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,10 @@ concurrency: env: UV_VERSION: "0.13.1" PYTHON_VERSION: "3.14.8" + # Use uv-managed CPython on every runner. A preinstalled runner Python can link + # an older SQLite (the Windows python.org build ships 3.50.4) than the 3.51.3 + # minimum; uv's managed builds bundle a current one. + UV_MANAGED_PYTHON: "1" jobs: contracts: From 6466e60c2349f28b5fc2d8b42da585d3ea225bd3 Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:43:24 +0200 Subject: [PATCH 7/9] Type-check cleanly on Windows mypy checks Windows runners as sys.platform == "win32". With warn_unreachable, the POSIX code after an `if sys.platform == "win32": return` block was reported unreachable, and resource.getrusage does not exist in the Windows stubs. Put the POSIX paths in explicit platform else-branches (which mypy treats as platform checks) in native_user_home, development_socket and _max_rss_bytes. Behavior is unchanged; mypy --platform win32/linux/darwin all pass. Co-Authored-By: Claude Opus 5.5 --- src/coinfly/development.py | 3 ++- src/coinfly/paths.py | 5 +++-- src/coinfly/research/optimization.py | 17 ++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/coinfly/development.py b/src/coinfly/development.py index b84b4c8..1a22452 100644 --- a/src/coinfly/development.py +++ b/src/coinfly/development.py @@ -62,7 +62,8 @@ def development_socket(checkout: Path) -> Path: from coinfly.ipc.windows_native import pipe_path, process_sid return Path(pipe_path(process_sid(os.getpid()), checkout=identity)) - return default_socket_path().with_name(f"development-{identity}.sock") + else: + return default_socket_path().with_name(f"development-{identity}.sock") # How long a launcher waits for a stale worker to retire (automation stopped, diff --git a/src/coinfly/paths.py b/src/coinfly/paths.py index 3444637..875d161 100644 --- a/src/coinfly/paths.py +++ b/src/coinfly/paths.py @@ -50,9 +50,10 @@ def native_user_home() -> Path: if function(None, 0x0028, None, 0, buffer) != 0 or not buffer.value: raise DomainError("NATIVE_HOME_UNAVAILABLE") return Path(buffer.value) - import pwd + else: + import pwd - return Path(pwd.getpwuid(os.getuid()).pw_dir) + return Path(pwd.getpwuid(os.getuid()).pw_dir) def is_link(path: Path) -> bool: diff --git a/src/coinfly/research/optimization.py b/src/coinfly/research/optimization.py index a51f8ea..59d1a8a 100644 --- a/src/coinfly/research/optimization.py +++ b/src/coinfly/research/optimization.py @@ -544,13 +544,16 @@ def evaluate_fold(resources: StudyResources, fold: Fold, profile: TrainingProfil def _max_rss_bytes() -> int | None: """Process peak RSS in bytes (Linux reports KiB, macOS bytes); None where unmeasured.""" - try: - import resource # POSIX only; Windows hosts report None rather than a guess. - - peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - except (ImportError, AttributeError, OSError, ValueError): - return None - return int(peak) if sys.platform == "darwin" else int(peak) * 1024 + if sys.platform == "win32": + return None # no POSIX resource module: reported as unmeasured rather than a guess + else: + try: + import resource + + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + except (ImportError, AttributeError, OSError, ValueError): + return None + return int(peak) if sys.platform == "darwin" else int(peak) * 1024 def comparable_metrics(evaluation: dict[str, Any]) -> dict[str, float | None]: From 57229e374573f5f27f81dd3a3e07115206a9f06c Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:58:04 +0200 Subject: [PATCH 8/9] Fix Windows source checkouts, dataset prep and launcher modes Found by the first Windows CI run: - Check text out with LF everywhere (.gitattributes). A CRLF checkout on Windows changed the bytes behind pinned digests, so the schema and manifest identity checks failed. No committed content changes. - Dataset preparation fsynced graph files through read-only handles; Windows flushes only writable ones (EBADF). Open them r+b. - The source archive took launcher execute bits from the filesystem, which Windows does not record; shell launchers are now always 0o755. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 8 ++++++++ scripts/build_release_archive.py | 5 +++-- src/coinfly/bootstrap/development.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3e4a727 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Check text out with LF on every platform. Pinned digests (schema, manifests, +# native sources) are computed over the committed bytes, so a CRLF checkout on +# Windows would not match them. +* text=auto eol=lf + +*.png binary +*.ico binary +*.icns binary diff --git a/scripts/build_release_archive.py b/scripts/build_release_archive.py index 0976fdd..1cbb26b 100644 --- a/scripts/build_release_archive.py +++ b/scripts/build_release_archive.py @@ -46,8 +46,9 @@ def main() -> None: entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) entry.compress_type = zipfile.ZIP_DEFLATED entry.create_system = 3 - # Keep launchers executable after extraction, with normalized modes. - permissions = 0o755 if path.stat().st_mode & 0o111 else 0o644 + # Keep launchers executable after extraction, with normalized modes. Windows + # file modes carry no execute bit, so shell launchers are named explicitly. + permissions = 0o755 if path.stat().st_mode & 0o111 or path.suffix == ".sh" else 0o644 entry.external_attr = (0o100000 | permissions) << 16 archive.writestr(entry, data) source_files += 1 diff --git a/src/coinfly/bootstrap/development.py b/src/coinfly/bootstrap/development.py index 4340f61..6fd64cb 100644 --- a/src/coinfly/bootstrap/development.py +++ b/src/coinfly/bootstrap/development.py @@ -166,7 +166,7 @@ def install(self, envelope: bytes | None, *, now: datetime, prepare_graph(staging, controls, progress) dataset = self._verified(staging, controls, progress) for name in ("graph.npz", "normalized/neurons.feather"): - with (staging / name).open("rb") as source: + with (staging / name).open("r+b") as source: # Windows flushes only writable handles os.fsync(source.fileno()) _fsync_directory(staging / "normalized") _fsync_directory(staging) From 7f4bdd10b5f8495962f4ef70ba0b443a65c3ec32 Mon Sep 17 00:00:00 2001 From: Baris Aydin Date: Fri, 16 Oct 2026 19:58:04 +0200 Subject: [PATCH 9/9] Make the test suite portable to Windows - The network guard now allows loopback: Windows asyncio wakes its event loop through a loopback socket pair. Internet addresses and DNS stay blocked. - Worker RPC fixtures identify the caller with tests.support.local_peer() (coinfly.ipc.auth.current_uid) instead of POSIX-only os.getuid/getgid. - Read sources as UTF-8, use a platform socketpair, and expect coinfly-worker.exe for the packaged worker on Windows. - Skip on Windows only what is inherently POSIX, each with its reason: Unix-domain socket IPC tests (Windows uses named pipes) and the macOS release tooling's rename-over-empty-directory test. Co-Authored-By: Claude Opus 5.5 --- tests/architecture/test_context_boundaries.py | 4 ++-- tests/bootstrap/test_development_bootstrap.py | 4 +++- .../test_development_distribution.py | 2 +- tests/conftest.py | 23 ++++++++++++++----- tests/ipc/test_handshake.py | 4 +++- tests/network/test_websocket_client.py | 4 ++-- tests/support.py | 15 ++++++++++++ tests/test_macos_release.py | 2 ++ tests/ui/test_startup_handoff.py | 3 ++- tests/worker/test_automation_lifecycle.py | 5 ++-- tests/worker/test_automation_real_feed.py | 5 ++-- tests/worker/test_database_bootstrap.py | 2 ++ .../worker/test_initialization_diagnostics.py | 4 ++-- tests/worker/test_live_arming.py | 5 ++-- tests/worker/test_worker_replacement.py | 8 ++++--- 15 files changed, 62 insertions(+), 28 deletions(-) create mode 100644 tests/support.py diff --git a/tests/architecture/test_context_boundaries.py b/tests/architecture/test_context_boundaries.py index 675dce3..e041c6c 100644 --- a/tests/architecture/test_context_boundaries.py +++ b/tests/architecture/test_context_boundaries.py @@ -76,7 +76,7 @@ def test_active_imports_do_not_include_ambient_sdk_or_dotenv(self): root = Path(__file__).resolve().parents[2] / "src" / "coinfly" bad = {"dotenv", "pydantic_settings", "coinbase_agentkit", "coinbase.rest"} for path in root.rglob("*.py"): - for node in ast.walk(ast.parse(path.read_text())): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): modules = ([a.name for a in node.names] if isinstance(node, ast.Import) else [node.module or ""] if isinstance(node, ast.ImportFrom) else []) for module in modules: @@ -87,7 +87,7 @@ def test_ui_cannot_import_worker_database_or_broker(self): root = Path(__file__).resolve().parents[2] / "src" / "coinfly" / "ui" forbidden = ("sqlite3", "coinfly.storage", "coinfly.execution", "coinfly.security", "coinfly.venues") for path in root.rglob("*.py"): - for node in ast.walk(ast.parse(path.read_text())): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): modules = ([a.name for a in node.names] if isinstance(node, ast.Import) else [node.module or ""] if isinstance(node, ast.ImportFrom) else []) self.assertFalse(any(m == f or m.startswith(f + ".") for m in modules for f in forbidden), diff --git a/tests/bootstrap/test_development_bootstrap.py b/tests/bootstrap/test_development_bootstrap.py index c7a61c5..8852184 100644 --- a/tests/bootstrap/test_development_bootstrap.py +++ b/tests/bootstrap/test_development_bootstrap.py @@ -2,6 +2,7 @@ import asyncio import hashlib +import sys import threading from datetime import datetime, timezone from pathlib import Path @@ -149,7 +150,8 @@ def test_development_state_and_socket_are_isolated_and_live_is_rejected(tmp_path paths = WorkerPaths(root) assert paths.database.is_relative_to(application.storage / "development") assert not (tmp_path / "checkout").exists() - assert development_socket(tmp_path) != default_socket_path() + if sys.platform != "win32": # Windows has no Unix default socket; checkouts still differ below + assert development_socket(tmp_path) != default_socket_path() assert development_socket(tmp_path) != development_socket(tmp_path / "another-checkout") config = default_config().model_copy(update={"environment": Environment.LIVE, "run_profile": RunProfile.STANDARD}) with pytest.raises(DomainError, match="DEVELOPMENT_PAPER_ONLY"): diff --git a/tests/bootstrap/test_development_distribution.py b/tests/bootstrap/test_development_distribution.py index 6da259b..f13240c 100644 --- a/tests/bootstrap/test_development_distribution.py +++ b/tests/bootstrap/test_development_distribution.py @@ -83,7 +83,7 @@ def test_frozen_development_uses_bundled_worker_without_checkout(packaged, tmp_p desktop = tmp_path / "coinfly.app/Contents/MacOS/coinfly" desktop.parent.mkdir(parents=True) - worker = desktop.with_name("coinfly-worker") + worker = desktop.with_name("coinfly-worker.exe" if sys.platform == "win32" else "coinfly-worker") worker.touch() monkeypatch.setattr(sys, "executable", str(desktop)) assert development.checkout_root() == desktop.parent diff --git a/tests/conftest.py b/tests/conftest.py index 14a7d1f..7487587 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Test-process network boundary: local IPC only, no exchange connectivity.""" +import ipaddress import socket import pytest @@ -11,26 +12,36 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) def deny_internet_sockets(monkeypatch): - # Synthetic transport tests inject in-process responses. AF_UNIX remains - # available for the real worker IPC tests. Neither live nor sandbox HTTP is - # needed by this suite, so reject Internet sockets before DNS or connection. + # Synthetic transport tests inject in-process responses. AF_UNIX and loopback + # remain available for the real worker IPC tests (and for Windows asyncio, + # which wakes its event loop through a loopback socket pair). Neither live nor + # sandbox HTTP is needed by this suite, so reject Internet sockets before DNS + # or connection. def blocked(*_args, **_kwargs): raise AssertionError("Tests must use synthetic transports; Internet access is forbidden") + def internet(sock, address): + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + try: + return not ipaddress.ip_address(address[0]).is_loopback + except (TypeError, ValueError, IndexError): + return True + connect, connect_ex, sendto = socket.socket.connect, socket.socket.connect_ex, socket.socket.sendto def local_connect(sock, address): - if sock.family in (socket.AF_INET, socket.AF_INET6): + if internet(sock, address): return blocked() return connect(sock, address) def local_connect_ex(sock, address): - if sock.family in (socket.AF_INET, socket.AF_INET6): + if internet(sock, address): return blocked() return connect_ex(sock, address) def local_sendto(sock, *args): - if sock.family in (socket.AF_INET, socket.AF_INET6): + if internet(sock, args[-1]): return blocked() return sendto(sock, *args) diff --git a/tests/ipc/test_handshake.py b/tests/ipc/test_handshake.py index bcd0bca..42d1807 100644 --- a/tests/ipc/test_handshake.py +++ b/tests/ipc/test_handshake.py @@ -8,12 +8,13 @@ import errno import os +import sys import tempfile import threading import time from pathlib import Path from typing import Any -from unittest import TestCase +from unittest import TestCase, skipIf from coinfly import __application_id__, __version__ from coinfly.ipc.auth import OwnerOnlySandboxPolicy @@ -25,6 +26,7 @@ from tests.worker.test_automation_lifecycle import TestOnlyBootstrap +@skipIf(sys.platform == "win32", "Unix-domain socket IPC; the Windows transport uses named pipes") class HandshakeTests(TestCase): def setUp(self) -> None: self.directory = tempfile.TemporaryDirectory(prefix="cf-hello-") diff --git a/tests/network/test_websocket_client.py b/tests/network/test_websocket_client.py index 2072276..d914a09 100644 --- a/tests/network/test_websocket_client.py +++ b/tests/network/test_websocket_client.py @@ -1,6 +1,6 @@ """RFC 6455 client: framing, masking rules, handshake validation, ping/pong, silence. -The server side is an in-process thread on an AF_UNIX socketpair; the test never +The server side is an in-process thread on a local socketpair; the test never opens an Internet socket (the suite forbids it), so ``socket.create_connection`` is replaced for the duration of each connection. """ @@ -65,7 +65,7 @@ class FakeServer: """Minimal WebSocket server on one end of a socketpair.""" def __init__(self, *, accept_correctly: bool = True, status: bytes = b"101 Switching Protocols") -> None: - self.server, self.client_end = socket.socketpair(socket.AF_UNIX) + self.server, self.client_end = socket.socketpair() # AF_UNIX, or loopback on Windows self.accept_correctly = accept_correctly self.status = status self.received: list[bytes] = [] diff --git a/tests/support.py b/tests/support.py new file mode 100644 index 0000000..f2e1097 --- /dev/null +++ b/tests/support.py @@ -0,0 +1,15 @@ +"""Shared test helpers that hold on every supported platform.""" + +import os + +from coinfly.ipc.auth import PeerIdentity, current_uid + + +def local_peer() -> PeerIdentity: + """This process as an IPC peer, as the platform transport identifies it. + + POSIX transports report uid/gid; the Windows named-pipe transport reports + the user's principal id for both. + """ + uid = current_uid() + return PeerIdentity(uid=uid, gid=os.getgid() if hasattr(os, "getgid") else uid, pid=os.getpid()) diff --git a/tests/test_macos_release.py b/tests/test_macos_release.py index 469952b..2c7e66f 100644 --- a/tests/test_macos_release.py +++ b/tests/test_macos_release.py @@ -5,6 +5,7 @@ import plistlib import runpy import subprocess +import sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -147,6 +148,7 @@ def test_explicit_invalid_publisher_never_falls_back(builder, tmp_path, failure) assert not (stage / "development-distribution").exists() +@pytest.mark.skipif(sys.platform == "win32", reason="macOS release tooling relies on POSIX rename over an empty directory") def test_development_initialization_is_idempotent_atomic_and_preserves_existing(builder, tmp_path): directory = tmp_path / "local metadata" directory.mkdir() # dev.sh's mkdir -p is supported. diff --git a/tests/ui/test_startup_handoff.py b/tests/ui/test_startup_handoff.py index ff0d364..9939c30 100644 --- a/tests/ui/test_startup_handoff.py +++ b/tests/ui/test_startup_handoff.py @@ -1,3 +1,4 @@ +import sys import threading import pytest @@ -11,7 +12,7 @@ def test_packaged_desktop_launches_sibling_worker(tmp_path, monkeypatch): executable = tmp_path / "coinfly.app/Contents/MacOS/coinfly" executable.parent.mkdir(parents=True) - worker = executable.with_name("coinfly-worker") + worker = executable.with_name("coinfly-worker.exe" if sys.platform == "win32" else "coinfly-worker") monkeypatch.setattr("coinfly.cli.sys.executable", str(executable)) monkeypatch.setattr("coinfly.cli.sys.frozen", True, raising=False) with pytest.raises(DomainError, match="PACKAGED_WORKER_MISSING"): diff --git a/tests/worker/test_automation_lifecycle.py b/tests/worker/test_automation_lifecycle.py index 5edee5e..18485b0 100644 --- a/tests/worker/test_automation_lifecycle.py +++ b/tests/worker/test_automation_lifecycle.py @@ -12,7 +12,6 @@ from __future__ import annotations -import os import time from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -29,7 +28,6 @@ from coinfly.domain.context import DomainError, EngineState, ProductTrack, RegionPolicy from coinfly.domain.products import ProductSubtype from coinfly.execution.intent_store import IntentStore -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import RPCRequest from coinfly.market.feed import fixture_catalog from coinfly.market.observations import ( @@ -44,6 +42,7 @@ from coinfly.worker.automation import AutomationState from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer _SENTINEL = object() @@ -166,7 +165,7 @@ def rpc(self, method: str, params: dict[str, Any] | None = None, *, request_id: epoch = self.worker.coordinator.status(self.worker.authority_id).transition_epoch request = RPCRequest.parse({"jsonrpc": "2.0", "id": request_id or method, "method": method, "params": params or {}, "epoch": epoch, "deadline_ms": int(time.time() * 1000) + 10000}) - result = self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid())) + result = self.worker.handle(request, local_peer()) return cast(dict[str, Any], result) def initialize(self) -> None: diff --git a/tests/worker/test_automation_real_feed.py b/tests/worker/test_automation_real_feed.py index c659771..6c55ba2 100644 --- a/tests/worker/test_automation_real_feed.py +++ b/tests/worker/test_automation_real_feed.py @@ -13,7 +13,6 @@ from __future__ import annotations import json -import os import time from collections.abc import Mapping from datetime import datetime, timezone @@ -28,13 +27,13 @@ from coinfly.domain.context import DomainError, ProductTrack, RegionPolicy from coinfly.domain.products import ProductSubtype from coinfly.execution.intent_store import IntentStore -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import RPCRequest from coinfly.market.coinbase_public import CoinbasePublicRestFeed from coinfly.market.observations import DataSource from coinfly.worker.automation import AutomationConfig, AutomationState from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer from .test_automation_lifecycle import TestOnlyBootstrap @@ -110,7 +109,7 @@ def rpc(self, method: str, params: dict[str, Any] | None = None, *, request_id: epoch = self.worker.coordinator.status(self.worker.authority_id).transition_epoch request = RPCRequest.parse({"jsonrpc": "2.0", "id": request_id or method, "method": method, "params": params or {}, "epoch": epoch, "deadline_ms": int(time.time() * 1000) + 10000}) - return cast(dict[str, Any], self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid()))) + return cast(dict[str, Any], self.worker.handle(request, local_peer())) def initialize(self, config: EngineConfig | None = None) -> None: config = config or default_config() diff --git a/tests/worker/test_database_bootstrap.py b/tests/worker/test_database_bootstrap.py index 90bc58e..16fc3f7 100644 --- a/tests/worker/test_database_bootstrap.py +++ b/tests/worker/test_database_bootstrap.py @@ -1,6 +1,7 @@ """Real SQLite contention and a responsive diagnostic boundary; no operator data.""" import sqlite3 +import sys import threading import time from pathlib import Path @@ -151,6 +152,7 @@ def factory(**kwargs): startup.close() +@pytest.mark.skipif(sys.platform == "win32", reason="Unix-domain socket IPC; the Windows transport uses named pipes (sockets under /tmp)") def test_diagnostic_ipc_remains_available_until_database_lock_is_released(tmp_path): # Short AF_UNIX path, with the same private directory checks as production. import tempfile diff --git a/tests/worker/test_initialization_diagnostics.py b/tests/worker/test_initialization_diagnostics.py index 1295cdb..37323bb 100644 --- a/tests/worker/test_initialization_diagnostics.py +++ b/tests/worker/test_initialization_diagnostics.py @@ -22,12 +22,12 @@ from coinfly.bootstrap.manifest import BootstrapError from coinfly.config.models import default_config from coinfly.control.transition_state import TransitionPhase -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import RPCRequest from coinfly.observability.diagnostics import redacted_chain, source_fingerprint from coinfly.ui.state import DesktopState from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer _SENTINEL = object() @@ -69,7 +69,7 @@ def rpc(self, method: str, params: dict[str, Any] | None = None, *, request_id: epoch = self.worker.coordinator.status(self.worker.authority_id).transition_epoch request = RPCRequest.parse({"jsonrpc": "2.0", "id": request_id or method, "method": method, "params": params or {}, "epoch": epoch, "deadline_ms": int(time.time() * 1000) + 10000}) - result = self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid())) + result = self.worker.handle(request, local_peer()) return cast(dict[str, Any], result) def apply(self, request_id: str) -> dict[str, Any]: diff --git a/tests/worker/test_live_arming.py b/tests/worker/test_live_arming.py index 0f9a9d2..5c7f86f 100644 --- a/tests/worker/test_live_arming.py +++ b/tests/worker/test_live_arming.py @@ -8,7 +8,6 @@ from __future__ import annotations -import os import time from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -21,7 +20,6 @@ from coinfly.config.models import EngineConfig, default_config from coinfly.domain.context import DomainError, Environment, ProductTrack, RunProfile from coinfly.domain.products import ProductSubtype -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.protocol import OWNER_METHODS, READ_METHODS, RPCRequest from coinfly.operations.live_arming import ( MANDATE_STATEMENT, @@ -32,6 +30,7 @@ from coinfly.venues.capability import LocalQualifications from coinfly.worker.paths import WorkerPaths from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer NOW = datetime(2026, 9, 21, 12, 0, tzinfo=timezone.utc) @@ -100,7 +99,7 @@ def setUp(self) -> None: def rpc(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]: request = RPCRequest.parse({"jsonrpc": "2.0", "id": method, "method": method, "params": params or {}, "epoch": 0, "deadline_ms": int(time.time() * 1000) + 10000}) - return cast(dict[str, Any], self.worker.handle(request, PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid()))) + return cast(dict[str, Any], self.worker.handle(request, local_peer())) def test_prerequisites_report_and_mandate_lifecycle(self) -> None: self.assertIn("live.prerequisites", READ_METHODS) diff --git a/tests/worker/test_worker_replacement.py b/tests/worker/test_worker_replacement.py index c6fa651..3730a5b 100644 --- a/tests/worker/test_worker_replacement.py +++ b/tests/worker/test_worker_replacement.py @@ -12,16 +12,16 @@ from __future__ import annotations import os +import sys import tempfile import threading import time from pathlib import Path from typing import Any -from unittest import TestCase +from unittest import TestCase, skipIf from unittest.mock import patch from coinfly.config.models import default_config -from coinfly.ipc.auth import PeerIdentity from coinfly.ipc.local import LocalRPCClient, LocalRPCServer from coinfly.ipc.protocol import PROTOCOL_VERSION, RPCRequest from coinfly.observability import diagnostics @@ -31,6 +31,7 @@ from coinfly.worker.paths import WorkerPaths from coinfly.worker.startup import WorkerStartup from coinfly.worker.supervisor import WorkerSupervisor +from tests.support import local_peer from .test_automation_lifecycle import TestOnlyBootstrap @@ -47,6 +48,7 @@ def hello_params(fingerprint: str | None = None) -> dict[str, Any]: return identity +@skipIf(sys.platform == "win32", "Unix-domain socket IPC; the Windows transport uses named pipes (sockets under /tmp)") class ReplacementFixture(TestCase): def setUp(self) -> None: # Short, private path: Unix socket paths are limited to ~104 bytes on macOS. @@ -78,7 +80,7 @@ def slow_factory(**kwargs: Any) -> WorkerSupervisor: startup = WorkerStartup(socket_path=self.socket, factory=slow_factory, timeout=30).start() self.addCleanup(startup.close) self.addCleanup(release.set) - peer = PeerIdentity(uid=os.getuid(), gid=os.getgid(), pid=os.getpid()) + peer = local_peer() hello = startup.handle(request("worker.hello", hello_params()), peer) self.assertEqual((hello["protocol_version"], hello["build_match"], hello["initializing"]), (PROTOCOL_VERSION, True, True)) stale = startup.handle(request("worker.hello", hello_params("0" * 64)), peer)