feat(engine+web): wire the discretionary journal — CLI-only in, self-reported everywhere out (#705)

keel/cli.py

@@ -158,6 +158,7 @@
 from keel.commands.fetch import run_fetch
 from keel.commands.insights import _parse_ts as _parse_since_until
 from keel.commands.insights import insights_group
+from keel.commands.journal import journal_group
 from keel.commands.mcp import mcp_cmd
 from keel.commands.monitor import run_monitor
 from keel.commands.orders import orders_cmd
@@ -1500,6 +1501,18 @@ def simulate(
 cli.add_command(insights_group)
 
 
+# -- journal (the discretionary journal: human-sourced, CLI-only, append-only) --------------------
+
+# #705. The `journal` table was declared in the schema from the beginning and had no repository
+# method and no caller -- dead schema. This is its only write path, and it is deliberately the
+# ONLY one: attestations are human-sourced or refused, `keel serve` has no route to it, and
+# `journal add` takes no value options so the entry cannot be scripted past the TTY gate.
+#
+# NOT `keel insights journal`, registered above, which is a filterable view of closed TRADES.
+# Two similar names over two different kinds of evidence, and both say which they are.
+cli.add_command(journal_group)
+
+
 # -- versions (the deploy check: every keel distribution, not just this one) ---------------------
 
 # `--version` above answers for `keel-trader` alone and therefore cannot see a partial upgrade;

keel/commands/timeline.py

@@ -1,8 +1,9 @@
 """One chronology over everything keel has done -- issue #703.
 
-Four stores record activity and none of them knew about the others: the engine's JSONL log
-(cycles), the `orders` table (fills), the `transactions` ledger (cash flows), and the attestation
-tables (what a human swore to). This module merges them into one timeline WITHOUT letting them
+Five stores record activity and none of them knew about the others: the engine's JSONL log
+(cycles), the `orders` table (fills), the `transactions` ledger (cash flows), the attestation
+tables (what a human swore to about the world), and the discretionary `journal` (#705 -- what the
+operator says about THEMSELVES). This module merges them into one timeline WITHOUT letting them
 blur, which is the whole difficulty: a venue-reported fill, a line imported from a venue's CSV,
 and a sentence a human typed are three different kinds of evidence, and a feed that presented
 them identically would be worse than four separate tables.
@@ -58,11 +59,18 @@
 #: - `imported-ledger` -- a `transactions` row, read out of a venue's own CSV export.
 #: - `human-attested` -- someone typed it and signed their name to it.
 #: - `engine-log` -- the agent's own structured log of what it did.
+#: - `self-reported` -- the operator's account of their OWN conduct (#705). A sixth word rather
+#:   than a reuse of `human-attested`, because the two are different kinds of claim: an asset
+#:   attestation says PAXG is backed by allocated gold, which a prospectus could contradict, while
+#:   "I felt rushed and broke my rule" has no external referent and cannot be checked by anyone,
+#:   ever. Filing a self-assessment under the word this feed uses for checkable human claims would
+#:   put the one unverifiable record in the database under a heading that implies otherwise.
 PROVENANCES: tuple[str, ...] = (
     "venue-reported",
     "simulated",
     "imported-ledger",
     "human-attested",
+    "self-reported",
     "engine-log",
 )
 
@@ -405,6 +413,69 @@ def _attestation_rows(
     return rows
 
 
+def _journal_rows(
+    repo: Repository, since_ts: int | None, chain: _Chain
+) -> list[TimelineRow]:
+    """`journal` -> attestation rows (#705).
+
+    Under the ATTESTATION chip, because that is the kind of thing this is -- something a person
+    put their name to -- and with `self-reported` provenance, because it is the one kind of
+    attestation nothing outside the operator's head produced. The chip groups it with the asset
+    and instrument attestations; the provenance column is what keeps it from being read as one.
+
+    The summary carries EVERY sentence the operator wrote, in a fixed order, because this row is
+    the journal's whole representation in the CSV -- there is no other column any of it could
+    reappear in. An entry whose only content is an emotion score still says something, and a row
+    reading only "journal entry" would make the feed's densest human content its least legible.
+    """
+    rows: list[TimelineRow] = []
+    for raw in repo.get_journal_entries(since_ts=since_ts):
+        entry_id = str(raw.get("id") or "")
+        rows.append(
+            TimelineRow(
+                ts=int(raw["ts"]),
+                kind="attestation",
+                provenance="self-reported",
+                source="journal",
+                reference=entry_id,
+                summary=_journal_summary(raw),
+                product_id="",
+                # The figure is what the operator SAYS the day cost them, and `amount_kind` names
+                # it as such: a self-reported impact and a venue-reported fee in one column, with
+                # nothing saying which is which, is a column that will be summed.
+                amount=raw.get("dollar_impact"),
+                amount_kind="self-reported impact" if raw.get("dollar_impact") is not None else "",
+                **chain.of("journal", entry_id),
+            )
+        )
+    return rows
+
+
+def _journal_summary(raw: dict[str, Any]) -> str:
+    """One line from an entry, leading with whatever the operator wrote.
+
+    `rules_followed` is THREE-valued and only one of the three is worth a chip: `False` is the
+    operator saying they broke their own rules, which is the single most consequential thing this
+    table can hold, and `None` is a question they skipped. `bool(None)` would print the confession
+    over the silence.
+    """
+    parts: list[str] = []
+    if raw.get("rules_followed") is False:
+        parts.append("BROKE RULES")
+    # EVERY sentence the operator wrote, not the first one found. The first cut used `elif`, so an
+    # entry carrying both an error and a chart note exported only the error -- and this row is the
+    # journal's whole representation in a file an operator hands to an auditor. There is no other
+    # column it could reappear in.
+    for field in ("errors_made", "chart_note", "screenshot_ref"):
+        written = str(raw.get(field) or "").strip()
+        if written:
+            parts.append(written)
+    emotion = str(raw.get("emotion_score") or "").strip()
+    if emotion:
+        parts.append(f"emotion {emotion}")
+    return " — ".join(parts) if parts else "journal entry (nothing written)"
+
+
 def _cycle_rows(cycles: Iterable[Any], since_ts: int | None) -> list[TimelineRow]:
     """`ActivityCycle`s -> system rows.
 
@@ -490,6 +561,7 @@ def gather_timeline(
     scoped.extend(_order_rows(repo, since, chained))
     scoped.extend(_transaction_rows(repo, since, chained))
     scoped.extend(_attestation_rows(repo, since, chained))
+    scoped.extend(_journal_rows(repo, since, chained))
     # No chain argument, and never one: `_cycle_rows` reads the engine's own log FILE, which is
     # not a chained store. A cycle row carrying a hash would be this module attesting to something
     # it merely read.

keel/data/audit.py

@@ -70,6 +70,12 @@
     "transaction_recorded": "transactions",
     "asset_attested": "asset_attestations",
     "instrument_attested": "instrument_attestations",
+    # #705. The journal is chained for the same reason the two attestation tables are -- it is
+    # something a human swore to, and it rides the same audit export. It is also the ONLY store
+    # here whose subject is the operator rather than the world, which is a difference the
+    # timeline's `provenance` column carries, not this one: the chain's job is that a row cannot
+    # be altered quietly, and that is the same job whatever the row claims.
+    "journal_recorded": "journal",
 }
 
 

keel/data/repository.py

@@ -226,6 +226,159 @@ class Repository:
     def __init__(self, conn: sqlite3.Connection) -> None:
         self._conn = conn
 
+    # -- the discretionary journal (#705) ---------------------------------
+    #
+    # The operator's own account of their own conduct: what they felt, whether they followed
+    # their rules, what it cost. Declared in the schema since the beginning with no method and no
+    # caller -- dead schema, which is worse than none, because a reader assumes a declared table
+    # is a used one.
+    #
+    # UNLIKE EVERY OTHER STORE HERE, none of this can be checked. An order is what a venue
+    # reported, a transaction is a line from a venue's own export, an asset attestation is a claim
+    # a prospectus could contradict. A self-assessment has no external referent at all, and the
+    # whole value of keeping it depends on it staying visibly separate from the things that do --
+    # which is why `commands/timeline.py` gives it its own provenance word rather than filing it
+    # under `human-attested` beside the attestations.
+    #
+    # APPEND-ONLY, and there is no update method by design. A journal you can go back and edit is
+    # a journal that records what you wish you had thought.
+
+    def append_journal_entry(
+        self,
+        *,
+        ts: int,
+        emotion_score: str | None = None,
+        rules_followed: bool | None = None,
+        errors_made: str | None = None,
+        dollar_impact: Decimal | None = None,
+        chart_note: str | None = None,
+        screenshot_ref: str | None = None,
+    ) -> int:
+        """Append one entry and return its `id`.
+
+        Every field but `ts` defaults to `None`, and `None` means DID NOT SAY -- never a zero, an
+        empty string or a `False`. An operator who wants to record one sentence about one day must
+        not have to invent an emotion score to do it, and `rules_followed=False` is a positive
+        confession that nobody should be able to make by omission.
+
+        The entry and its audit-chain row land in one transaction (#721), the same discipline
+        every other writer here follows.
+        """
+        with write_transaction(self._conn):
+            cursor = self._conn.execute(
+                """
+                INSERT INTO journal
+                    (ts, emotion_score, rules_followed, errors_made, dollar_impact, chart_note,
+                     screenshot_ref)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    ts,
+                    emotion_score,
+                    None if rules_followed is None else int(rules_followed),
+                    errors_made,
+                    _dec_to_text(dollar_impact),
+                    chart_note,
+                    screenshot_ref,
+                ),
+            )
+            assert cursor.lastrowid is not None
+            entry_id = cursor.lastrowid
+            # The row id, because `journal` has no natural key -- no `coinbase_id`, no asset, no
+            # venue pair -- and it is what `commands/timeline.py` prints as the row's reference.
+            append_event(
+                self._conn,
+                ts=ts,
+                event_type="journal_recorded",
+                entity_id=str(entry_id),
+                payload={
+                    "id": entry_id,
+                    "ts": ts,
+                    "emotion_score": emotion_score,
+                    "rules_followed": rules_followed,
+                    "errors_made": errors_made,
+                    "dollar_impact": dollar_impact,
+                    "chart_note": chart_note,
+                    "screenshot_ref": screenshot_ref,
+                },
+            )
+        return entry_id
+
+    def get_journal_entries(
+        self,
+        *,
+        since_ts: int | None = None,
+        until_ts: int | None = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        """Entries in the window, OLDEST FIRST -- a journal reads forwards.
+
+        The window is half-open (`since_ts <= ts < until_ts`), matching `get_candles` and
+        `commands/orders.py::scope_start_ts`, so two adjacent windows cover a range without
+        double-counting the seam.
+
+        `limit` keeps the NEWEST entries and still returns them oldest-first: a capped read of a
+        journal wants the recent end, and the cap must change how many entries a caller sees
+        rather than which way they read. `id` breaks a timestamp tie, so two notes written in one
+        second keep a stable order across reads.
+        """
+        where, params = self._journal_where(since_ts, until_ts)
+        if limit is None:
+            query = f"SELECT * FROM journal{where} ORDER BY ts, id"
+        else:
+            # The SUBQUERY, not a DESC read reversed in Python -- the identical shape
+            # `get_equity_points` and `get_cycle_balances` use, and `get_equity_points`' docstring
+            # is where the reasoning lives: the ordering is the caller's contract rather than an
+            # artefact of how the rows were selected, so a bounded read and an unbounded one
+            # differ only in how much they return. `id` breaks the tie in BOTH directions, so the
+            # newest-N and the oldest-first re-order agree about which of two same-second entries
+            # is the newer.
+            query = (
+                f"SELECT * FROM (SELECT * FROM journal{where} "
+                "ORDER BY ts DESC, id DESC LIMIT ?) ORDER BY ts, id"
+            )
+            params = [*params, limit]
+        rows = self._conn.execute(query, params).fetchall()
+        return [self._journal_row_to_dict(row) for row in rows]
+
+    def count_journal_entries(
+        self, *, since_ts: int | None = None, until_ts: int | None = None
+    ) -> int:
+        """How many entries the window holds, BEFORE any `limit` truncated it.
+
+        The sibling `count_equity_points` exists for the same reason and its docstring states the
+        rule: a caller that bounds a read is showing a WINDOW of the record and must say so. The
+        console showed a capped journal with nothing on the page distinguishing it from a complete
+        one, which is the failure that rule exists to prevent.
+        """
+        where, params = self._journal_where(since_ts, until_ts)
+        row = self._conn.execute(f"SELECT COUNT(*) AS n FROM journal{where}", params).fetchone()
+        return int(row["n"])
+
+    @staticmethod
+    def _journal_where(since_ts: int | None, until_ts: int | None) -> tuple[str, list[Any]]:
+        """The half-open window, shared by the read and the count so the two cannot disagree
+        about which entries are in it."""
+        clauses: list[str] = []
+        params: list[Any] = []
+        if since_ts is not None:
+            clauses.append("ts >= ?")
+            params.append(since_ts)
+        if until_ts is not None:
+            clauses.append("ts < ?")
+            params.append(until_ts)
+        return ((" WHERE " + " AND ".join(clauses)) if clauses else "", params)
+
+    def _journal_row_to_dict(self, row: sqlite3.Row) -> dict[str, Any]:
+        entry = dict(row)
+        entry["dollar_impact"] = _text_to_dec(entry.get("dollar_impact"))
+        raw = entry.get("rules_followed")
+        # THREE-VALUED. `bool(None)` is `False`, and `False` on this column is the operator
+        # saying they broke their rules -- a confession nobody should make by leaving a prompt
+        # blank.
+        entry["rules_followed"] = None if raw is None else bool(raw)
+        return entry
+
     # -- the audit chain ------------------------------------------------
 
     def rollback(self) -> None:

keel/web/api.py

@@ -710,14 +710,34 @@ def read_journal(cfg: ServeConfig, query: Query, _state: Any, now_ts: int) -> di
     redrawn in `pnl` order would be a cumulative total of a sequence that never happened.
     """
     from keel.commands.insights import build_equity_curve, build_journal_report
+    from keel.commands.journal import DEFAULT_NOTES_LIMIT, gather_journal
 
     limit = _journal_limit(query)
     repo = open_repo(cfg.db_path)
     try:
         report = build_journal_report(repo, _status_report(cfg, now_ts), now_ts, limit=limit)
+        # #705's DISCRETIONARY journal, on this route rather than one of its own. Two things
+        # called a journal, and this is the page where the distinction has to be visible: the
+        # table above is closed trades as a venue reported them, and these are sentences the
+        # operator wrote about themselves. Putting them on separate pages would let a reader meet
+        # one without ever learning the other exists.
+        #
+        # It is NOT this route's `collection`, so `?sort=` reorders the trades and leaves these
+        # alone -- and that is a refusal, not an omission. A journal reads forwards; sorted by
+        # dollar impact it becomes a ranking of your own worst days, which is the shape this
+        # codebase refuses everywhere else it appears.
+        #
+        # ITS OWN CAP, not the trades' `?limit=`. The first cut passed `limit` through, so
+        # narrowing to one closed trade silently hid 300 of an operator's 301 notes -- one
+        # record's page control truncating a different record, by the coincidence of their
+        # sharing a route. `total_count` rides the payload either way, so the page can say what
+        # it is not showing.
+        notes = gather_journal(repo, now_ts=now_ts, limit=DEFAULT_NOTES_LIMIT)
     finally:
         close_repo(repo)
-    return payload.journal_payload(report, curve=build_equity_curve(report.entries))
+    return payload.journal_payload(
+        report, curve=build_equity_curve(report.entries), notes=notes
+    )
 
 
 def read_rules(cfg: ServeConfig, _query: Query, _state: Any, _now_ts: int) -> dict[str, Any]: