From 9d8290a4cc9223649b2f7036c7406d1dc0d7d409 Mon Sep 17 00:00:00 2001 From: Shawn <176741516+Algorithm-zh@users.noreply.github.com> Date: Fri, 4 Sep 2026 06:15:29 -0500 Subject: [PATCH] Avoid tool JSON in utility history summaries --- helpers/history.py | 53 +++++++++++++++- helpers/history.py.dox.md | 2 + plugins/_chat_compaction/AGENTS.md | 1 + plugins/_chat_compaction/helpers/compactor.py | 24 ++++--- tests/test_chat_compaction.py | 62 +++++++++++++++++++ tests/test_history.py | 51 ++++++++++++++- 6 files changed, 183 insertions(+), 10 deletions(-) diff --git a/helpers/history.py b/helpers/history.py index c980e8f135..6a4b3dc7d7 100644 --- a/helpers/history.py +++ b/helpers/history.py @@ -278,7 +278,7 @@ async def compress_attention(self, ratio: float = CURRENT_TOPIC_ATTENTION_COMPRE return True async def summarize_messages(self, messages: list[Message]): - msg_txt = [m.output_text() for m in messages] + msg_txt = [output_text_for_summary(m.output()) for m in messages] summary = await self.history.agent.call_utility_model( system=self.history.agent.read_prompt("fw.topic_summary.sys.md"), message=self.history.agent.read_prompt( @@ -332,7 +332,8 @@ async def summarize(self): self.summary = await self.history.agent.call_utility_model( system=self.history.agent.read_prompt("fw.topic_summary.sys.md"), message=self.history.agent.read_prompt( - "fw.topic_summary.msg.md", content=self.output_text() + "fw.topic_summary.msg.md", + content=output_text_for_summary(self.output()), ), ) return self.summary @@ -656,6 +657,45 @@ def _stringify_output(output: OutputMessage, ai_label="ai", human_label="human") return f'{ai_label if output["ai"] else human_label}: {_stringify_content(output["content"])}' +def _stringify_summary_output( + output: OutputMessage, ai_label="ai", human_label="human" +) -> str: + content = output["content"] + if output["ai"] and isinstance(content, str): + content = _assistant_summary_content(content) + return f'{ai_label if output["ai"] else human_label}: {_stringify_content(content)}' + + +def _assistant_summary_content(content: str) -> str: + """Render assistant tool envelopes as prose for summary-model inputs. + + The stored history remains unchanged. This view intentionally omits reasoning and + tool arguments so utility models do not imitate the agent's JSON protocol or ingest + large command payloads while summarizing a conversation. + """ + from helpers import extract_tools + + request = extract_tools.extract_tool_request(content) + if request is None: + return content + + try: + tool_name, tool_args = extract_tools.normalize_tool_request(request) + except ValueError: + return content + + if tool_name == "response": + for key in ("text", "message"): + response = tool_args.get(key) + if isinstance(response, str) and response.strip(): + return response + + headline = request.get("headline") + if isinstance(headline, str) and headline.strip(): + return f"{headline.strip()} (tool: {tool_name})" + return f"Used tool {tool_name}." + + def _stringify_content(content: MessageContent) -> str: # already a string if isinstance(content, str): @@ -733,6 +773,15 @@ def output_text(messages: list[OutputMessage], ai_label="ai", human_label="human return "\n".join(_stringify_output(o, ai_label, human_label) for o in messages) +def output_text_for_summary( + messages: list[OutputMessage], ai_label="ai", human_label="human" +) -> str: + """Return a utility-model transcript without assistant tool-protocol JSON.""" + return "\n".join( + _stringify_summary_output(o, ai_label, human_label) for o in messages + ) + + def clear_responses_provider_state(agent) -> None: key = getattr(agent, "DATA_NAME_RESPONSES_STATE", "responses_state") get_data = getattr(agent, "get_data", None) diff --git a/helpers/history.py.dox.md b/helpers/history.py.dox.md index ac1c2b03fb..86ababc8e9 100644 --- a/helpers/history.py.dox.md +++ b/helpers/history.py.dox.md @@ -65,6 +65,7 @@ - `group_messages_abab(messages: list[BaseMessage]) -> list[BaseMessage]` - `output_langchain(messages: list[OutputMessage])` - `output_text(messages: list[OutputMessage], ai_label=..., human_label=...)` +- `output_text_for_summary(messages: list[OutputMessage], ai_label=..., human_label=...)` - `clear_responses_provider_state(agent) -> None` - `_merge_outputs(a: MessageContent, b: MessageContent) -> MessageContent` - `_merge_properties(a: Dict[str, MessageContent], b: Dict[str, MessageContent]) -> Dict[str, MessageContent]` @@ -82,6 +83,7 @@ - `Message.from_dict()` normalizes legacy AI Responses metadata through `LLMResult.metadata()` so loaded chats shed transient payloads while unrelated metadata and non-AI tool-result inputs remain intact. - `output_langchain()` removes leading assistant messages after grouping so provider histories always begin with a user turn; the WebUI greeting remains persisted and displayed but is not sent as an orphaned assistant message. - `_json_dumps()` emits compact JSON (`","`, `":"` separators) for serialized history and generated non-string user-turn content. +- `output_text_for_summary()` is a non-persistent utility-model view. It renders valid assistant tool envelopes as prose, preserves final response text, and omits reasoning plus tool arguments; `output_text()` remains the lossless text representation used by APIs and backups. - Observed side-effect areas: filesystem writes, filesystem deletion, model calls, plugin state, settings/state persistence, secret handling. - Imported dependency areas include: `abc`, `asyncio`, `collections`, `collections.abc`, `enum`, `helpers`, `json`, `langchain_core.messages`, `math`, `plugins._model_config.helpers.model_config`, `typing`, `uuid`. diff --git a/plugins/_chat_compaction/AGENTS.md b/plugins/_chat_compaction/AGENTS.md index d0c985afbc..6b9e0ac822 100644 --- a/plugins/_chat_compaction/AGENTS.md +++ b/plugins/_chat_compaction/AGENTS.md @@ -22,6 +22,7 @@ - Store the compacted summary as non-assistant history context so provider-history normalization cannot discard it as an orphaned assistant turn. - Preserve loaded skill names from `skill_instructions` metadata without copying full skill bodies into compacted summaries. - Preserve only secret references such as names, aliases, purposes, or storage locations; never preserve secret values. +- Build model-facing compaction input from the summary transcript view so assistant tool-protocol JSON, reasoning, and large tool arguments are not copied into the summarizer prompt; keep the pre-compaction text backup lossless. - Clear the cached context window after replacing history so stale transcript content is not persisted as active resumable state; the cache rebuilds on the next model turn. - After replacing local history, clear the active Responses provider continuation while preserving stored response IDs for cleanup. - Do not discard original context data unless the compaction flow explicitly owns that behavior. diff --git a/plugins/_chat_compaction/helpers/compactor.py b/plugins/_chat_compaction/helpers/compactor.py index b9285e4aa7..be67bd4f64 100644 --- a/plugins/_chat_compaction/helpers/compactor.py +++ b/plugins/_chat_compaction/helpers/compactor.py @@ -5,7 +5,12 @@ import models as models_module from agent import Agent from helpers import files, tokens -from helpers.history import History, clear_responses_provider_state, output_text +from helpers.history import ( + History, + clear_responses_provider_state, + output_text, + output_text_for_summary, +) from helpers.persist_chat import ( export_json_chat, get_chat_folder_path, @@ -99,12 +104,15 @@ async def run_compaction( # Step 1: Extract full conversation text history_output = agent.history.output() full_text = output_text(history_output, ai_label="assistant", human_label="user") + summary_text = output_text_for_summary( + history_output, ai_label="assistant", human_label="user" + ) - if not full_text.strip(): + if not summary_text.strip(): raise ValueError("No conversation content to compact") # Step 2: Estimate tokens, resolve model, and compute context budget - token_count = tokens.approximate_tokens(full_text) + token_count = tokens.approximate_tokens(summary_text) resolved_cfg, model = _build_model(use_chat_model, preset_name, agent) ctx_length = int(resolved_cfg.get("ctx_length", 128000)) if resolved_cfg else 128000 @@ -122,11 +130,11 @@ async def run_compaction( # Step 4: Handle large histories by chunking if necessary if token_count > max_input_tokens: summary = await _compact_large_history( - agent, full_text, token_count, max_input_tokens, log_item, model + agent, summary_text, token_count, max_input_tokens, log_item, model ) else: summary = await _compact_single_pass( - agent, full_text, log_item, model + agent, summary_text, log_item, model ) if not summary or not summary.strip(): @@ -347,8 +355,10 @@ async def get_compaction_stats(context) -> dict: # Estimate tokens history_output = agent.history.output() - full_text = output_text(history_output, ai_label="assistant", human_label="user") - token_count = tokens.approximate_tokens(full_text) if full_text else 0 + summary_text = output_text_for_summary( + history_output, ai_label="assistant", human_label="user" + ) + token_count = tokens.approximate_tokens(summary_text) if summary_text else 0 # Get model names for both chat and utility chat_cfg = get_chat_model_config(agent) diff --git a/tests/test_chat_compaction.py b/tests/test_chat_compaction.py index 87a7ad00ad..6a9e71c090 100644 --- a/tests/test_chat_compaction.py +++ b/tests/test_chat_compaction.py @@ -49,6 +49,22 @@ def output(self): return [{"ai": False, "content": "hello"}] +class _StructuredCompactionHistory: + def output(self): + return [ + {"ai": False, "content": "Inspect the service."}, + { + "ai": True, + "content": ( + '{"thoughts":["Read the credentials"],' + '"headline":"Checking service status",' + '"tool_name":"code_execution_tool",' + '"tool_args":{"code":"print(SECRET_VALUE)"}}' + ), + }, + ] + + class _CompactionLog: def __init__(self): self.logs = [] @@ -231,3 +247,49 @@ async def fake_single_pass(*args, **kwargs): assert "previous_response_id" not in state assert state["response_ids"] == ["resp_previous", "resp_current"] assert "ctx_window" not in agent.data + + +@pytest.mark.asyncio +async def test_manual_compaction_summarizes_plain_view_but_backs_up_raw_history( + monkeypatch, +): + captured = {} + + async def fake_single_pass(_agent, transcript, *_args, **_kwargs): + captured["summary_transcript"] = transcript + return "summary" + + def fake_backup(_context, transcript): + captured["backup_transcript"] = transcript + return {"txt": "/tmp/pre.txt"} + + agent = _CompactionAgent() + agent.history = _StructuredCompactionHistory() + context = SimpleNamespace( + id="compact-structured-chat", + agent0=agent, + log=_CompactionLog(), + streaming_agent=object(), + ) + + monkeypatch.setattr( + compactor, + "_build_model", + lambda *args: ({"ctx_length": 128000}, _RecordingModel()), + ) + monkeypatch.setattr(compactor, "_compact_single_pass", fake_single_pass) + monkeypatch.setattr(compactor, "_save_pre_compaction_backup", fake_backup) + monkeypatch.setattr(compactor, "save_tmp_chat", lambda *args: None) + monkeypatch.setattr(compactor, "remove_msg_files", lambda *args: None) + monkeypatch.setattr(compactor, "mark_dirty_all", lambda *args, **kwargs: None) + + await compactor.run_compaction(context) + + assert captured["summary_transcript"] == ( + "user: Inspect the service.\n" + "assistant: Checking service status (tool: code_execution_tool)" + ) + assert "Read the credentials" not in captured["summary_transcript"] + assert "SECRET_VALUE" not in captured["summary_transcript"] + assert '"thoughts"' in captured["backup_transcript"] + assert "SECRET_VALUE" in captured["backup_transcript"] diff --git a/tests/test_history.py b/tests/test_history.py index 6357f63150..f0bd18fa09 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -1,4 +1,4 @@ -from helpers.history import output_langchain +from helpers.history import output_langchain, output_text, output_text_for_summary from langchain_core.messages import AIMessage, HumanMessage @@ -12,3 +12,52 @@ def test_output_langchain_omits_leading_assistant_messages(): ) assert messages == [HumanMessage("Hello"), AIMessage("Hi")] + + +def test_output_text_for_summary_flattens_assistant_tool_protocol(): + messages = [ + {"ai": False, "content": "Check the database connection."}, + { + "ai": True, + "content": ( + '{"thoughts":["Inspect credentials"],' + '"headline":"Checking database configuration",' + '"tool_name":"code_execution_tool",' + '"tool_args":{"runtime":"terminal","code":"psql -l"}}' + ), + }, + ] + + raw = output_text(messages, ai_label="assistant", human_label="user") + summary = output_text_for_summary( + messages, ai_label="assistant", human_label="user" + ) + + assert '"thoughts"' in raw + assert '"tool_args"' in raw + assert summary == ( + "user: Check the database connection.\n" + "assistant: Checking database configuration (tool: code_execution_tool)" + ) + assert "Inspect credentials" not in summary + assert "psql -l" not in summary + + +def test_output_text_for_summary_keeps_final_response_text(): + messages = [ + { + "ai": True, + "content": ( + '{"thoughts":["Done"],"headline":"Responding",' + '"tool_name":"response","tool_args":{"text":"The fix is ready."}}' + ), + } + ] + + assert output_text_for_summary(messages) == "ai: The fix is ready." + + +def test_output_text_for_summary_preserves_plain_assistant_content(): + messages = [{"ai": True, "content": "A plain assistant response."}] + + assert output_text_for_summary(messages) == "ai: A plain assistant response."