From 5b73f4dfad52262333bbe895ad525187dfac36e8 Mon Sep 17 00:00:00 2001 From: Konstantina Kontoudi Date: Fri, 31 Jul 2026 10:50:53 +0200 Subject: [PATCH] add trace summaries doc Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/marketplace.json | 2 +- plugins/tsuga/.claude-plugin/plugin.json | 2 +- .../tsuga-analyze-trace-latency/SKILL.md | 35 +++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 8009fec..8cf2cc2 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,7 +6,7 @@ }, "metadata": { "description": "Tsuga toolkit for AI coding agents: one plugin with the tsuga CLI driver, live-platform investigation, dashboards, incident workflows, OpenTelemetry instrumentation, Collector, signal-choice, telemetry debug, and audit skills.", - "version": "0.9.0" + "version": "0.9.1" }, "plugins": [ { diff --git a/plugins/tsuga/.claude-plugin/plugin.json b/plugins/tsuga/.claude-plugin/plugin.json index 0a0b768..19326ce 100644 --- a/plugins/tsuga/.claude-plugin/plugin.json +++ b/plugins/tsuga/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "tsuga", "description": "Tsuga observability plugin: the `tsuga` CLI driver (commands, TQL syntax, aggregation bodies, counter math, deep links, cloud/k8s translators); live-platform investigation for service health, errors, latency, and monitor coverage; dashboard building; incident orchestration; OpenTelemetry SDK, Collector, OTTL, signal-choice, telemetry debug, and audit skills; and meta-skills for building and validating skill bundles.", - "version": "0.9.0", + "version": "0.9.1", "author": { "name": "Tsuga Engineering", "email": "engineering@tsuga.com" diff --git a/plugins/tsuga/skills/tsuga-analyze-trace-latency/SKILL.md b/plugins/tsuga/skills/tsuga-analyze-trace-latency/SKILL.md index a457bb6..9494a69 100644 --- a/plugins/tsuga/skills/tsuga-analyze-trace-latency/SKILL.md +++ b/plugins/tsuga/skills/tsuga-analyze-trace-latency/SKILL.md @@ -1,6 +1,6 @@ --- name: tsuga-analyze-trace-latency -description: "Use when asked about slow requests, high latency, latency spikes, p95 or p99 trace duration, slow spans, top slow operations, peak latency windows, sustained vs transient latency degradation, trace-log correlation, span count by operation, downstream latency suspicion, which operations are slow for a specific service, or whether latency correlates with errors." +description: "Use when asked about slow requests, high latency, latency spikes, p95 or p99 trace duration, slow spans, top slow operations, peak latency windows, sustained vs transient latency degradation, trace-log correlation, span count by operation, downstream latency suspicion, which operations are slow for a specific service, or whether latency correlates with errors. Also covers per-trace drill-down: where wall-clock time went inside one trace and per-service latency ownership (trace latency summary / latency summary), and collapsing a large or repetitive trace into summary spans (trace summarize)." --- # Analyze Trace Latency @@ -13,6 +13,8 @@ description: "Use when asked about slow requests, high latency, latency spikes, - "Which operations in X are slow?" - "Trace performance investigation for X" - "p95 spike in X" +- "Where did the time go in this trace?" / "which service is slow in trace ?" +- "Summarize this trace" / "this trace has thousands of spans" ## Required Inputs @@ -67,6 +69,32 @@ tsuga logs search --query "trace_id:" --from --to ``` 0 results is a valid outcome — not all services emit both signals. +## Drilling Into One Trace (optional) + +The steps above find _which operation_ is slow across many traces. To understand _where the time went inside a single slow trace_ — after picking a `trace_id` from `tsuga traces search` — Tsuga has two per-trace CLI commands. Both take a `--trace-id` and a `--from`/`--to` window that must cover the trace, and both are read-only. Use them to decide which service or operation to investigate next; a single trace is one sample, not proof of a sustained pattern. + +### `tsuga traces latency-summary` — where wall-clock time is spent, per service + +```bash +tsuga traces latency-summary --trace-id --from --to [--min-range-duration-ms ] [--no-ranges] +``` + +This is the **trace latency summary** / **latency summary**. It attributes the trace's real elapsed time across the services that participated, instead of summing per-span durations (which overlap and double-count in a concurrent trace). Read the result like this: + +- **`serviceTotals[]` is the headline and is authoritative** — per service: `durationNs` (nanoseconds, a **string**) and `share` (`0`–`1`; multiply by 100 for a percent), sorted largest first. The top entry is where the wall-clock time went in this trace → the next service to run this skill against. +- **Attribution is leaf-only.** At any instant only the deepest active spans (no active descendant) are credited, so a parent is not counted for time its own downstream is doing the work. Within a range the time is split evenly across active leaves; leaves of the same service combine (2 `pay` + 1 `db` in parallel → `pay` 2/3, `db` 1/3). +- **`ranges[]`** is the moment-by-moment timeline (each: `fromNs`/`toNs`/`durationNs` as strings, `contributions[]` with `serviceKey` + `weight` + `durationNs`, and `spanIds[]`). Ranges shorter than `--min-range-duration-ms` are collapsed into neighbors to cut noise; default is `max(1ms, 1% of trace duration)`, `0` disables it. Pass `--no-ranges` for just `services` + `serviceTotals`. +- **`services[]`** enriches each `serviceKey` with catalog `id`/`namespace` when Tsuga resolves the service unambiguously (absent otherwise). Unresolvable `service.name` buckets into a synthetic `unknown` service. +- **Units:** all durations are **nanoseconds transmitted as strings** (`totalDurationNs`, `durationNs`) to preserve precision — convert to ms for output. `truncated: true` means the span fetch hit its cap and the attribution may be incomplete; say so in any finding. + +### `tsuga traces summarize` — compact view of a large/wide trace + +```bash +tsuga traces summarize --trace-id --from --to +``` + +This is the **trace summary**. It collapses groups of similar spans into synthetic **summary spans** so a trace with thousands of repetitive spans (fan-out loops, per-row DB calls) is readable. Summary spans are marked `spanAttributes.aggregation.is_summary: true` and carry a span count and duration statistics. Use it to spot a repeated operation dominating a trace; it does **not** attribute wall-clock time per service — use `latency-summary` for that. + ## Evidence Requirements - "Latency degraded" = selected percentile > threshold **and** sustained over ≥ 2 consecutive 5-minute windows. A single window = "transient spike," not confirmed degradation. @@ -100,6 +128,7 @@ Trace-log correlation: matching traces found via trace_id / not attempted (s ## Recommended Actions 1. Investigate further — if this spans a downstream service, run `tsuga-analyze-trace-latency` for that service +2. For a specific slow trace, run `tsuga traces latency-summary --trace-id ` to see per-service wall-clock ownership (and `tsuga traces summarize --trace-id ` if the trace has many repetitive spans) ## Limitations - No service topology map — downstream attribution requires running this skill per suspected downstream service @@ -108,6 +137,7 @@ Trace-log correlation: matching traces found via trace_id / not attempted (s - Percentile groupBy is limited to top 10 operations; additional operations may exist beyond this limit - Duration values are milliseconds throughout - `services list` counters are snapshot state, not proof that traces exist or do not exist in a historical window +- `traces latency-summary` and `traces summarize` describe a single trace — one sample, not a sustained pattern. `latency-summary` durations are nanoseconds-as-strings (not ms), and a `truncated` summary attributes an incomplete trace ``` ## Safety Rules @@ -116,7 +146,8 @@ Trace-log correlation: matching traces found via trace_id / not attempted (s - Use explicit `--from`/`--to` or state the CLI default; ask for exact bounds on ambiguous natural-language windows. - Resolve ownership with `tsuga services list` plus `tsuga teams list/get`; never infer ownership from names. - Do not attribute latency to a downstream service without running this skill against that service explicitly. -- Duration values are milliseconds — always state units in output. +- Duration values are milliseconds — always state units in output. Exception: `traces latency-summary` returns nanoseconds as strings (`durationNs`, `totalDurationNs`); convert before reporting. +- `traces latency-summary` / `traces summarize` describe one trace; do not generalize a single trace to a service-wide pattern without the aggregation steps above. - Single-window percentile spike = "transient"; requires ≥ 2 consecutive windows to call it "sustained degradation." - Correlated errors are only consistent with a hypothesis; root cause requires at least two corroborating signals. - No create/update/delete/push/upsert/API writes from this skill.