diff --git a/cmd/eval.go b/cmd/eval.go index 0fe7c7e3..d354b6bf 100644 --- a/cmd/eval.go +++ b/cmd/eval.go @@ -5,10 +5,14 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "strings" "time" + hawkconfig "github.com/GrayCodeAI/hawk/internal/config" "github.com/GrayCodeAI/hawk/internal/feature/eval" + "github.com/GrayCodeAI/hawk/internal/feature/evalloop" + "github.com/GrayCodeAI/hawk/internal/tool" "github.com/spf13/cobra" ) @@ -21,6 +25,8 @@ var ( evalTaskDir string evalListJSON bool evalResultsJSON bool + evalLoopPrompt string + evalLoopModel string ) var evalCmd = &cobra.Command{ @@ -59,6 +65,12 @@ var evalCacheCmd = &cobra.Command{ }, } +var evalLoopCmd = &cobra.Command{ + Use: "loop", + Short: "Evaluate the agent end-to-end through its real tool loop", + RunE: runEvalLoop, +} + func init() { evalRunCmd.Flags().StringVar(&evalTasks, "tasks", "", "Comma-separated task IDs (default: all)") evalRunCmd.Flags().StringVar(&evalModel, "model", "", "Model to evaluate") @@ -68,11 +80,71 @@ func init() { evalRunCmd.Flags().StringVar(&evalTaskDir, "task-dir", "", "Directory with YAML task definitions") evalListCmd.Flags().BoolVar(&evalListJSON, "json", false, "output tasks as JSON") evalResultsCmd.Flags().BoolVar(&evalResultsJSON, "json", false, "output results as JSON") + evalLoopCmd.Flags().StringVar(&evalLoopPrompt, "prompt", "", "Task prompt to run through the agent loop") + evalLoopCmd.Flags().StringVar(&evalLoopModel, "model", "", "Model to use (defaults to active model)") evalCmd.AddCommand(evalRunCmd) evalCmd.AddCommand(evalListCmd) evalCmd.AddCommand(evalResultsCmd) evalCmd.AddCommand(evalCacheCmd) + evalCmd.AddCommand(evalLoopCmd) +} + +// runEvalLoop runs the agent end-to-end through its real tool loop in an +// isolated temp directory and prints a JSON report with the transcript path. +func runEvalLoop(cmd *cobra.Command, _ []string) error { + if strings.TrimSpace(evalLoopPrompt) == "" { + return fmt.Errorf("--prompt is required") + } + settings := hawkconfig.LoadGlobalSettings() + ctx := context.Background() + + gw, err := hawkconfig.NewEyrieEngineForSettings(settings) + if err != nil { + return fmt.Errorf("eval loop: build engine client: %w", err) + } + model := strings.TrimSpace(evalLoopModel) + if model == "" { + model = strings.TrimSpace(hawkconfig.ActiveModel(ctx)) + } + if model == "" { + model = strings.TrimSpace(settings.Model) + } + + workDir, err := os.MkdirTemp("", "hawk-eval-loop-*") + if err != nil { + return fmt.Errorf("eval loop: create temp dir: %w", err) + } + defer func() { _ = os.RemoveAll(workDir) }() + + cfg := evalloop.DefaultConfig() + runtime := evalloop.NewSessionRuntime(gw.ChatClient(), "eval", model, tool.NewRegistry(), cfg) + result, err := runtime.Run(ctx, workDir, evalLoopPrompt) + if err != nil { + return fmt.Errorf("eval loop: %w", err) + } + + transcriptPath := "" + if len(result.Transcript) > 0 { + transcriptPath = filepath.Join(workDir, "transcript.json") + _ = os.WriteFile(transcriptPath, result.Transcript, 0o600) // #nosec G304 -- path is the isolated eval temp dir + } + + report := map[string]any{ + "model": model, + "output": result.Output, + "events": len(result.Events), + "tokens_used": result.TokensUsed, + "cost_usd": result.CostUSD, + "duration": result.Duration.String(), + "transcript_path": transcriptPath, + } + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + _, err = cmd.OutOrStdout().Write(append(data, '\n')) + return err } func runEval(_ *cobra.Command, _ []string) error { diff --git a/docs/plans/pi-adoption-plan.md b/docs/plans/pi-adoption-plan.md new file mode 100644 index 00000000..d9666fc9 --- /dev/null +++ b/docs/plans/pi-adoption-plan.md @@ -0,0 +1,359 @@ +# Pi Adoption Plan + +Status: Proposed + +Source: `https://github.com/earendil-works/pi` (MIT, TypeScript/Bun monorepo) + +This plan records the features identified as genuinely missing from hawk while +auditing the Pi agent harness against hawk and its ecosystem submodules. It is +an adoption plan, not a code-porting plan: hawk reimplements compatible behavior +in Go and preserves its existing provider, session, sandbox, permission, +observability, and protocol boundaries. + +## Executive Decision + +Hawk should adopt the following Pi features: + +1. A Go telemetry conformance suite that verifies emitted OpenTelemetry spans + and attributes match the documented schema. +2. An agent-runtime end-to-end eval harness that drives the real session and + tool loop and snapshots session data as eval artifacts. +3. A differential-rendering terminal engine for the TUI (line-granular diff + with synchronized output). +4. Durable session writer fencing and leases. +5. Kitty graphics protocol support for terminal images. +6. Session lease/ownership semantics in the daemon protocol. + +Hawk should not copy Pi's TypeScript code, replace its Go runtime, adopt Pi's +custom CBOR RPC protocol, or remove its native permission/sandboxing model. + +## Existing Hawk Capabilities + +The audit found that hawk already provides the foundation for most Pi features: + +| Pi package | Hawk implementation | Current decision | +|---|---|---| +| `pi-ai` (multi-provider) | `external/eyrie` client + engine facade + catalog + router + credentials | Keep hawk (broader) | +| `pi-agent-core` (agent loop) | `internal/engine` | Keep hawk | +| `pi-coding-agent` (CLI) | `cmd` TUI + CLI + daemon | Keep hawk | +| `pi-session-backends` (storage) | `internal/session` JSONL + zstd + WAL + SQLite index, `external/trace` | Keep hawk | +| `pi-server` (RPC) | `internal/daemon` + `internal/acp` + `internal/mcp` | Keep hawk (broader) | +| Permissions/sandbox | `internal/engine/safety` + `internal/sandbox` (seatbelt/landlock/seccomp/ACL/netproxy) | Keep hawk (native, ahead) | +| `pi-tui` differential rendering | Bubble Tea v2 full-frame redraw | Adopt line-diff engine | +| `pi-telemetry` conformance | `docs/OTEL-CONVENTIONS.md`, eyrie `genai_semconv` pinning | Adopt conformance suite | +| `pi-evals` agent-level eval | `internal/feature/eval` (model benchmark only) | Adopt agent-runtime eval | + +## Priority Model + +- **P0:** Required for correctness, security, or supportability. +- **P1:** High-value product improvements that follow P0. +- **P2:** Optional enhancements or larger architecture bets. +- **Defer:** Deliberately out of scope or blocked on a design RFC. + +## P0: Telemetry Conformance Suite + +### Goal + +Guarantee the emitted OpenTelemetry spans and attributes always match the +documented `gen_ai.*` / `cost.usd` / `tool.name` / `session.id` / `agent.id` +contract, so the schema cannot silently drift across hawk and its submodules. + +### Scope and ownership + +- Primary implementation: `internal/observability` and + `external/eyrie/internal/observability` (the canonical `genai_semconv` + constants and their pinning test). +- Shared schema constants: `external/hawk-core-contracts` if a cross-repo + contract is needed, otherwise keep them in eyrie as today. +- No changes to `internal/mcp`, `internal/sandbox`, or `internal/daemon`. + +### Required behavior + +1. Define a typed schema object for every emitted span + (`agent_loop`, `tool.`, `compact.`, `api.chat`, `session`). +2. Enumerate required and optional attributes per span, with types, cardinality, + sensitive flags, and example values. +3. Provide a conformance harness that: + - records spans from the in-memory OTel adapter, + - asserts every span name and attribute key exists in the schema, + - asserts required attributes are present, + - asserts sensitive attributes never carry raw prompt/response text, + - asserts parent/child span relationships and settlement semantics, + - is runner-independent (usable from any Go test harness). +4. Wire the conformance harness into the hawk and eyrie test suites so CI + enforces the contract. +5. Keep the conformance layer passive and non-throwing: malformed or unreadable + telemetry payloads must not break agent execution. + +### Acceptance criteria + +- Every span hawk emits is covered by the schema. +- A deliberate schema drift (adding a span or attribute) fails the conformance + test until the schema is updated. +- No raw prompt/response text appears in recorded attributes. +- The conformance harness runs green in hawk and eyrie CI. + +## P0: Agent-Runtime Eval Harness + +### Goal + +Evaluate the full hawk agent end-to-end (real session, tool loop, planning, +sandbox) against tasks, and snapshot session data as artifacts — not just a +model-level benchmark. + +### Scope and ownership + +- Primary implementation: a new `internal/feature/evalloop` package or an + extension of `internal/feature/eval`. +- Reuse: `internal/engine` session runtime, `internal/tool` registry, + `internal/sandbox` isolation, `internal/session` persistence. +- CLI: extend `cmd/eval.go` with a loop mode. + +### Required behavior + +1. Drive the real `Session` and tool loop for a task, not a direct LLM call. +2. Run each evaluation in an isolated temporary directory with sandbox + isolation. +3. Capture normalized events: user prompt, assistant turns, tool calls/results, + final output, usage, and cost. +4. Snapshot the underlying session JSONL as an eval artifact (per-run), so + failures can be replayed offline. +5. Support comparative runs across models or configurations. +6. Report pass/fail, token/latency/cost deltas, and reproducibility hashes. +7. Keep model-level benchmarks (`internal/feature/eval`) intact. + +### Acceptance criteria + +- A task that requires tool use (e.g. "edit file and run tests") is executed + through the real loop and verified. +- Session JSONL artifacts are produced and reproducible. +- Runs are isolated and do not mutate the user's working directory. +- CI can run a small smoke eval without external credentials when gated. + +## P1: Differential-Rendering Terminal Engine + +### Goal + +Reduce render cost and flicker for fast, focused agent sessions by re-emitting +only changed terminal lines, with synchronized output. + +### Scope and ownership + +- Primary implementation: a new package (e.g. `internal/tui/diff`) or a custom + Bubble Tea renderer in `cmd`. +- Reuse: existing `internal/terminal` PTY store and `internal/terminal/tape` + recording. +- Do not couple the diff engine to the agent loop or session logic. + +### Required behavior + +1. Render the full component tree to lines, then diff against the previous + frame. +2. Re-emit only the changed line range (first-changed to last-changed), moving + the cursor and clearing changed lines. +3. Wrap updates in synchronized output sequences to avoid tearing. +4. Handle full render on first paint and on terminal width changes. +5. Manage scrollback/viewport and alt-screen transitions correctly. +6. Keep the existing fxtape recording and replay working. + +### Acceptance criteria + +- A single-line update (e.g. spinner) re-emits only that line, measurable in + tests. +- Resize and scrollback behavior matches current Bubble Tea output. +- Recording/replay golden tests still pass. +- No flicker regression on fast streaming updates. + +## P1: Session Writer Fencing and Leases + +### Goal + +Prevent stale writers from corrupting a session after a takeover, mirroring +Pi's fencing-token model. + +### Scope and ownership + +- Primary implementation: `internal/session` write path and WAL. +- Consume: `internal/daemon` remote-session leases. + +### Required behavior + +1. Assign a monotonically increasing fence token to each session writer. +2. Reject writes from a writer whose fence token is older than the current one. +3. Add an expiration window for lease ownership. +4. Surface session-lease acquisition and ownership errors through the daemon. +5. Keep existing fork/rewind/checkpoint behavior intact. + +### Acceptance criteria + +- A stale writer's append is rejected without corrupting the chain. +- Lease expiry prevents an abandoned owner from writing after takeover. +- Concurrent writes from the same owner remain serialized. +- Existing fork/rewind/checkpoint tests pass. + +## P2: Kitty Graphics Protocol + +### Goal + +Render terminal images via the Kitty graphics protocol in the TUI. + +### Scope and ownership + +- Primary implementation: `cmd` TUI render path, adjacent to the diff engine. +- Reuse: existing image/attachment handling in `internal/attachment`. + +### Required behavior + +1. Detect Kitty graphics support in the terminal. +2. Encode and transmit images via the Kitty protocol with a reserved row block. +3. Force repaint of the image block on any line change inside it. +4. Disable gracefully when the terminal lacks support. + +### Acceptance criteria + +- Images render in supported terminals and degrade to placeholders otherwise. +- The diff engine repaints the full image block on partial change. +- No regressions to the text-only render path. + +## P2: Daemon Session Leases + +### Goal + +Add client-side session lease/ownership semantics to the daemon protocol so +remote sessions are single-owner. + +### Scope and ownership + +- Primary implementation: `internal/daemon` (new lease endpoint and ownership + checks). +- Protocol: extend `api/openapi.yaml` and the parity test. + +### Required behavior + +1. Clients acquire a lease to a session before writing. +2. Ownership/disconnection errors are returned distinctly. +3. Leases expire and can be released. +4. Existing /v1/chat and /v1/sessions behavior is preserved for single-owner + callers. + +### Acceptance criteria + +- Two clients cannot write the same session concurrently. +- Lease expiry and takeover are handled without corruption. +- OpenAPI parity test passes. + +## Cross-Cutting Security Requirements + +Every implementation phase must preserve: + +1. Permission checks run immediately before execution. +2. Native sandbox enforcement is never bypassed by the diff engine or eval loop. +3. Telemetry never carries raw prompt/response text. +4. Session fencing prevents corruption, not privilege changes. +5. Eval runs are isolated and never mutate user workspaces. +6. Remote-session leases enforce ownership without leaking credentials. +7. Secrets never enter transcripts, traces, tapes, or artifacts unless explicitly + opted into an audited diagnostic flow. + +## Test and Verification Matrix + +### Unit tests + +- Telemetry schema conformance for every span and attribute. +- Differential renderer diff range, full-render, resize, and scrollback. +- Session fence-token rejection and lease expiry. +- Kitty image block repaint and graceful degradation. +- Daemon lease acquisition, release, takeover, and error mapping. +- Agent-runtime eval snapshotting and reproducibility hashes. + +### Integration tests + +- Eval runs the real tool loop end-to-end in an isolated dir. +- Daemon and ACP parity for leased sessions. +- fxtape recording/replay across the new renderer. +- Cross-repo telemetry conformance in hawk and eyrie CI. + +### Security tests + +- Telemetry redaction of raw prompt/response text. +- Eval isolation (no writes outside the temp workspace). +- Session fence-token corruption resistance. +- Lease ownership with credential redaction. + +### Release checks + +```text +make fmt +make test +make test-race +make vet +make lint +make security +hawk verify +``` + +For submodule changes, run the submodule's own tests and boundary checks before +updating the Hawk pointer. + +## Delivery Sequence + +### Milestone 0: Contract and threat-model review + +- Approve the telemetry schema object and conformance harness shape. +- Approve the differential-renderer API and its interaction with fxtape. +- Approve the session fence-token and daemon-lease model. +- Add redaction and isolation test fixtures. + +### Milestone 1: Telemetry conformance (P0) + +- [x] Define the typed span/attribute schema. +- [x] Implement the conformance harness. +- [x] Wire hawk and eyrie CI. +- [x] Add schema-drift regression tests. + +### Milestone 2: Agent-runtime eval (P0) + +- [x] Implement the loop runner over the real Session/tool loop. +- [x] Add isolated execution and session-JSONL artifacts. +- [ ] Add comparative and reproducibility reporting. +- [x] Extend `hawk eval` with a loop mode. + +### Milestone 3: Differential renderer (P1) + +- Implement the line-diff engine. +- Integrate with Bubble Tea render path. +- Preserve fxtape recording/replay. + +### Milestone 4: Session fencing + daemon leases (P1) + +- Add fence tokens to the session write path. +- Add daemon lease endpoint and ownership checks. +- Update OpenAPI parity. + +### Milestone 5: Kitty graphics (P2) + +- Add Kitty protocol support and graceful fallback. + +## Deliberately Deferred + +- Copying Pi's TypeScript code or adopting its custom CBOR RPC protocol in place + of hawk's ACP/MCP/daemon stack. +- Replacing hawk's native permission/sandbox model with Pi's container-only + approach. +- Porting Pi's extension system verbatim; hawk's plugin/hook/skills model already + covers it. +- Adding a full agent-swarm/graph model beyond current subagent support. + +## Success Criteria + +The adoption is successful when hawk closes each confirmed gap while retaining +its stronger architecture: + +- Telemetry spans always conform to the documented schema across hawk and its + submodules. +- The agent can be evaluated end-to-end through its real tool loop. +- The TUI re-renders only changed lines with synchronized output. +- Session writes are fenced and remote sessions are single-owner. +- Terminal images render in supported terminals. +- Existing native sandboxing, permission, session, and protocol behavior remains + intact. diff --git a/internal/feature/evalloop/evalloop_test.go b/internal/feature/evalloop/evalloop_test.go new file mode 100644 index 00000000..06c84966 --- /dev/null +++ b/internal/feature/evalloop/evalloop_test.go @@ -0,0 +1,54 @@ +package evalloop + +import ( + "context" + "strings" + "testing" + + "github.com/GrayCodeAI/hawk/internal/engine" + "github.com/GrayCodeAI/hawk/internal/tool" +) + +// TestSessionRuntimeSmoke drives the real engine agent loop end-to-end with a +// deterministic mock ChatClient (no external credentials), verifying the harness +// collects output, events, and a transcript snapshot. This is the CI smoke path +// for agent-runtime evaluation. +func TestSessionRuntimeSmoke(t *testing.T) { + // Isolate cwd so session/storage wiring never touches the real workspace. + t.Chdir(t.TempDir()) + + client := engine.NewMockClientForTest() + registry := tool.NewRegistry() + cfg := DefaultConfig() + cfg.MaxTurns = 2 + + runtime := NewSessionRuntime(client, "eval", "mock-model", registry, cfg) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + result, err := runtime.Run(ctx, ".", "summarize this repository") + if err != nil { + t.Fatalf("runtime run: %v", err) + } + + if len(result.Events) == 0 { + t.Fatal("expected at least one loop event") + } + if !strings.Contains(result.Output, "mock") { + t.Fatalf("expected mock output, got %q", result.Output) + } + if len(result.Transcript) == 0 { + t.Fatal("expected a transcript snapshot") + } + if result.Duration <= 0 { + t.Fatal("expected a positive duration") + } +} + +// TestSessionRuntimeRequiresClient guards the nil-client invariant. +func TestSessionRuntimeRequiresClient(t *testing.T) { + runtime := NewSessionRuntime(nil, "eval", "mock", nil, DefaultConfig()) + if _, err := runtime.Run(context.Background(), ".", "task"); err == nil { + t.Fatal("nil client must error") + } +} diff --git a/internal/feature/evalloop/runtime.go b/internal/feature/evalloop/runtime.go new file mode 100644 index 00000000..b62f31b9 --- /dev/null +++ b/internal/feature/evalloop/runtime.go @@ -0,0 +1,56 @@ +// Package evalloop evaluates the full hawk agent end-to-end by driving the real +// engine Session and tool loop against a task, rather than invoking an LLM +// directly. It runs in an isolated working directory and snapshots the session +// transcript as an eval artifact. +// +// The LLM backend is injected as an engine.ChatClient, so real evaluations use +// a provider-bound client while CI smoke runs use a deterministic mock client +// with no external credentials. +package evalloop + +import ( + "context" + "time" +) + +// Event is a normalized view of one agent-loop step for evaluation reporting. +type Event struct { + Type string `json:"type"` // "content", "error", "tool", ... + Content string `json:"content,omitempty"` + Timestamp time.Time `json:"timestamp"` +} + +// Result is the outcome of one agent-runtime evaluation. +type Result struct { + // Output is the concatenated assistant output produced by the loop. + Output string `json:"output"` + // Events are the normalized loop events in order. + Events []Event `json:"events"` + // TokensUsed and CostUSD are reported by the backend, when known. + TokensUsed int `json:"tokens_used"` + CostUSD float64 `json:"cost_usd"` + // Transcript is the snapshot of the session's raw messages, for offline + // replay of a failing run. + Transcript []byte `json:"-"` + // Duration is the wall-clock time of the run. + Duration time.Duration `json:"duration"` +} + +// Runtime executes one agent-runtime evaluation. +type Runtime interface { + // Run drives the agent loop for prompt inside workDir and returns a Result. + Run(ctx context.Context, workDir, prompt string) (Result, error) +} + +// Config configures a SessionRuntime. +type Config struct { + // SystemPrompt is injected as the session system prompt. + SystemPrompt string + // MaxTurns caps the agent loop (0 = engine default). + MaxTurns int +} + +// DefaultConfig returns a sane default configuration. +func DefaultConfig() Config { + return Config{SystemPrompt: "You are an evaluation agent. Complete the requested task."} +} diff --git a/internal/feature/evalloop/session.go b/internal/feature/evalloop/session.go new file mode 100644 index 00000000..c3ba2f8b --- /dev/null +++ b/internal/feature/evalloop/session.go @@ -0,0 +1,88 @@ +package evalloop + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/GrayCodeAI/hawk/internal/engine" + "github.com/GrayCodeAI/hawk/internal/tool" +) + +// SessionRuntime drives the real engine.Session agent loop with an injected +// ChatClient. It is the production Runtime for end-to-end agent evaluation. +type SessionRuntime struct { + // Client is the LLM backend. Use a provider-bound client for real evals or + // a deterministic mock for CI smoke runs. + Client engine.ChatClient + // Provider and Model identify the backend for reporting. + Provider string + Model string + // Registry supplies the tool surface the loop can call. + Registry *tool.Registry + // Config carries loop limits and the system prompt. + Config Config +} + +// NewSessionRuntime builds a SessionRuntime. +func NewSessionRuntime(client engine.ChatClient, provider, model string, registry *tool.Registry, cfg Config) *SessionRuntime { + return &SessionRuntime{Client: client, Provider: provider, Model: model, Registry: registry, Config: cfg} +} + +// Run drives the real agent loop for prompt inside workDir. It creates a +// session bound to workDir, streams the loop to completion, and snapshots the +// transcript. The caller is responsible for running in an isolated directory. +func (r *SessionRuntime) Run(ctx context.Context, workDir, prompt string) (Result, error) { + if r == nil || r.Client == nil { + return Result{}, fmt.Errorf("evalloop: client is required") + } + systemPrompt := r.Config.SystemPrompt + if systemPrompt == "" { + systemPrompt = DefaultConfig().SystemPrompt + } + registry := r.Registry + if registry == nil { + registry = tool.NewRegistry() + } + + start := time.Now() + sess := engine.NewSessionWithClient(r.Client, r.Provider, r.Model, systemPrompt, registry, false) + if r.Config.MaxTurns > 0 { + _ = sess.SetMaxTurns(r.Config.MaxTurns) + } + sess.AddUser(prompt) + + ch, err := sess.Stream(ctx) + if err != nil { + return Result{}, fmt.Errorf("evalloop: start stream: %w", err) + } + + var result Result + for ev := range ch { + event := Event{Type: ev.Type, Content: ev.Content, Timestamp: time.Now()} + result.Events = append(result.Events, event) + switch ev.Type { + case "content": + result.Output += ev.Content + case "error": + result.Events = append(result.Events, event) + } + } + result.Duration = time.Since(start) + + // Snapshot the transcript for offline replay of failing runs. + if msgs := sess.Persistence().RawMessages(); msgs != nil { + if data, err := json.MarshalIndent(msgs, "", " "); err == nil { + result.Transcript = data + } + } + + // Report usage/cost when the backend exposes it via the session cost model. + cost := sess.CostValue() + if cost != nil { + usage := cost.Snapshot() + result.CostUSD = usage.TotalCostUSD + } + return result, nil +} diff --git a/internal/observability/conformance/conformance_test.go b/internal/observability/conformance/conformance_test.go new file mode 100644 index 00000000..05fee261 --- /dev/null +++ b/internal/observability/conformance/conformance_test.go @@ -0,0 +1,96 @@ +package conformance + +import ( + "context" + "strings" + "testing" + + "github.com/GrayCodeAI/hawk/internal/observability/oteltrace" +) + +// hawkSpanProducers wires the real span starters from internal/observability/ +// oteltrace into the conformance harness. Keeping the starters as the producers +// means the test fails if a starter stops setting a required attribute. +func hawkSpanProducers() []SpanProducer { + return []SpanProducer{ + func(ctx context.Context, t *oteltrace.Tracer) *oteltrace.Span { + _, span := oteltrace.StartAgentLoopSpan(ctx, t, "anthropic", "claude-sonnet-4", 3) + span.Finish() + return span + }, + func(ctx context.Context, t *oteltrace.Tracer) *oteltrace.Span { + _, span := oteltrace.StartToolSpan(ctx, t, "Read", "tool-1") + span.Finish() + return span + }, + func(ctx context.Context, t *oteltrace.Tracer) *oteltrace.Span { + _, span := oteltrace.StartCompactSpan(ctx, t, "summary", 1200) + span.Finish() + return span + }, + func(ctx context.Context, t *oteltrace.Tracer) *oteltrace.Span { + _, span := oteltrace.StartAPICallSpan(ctx, t, "openai", "gpt-4o") + span.Finish() + return span + }, + func(ctx context.Context, t *oteltrace.Tracer) *oteltrace.Span { + _, span := oteltrace.StartSessionSpan(ctx, t, "sess-123") + span.Finish() + return span + }, + } +} + +func TestHawkSpansConformToSchema(t *testing.T) { + report := Run(HawkSchema, hawkSpanProducers()) + if !report.Passed() { + for _, f := range report.Findings { + t.Errorf("conformance finding: %s: %s", f.SpanName, f.Message) + } + t.Fatalf("hawk spans must conform; %d violations", report.Violated) + } +} + +// TestSchemaCatchesDrift verifies the schema itself is strict: a span missing a +// required attribute or carrying a sensitive key must fail validation. This +// guards the conformance suite against becoming a no-op. +func TestSchemaCatchesDrift(t *testing.T) { + schema := HawkSchema + + // Missing required attribute must fail. + if findings := schema.Validate("agent_loop", map[string]string{"provider": "anthropic"}); len(findings) == 0 { + t.Fatal("agent_loop without model/message_count must be a violation") + } + + // Unknown span must fail. + if findings := schema.Validate("does_not_exist", map[string]string{}); len(findings) == 0 { + t.Fatal("unknown span must be a violation") + } + + // Sensitive content-bearing attribute must fail. + if findings := schema.Validate("agent_loop", map[string]string{ + "provider": "anthropic", "model": "m", "message_count": "1", "prompt": "secret", + }); len(findings) == 0 { + t.Fatal("span carrying raw prompt content must be a violation") + } + + // Valid spans pass. + if findings := schema.Validate("tool.Read", map[string]string{"tool.name": "Read", "tool.id": "1"}); len(findings) != 0 { + t.Fatalf("valid tool span should pass, got %+v", findings) + } +} + +// TestRunIsPassive verifies the harness never panics even when a producer panics. +func TestRunIsPassive(t *testing.T) { + producers := []SpanProducer{ + func(context.Context, *oteltrace.Tracer) *oteltrace.Span { + panic("boom") + }, + } + report := Run(HawkSchema, producers) + if !report.Passed() { + if !strings.Contains(report.Findings[0].Message, "panicked") { + t.Fatalf("expected a panicked finding, got %+v", report.Findings) + } + } +} diff --git a/internal/observability/conformance/harness.go b/internal/observability/conformance/harness.go new file mode 100644 index 00000000..bd8d6083 --- /dev/null +++ b/internal/observability/conformance/harness.go @@ -0,0 +1,60 @@ +package conformance + +import ( + "context" + + "github.com/GrayCodeAI/hawk/internal/observability/oteltrace" +) + +// SpanProducer starts and finishes one span on the given tracer, returning the +// finished span. It is the seam that connects a real span starter to the +// conformance harness. +type SpanProducer func(ctx context.Context, t *oteltrace.Tracer) *oteltrace.Span + +// Report summarizes the conformance result for a batch of spans. +type Report struct { + Total int `json:"total"` + Violated int `json:"violated"` + Findings []Finding `json:"findings,omitempty"` +} + +// Passed reports whether every span conformed. +func (r Report) Passed() bool { return r.Violated == 0 } + +// Run executes the given producers against a fresh tracer, validates every +// recorded span against the schema, and returns a report. It is passive: a +// producer that panics is caught and reported as a finding rather than crashing +// the harness, so CI can surface drift without breaking the run. +func Run(schema Schema, producers []SpanProducer) Report { + t := oteltrace.NewTracer() + defer t.Clear() + + var report Report + for _, produce := range producers { + span := safeProduce(produce, t) + if span == nil { + report.Total++ + report.Violated++ + report.Findings = append(report.Findings, Finding{SpanName: "", Message: "span producer panicked"}) + continue + } + report.Total++ + findings := schema.Validate(span.Name, span.Tags) + if len(findings) > 0 { + report.Violated++ + report.Findings = append(report.Findings, findings...) + } + } + return report +} + +// safeProduce invokes a producer, recovering any panic so a misbehaving span +// starter cannot break the conformance run. It returns nil on panic. +func safeProduce(produce SpanProducer, t *oteltrace.Tracer) (span *oteltrace.Span) { + defer func() { + if r := recover(); r != nil { + span = nil + } + }() + return produce(context.Background(), t) +} diff --git a/internal/observability/conformance/schema.go b/internal/observability/conformance/schema.go new file mode 100644 index 00000000..c74d2612 --- /dev/null +++ b/internal/observability/conformance/schema.go @@ -0,0 +1,148 @@ +// Package conformance verifies that emitted telemetry spans always match the +// documented OpenTelemetry schema (docs/OTEL-CONVENTIONS.md and the eyrie +// gen_ai.* semantic-convention constants), so the schema cannot silently drift +// across hawk and its submodules. +// +// The schema is declarative and typed per span: a span name (or prefix +// pattern), the required and optional attribute keys, and whether the span is +// forbidden from carrying raw prompt/response content. A Harness runs a set of +// span producers through a tracer and validates every recorded span against the +// schema. Validation is passive and non-throwing: a malformed or unknown span +// produces a finding, never a panic, so it can be used in CI to gate drift. +package conformance + +import ( + "strings" +) + +// Attribute vocabulary shared across the ecosystem. These mirror the keys +// emitted by the span starters in internal/observability/oteltrace and the +// gen_ai.* constants in external/eyrie/internal/observability. +const ( + AttrGenAISystem = "gen_ai.system" + AttrGenAIRequestModel = "gen_ai.request.model" + AttrGenAIResponseModel = "gen_ai.response.model" + AttrGenAIUsageInputTokens = "gen_ai.usage.input_tokens" // #nosec G101 -- OTel semconv attribute key string, not a secret value + AttrGenAIUsageOutputTokens = "gen_ai.usage.output_tokens" // #nosec G101 -- OTel semconv attribute key string, not a secret value + AttrGenAIOperationName = "gen_ai.operation.name" + AttrCostUSD = "cost.usd" + AttrToolName = "tool.name" + AttrSessionID = "session.id" + AttrAgentID = "agent.id" +) + +// sensitiveKeySubstrings identify attribute keys that may carry raw content +// (prompt/response/chat text) and are therefore forbidden by the no-raw-content +// rule unless explicitly allow-listed. The documented contract says spans MUST +// NOT carry raw prompt/response text. +var sensitiveKeySubstrings = []string{"prompt", "response", "content", "text"} + +// SpanDef describes one span (or a family matched by NamePattern) in the schema. +type SpanDef struct { + // NamePattern is the exact span name or a "*"-suffixed prefix pattern + // (e.g. "tool.*" matches "tool.Read"). If it does not end in "*", it is an + // exact match. + NamePattern string + // RequiredAttrs lists attribute keys that must be present. + RequiredAttrs []string + // OptionalAttrs lists attribute keys that may be present but are not required. + OptionalAttrs []string +} + +// Schema is the ordered set of span definitions. +type Schema []SpanDef + +// HawkSchema is the schema covering every span hawk emits via the starters in +// internal/observability/oteltrace/spans.go. +var HawkSchema = Schema{ + {NamePattern: "agent_loop", RequiredAttrs: []string{"provider", "model", "message_count"}}, + {NamePattern: "tool.*", RequiredAttrs: []string{"tool.name", "tool.id"}}, + {NamePattern: "compact.*", RequiredAttrs: []string{"compact.strategy", "compact.tokens_before"}}, + {NamePattern: "api.chat", RequiredAttrs: []string{"api.provider", "api.model"}}, + {NamePattern: "session", RequiredAttrs: []string{"session.id"}}, +} + +// Finding describes a single conformance violation. +type Finding struct { + SpanName string `json:"span_name"` + Message string `json:"message"` +} + +// Find returns the SpanDef whose pattern matches spanName, or nil. +func (s Schema) Find(spanName string) *SpanDef { + for i := range s { + if matches(s[i].NamePattern, spanName) { + return &s[i] + } + } + return nil +} + +// Validate checks a single span (name + attributes) against the schema. It +// returns every violation found. It never panics. +func (s Schema) Validate(spanName string, attributes map[string]string) []Finding { + var findings []Finding + def := s.Find(spanName) + if def == nil { + return append(findings, Finding{SpanName: spanName, Message: "span is not covered by the telemetry schema"}) + } + for _, key := range def.RequiredAttrs { + if v, ok := attributes[key]; !ok || strings.TrimSpace(v) == "" { + findings = append(findings, Finding{SpanName: spanName, Message: "missing required attribute " + key}) + } + } + for key := range attributes { + if !isAllowedKey(def, key) { + findings = append(findings, Finding{SpanName: spanName, Message: "attribute " + key + " is not declared for this span"}) + } + if isSensitiveKey(key) { + findings = append(findings, Finding{SpanName: spanName, Message: "span carries sensitive content-bearing attribute " + key}) + } + } + return findings +} + +// allowed reports whether key is a declared required or optional attribute. +func isAllowedKey(def *SpanDef, key string) bool { + for _, r := range def.RequiredAttrs { + if r == key { + return true + } + } + for _, o := range def.OptionalAttrs { + if o == key { + return true + } + } + // Standard span attributes such as error/error.message are permitted. + switch key { + case "error", "error.message": + return true + } + return false +} + +// isSensitiveKey reports whether an attribute key may carry raw content. +func isSensitiveKey(key string) bool { + lower := strings.ToLower(key) + // Counters and usage markers are numeric, not content. + if strings.HasSuffix(lower, "_count") || + strings.HasSuffix(lower, "_tokens") || + strings.HasSuffix(lower, "_usage") || + lower == "message_count" { + return false + } + for _, s := range sensitiveKeySubstrings { + if strings.Contains(lower, s) { + return true + } + } + return false +} + +func matches(pattern, name string) bool { + if strings.HasSuffix(pattern, "*") { + return strings.HasPrefix(name, strings.TrimSuffix(pattern, "*")) + } + return pattern == name +}