From 1a69b3fad20d0f82518911a7a6e3054567b217a6 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Mon, 24 Aug 2026 10:09:59 +0530 Subject: [PATCH] feat(tool): add SearchX and ComputerUse tools --- README.md | 2 + cmd/chat_tools.go | 2 + internal/tool/computer_use.go | 141 +++++++++++++++++++++++++++++ internal/tool/computer_use_test.go | 89 ++++++++++++++++++ internal/tool/search_x.go | 127 ++++++++++++++++++++++++++ internal/tool/search_x_test.go | 99 ++++++++++++++++++++ 6 files changed, 460 insertions(+) create mode 100644 internal/tool/computer_use.go create mode 100644 internal/tool/computer_use_test.go create mode 100644 internal/tool/search_x.go create mode 100644 internal/tool/search_x_test.go diff --git a/README.md b/README.md index 247db7bc..b95dcb9e 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,8 @@ Features adopted from open-source agent projects. All are off by default unless | Agent family messaging | `internal/multiagent` (`FamilyMessenger`) | Direct parent/sibling/child messages with pending caps + rate limits | | Path reservations | `internal/multiagent` ledger | Detect overlapping-file changes between parallel branches | | Live agent status | `GET /v1/agent/status` (daemon) | Machine-readable working/idle/stale per session | +| X/Twitter search | `SearchX` tool | Live X search by forwarding a query to an xAI endpoint with server-side search; returns a cited summary. Requires `XAI_API_KEY` (or `GROK_API_KEY`) | +| Desktop computer-use | `ComputerUse` tool | snapshot/click/type/scroll/press/screenshot via a pluggable `tool.SetComputerBackend` seam (host wires a native macOS accessibility backend) | ## Usage diff --git a/cmd/chat_tools.go b/cmd/chat_tools.go index 9fc8c3a8..bb8a64bd 100644 --- a/cmd/chat_tools.go +++ b/cmd/chat_tools.go @@ -173,6 +173,8 @@ func optionalTools() []tool.Tool { tool.CodeMatchTool{}, tool.FuzzyFindTool{}, tool.BatchExecTool{}, + tool.SearchXTool{}, + tool.ComputerUseTool{}, tool.ToolsetTool{}, tool.CoreMemoryAppendTool{}, tool.CoreMemoryReplaceTool{}, diff --git a/internal/tool/computer_use.go b/internal/tool/computer_use.go new file mode 100644 index 00000000..40cde2b4 --- /dev/null +++ b/internal/tool/computer_use.go @@ -0,0 +1,141 @@ +package tool + +import ( + "context" + "encoding/json" + "fmt" + "strings" +) + +// ComputerUseTool exposes host desktop automation to the agent through a +// pluggable backend. Adopted from Prime Agent / Orca computer-use: the tool +// surface (snapshot, click, type, scroll, press, screenshot) is provider- +// neutral; a host wires the actual backend (e.g. a native macOS accessibility +// backend via SetComputerBackend). Without a wired backend the tool reports a +// clear error — the seam is the deliverable. +type ComputerUseTool struct{} + +func (ComputerUseTool) Name() string { return "ComputerUse" } +func (ComputerUseTool) RiskLevel() string { return "high" } +func (ComputerUseTool) Aliases() []string { return []string{"computer_use", "computer"} } +func (ComputerUseTool) Description() string { + return "Operate the host desktop (snapshot UI, click, type, scroll, keypress, screenshot) via a pluggable backend. Requires a wired computer backend (see SetComputerBackend)." +} + +func (ComputerUseTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "action": map[string]interface{}{ + "type": "string", + "enum": []string{"snapshot", "click", "type", "scroll", "press", "screenshot"}, + "description": "snapshot: dump the UI; click: click an element/ref; type: enter text; scroll: scroll; press: send key chord; screenshot: capture screen.", + }, + "target": map[string]interface{}{ + "type": "string", + "description": "Element ref (e.g. @e1) or label for click/type/scroll.", + }, + "text": map[string]interface{}{ + "type": "string", + "description": "Text to type or key chord for press.", + }, + }, + "required": []string{"action"}, + } +} + +// ComputerBackend is the pluggable host-desktop automation backend. +type ComputerBackend interface { + // Name identifies the backend for provenance. + Name() string + // Snapshot returns a representation of the current UI. + Snapshot(ctx context.Context) (string, error) + // Click activates the element identified by ref. + Click(ctx context.Context, ref string) error + // Type enters text into the focused field. + Type(ctx context.Context, text string) error + // Scroll scrolls (direction: up/down/left/right). + Scroll(ctx context.Context, ref, direction string) error + // Press sends a key chord (e.g. cmd+k). + Press(ctx context.Context, chord string) error + // Screenshot captures the screen and returns a path or data URL. + Screenshot(ctx context.Context) (string, error) +} + +var computerBackend ComputerBackend + +// SetComputerBackend installs the host-desktop backend. Nil by default; the +// tool reports a clear error until wired. +func SetComputerBackend(b ComputerBackend) { computerBackend = b } + +// ComputerBackendName returns the active backend name, or "" when none. +func ComputerBackendName() string { + if computerBackend == nil { + return "" + } + return computerBackend.Name() +} + +func (ComputerUseTool) Execute(ctx context.Context, input json.RawMessage) (string, error) { + var p struct { + Action string `json:"action"` + Target string `json:"target"` + Text string `json:"text"` + Direction string `json:"direction"` + } + if err := json.Unmarshal(input, &p); err != nil { + return "", fmt.Errorf("invalid input: %w", err) + } + p.Action = strings.ToLower(strings.TrimSpace(p.Action)) + if p.Action == "" { + return "", fmt.Errorf("action is required") + } + if computerBackend == nil { + return "", fmt.Errorf("no computer backend installed — wire one via SetComputerBackend") + } + switch p.Action { + case "snapshot": + out, err := computerBackend.Snapshot(ctx) + if err != nil { + return "", fmt.Errorf("computer snapshot: %w", err) + } + return out, nil + case "click": + if p.Target == "" { + return "", fmt.Errorf("click requires a target ref") + } + if err := computerBackend.Click(ctx, p.Target); err != nil { + return "", fmt.Errorf("computer click: %w", err) + } + return fmt.Sprintf("Clicked %s", p.Target), nil + case "type": + if p.Text == "" { + return "", fmt.Errorf("type requires text") + } + if err := computerBackend.Type(ctx, p.Text); err != nil { + return "", fmt.Errorf("computer type: %w", err) + } + return fmt.Sprintf("Typed %d characters", len(p.Text)), nil + case "scroll": + if err := computerBackend.Scroll(ctx, p.Target, p.Direction); err != nil { + return "", fmt.Errorf("computer scroll: %w", err) + } + return "Scrolled", nil + case "press": + if p.Text == "" { + return "", fmt.Errorf("press requires a key chord") + } + if err := computerBackend.Press(ctx, p.Text); err != nil { + return "", fmt.Errorf("computer press: %w", err) + } + return fmt.Sprintf("Pressed %s", p.Text), nil + case "screenshot": + out, err := computerBackend.Screenshot(ctx) + if err != nil { + return "", fmt.Errorf("computer screenshot: %w", err) + } + return out, nil + default: + return "", fmt.Errorf("unsupported action %q (use snapshot, click, type, scroll, press, or screenshot)", p.Action) + } +} diff --git a/internal/tool/computer_use_test.go b/internal/tool/computer_use_test.go new file mode 100644 index 00000000..5d90ce96 --- /dev/null +++ b/internal/tool/computer_use_test.go @@ -0,0 +1,89 @@ +package tool + +import ( + "context" + "encoding/json" + "strings" + "testing" +) + +type mockComputer struct{} + +func (mockComputer) Name() string { return "mock" } +func (mockComputer) Snapshot(ctx context.Context) (string, error) { + return "UI: button @e1 Submit", nil +} +func (mockComputer) Click(ctx context.Context, ref string) error { return nil } +func (mockComputer) Type(ctx context.Context, text string) error { return nil } +func (mockComputer) Scroll(ctx context.Context, ref, dir string) error { return nil } +func (mockComputer) Press(ctx context.Context, chord string) error { return nil } +func (mockComputer) Screenshot(ctx context.Context) (string, error) { return "/tmp/shot.png", nil } + +func TestComputerUseRequiresBackend(t *testing.T) { + SetComputerBackend(nil) + tool := ComputerUseTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage(`{"action":"snapshot"}`)); err == nil || !strings.Contains(err.Error(), "no computer backend") { + t.Fatalf("err = %v", err) + } +} + +func TestComputerUseSnapshot(t *testing.T) { + SetComputerBackend(mockComputer{}) + defer SetComputerBackend(nil) + out, err := ComputerUseTool{}.Execute(context.Background(), json.RawMessage(`{"action":"snapshot"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "@e1 Submit") { + t.Fatalf("out = %q", out) + } +} + +func TestComputerUseClickRequiresTarget(t *testing.T) { + SetComputerBackend(mockComputer{}) + defer SetComputerBackend(nil) + tool := ComputerUseTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage(`{"action":"click"}`)); err == nil || !strings.Contains(err.Error(), "requires a target") { + t.Fatalf("err = %v", err) + } +} + +func TestComputerUseTypeAndPress(t *testing.T) { + SetComputerBackend(mockComputer{}) + defer SetComputerBackend(nil) + out, err := ComputerUseTool{}.Execute(context.Background(), json.RawMessage(`{"action":"type","text":"hello"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "5 characters") { + t.Fatalf("out = %q", out) + } + out, err = ComputerUseTool{}.Execute(context.Background(), json.RawMessage(`{"action":"press","text":"cmd+k"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "cmd+k") { + t.Fatalf("out = %q", out) + } +} + +func TestComputerUseScreenshot(t *testing.T) { + SetComputerBackend(mockComputer{}) + defer SetComputerBackend(nil) + out, err := ComputerUseTool{}.Execute(context.Background(), json.RawMessage(`{"action":"screenshot"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "shot.png") { + t.Fatalf("out = %q", out) + } +} + +func TestComputerUseInvalidAction(t *testing.T) { + SetComputerBackend(mockComputer{}) + defer SetComputerBackend(nil) + tool := ComputerUseTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage(`{"action":"nope"}`)); err == nil { + t.Fatal("expected error for invalid action") + } +} diff --git a/internal/tool/search_x.go b/internal/tool/search_x.go new file mode 100644 index 00000000..86d6ebfd --- /dev/null +++ b/internal/tool/search_x.go @@ -0,0 +1,127 @@ +package tool + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +// SearchXTool performs live X/Twitter search by forwarding a query to an +// xAI-compatible chat-completions endpoint with server-side X search enabled, +// returning the model's summarized answer. Adopted from grok-cli's search_x +// (server-side search tooling rather than a bespoke crawler). Inline HTTP, +// boundary-compliant (no eyrie/client import), httptest-testable. +type SearchXTool struct{} + +func (SearchXTool) Name() string { return "SearchX" } +func (SearchXTool) RiskLevel() string { return "medium" } +func (SearchXTool) Aliases() []string { return []string{"search_x", "x_search", "xsearch"} } +func (SearchXTool) Description() string { + return "Search X/Twitter for live posts matching a query. Returns a summarized answer incorporating current X results. Requires XAI_API_KEY." +} + +func (SearchXTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "query": map[string]interface{}{ + "type": "string", + "description": "The X/Twitter search query (topic, hashtag, account, etc.).", + }, + "model": map[string]interface{}{ + "type": "string", + "description": "Model to use (default grok-4.20-non-reasoning).", + }, + }, + "required": []string{"query"}, + } +} + +// xAISearchSystemPrompt directs the model to use its built-in X search and +// return a concise, cited summary. +const xAISearchSystemPrompt = "You are a live X/Twitter search assistant. Use your built-in X search tool to find current posts matching the user's query, then summarize the most relevant results with attribution. If X search is unavailable, say so clearly." + +// xAISearchBaseURL is the xAI API host; a var so tests can redirect. +var xAISearchBaseURL = "https://api.x.ai" + +// xAISearchKey reads the API key (XAI_API_KEY, with GROK_API_KEY fallback). +func xAISearchKey() string { + if k := os.Getenv("XAI_API_KEY"); k != "" { + return k + } + return os.Getenv("GROK_API_KEY") +} + +// xAISearchModel is the default model. +func xAISearchModel() string { return "grok-4.20-non-reasoning" } + +func (SearchXTool) Execute(ctx context.Context, input json.RawMessage) (string, error) { + var p struct { + Query string `json:"query"` + Model string `json:"model"` + } + if err := json.Unmarshal(input, &p); err != nil { + return "", fmt.Errorf("invalid input: %w", err) + } + query := strings.TrimSpace(p.Query) + if query == "" { + return "", fmt.Errorf("query is required") + } + model := strings.TrimSpace(p.Model) + if model == "" { + model = xAISearchModel() + } + apiKey := xAISearchKey() + if apiKey == "" { + return "", fmt.Errorf("XAI_API_KEY not set — required for X search") + } + + body, err := json.Marshal(map[string]interface{}{ + "model": model, + "max_tokens": 1000, + "messages": []map[string]string{ + {"role": "system", "content": xAISearchSystemPrompt}, + {"role": "user", "content": query}, + }, + }) + if err != nil { + return "", err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + xAISearchBaseURL+"/v1/chat/completions", bytes.NewReader(body)) // #nosec G107 -- fixed API host + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+apiKey) + client := &http.Client{Timeout: 60 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("x search: %w", err) + } + defer func() { _ = resp.Body.Close() }() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("x search API %d: %s", resp.StatusCode, strings.TrimSpace(string(raw))) + } + var result struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(raw, &result); err != nil { + return "", fmt.Errorf("x search decode: %w", err) + } + if len(result.Choices) == 0 || strings.TrimSpace(result.Choices[0].Message.Content) == "" { + return "X search returned no results.", nil + } + return strings.TrimSpace(result.Choices[0].Message.Content), nil +} diff --git a/internal/tool/search_x_test.go b/internal/tool/search_x_test.go new file mode 100644 index 00000000..a05acd82 --- /dev/null +++ b/internal/tool/search_x_test.go @@ -0,0 +1,99 @@ +package tool + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestSearchXBasic(t *testing.T) { + var sawAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + t.Fatalf("path = %s", r.URL.Path) + } + sawAuth = r.Header.Get("Authorization") + fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"summary of X results"}}]}`) + })) + defer srv.Close() + oldBase := xAISearchBaseURL + xAISearchBaseURL = srv.URL + defer func() { xAISearchBaseURL = oldBase }() + t.Setenv("XAI_API_KEY", "k") + + out, err := SearchXTool{}.Execute(context.Background(), json.RawMessage(`{"query":"hawk ai"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "summary of X results") { + t.Fatalf("out = %q", out) + } + if !strings.Contains(sawAuth, "Bearer k") { + t.Fatalf("auth = %q", sawAuth) + } +} + +func TestSearchXRequiresQuery(t *testing.T) { + t.Setenv("XAI_API_KEY", "k") + tool := SearchXTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage(`{"query":""}`)); err == nil { + t.Fatal("expected error for empty query") + } +} + +func TestSearchXRequiresKey(t *testing.T) { + t.Setenv("XAI_API_KEY", "") + t.Setenv("GROK_API_KEY", "") + tool := SearchXTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage(`{"query":"x"}`)); err == nil || !strings.Contains(err.Error(), "XAI_API_KEY") { + t.Fatalf("err = %v", err) + } +} + +func TestSearchXErrorStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprint(w, `{"error":"bad key"}`) + })) + defer srv.Close() + oldBase := xAISearchBaseURL + xAISearchBaseURL = srv.URL + defer func() { xAISearchBaseURL = oldBase }() + t.Setenv("XAI_API_KEY", "k") + + tool := SearchXTool{} + if _, err := tool.Execute(context.Background(), json.RawMessage(`{"query":"x"}`)); err == nil || !strings.Contains(err.Error(), "401") { + t.Fatalf("err = %v", err) + } +} + +func TestSearchXNoChoices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"choices":[]}`) + })) + defer srv.Close() + oldBase := xAISearchBaseURL + xAISearchBaseURL = srv.URL + defer func() { xAISearchBaseURL = oldBase }() + t.Setenv("XAI_API_KEY", "k") + + out, err := SearchXTool{}.Execute(context.Background(), json.RawMessage(`{"query":"x"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "no results") { + t.Fatalf("out = %q", out) + } +} + +func TestSearchXGrokKeyFallback(t *testing.T) { + t.Setenv("XAI_API_KEY", "") + t.Setenv("GROK_API_KEY", "grok-key") + if got := xAISearchKey(); got != "grok-key" { + t.Fatalf("key = %q", got) + } +}