llm and everything: Update ToolResult to use []Content instead of string for multimodal support
This was a journey. The sketch-generated summary below is acceptable,
but I want to tell you about it in my voice too. The goal was to send
screenshots to Claude, so that it could... look at them. Currently
the take screenshot and read screenshot tools are different, and they'll
need to be renamed/prompt-engineered a bit, but that's all fine.
The miserable part was that we had to change the return value
of tool from string to Content[], and this crosses several layers:
- llm.Tool
- llm.Content
- ant.Content & openai and gemini friends
- AgentMessage [we left this alone]
Extra fun is that Claude's API for sending images has nested Content
fields, and empty string and missing needs to be distinguished for the
Text field (because lots of shell commands return the empty string!).
For the UI, I made us transform the results into a string, dropping
images. This would have been yet more churn for not much obvious
benefit. Plus, it was going to break skaband's compatibility, and ...
yet more work.
OpenAI and Gemini don't obviously support images in this same way,
so they just don't get the tools.
~~~~~~~~~~ Sketch said:
This architectural change transforms tool results from plain strings to []Content arrays, enabling multimodal interaction in the system. Key changes include:
- Core structural changes:
- Modified ToolResult type from string to []Content across all packages
- Added MediaType field to Content struct for MIME type support
- Created TextContent and ImageContent helper functions
- Updated all tool.Run implementations to return []Content
- Image handling:
- Implemented base64 image support in Anthropic adapter
- Added proper media type detection and content formatting
- Created browser_read_image tool for displaying screenshots
- Updated browser_screenshot to provide usable image paths
- Adapter improvements:
- Updated all LLM adapters (ANT, OAI, GEM) to handle content arrays
- Added specialized image content handling in the Anthropic adapter
- Ensured proper JSON serialization/deserialization for all content types
- Improved test coverage for content arrays
- UI enhancements:
- Added omitempty tags to reduce JSON response size
- Updated TypeScript types to handle array content
- Made field naming consistent (tool_error vs is_error)
- Preserved backward compatibility for existing consumers
Co-Authored-By: sketch <hello@sketch.dev>
Change-ID: s1a2b3c4d5e6f7g8h
diff --git a/loop/agent.go b/loop/agent.go
index c103919..2c8eec9 100644
--- a/loop/agent.go
+++ b/loop/agent.go
@@ -26,6 +26,7 @@
"sketch.dev/claudetool/onstart"
"sketch.dev/experiment"
"sketch.dev/llm"
+ "sketch.dev/llm/ant"
"sketch.dev/llm/conversation"
)
@@ -228,8 +229,8 @@
if a.TurnDuration != nil {
attrs = append(attrs, slog.Int64("turnDuration", a.TurnDuration.Nanoseconds()))
}
- if a.ToolResult != "" {
- attrs = append(attrs, slog.String("tool_result", a.ToolResult))
+ if len(a.ToolResult) > 0 {
+ attrs = append(attrs, slog.Any("tool_result", a.ToolResult))
}
if a.ToolError {
attrs = append(attrs, slog.Bool("tool_error", a.ToolError))
@@ -554,6 +555,33 @@
a.mu.Unlock()
}
+// contentToString converts []llm.Content to a string, concatenating all text content and skipping non-text types.
+// If there's only one element in the array and it's a text type, it returns that text directly.
+// It also processes nested ToolResult arrays recursively.
+func contentToString(contents []llm.Content) string {
+ if len(contents) == 0 {
+ return ""
+ }
+
+ // If there's only one element and it's a text type, return it directly
+ if len(contents) == 1 && contents[0].Type == llm.ContentTypeText {
+ return contents[0].Text
+ }
+
+ // Otherwise, concatenate all text content
+ var result strings.Builder
+ for _, content := range contents {
+ if content.Type == llm.ContentTypeText {
+ result.WriteString(content.Text)
+ } else if content.Type == llm.ContentTypeToolResult && len(content.ToolResult) > 0 {
+ // Recursively process nested tool results
+ result.WriteString(contentToString(content.ToolResult))
+ }
+ }
+
+ return result.String()
+}
+
// OnToolResult implements ant.Listener.
func (a *Agent) OnToolResult(ctx context.Context, convo *conversation.Convo, toolID string, toolName string, toolInput json.RawMessage, content llm.Content, result *string, err error) {
// Remove the tool call from outstanding calls
@@ -564,7 +592,7 @@
m := AgentMessage{
Type: ToolUseMessageType,
Content: content.Text,
- ToolResult: content.ToolResult,
+ ToolResult: contentToString(content.ToolResult),
ToolError: content.ToolError,
ToolName: toolName,
ToolInput: string(toolInput),
@@ -879,7 +907,8 @@
// Add browser tools if enabled
// if experiment.Enabled("browser") {
if true {
- bTools, browserCleanup := browse.RegisterBrowserTools(a.config.Context)
+ _, supportsScreenshots := a.config.Service.(*ant.Service)
+ bTools, browserCleanup := browse.RegisterBrowserTools(a.config.Context, supportsScreenshots)
// Add cleanup function to context cancel
go func() {
<-a.config.Context.Done()
@@ -943,13 +972,13 @@
},
"required": ["question", "responseOptions"]
}`),
- Run: func(ctx context.Context, input json.RawMessage) (string, error) {
+ Run: func(ctx context.Context, input json.RawMessage) ([]llm.Content, error) {
// The Run logic for "multiplchoice" tool is a no-op on the server.
// The UI will present a list of options for the user to select from,
// and that's it as far as "executing" the tool_use goes.
// When the user *does* select one of the presented options, that
// responseText gets sent as a chat message on behalf of the user.
- return "end your turn and wait for the user to respond", nil
+ return llm.TextContent("end your turn and wait for the user to respond"), nil
},
}
return ret
@@ -997,28 +1026,28 @@
},
"required": ["title"]
}`),
- Run: func(ctx context.Context, input json.RawMessage) (string, error) {
+ Run: func(ctx context.Context, input json.RawMessage) ([]llm.Content, error) {
var params struct {
Title string `json:"title"`
}
if err := json.Unmarshal(input, ¶ms); err != nil {
- return "", err
+ return nil, err
}
// We don't allow changing the title once set to be consistent with the previous behavior
// and to prevent accidental title changes
t := a.Title()
if t != "" {
- return "", fmt.Errorf("title already set to: %s", t)
+ return nil, fmt.Errorf("title already set to: %s", t)
}
if params.Title == "" {
- return "", fmt.Errorf("title parameter cannot be empty")
+ return nil, fmt.Errorf("title parameter cannot be empty")
}
a.SetTitle(params.Title)
response := fmt.Sprintf("Title set to %q", params.Title)
- return response, nil
+ return llm.TextContent(response), nil
},
}
return titleTool
@@ -1039,28 +1068,28 @@
},
"required": ["branch_name"]
}`),
- Run: func(ctx context.Context, input json.RawMessage) (string, error) {
+ Run: func(ctx context.Context, input json.RawMessage) ([]llm.Content, error) {
var params struct {
BranchName string `json:"branch_name"`
}
if err := json.Unmarshal(input, ¶ms); err != nil {
- return "", err
+ return nil, err
}
b := a.BranchName()
if b != "" {
- return "", fmt.Errorf("branch already set to: %s", b)
+ return nil, fmt.Errorf("branch already set to: %s", b)
}
if params.BranchName == "" {
- return "", fmt.Errorf("branch_name parameter cannot be empty")
+ return nil, fmt.Errorf("branch_name must not be empty")
}
if params.BranchName != cleanBranchName(params.BranchName) {
- return "", fmt.Errorf("branch_name parameter must be alphanumeric hyphenated slug")
+ return nil, fmt.Errorf("branch_name parameter must be alphanumeric hyphenated slug")
}
branchName := "sketch/" + params.BranchName
if branchExists(a.workingDir, branchName) {
- return "", fmt.Errorf("branch %q already exists; please choose a different branch name", branchName)
+ return nil, fmt.Errorf("branch %q already exists; please choose a different branch name", branchName)
}
a.SetBranch(branchName)
@@ -1074,7 +1103,7 @@
response += "\n\n" + styleHint
}
- return response, nil
+ return llm.TextContent(response), nil
},
}
return preCommit
@@ -1089,11 +1118,6 @@
a.inbox <- msg
}
-func (a *Agent) ToolResultMessage(ctx context.Context, toolCallID, msg string) {
- a.pushToOutbox(ctx, AgentMessage{Type: UserMessageType, Content: msg, ToolCallId: toolCallID})
- a.inbox <- msg
-}
-
func (a *Agent) CancelToolUse(toolUseID string, cause error) error {
return a.convo.CancelToolUse(toolUseID, cause)
}
@@ -1137,6 +1161,11 @@
m.Timestamp = time.Now()
}
+ // If this is a ToolUseMessage and ToolResult is set but Content is not, copy the ToolResult to Content
+ if m.Type == ToolUseMessageType && m.ToolResult != "" && m.Content == "" {
+ m.Content = m.ToolResult
+ }
+
// If this is an end-of-turn message, calculate the turn duration and add it to the message
if m.EndOfTurn && m.Type == AgentMessageType {
turnDuration := time.Since(a.startOfTurn)