blob: d1b1eece94785a9a6560a685dab468ae7dfb8e24 [file] [log] [blame]
Philip Zeyliger33d282f2025-05-03 04:01:54 +00001package browse
2
3import (
4 "context"
5 "encoding/json"
Philip Zeyliger72252cb2025-05-10 17:00:08 -07006 "fmt"
Philip Zeyliger33d282f2025-05-03 04:01:54 +00007 "os"
Philip Zeyliger72252cb2025-05-10 17:00:08 -07008 "path/filepath"
Philip Zeyliger33d282f2025-05-03 04:01:54 +00009 "slices"
10 "strings"
11 "testing"
12 "time"
13
14 "github.com/chromedp/chromedp"
15 "sketch.dev/llm"
16)
17
18func TestToolCreation(t *testing.T) {
19 // Create browser tools instance
20 tools := NewBrowseTools(context.Background())
Philip Zeyligera35de5f2025-06-14 12:00:48 -070021 t.Cleanup(func() {
22 tools.Close()
23 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +000024
25 // Test each tool has correct name and description
26 toolTests := []struct {
27 tool *llm.Tool
28 expectedName string
29 shortDesc string
30 requiredProps []string
31 }{
32 {tools.NewNavigateTool(), "browser_navigate", "Navigate", []string{"url"}},
33 {tools.NewClickTool(), "browser_click", "Click", []string{"selector"}},
34 {tools.NewTypeTool(), "browser_type", "Type", []string{"selector", "text"}},
35 {tools.NewWaitForTool(), "browser_wait_for", "Wait", []string{"selector"}},
36 {tools.NewGetTextTool(), "browser_get_text", "Get", []string{"selector"}},
37 {tools.NewEvalTool(), "browser_eval", "Evaluate", []string{"expression"}},
Philip Zeyliger80b488d2025-05-10 18:21:54 -070038 {tools.NewScreenshotTool(), "browser_take_screenshot", "Take", nil},
Philip Zeyliger33d282f2025-05-03 04:01:54 +000039 {tools.NewScrollIntoViewTool(), "browser_scroll_into_view", "Scroll", []string{"selector"}},
40 }
41
42 for _, tt := range toolTests {
43 t.Run(tt.expectedName, func(t *testing.T) {
44 if tt.tool.Name != tt.expectedName {
45 t.Errorf("expected name %q, got %q", tt.expectedName, tt.tool.Name)
46 }
47
48 if !strings.Contains(tt.tool.Description, tt.shortDesc) {
49 t.Errorf("description %q should contain %q", tt.tool.Description, tt.shortDesc)
50 }
51
52 // Verify schema has required properties
53 if len(tt.requiredProps) > 0 {
54 var schema struct {
55 Required []string `json:"required"`
56 }
57 if err := json.Unmarshal(tt.tool.InputSchema, &schema); err != nil {
58 t.Fatalf("failed to unmarshal schema: %v", err)
59 }
60
61 for _, prop := range tt.requiredProps {
62 if !slices.Contains(schema.Required, prop) {
63 t.Errorf("property %q should be required", prop)
64 }
65 }
66 }
67 })
68 }
69}
70
Philip Zeyliger72252cb2025-05-10 17:00:08 -070071func TestGetTools(t *testing.T) {
Philip Zeyliger33d282f2025-05-03 04:01:54 +000072 // Create browser tools instance
73 tools := NewBrowseTools(context.Background())
Philip Zeyligera35de5f2025-06-14 12:00:48 -070074 t.Cleanup(func() {
75 tools.Close()
76 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +000077
Philip Zeyliger72252cb2025-05-10 17:00:08 -070078 // Test with screenshot tools included
79 t.Run("with screenshots", func(t *testing.T) {
80 toolsWithScreenshots := tools.GetTools(true)
Philip Zeyliger18e33682025-05-13 16:34:21 -070081 if len(toolsWithScreenshots) != 12 {
82 t.Errorf("expected 12 tools with screenshots, got %d", len(toolsWithScreenshots))
Philip Zeyliger33d282f2025-05-03 04:01:54 +000083 }
Philip Zeyliger72252cb2025-05-10 17:00:08 -070084
85 // Check tool naming convention
86 for _, tool := range toolsWithScreenshots {
Philip Zeyliger542bda32025-06-11 18:31:03 -070087 // Most tools have browser_ prefix, except for read_image
88 if tool.Name != "read_image" && !strings.HasPrefix(tool.Name, "browser_") {
Philip Zeyliger72252cb2025-05-10 17:00:08 -070089 t.Errorf("tool name %q does not have prefix 'browser_'", tool.Name)
90 }
91 }
92 })
93
94 // Test without screenshot tools
95 t.Run("without screenshots", func(t *testing.T) {
96 noScreenshotTools := tools.GetTools(false)
Philip Zeyliger18e33682025-05-13 16:34:21 -070097 if len(noScreenshotTools) != 10 {
98 t.Errorf("expected 10 tools without screenshots, got %d", len(noScreenshotTools))
Philip Zeyliger72252cb2025-05-10 17:00:08 -070099 }
100 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000101}
102
103// TestBrowserInitialization verifies that the browser can start correctly
104func TestBrowserInitialization(t *testing.T) {
105 // Skip long tests in short mode
106 if testing.Short() {
107 t.Skip("skipping browser initialization test in short mode")
108 }
109
110 // Create browser tools instance
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700111 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000112 defer cancel()
113
114 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700115 t.Cleanup(func() {
116 tools.Close()
117 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000118
119 // Initialize the browser
120 err := tools.Initialize()
121 if err != nil {
122 // If browser automation is not available, skip the test
123 if strings.Contains(err.Error(), "browser automation not available") {
124 t.Skip("Browser automation not available in this environment")
125 } else {
126 t.Fatalf("Failed to initialize browser: %v", err)
127 }
128 }
129
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000130 // Get browser context to verify it's working
131 browserCtx, err := tools.GetBrowserContext()
132 if err != nil {
133 t.Fatalf("Failed to get browser context: %v", err)
134 }
135
136 // Try to navigate to a simple page
137 var title string
138 err = chromedp.Run(browserCtx,
139 chromedp.Navigate("about:blank"),
140 chromedp.Title(&title),
141 )
142 if err != nil {
143 t.Fatalf("Failed to navigate to about:blank: %v", err)
144 }
145
146 t.Logf("Successfully navigated to about:blank, title: %q", title)
147}
148
149// TestNavigateTool verifies that the navigate tool works correctly
150func TestNavigateTool(t *testing.T) {
151 // Skip long tests in short mode
152 if testing.Short() {
153 t.Skip("skipping navigate tool test in short mode")
154 }
155
156 // Create browser tools instance
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700157 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000158 defer cancel()
159
160 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700161 t.Cleanup(func() {
162 tools.Close()
163 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000164
165 // Check if browser initialization works
166 if err := tools.Initialize(); err != nil {
167 if strings.Contains(err.Error(), "browser automation not available") {
168 t.Skip("Browser automation not available in this environment")
169 }
170 }
171
172 // Get the navigate tool
173 navTool := tools.NewNavigateTool()
174
175 // Create input for the navigate tool
176 input := map[string]string{"url": "https://example.com"}
177 inputJSON, _ := json.Marshal(input)
178
179 // Call the tool
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700180 toolOut := navTool.Run(ctx, json.RawMessage(inputJSON))
181 if toolOut.Error != nil {
182 t.Fatalf("Error running navigate tool: %v", toolOut.Error)
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000183 }
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700184 result := toolOut.LLMContent
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000185
186 // Verify the response is successful
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700187 resultText := result[0].Text
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000188 if !strings.Contains(resultText, "done") {
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000189 // If browser automation is not available, skip the test
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000190 if strings.Contains(resultText, "browser automation not available") {
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000191 t.Skip("Browser automation not available in this environment")
192 } else {
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000193 t.Fatalf("Expected done in result text, got: %s", resultText)
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000194 }
195 }
196
197 // Try to get the page title to verify the navigation worked
198 browserCtx, err := tools.GetBrowserContext()
199 if err != nil {
200 // If browser automation is not available, skip the test
201 if strings.Contains(err.Error(), "browser automation not available") {
202 t.Skip("Browser automation not available in this environment")
203 } else {
204 t.Fatalf("Failed to get browser context: %v", err)
205 }
206 }
207
208 var title string
209 err = chromedp.Run(browserCtx, chromedp.Title(&title))
210 if err != nil {
211 t.Fatalf("Failed to get page title: %v", err)
212 }
213
214 t.Logf("Successfully navigated to example.com, title: %q", title)
215 if title != "Example Domain" {
216 t.Errorf("Expected title 'Example Domain', got '%s'", title)
217 }
218}
219
220// TestScreenshotTool tests that the screenshot tool properly saves files
221func TestScreenshotTool(t *testing.T) {
222 // Create browser tools instance
223 ctx := context.Background()
224 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700225 t.Cleanup(func() {
226 tools.Close()
227 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000228
229 // Test SaveScreenshot function directly
230 testData := []byte("test image data")
231 id := tools.SaveScreenshot(testData)
232 if id == "" {
233 t.Fatal("SaveScreenshot returned empty ID")
234 }
235
236 // Get the file path and check if the file exists
237 filePath := GetScreenshotPath(id)
238 _, err := os.Stat(filePath)
239 if err != nil {
240 t.Fatalf("Failed to find screenshot file: %v", err)
241 }
242
243 // Read the file contents
244 contents, err := os.ReadFile(filePath)
245 if err != nil {
246 t.Fatalf("Failed to read screenshot file: %v", err)
247 }
248
249 // Check the file contents
250 if string(contents) != string(testData) {
251 t.Errorf("File contents don't match: expected %q, got %q", string(testData), string(contents))
252 }
253
254 // Clean up the test file
255 os.Remove(filePath)
256}
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700257
258func TestReadImageTool(t *testing.T) {
259 // Create a test BrowseTools instance
260 ctx := context.Background()
261 browseTools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700262 t.Cleanup(func() {
263 browseTools.Close()
264 })
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700265
266 // Create a test image
267 testDir := t.TempDir()
268 testImagePath := filepath.Join(testDir, "test_image.png")
269
270 // Create a small 1x1 black PNG image
271 smallPng := []byte{
272 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
273 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
274 0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, 0x54, 0x08, 0xD7, 0x63, 0x60, 0x00, 0x00, 0x00,
275 0x02, 0x00, 0x01, 0xE2, 0x21, 0xBC, 0x33, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE,
276 0x42, 0x60, 0x82,
277 }
278
279 // Write the test image
280 err := os.WriteFile(testImagePath, smallPng, 0o644)
281 if err != nil {
282 t.Fatalf("Failed to create test image: %v", err)
283 }
284
285 // Create the tool
286 readImageTool := browseTools.NewReadImageTool()
287
288 // Prepare input
289 input := fmt.Sprintf(`{"path": "%s"}`, testImagePath)
290
291 // Run the tool
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700292 toolOut := readImageTool.Run(ctx, json.RawMessage(input))
293 if toolOut.Error != nil {
294 t.Fatalf("Read image tool failed: %v", toolOut.Error)
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700295 }
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700296 result := toolOut.LLMContent
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700297
298 // In the updated code, result is already a []llm.Content
299 contents := result
300
301 // Check that we got at least two content objects
302 if len(contents) < 2 {
303 t.Fatalf("Expected at least 2 content objects, got %d", len(contents))
304 }
305
306 // Check that the second content has image data
307 if contents[1].MediaType == "" {
308 t.Errorf("Expected MediaType in second content")
309 }
310
311 if contents[1].Data == "" {
312 t.Errorf("Expected Data in second content")
313 }
314}
Philip Zeyliger05224842025-05-10 18:26:08 -0700315
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000316// TestDefaultViewportSize verifies that the browser starts with the correct default viewport size
317func TestDefaultViewportSize(t *testing.T) {
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700318 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000319 defer cancel()
320
321 // Skip if CI or headless testing environment
322 if os.Getenv("CI") != "" || os.Getenv("HEADLESS_TEST") != "" {
323 t.Skip("Skipping browser test in CI/headless environment")
324 }
325
326 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700327 t.Cleanup(func() {
328 tools.Close()
329 })
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000330
331 // Initialize browser (which should set default viewport to 1280x720)
332 err := tools.Initialize()
333 if err != nil {
334 if strings.Contains(err.Error(), "browser automation not available") {
335 t.Skip("Browser automation not available in this environment")
336 } else {
337 t.Fatalf("Failed to initialize browser: %v", err)
338 }
339 }
340
341 // Navigate to a simple page to ensure the browser is ready
342 navInput := json.RawMessage(`{"url": "about:blank"}`)
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700343 toolOut := tools.NewNavigateTool().Run(ctx, navInput)
344 if toolOut.Error != nil {
345 t.Fatalf("Navigation error: %v", toolOut.Error)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000346 }
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700347 content := toolOut.LLMContent
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000348 if !strings.Contains(content[0].Text, "done") {
349 t.Fatalf("Expected done in navigation response, got: %s", content[0].Text)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000350 }
351
352 // Check default viewport dimensions via JavaScript
353 evalInput := json.RawMessage(`{"expression": "({width: window.innerWidth, height: window.innerHeight})"}`)
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700354 toolOut = tools.NewEvalTool().Run(ctx, evalInput)
355 if toolOut.Error != nil {
356 t.Fatalf("Evaluation error: %v", toolOut.Error)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000357 }
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700358 content = toolOut.LLMContent
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000359
360 // Parse the result to verify dimensions
361 var response struct {
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700362 Width float64 `json:"width"`
363 Height float64 `json:"height"`
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000364 }
365
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700366 text := content[0].Text
367 text = strings.TrimPrefix(text, "<javascript_result>")
368 text = strings.TrimSuffix(text, "</javascript_result>")
369
370 if err := json.Unmarshal([]byte(text), &response); err != nil {
371 t.Fatalf("Failed to parse evaluation response (%q => %q): %v", content[0].Text, text, err)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000372 }
373
374 // Verify the default viewport size is 1280x720
375 expectedWidth := 1280.0
376 expectedHeight := 720.0
377
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700378 if response.Width != expectedWidth {
379 t.Errorf("Expected default width %v, got %v", expectedWidth, response.Width)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000380 }
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700381 if response.Height != expectedHeight {
382 t.Errorf("Expected default height %v, got %v", expectedHeight, response.Height)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000383 }
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000384}
385
Philip Zeyliger05224842025-05-10 18:26:08 -0700386// TestResizeTool tests the browser resize functionality
387func TestResizeTool(t *testing.T) {
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700388 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Philip Zeyliger05224842025-05-10 18:26:08 -0700389 defer cancel()
390
391 // Skip if CI or headless testing environment
392 if os.Getenv("CI") != "" || os.Getenv("HEADLESS_TEST") != "" {
393 t.Skip("Skipping browser test in CI/headless environment")
394 }
395
396 t.Run("ResizeWindow", func(t *testing.T) {
397 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700398 t.Cleanup(func() {
399 tools.Close()
400 })
Philip Zeyliger05224842025-05-10 18:26:08 -0700401
402 // Resize to mobile dimensions
403 resizeTool := tools.NewResizeTool()
404 input := json.RawMessage(`{"width": 375, "height": 667}`)
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700405 toolOut := resizeTool.Run(ctx, input)
406 if toolOut.Error != nil {
407 t.Fatalf("Error: %v", toolOut.Error)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000408 }
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700409 content := toolOut.LLMContent
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000410 if !strings.Contains(content[0].Text, "done") {
411 t.Fatalf("Expected done in response, got: %s", content[0].Text)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000412 }
Philip Zeyliger05224842025-05-10 18:26:08 -0700413
414 // Navigate to a test page and verify using JavaScript to get window dimensions
415 navInput := json.RawMessage(`{"url": "https://example.com"}`)
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700416 toolOut = tools.NewNavigateTool().Run(ctx, navInput)
417 if toolOut.Error != nil {
418 t.Fatalf("Error: %v", toolOut.Error)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000419 }
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700420 content = toolOut.LLMContent
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000421 if !strings.Contains(content[0].Text, "done") {
422 t.Fatalf("Expected done in response, got: %s", content[0].Text)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000423 }
Philip Zeyliger05224842025-05-10 18:26:08 -0700424
425 // Check dimensions via JavaScript
426 evalInput := json.RawMessage(`{"expression": "({width: window.innerWidth, height: window.innerHeight})"}`)
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700427 toolOut = tools.NewEvalTool().Run(ctx, evalInput)
428 if toolOut.Error != nil {
429 t.Fatalf("Error: %v", toolOut.Error)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000430 }
Josh Bleecher Snyder43b60b92025-07-21 14:57:10 -0700431 content = toolOut.LLMContent
Philip Zeyliger05224842025-05-10 18:26:08 -0700432
433 // The dimensions might not be exactly what we set (browser chrome, etc.)
434 // but they should be close
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000435 if !strings.Contains(content[0].Text, "width") {
436 t.Fatalf("Expected width in response, got: %s", content[0].Text)
437 }
438 if !strings.Contains(content[0].Text, "height") {
439 t.Fatalf("Expected height in response, got: %s", content[0].Text)
440 }
Philip Zeyliger05224842025-05-10 18:26:08 -0700441 })
442}