blob: ff17055ae20e09dde02e4814d42fe0972e3b01bb [file] [log] [blame]
Philip Zeyliger33d282f2025-05-03 04:01:54 +00001package browse
2
3import (
4 "context"
5 "encoding/json"
Philip Zeyliger72252cb2025-05-10 17:00:08 -07006 "fmt"
Philip Zeyliger33d282f2025-05-03 04:01:54 +00007 "os"
Philip Zeyliger72252cb2025-05-10 17:00:08 -07008 "path/filepath"
Philip Zeyliger33d282f2025-05-03 04:01:54 +00009 "slices"
10 "strings"
11 "testing"
12 "time"
13
14 "github.com/chromedp/chromedp"
15 "sketch.dev/llm"
16)
17
18func TestToolCreation(t *testing.T) {
19 // Create browser tools instance
20 tools := NewBrowseTools(context.Background())
Philip Zeyligera35de5f2025-06-14 12:00:48 -070021 t.Cleanup(func() {
22 tools.Close()
23 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +000024
25 // Test each tool has correct name and description
26 toolTests := []struct {
27 tool *llm.Tool
28 expectedName string
29 shortDesc string
30 requiredProps []string
31 }{
32 {tools.NewNavigateTool(), "browser_navigate", "Navigate", []string{"url"}},
33 {tools.NewClickTool(), "browser_click", "Click", []string{"selector"}},
34 {tools.NewTypeTool(), "browser_type", "Type", []string{"selector", "text"}},
35 {tools.NewWaitForTool(), "browser_wait_for", "Wait", []string{"selector"}},
36 {tools.NewGetTextTool(), "browser_get_text", "Get", []string{"selector"}},
37 {tools.NewEvalTool(), "browser_eval", "Evaluate", []string{"expression"}},
Philip Zeyliger80b488d2025-05-10 18:21:54 -070038 {tools.NewScreenshotTool(), "browser_take_screenshot", "Take", nil},
Philip Zeyliger33d282f2025-05-03 04:01:54 +000039 {tools.NewScrollIntoViewTool(), "browser_scroll_into_view", "Scroll", []string{"selector"}},
40 }
41
42 for _, tt := range toolTests {
43 t.Run(tt.expectedName, func(t *testing.T) {
44 if tt.tool.Name != tt.expectedName {
45 t.Errorf("expected name %q, got %q", tt.expectedName, tt.tool.Name)
46 }
47
48 if !strings.Contains(tt.tool.Description, tt.shortDesc) {
49 t.Errorf("description %q should contain %q", tt.tool.Description, tt.shortDesc)
50 }
51
52 // Verify schema has required properties
53 if len(tt.requiredProps) > 0 {
54 var schema struct {
55 Required []string `json:"required"`
56 }
57 if err := json.Unmarshal(tt.tool.InputSchema, &schema); err != nil {
58 t.Fatalf("failed to unmarshal schema: %v", err)
59 }
60
61 for _, prop := range tt.requiredProps {
62 if !slices.Contains(schema.Required, prop) {
63 t.Errorf("property %q should be required", prop)
64 }
65 }
66 }
67 })
68 }
69}
70
Philip Zeyliger72252cb2025-05-10 17:00:08 -070071func TestGetTools(t *testing.T) {
Philip Zeyliger33d282f2025-05-03 04:01:54 +000072 // Create browser tools instance
73 tools := NewBrowseTools(context.Background())
Philip Zeyligera35de5f2025-06-14 12:00:48 -070074 t.Cleanup(func() {
75 tools.Close()
76 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +000077
Philip Zeyliger72252cb2025-05-10 17:00:08 -070078 // Test with screenshot tools included
79 t.Run("with screenshots", func(t *testing.T) {
80 toolsWithScreenshots := tools.GetTools(true)
Philip Zeyliger18e33682025-05-13 16:34:21 -070081 if len(toolsWithScreenshots) != 12 {
82 t.Errorf("expected 12 tools with screenshots, got %d", len(toolsWithScreenshots))
Philip Zeyliger33d282f2025-05-03 04:01:54 +000083 }
Philip Zeyliger72252cb2025-05-10 17:00:08 -070084
85 // Check tool naming convention
86 for _, tool := range toolsWithScreenshots {
Philip Zeyliger542bda32025-06-11 18:31:03 -070087 // Most tools have browser_ prefix, except for read_image
88 if tool.Name != "read_image" && !strings.HasPrefix(tool.Name, "browser_") {
Philip Zeyliger72252cb2025-05-10 17:00:08 -070089 t.Errorf("tool name %q does not have prefix 'browser_'", tool.Name)
90 }
91 }
92 })
93
94 // Test without screenshot tools
95 t.Run("without screenshots", func(t *testing.T) {
96 noScreenshotTools := tools.GetTools(false)
Philip Zeyliger18e33682025-05-13 16:34:21 -070097 if len(noScreenshotTools) != 10 {
98 t.Errorf("expected 10 tools without screenshots, got %d", len(noScreenshotTools))
Philip Zeyliger72252cb2025-05-10 17:00:08 -070099 }
100 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000101}
102
103// TestBrowserInitialization verifies that the browser can start correctly
104func TestBrowserInitialization(t *testing.T) {
105 // Skip long tests in short mode
106 if testing.Short() {
107 t.Skip("skipping browser initialization test in short mode")
108 }
109
110 // Create browser tools instance
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700111 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000112 defer cancel()
113
114 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700115 t.Cleanup(func() {
116 tools.Close()
117 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000118
119 // Initialize the browser
120 err := tools.Initialize()
121 if err != nil {
122 // If browser automation is not available, skip the test
123 if strings.Contains(err.Error(), "browser automation not available") {
124 t.Skip("Browser automation not available in this environment")
125 } else {
126 t.Fatalf("Failed to initialize browser: %v", err)
127 }
128 }
129
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000130 // Get browser context to verify it's working
131 browserCtx, err := tools.GetBrowserContext()
132 if err != nil {
133 t.Fatalf("Failed to get browser context: %v", err)
134 }
135
136 // Try to navigate to a simple page
137 var title string
138 err = chromedp.Run(browserCtx,
139 chromedp.Navigate("about:blank"),
140 chromedp.Title(&title),
141 )
142 if err != nil {
143 t.Fatalf("Failed to navigate to about:blank: %v", err)
144 }
145
146 t.Logf("Successfully navigated to about:blank, title: %q", title)
147}
148
149// TestNavigateTool verifies that the navigate tool works correctly
150func TestNavigateTool(t *testing.T) {
151 // Skip long tests in short mode
152 if testing.Short() {
153 t.Skip("skipping navigate tool test in short mode")
154 }
155
156 // Create browser tools instance
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700157 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000158 defer cancel()
159
160 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700161 t.Cleanup(func() {
162 tools.Close()
163 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000164
165 // Check if browser initialization works
166 if err := tools.Initialize(); err != nil {
167 if strings.Contains(err.Error(), "browser automation not available") {
168 t.Skip("Browser automation not available in this environment")
169 }
170 }
171
172 // Get the navigate tool
173 navTool := tools.NewNavigateTool()
174
175 // Create input for the navigate tool
176 input := map[string]string{"url": "https://example.com"}
177 inputJSON, _ := json.Marshal(input)
178
179 // Call the tool
180 result, err := navTool.Run(ctx, json.RawMessage(inputJSON))
181 if err != nil {
182 t.Fatalf("Error running navigate tool: %v", err)
183 }
184
185 // Verify the response is successful
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700186 resultText := result[0].Text
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000187 if !strings.Contains(resultText, "done") {
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000188 // If browser automation is not available, skip the test
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000189 if strings.Contains(resultText, "browser automation not available") {
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000190 t.Skip("Browser automation not available in this environment")
191 } else {
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000192 t.Fatalf("Expected done in result text, got: %s", resultText)
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000193 }
194 }
195
196 // Try to get the page title to verify the navigation worked
197 browserCtx, err := tools.GetBrowserContext()
198 if err != nil {
199 // If browser automation is not available, skip the test
200 if strings.Contains(err.Error(), "browser automation not available") {
201 t.Skip("Browser automation not available in this environment")
202 } else {
203 t.Fatalf("Failed to get browser context: %v", err)
204 }
205 }
206
207 var title string
208 err = chromedp.Run(browserCtx, chromedp.Title(&title))
209 if err != nil {
210 t.Fatalf("Failed to get page title: %v", err)
211 }
212
213 t.Logf("Successfully navigated to example.com, title: %q", title)
214 if title != "Example Domain" {
215 t.Errorf("Expected title 'Example Domain', got '%s'", title)
216 }
217}
218
219// TestScreenshotTool tests that the screenshot tool properly saves files
220func TestScreenshotTool(t *testing.T) {
221 // Create browser tools instance
222 ctx := context.Background()
223 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700224 t.Cleanup(func() {
225 tools.Close()
226 })
Philip Zeyliger33d282f2025-05-03 04:01:54 +0000227
228 // Test SaveScreenshot function directly
229 testData := []byte("test image data")
230 id := tools.SaveScreenshot(testData)
231 if id == "" {
232 t.Fatal("SaveScreenshot returned empty ID")
233 }
234
235 // Get the file path and check if the file exists
236 filePath := GetScreenshotPath(id)
237 _, err := os.Stat(filePath)
238 if err != nil {
239 t.Fatalf("Failed to find screenshot file: %v", err)
240 }
241
242 // Read the file contents
243 contents, err := os.ReadFile(filePath)
244 if err != nil {
245 t.Fatalf("Failed to read screenshot file: %v", err)
246 }
247
248 // Check the file contents
249 if string(contents) != string(testData) {
250 t.Errorf("File contents don't match: expected %q, got %q", string(testData), string(contents))
251 }
252
253 // Clean up the test file
254 os.Remove(filePath)
255}
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700256
257func TestReadImageTool(t *testing.T) {
258 // Create a test BrowseTools instance
259 ctx := context.Background()
260 browseTools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700261 t.Cleanup(func() {
262 browseTools.Close()
263 })
Philip Zeyliger72252cb2025-05-10 17:00:08 -0700264
265 // Create a test image
266 testDir := t.TempDir()
267 testImagePath := filepath.Join(testDir, "test_image.png")
268
269 // Create a small 1x1 black PNG image
270 smallPng := []byte{
271 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
272 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53,
273 0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, 0x54, 0x08, 0xD7, 0x63, 0x60, 0x00, 0x00, 0x00,
274 0x02, 0x00, 0x01, 0xE2, 0x21, 0xBC, 0x33, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE,
275 0x42, 0x60, 0x82,
276 }
277
278 // Write the test image
279 err := os.WriteFile(testImagePath, smallPng, 0o644)
280 if err != nil {
281 t.Fatalf("Failed to create test image: %v", err)
282 }
283
284 // Create the tool
285 readImageTool := browseTools.NewReadImageTool()
286
287 // Prepare input
288 input := fmt.Sprintf(`{"path": "%s"}`, testImagePath)
289
290 // Run the tool
291 result, err := readImageTool.Run(ctx, json.RawMessage(input))
292 if err != nil {
293 t.Fatalf("Read image tool failed: %v", err)
294 }
295
296 // In the updated code, result is already a []llm.Content
297 contents := result
298
299 // Check that we got at least two content objects
300 if len(contents) < 2 {
301 t.Fatalf("Expected at least 2 content objects, got %d", len(contents))
302 }
303
304 // Check that the second content has image data
305 if contents[1].MediaType == "" {
306 t.Errorf("Expected MediaType in second content")
307 }
308
309 if contents[1].Data == "" {
310 t.Errorf("Expected Data in second content")
311 }
312}
Philip Zeyliger05224842025-05-10 18:26:08 -0700313
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000314// TestDefaultViewportSize verifies that the browser starts with the correct default viewport size
315func TestDefaultViewportSize(t *testing.T) {
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700316 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000317 defer cancel()
318
319 // Skip if CI or headless testing environment
320 if os.Getenv("CI") != "" || os.Getenv("HEADLESS_TEST") != "" {
321 t.Skip("Skipping browser test in CI/headless environment")
322 }
323
324 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700325 t.Cleanup(func() {
326 tools.Close()
327 })
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000328
329 // Initialize browser (which should set default viewport to 1280x720)
330 err := tools.Initialize()
331 if err != nil {
332 if strings.Contains(err.Error(), "browser automation not available") {
333 t.Skip("Browser automation not available in this environment")
334 } else {
335 t.Fatalf("Failed to initialize browser: %v", err)
336 }
337 }
338
339 // Navigate to a simple page to ensure the browser is ready
340 navInput := json.RawMessage(`{"url": "about:blank"}`)
341 content, err := tools.NewNavigateTool().Run(ctx, navInput)
342 if err != nil {
343 t.Fatalf("Navigation error: %v", err)
344 }
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000345 if !strings.Contains(content[0].Text, "done") {
346 t.Fatalf("Expected done in navigation response, got: %s", content[0].Text)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000347 }
348
349 // Check default viewport dimensions via JavaScript
350 evalInput := json.RawMessage(`{"expression": "({width: window.innerWidth, height: window.innerHeight})"}`)
351 content, err = tools.NewEvalTool().Run(ctx, evalInput)
352 if err != nil {
353 t.Fatalf("Evaluation error: %v", err)
354 }
355
356 // Parse the result to verify dimensions
357 var response struct {
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700358 Width float64 `json:"width"`
359 Height float64 `json:"height"`
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000360 }
361
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700362 text := content[0].Text
363 text = strings.TrimPrefix(text, "<javascript_result>")
364 text = strings.TrimSuffix(text, "</javascript_result>")
365
366 if err := json.Unmarshal([]byte(text), &response); err != nil {
367 t.Fatalf("Failed to parse evaluation response (%q => %q): %v", content[0].Text, text, err)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000368 }
369
370 // Verify the default viewport size is 1280x720
371 expectedWidth := 1280.0
372 expectedHeight := 720.0
373
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700374 if response.Width != expectedWidth {
375 t.Errorf("Expected default width %v, got %v", expectedWidth, response.Width)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000376 }
Josh Bleecher Snyder553cc842025-07-07 19:05:22 -0700377 if response.Height != expectedHeight {
378 t.Errorf("Expected default height %v, got %v", expectedHeight, response.Height)
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000379 }
Josh Bleecher Snyder7fbc8e42025-05-29 19:42:25 +0000380}
381
Philip Zeyliger05224842025-05-10 18:26:08 -0700382// TestResizeTool tests the browser resize functionality
383func TestResizeTool(t *testing.T) {
Philip Zeyliger29c481c2025-06-15 21:25:45 -0700384 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
Philip Zeyliger05224842025-05-10 18:26:08 -0700385 defer cancel()
386
387 // Skip if CI or headless testing environment
388 if os.Getenv("CI") != "" || os.Getenv("HEADLESS_TEST") != "" {
389 t.Skip("Skipping browser test in CI/headless environment")
390 }
391
392 t.Run("ResizeWindow", func(t *testing.T) {
393 tools := NewBrowseTools(ctx)
Philip Zeyligera35de5f2025-06-14 12:00:48 -0700394 t.Cleanup(func() {
395 tools.Close()
396 })
Philip Zeyliger05224842025-05-10 18:26:08 -0700397
398 // Resize to mobile dimensions
399 resizeTool := tools.NewResizeTool()
400 input := json.RawMessage(`{"width": 375, "height": 667}`)
401 content, err := resizeTool.Run(ctx, input)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000402 if err != nil {
403 t.Fatalf("Error: %v", err)
404 }
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000405 if !strings.Contains(content[0].Text, "done") {
406 t.Fatalf("Expected done in response, got: %s", content[0].Text)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000407 }
Philip Zeyliger05224842025-05-10 18:26:08 -0700408
409 // Navigate to a test page and verify using JavaScript to get window dimensions
410 navInput := json.RawMessage(`{"url": "https://example.com"}`)
411 content, err = tools.NewNavigateTool().Run(ctx, navInput)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000412 if err != nil {
413 t.Fatalf("Error: %v", err)
414 }
Josh Bleecher Snydercb557262025-06-30 23:55:20 +0000415 if !strings.Contains(content[0].Text, "done") {
416 t.Fatalf("Expected done in response, got: %s", content[0].Text)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000417 }
Philip Zeyliger05224842025-05-10 18:26:08 -0700418
419 // Check dimensions via JavaScript
420 evalInput := json.RawMessage(`{"expression": "({width: window.innerWidth, height: window.innerHeight})"}`)
421 content, err = tools.NewEvalTool().Run(ctx, evalInput)
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000422 if err != nil {
423 t.Fatalf("Error: %v", err)
424 }
Philip Zeyliger05224842025-05-10 18:26:08 -0700425
426 // The dimensions might not be exactly what we set (browser chrome, etc.)
427 // but they should be close
Josh Bleecher Snyder6e353322025-05-21 18:21:24 +0000428 if !strings.Contains(content[0].Text, "width") {
429 t.Fatalf("Expected width in response, got: %s", content[0].Text)
430 }
431 if !strings.Contains(content[0].Text, "height") {
432 t.Fatalf("Expected height in response, got: %s", content[0].Text)
433 }
Philip Zeyliger05224842025-05-10 18:26:08 -0700434 })
435}