blob: 966373b9d969a8c1e77bb454b6714724656cd809 [file] [log] [blame]
Earl Lee2e463fb2025-04-17 11:22:22 -07001// Package dockerimg
2package dockerimg
3
4import (
5 "bytes"
6 "context"
Philip Zeyliger5e227dd2025-04-21 15:55:29 -07007 "crypto/rand"
Earl Lee2e463fb2025-04-17 11:22:22 -07008 "crypto/sha256"
9 "encoding/hex"
10 "encoding/json"
11 "fmt"
12 "io"
13 "log/slog"
14 "net"
15 "net/http"
16 "os"
17 "os/exec"
18 "path/filepath"
19 "runtime"
20 "strings"
Josh Bleecher Snyder99570462025-05-05 10:26:14 -070021 "sync/atomic"
Earl Lee2e463fb2025-04-17 11:22:22 -070022 "time"
23
Sean McCullough7013e9e2025-05-14 02:03:58 +000024 "golang.org/x/crypto/ssh"
Josh Bleecher Snyder78707d62025-04-30 21:06:49 +000025 "sketch.dev/browser"
Josh Bleecher Snyder4f84ab72025-04-22 16:40:54 -070026 "sketch.dev/llm/ant"
Sean McCulloughbaa2b592025-04-23 10:40:08 -070027 "sketch.dev/loop/server"
Earl Lee2e463fb2025-04-17 11:22:22 -070028 "sketch.dev/skribe"
Philip Zeyliger5d6af872025-04-23 19:48:34 -070029 "sketch.dev/webui"
Earl Lee2e463fb2025-04-17 11:22:22 -070030)
31
32// ContainerConfig holds all configuration for launching a container
33type ContainerConfig struct {
34 // SessionID is the unique identifier for this session
35 SessionID string
36
37 // LocalAddr is the initial address to use (though it may be overwritten later)
38 LocalAddr string
39
40 // SkabandAddr is the address of the skaband service if available
41 SkabandAddr string
42
David Crawshaw5a7b3692025-05-05 16:49:15 -070043 // Model is the name of the LLM model to use.
44 Model string
Earl Lee2e463fb2025-04-17 11:22:22 -070045
David Crawshaw5a7b3692025-05-05 16:49:15 -070046 // ModelURL is the URL of the LLM service.
47 ModelURL string
48
49 // ModelAPIKey is the API key for LLM service.
50 ModelAPIKey string
Earl Lee2e463fb2025-04-17 11:22:22 -070051
52 // Path is the local filesystem path to use
53 Path string
54
55 // GitUsername is the username to use for git operations
56 GitUsername string
57
58 // GitEmail is the email to use for git operations
59 GitEmail string
60
61 // OpenBrowser determines whether to open a browser automatically
62 OpenBrowser bool
63
64 // NoCleanup prevents container cleanup when set to true
65 NoCleanup bool
66
67 // ForceRebuild forces rebuilding of the Docker image even if it exists
68 ForceRebuild bool
69
70 // Host directory to copy container logs into, if not set to ""
71 ContainerLogDest string
72
73 // Path to pre-built linux sketch binary, or build a new one if set to ""
74 SketchBinaryLinux string
75
76 // Sketch client public key.
77 SketchPubKey string
Philip Zeyligerd1402952025-04-23 03:54:37 +000078
Sean McCulloughbaa2b592025-04-23 10:40:08 -070079 // Host port for the container's ssh server
80 SSHPort int
81
Philip Zeyliger18532b22025-04-23 21:11:46 +000082 // Outside information to pass to the container
83 OutsideHostname string
84 OutsideOS string
85 OutsideWorkingDir string
Philip Zeyligerb74c4f62025-04-25 19:18:49 -070086
Pokey Rule0dcebe12025-04-28 14:51:04 +010087 // If true, exit after the first turn
88 OneShot bool
89
90 // Initial prompt
91 Prompt string
Philip Zeyliger1b47aa22025-04-28 19:25:38 +000092
93 // Initial commit to use as starting point
94 InitialCommit string
David Crawshawb5f6a002025-05-05 08:27:16 -070095
96 // Verbose enables verbose output
97 Verbose bool
Philip Zeyliger1dc21372025-05-05 19:54:44 +000098
99 // DockerArgs are additional arguments to pass to the docker create command
100 DockerArgs string
Josh Bleecher Snyderb1cca6f2025-05-06 01:52:55 +0000101
102 // ExperimentFlag contains the experimental features to enable
103 ExperimentFlag string
Earl Lee2e463fb2025-04-17 11:22:22 -0700104}
105
106// LaunchContainer creates a docker container for a project, installs sketch and opens a connection to it.
107// It writes status to stdout.
David Crawshawb5f6a002025-05-05 08:27:16 -0700108func LaunchContainer(ctx context.Context, config ContainerConfig) error {
Earl Lee2e463fb2025-04-17 11:22:22 -0700109 if _, err := exec.LookPath("docker"); err != nil {
Philip Zeyliger5e227dd2025-04-21 15:55:29 -0700110 if runtime.GOOS == "darwin" {
111 return fmt.Errorf("cannot find `docker` binary; run: brew install docker colima && colima start")
112 } else {
113 return fmt.Errorf("cannot find `docker` binary; install docker (e.g., apt-get install docker.io)")
114 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700115 }
116
117 if out, err := combinedOutput(ctx, "docker", "ps"); err != nil {
118 // `docker ps` provides a good error message here that can be
119 // easily chatgpt'ed by users, so send it to the user as-is:
120 // Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?
121 return fmt.Errorf("docker ps: %s (%w)", out, err)
122 }
123
124 _, hostPort, err := net.SplitHostPort(config.LocalAddr)
125 if err != nil {
126 return err
127 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700128 gitRoot, err := findGitRoot(ctx, config.Path)
129 if err != nil {
130 return err
131 }
132
David Crawshaw5a7b3692025-05-05 16:49:15 -0700133 imgName, err := findOrBuildDockerImage(ctx, config.Path, gitRoot, config.Model, config.ModelURL, config.ModelAPIKey, config.ForceRebuild, config.Verbose)
Earl Lee2e463fb2025-04-17 11:22:22 -0700134 if err != nil {
135 return err
136 }
137
138 linuxSketchBin := config.SketchBinaryLinux
139 if linuxSketchBin == "" {
David Crawshawb5f6a002025-05-05 08:27:16 -0700140 linuxSketchBin, err = buildLinuxSketchBin(ctx)
Earl Lee2e463fb2025-04-17 11:22:22 -0700141 if err != nil {
142 return err
143 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700144 }
145
Philip Zeyligerc72fff52025-04-29 20:17:54 +0000146 cntrName := "sketch-" + config.SessionID
Earl Lee2e463fb2025-04-17 11:22:22 -0700147 defer func() {
148 if config.NoCleanup {
149 return
150 }
151 if out, err := combinedOutput(ctx, "docker", "kill", cntrName); err != nil {
152 // TODO: print in verbose mode? fmt.Fprintf(os.Stderr, "docker kill: %s: %v\n", out, err)
153 _ = out
154 }
155 if out, err := combinedOutput(ctx, "docker", "rm", cntrName); err != nil {
156 // TODO: print in verbose mode? fmt.Fprintf(os.Stderr, "docker kill: %s: %v\n", out, err)
157 _ = out
158 }
159 }()
160
161 // errCh receives errors from operations that this function calls in separate goroutines.
162 errCh := make(chan error)
163
164 // Start the git server
165 gitSrv, err := newGitServer(gitRoot)
166 if err != nil {
167 return fmt.Errorf("failed to start git server: %w", err)
168 }
169 defer gitSrv.shutdown(ctx)
170
171 go func() {
172 errCh <- gitSrv.serve(ctx)
173 }()
174
175 // Get the current host git commit
176 var commit string
Philip Zeyliger1b47aa22025-04-28 19:25:38 +0000177 if out, err := combinedOutput(ctx, "git", "rev-parse", config.InitialCommit); err != nil {
178 return fmt.Errorf("git rev-parse %s: %w", config.InitialCommit, err)
Earl Lee2e463fb2025-04-17 11:22:22 -0700179 } else {
180 commit = strings.TrimSpace(string(out))
181 }
182 if out, err := combinedOutput(ctx, "git", "config", "http.receivepack", "true"); err != nil {
183 return fmt.Errorf("git config http.receivepack true: %s: %w", out, err)
184 }
185
186 relPath, err := filepath.Rel(gitRoot, config.Path)
187 if err != nil {
188 return err
189 }
190
191 // Create the sketch container
192 if err := createDockerContainer(ctx, cntrName, hostPort, relPath, imgName, config); err != nil {
Josh Bleecher Snyder2772f632025-05-01 21:42:35 +0000193 return fmt.Errorf("failed to create docker container: %w", err)
Earl Lee2e463fb2025-04-17 11:22:22 -0700194 }
195
196 // Copy the sketch linux binary into the container
197 if out, err := combinedOutput(ctx, "docker", "cp", linuxSketchBin, cntrName+":/bin/sketch"); err != nil {
198 return fmt.Errorf("docker cp: %s, %w", out, err)
199 }
Sean McCulloughf5bb3d32025-04-18 10:47:59 -0700200
201 // Make sure that the webui is built so we can copy the results to the container.
202 _, err = webui.Build()
203 if err != nil {
204 return fmt.Errorf("failed to build webui: %w", err)
205 }
206
David Crawshaw8bff16a2025-04-18 01:16:49 -0700207 webuiZipPath, err := webui.ZipPath()
208 if err != nil {
209 return err
210 }
211 if out, err := combinedOutput(ctx, "docker", "cp", webuiZipPath, cntrName+":/root/.cache/sketch/webui/"+filepath.Base(webuiZipPath)); err != nil {
212 return fmt.Errorf("docker cp: %s, %w", out, err)
213 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700214
David Crawshaw53786ef2025-04-24 12:52:51 -0700215 fmt.Printf("📦 running in container %s\n", cntrName)
Earl Lee2e463fb2025-04-17 11:22:22 -0700216
217 // Start the sketch container
218 if out, err := combinedOutput(ctx, "docker", "start", cntrName); err != nil {
219 return fmt.Errorf("docker start: %s, %w", out, err)
220 }
221
222 // Copies structured logs from the container to the host.
223 copyLogs := func() {
224 if config.ContainerLogDest == "" {
225 return
226 }
227 out, err := combinedOutput(ctx, "docker", "logs", cntrName)
228 if err != nil {
229 fmt.Fprintf(os.Stderr, "docker logs failed: %v\n", err)
230 return
231 }
Josh Bleecher Snyder7660e4e2025-04-24 10:34:17 -0700232 prefix := []byte("structured logs:")
233 for line := range bytes.Lines(out) {
234 rest, ok := bytes.CutPrefix(line, prefix)
235 if !ok {
Earl Lee2e463fb2025-04-17 11:22:22 -0700236 continue
237 }
Josh Bleecher Snyder7660e4e2025-04-24 10:34:17 -0700238 logFile := string(bytes.TrimSpace(rest))
Earl Lee2e463fb2025-04-17 11:22:22 -0700239 srcPath := fmt.Sprintf("%s:%s", cntrName, logFile)
240 logFileName := filepath.Base(logFile)
241 dstPath := filepath.Join(config.ContainerLogDest, logFileName)
242 _, err := combinedOutput(ctx, "docker", "cp", srcPath, dstPath)
243 if err != nil {
244 fmt.Fprintf(os.Stderr, "docker cp %s %s failed: %v\n", srcPath, dstPath, err)
245 }
246 fmt.Fprintf(os.Stderr, "\ncopied container log %s to %s\n", srcPath, dstPath)
247 }
248 }
249
250 // NOTE: we want to see what the internal sketch binary prints
251 // regardless of the setting of the verbosity flag on the external
252 // binary, so reading "docker logs", which is the stdout/stderr of
253 // the internal binary is not conditional on the verbose flag.
254 appendInternalErr := func(err error) error {
255 if err == nil {
256 return nil
257 }
258 out, logsErr := combinedOutput(ctx, "docker", "logs", cntrName)
Philip Zeyligerd1402952025-04-23 03:54:37 +0000259 if logsErr != nil {
Earl Lee2e463fb2025-04-17 11:22:22 -0700260 return fmt.Errorf("%w; and docker logs failed: %s, %v", err, out, logsErr)
261 }
262 out = bytes.TrimSpace(out)
263 if len(out) > 0 {
264 return fmt.Errorf("docker logs: %s;\n%w", out, err)
265 }
266 return err
267 }
268
269 // Get the sketch server port from the container
Sean McCulloughae3480f2025-04-23 15:28:20 -0700270 localAddr, err := getContainerPort(ctx, cntrName, "80")
Earl Lee2e463fb2025-04-17 11:22:22 -0700271 if err != nil {
272 return appendInternalErr(err)
273 }
274
Philip Zeyliger00442412025-05-14 11:03:23 -0700275 if config.Verbose {
276 fmt.Fprintf(os.Stderr, "Host web server: http://%s/\n", localAddr)
277 }
278
Sean McCulloughae3480f2025-04-23 15:28:20 -0700279 localSSHAddr, err := getContainerPort(ctx, cntrName, "22")
280 if err != nil {
281 return appendInternalErr(err)
282 }
283 sshHost, sshPort, err := net.SplitHostPort(localSSHAddr)
284 if err != nil {
David Crawshawb5f6a002025-05-05 08:27:16 -0700285 return appendInternalErr(fmt.Errorf("failed to split ssh host and port: %w", err))
Sean McCulloughae3480f2025-04-23 15:28:20 -0700286 }
Sean McCullough4854c652025-04-24 18:37:02 -0700287
Sean McCullough7013e9e2025-05-14 02:03:58 +0000288 var sshServerIdentity, sshUserIdentity, containerCAPublicKey, hostCertificate []byte
Sean McCullough4854c652025-04-24 18:37:02 -0700289
Sean McCullough078e85a2025-05-08 17:28:34 -0700290 cst, err := NewSSHTheater(cntrName, sshHost, sshPort)
291 if err != nil {
292 return appendInternalErr(fmt.Errorf("NewContainerSSHTheather: %w", err))
293 }
294
295 sshErr := CheckSSHReachability(cntrName)
Sean McCullough15c95282025-05-08 16:48:38 -0700296 sshAvailable := false
297 sshErrMsg := ""
298 if sshErr != nil {
299 fmt.Println(sshErr.Error())
300 sshErrMsg = sshErr.Error()
Sean McCulloughf5e28f62025-04-25 10:48:00 -0700301 // continue - ssh config is not required for the rest of sketch to function locally.
302 } else {
Sean McCullough15c95282025-05-08 16:48:38 -0700303 sshAvailable = true
Sean McCulloughea3fc202025-04-28 12:53:37 -0700304 // Note: The vscode: link uses an undocumented request parameter that I really had to dig to find:
305 // https://github.com/microsoft/vscode/blob/2b9486161abaca59b5132ce3c59544f3cc7000f6/src/vs/code/electron-main/app.ts#L878
Sean McCulloughf5e28f62025-04-25 10:48:00 -0700306 fmt.Printf(`Connect to this container via any of these methods:
Sean McCullough4854c652025-04-24 18:37:02 -0700307🖥️ ssh %s
308🖥️ code --remote ssh-remote+root@%s /app -n
Sean McCulloughea3fc202025-04-28 12:53:37 -0700309🔗 vscode://vscode-remote/ssh-remote+root@%s/app?windowId=_blank
Sean McCullough4854c652025-04-24 18:37:02 -0700310`, cntrName, cntrName, cntrName)
Sean McCulloughf5e28f62025-04-25 10:48:00 -0700311 sshUserIdentity = cst.userIdentity
312 sshServerIdentity = cst.serverIdentity
Sean McCullough7013e9e2025-05-14 02:03:58 +0000313
314 // Get the Container CA public key for mutual auth
315 if cst.containerCAPublicKey != nil {
316 containerCAPublicKey = ssh.MarshalAuthorizedKey(cst.containerCAPublicKey)
317 fmt.Println("🔒 SSH Mutual Authentication enabled (container will verify host)")
318 }
319
320 // Get the host certificate for mutual auth
321 hostCertificate = cst.hostCertificate
322
Sean McCulloughf5e28f62025-04-25 10:48:00 -0700323 defer func() {
324 if err := cst.Cleanup(); err != nil {
325 appendInternalErr(err)
326 }
327 }()
328 }
Sean McCulloughae3480f2025-04-23 15:28:20 -0700329
Earl Lee2e463fb2025-04-17 11:22:22 -0700330 // Tell the sketch container which git server port and commit to initialize with.
331 go func() {
332 // TODO: Why is this called in a goroutine? I have found that when I pull this out
333 // of the goroutine and call it inline, then the terminal UI clears itself and all
334 // the scrollback (which is not good, but also not fatal). I can't see why it does this
335 // though, since none of the calls in postContainerInitConfig obviously write to stdout
336 // or stderr.
Sean McCullough7013e9e2025-05-14 02:03:58 +0000337 if err := postContainerInitConfig(ctx, localAddr, commit, gitSrv.gitPort, gitSrv.pass, sshAvailable, sshErrMsg, sshServerIdentity, sshUserIdentity, containerCAPublicKey, hostCertificate); err != nil {
Earl Lee2e463fb2025-04-17 11:22:22 -0700338 slog.ErrorContext(ctx, "LaunchContainer.postContainerInitConfig", slog.String("err", err.Error()))
339 errCh <- appendInternalErr(err)
340 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700341
Philip Zeyliger6ed6adb2025-04-23 19:56:38 -0700342 // We open the browser after the init config because the above waits for the web server to be serving.
Josh Bleecher Snyder99570462025-05-05 10:26:14 -0700343 ps1URL := "http://" + localAddr
344 if config.SkabandAddr != "" {
345 ps1URL = fmt.Sprintf("%s/s/%s", config.SkabandAddr, config.SessionID)
Philip Zeyliger6ed6adb2025-04-23 19:56:38 -0700346 }
Josh Bleecher Snyder99570462025-05-05 10:26:14 -0700347 if config.OpenBrowser {
348 browser.Open(ps1URL)
349 }
350 gitSrv.ps1URL.Store(&ps1URL)
Philip Zeyliger6ed6adb2025-04-23 19:56:38 -0700351 }()
Earl Lee2e463fb2025-04-17 11:22:22 -0700352
353 go func() {
354 cmd := exec.CommandContext(ctx, "docker", "attach", cntrName)
355 cmd.Stdin = os.Stdin
356 cmd.Stdout = os.Stdout
357 cmd.Stderr = os.Stderr
358 errCh <- run(ctx, "docker attach", cmd)
359 }()
360
361 defer copyLogs()
362
363 for {
364 select {
365 case <-ctx.Done():
366 return ctx.Err()
367 case err := <-errCh:
368 if err != nil {
369 return appendInternalErr(fmt.Errorf("container process: %w", err))
370 }
371 return nil
372 }
373 }
374}
375
376func combinedOutput(ctx context.Context, cmdName string, args ...string) ([]byte, error) {
377 cmd := exec.CommandContext(ctx, cmdName, args...)
Earl Lee2e463fb2025-04-17 11:22:22 -0700378 start := time.Now()
379
380 out, err := cmd.CombinedOutput()
381 if err != nil {
David Crawshawc7e77962025-05-03 13:20:18 -0700382 slog.ErrorContext(ctx, cmdName, slog.Duration("elapsed", time.Since(start)), slog.String("err", err.Error()), slog.String("path", cmd.Path), slog.String("args", fmt.Sprintf("%v", skribe.Redact(cmd.Args))))
Earl Lee2e463fb2025-04-17 11:22:22 -0700383 } else {
David Crawshawc7e77962025-05-03 13:20:18 -0700384 slog.DebugContext(ctx, cmdName, slog.Duration("elapsed", time.Since(start)), slog.String("path", cmd.Path), slog.String("args", fmt.Sprintf("%v", skribe.Redact(cmd.Args))))
Earl Lee2e463fb2025-04-17 11:22:22 -0700385 }
386 return out, err
387}
388
389func run(ctx context.Context, cmdName string, cmd *exec.Cmd) error {
390 start := time.Now()
391 err := cmd.Run()
392 if err != nil {
David Crawshawc7e77962025-05-03 13:20:18 -0700393 slog.ErrorContext(ctx, cmdName, slog.Duration("elapsed", time.Since(start)), slog.String("err", err.Error()), slog.String("path", cmd.Path), slog.String("args", fmt.Sprintf("%v", skribe.Redact(cmd.Args))))
Earl Lee2e463fb2025-04-17 11:22:22 -0700394 } else {
David Crawshawc7e77962025-05-03 13:20:18 -0700395 slog.DebugContext(ctx, cmdName, slog.Duration("elapsed", time.Since(start)), slog.String("path", cmd.Path), slog.String("args", fmt.Sprintf("%v", skribe.Redact(cmd.Args))))
Earl Lee2e463fb2025-04-17 11:22:22 -0700396 }
397 return err
398}
399
400type gitServer struct {
401 gitLn net.Listener
402 gitPort string
403 srv *http.Server
Philip Zeyliger5e227dd2025-04-21 15:55:29 -0700404 pass string
Josh Bleecher Snyder99570462025-05-05 10:26:14 -0700405 ps1URL atomic.Pointer[string]
Earl Lee2e463fb2025-04-17 11:22:22 -0700406}
407
408func (gs *gitServer) shutdown(ctx context.Context) {
409 gs.srv.Shutdown(ctx)
410 gs.gitLn.Close()
411}
412
413// Serve a git remote from the host for the container to fetch from and push to.
414func (gs *gitServer) serve(ctx context.Context) error {
415 slog.DebugContext(ctx, "starting git server", slog.String("git_remote_addr", "http://host.docker.internal:"+gs.gitPort+"/.git"))
416 return gs.srv.Serve(gs.gitLn)
417}
418
419func newGitServer(gitRoot string) (*gitServer, error) {
Josh Bleecher Snyder9f6a9982025-04-22 17:34:15 -0700420 ret := &gitServer{
421 pass: rand.Text(),
422 }
Philip Zeyliger5e227dd2025-04-21 15:55:29 -0700423
Earl Lee2e463fb2025-04-17 11:22:22 -0700424 gitLn, err := net.Listen("tcp4", ":0")
425 if err != nil {
426 return nil, fmt.Errorf("git listen: %w", err)
427 }
428 ret.gitLn = gitLn
429
Josh Bleecher Snyder99570462025-05-05 10:26:14 -0700430 browserC := make(chan bool, 1) // channel of browser open requests
431
Josh Bleecher Snyder3e2111b2025-04-30 17:53:28 +0000432 go func() {
Josh Bleecher Snyder99570462025-05-05 10:26:14 -0700433 for range browserC {
434 browser.Open(*ret.ps1URL.Load())
Josh Bleecher Snyder3e2111b2025-04-30 17:53:28 +0000435 }
436 }()
437
438 srv := http.Server{Handler: &gitHTTP{gitRepoRoot: gitRoot, pass: []byte(ret.pass), browserC: browserC}}
Earl Lee2e463fb2025-04-17 11:22:22 -0700439 ret.srv = &srv
440
441 _, gitPort, err := net.SplitHostPort(gitLn.Addr().String())
442 if err != nil {
443 return nil, fmt.Errorf("git port: %w", err)
444 }
445 ret.gitPort = gitPort
446 return ret, nil
447}
448
449func createDockerContainer(ctx context.Context, cntrName, hostPort, relPath, imgName string, config ContainerConfig) error {
David Crawshaw69c67312025-04-17 13:42:00 -0700450 cmdArgs := []string{
451 "create",
David Crawshaw66cf74e2025-05-05 08:48:39 -0700452 "-i",
Earl Lee2e463fb2025-04-17 11:22:22 -0700453 "--name", cntrName,
454 "-p", hostPort + ":80", // forward container port 80 to a host port
David Crawshaw3659d872025-05-05 17:52:23 -0700455 "-e", "SKETCH_MODEL_API_KEY=" + config.ModelAPIKey,
Earl Lee2e463fb2025-04-17 11:22:22 -0700456 }
David Crawshaw66cf74e2025-05-05 08:48:39 -0700457 if !config.OneShot {
458 cmdArgs = append(cmdArgs, "-t")
459 }
Josh Bleecher Snyder2772f632025-05-01 21:42:35 +0000460
461 for _, envVar := range getEnvForwardingFromGitConfig(ctx) {
462 cmdArgs = append(cmdArgs, "-e", envVar)
463 }
David Crawshaw5a7b3692025-05-05 16:49:15 -0700464 if config.ModelURL != "" {
David Crawshaw3659d872025-05-05 17:52:23 -0700465 cmdArgs = append(cmdArgs, "-e", "SKETCH_MODEL_URL="+config.ModelURL)
Earl Lee2e463fb2025-04-17 11:22:22 -0700466 }
467 if config.SketchPubKey != "" {
468 cmdArgs = append(cmdArgs, "-e", "SKETCH_PUB_KEY="+config.SketchPubKey)
469 }
Sean McCulloughae3480f2025-04-23 15:28:20 -0700470 if config.SSHPort > 0 {
471 cmdArgs = append(cmdArgs, "-p", fmt.Sprintf("%d:22", config.SSHPort)) // forward container ssh port to host ssh port
472 } else {
473 cmdArgs = append(cmdArgs, "-p", "22") // use an ephemeral host port for ssh.
Sean McCulloughbaa2b592025-04-23 10:40:08 -0700474 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700475 if relPath != "." {
476 cmdArgs = append(cmdArgs, "-w", "/app/"+relPath)
477 }
Philip Zeyliger5e227dd2025-04-21 15:55:29 -0700478 // colima does this by default, but Linux docker seems to need this set explicitly
479 cmdArgs = append(cmdArgs, "--add-host", "host.docker.internal:host-gateway")
Earl Lee2e463fb2025-04-17 11:22:22 -0700480 cmdArgs = append(
481 cmdArgs,
482 imgName,
483 "/bin/sketch",
484 "-unsafe",
485 "-addr=:80",
486 "-session-id="+config.SessionID,
Philip Zeyligerd1402952025-04-23 03:54:37 +0000487 "-git-username="+config.GitUsername,
488 "-git-email="+config.GitEmail,
Philip Zeyliger18532b22025-04-23 21:11:46 +0000489 "-outside-hostname="+config.OutsideHostname,
490 "-outside-os="+config.OutsideOS,
491 "-outside-working-dir="+config.OutsideWorkingDir,
Josh Bleecher Snyder3cae7d92025-04-30 09:54:29 -0700492 "-open=false",
Josh Bleecher Snyderb1cca6f2025-05-06 01:52:55 +0000493 "-x="+config.ExperimentFlag,
Earl Lee2e463fb2025-04-17 11:22:22 -0700494 )
David Crawshaw5a7b3692025-05-05 16:49:15 -0700495 if config.Model != "" {
496 cmdArgs = append(cmdArgs, "-model="+config.Model)
497 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700498 if config.SkabandAddr != "" {
499 cmdArgs = append(cmdArgs, "-skaband-addr="+config.SkabandAddr)
500 }
Pokey Rule0dcebe12025-04-28 14:51:04 +0100501 if config.Prompt != "" {
502 cmdArgs = append(cmdArgs, "-prompt", config.Prompt)
503 }
504 if config.OneShot {
505 cmdArgs = append(cmdArgs, "-one-shot")
Philip Zeyligerb74c4f62025-04-25 19:18:49 -0700506 }
Philip Zeyliger1dc21372025-05-05 19:54:44 +0000507
508 // Add additional docker arguments if provided
509 if config.DockerArgs != "" {
510 // Parse space-separated docker arguments with support for quotes and escaping
511 args := parseDockerArgs(config.DockerArgs)
512 // Insert arguments after "create" but before other arguments
513 for i := len(args) - 1; i >= 0; i-- {
514 cmdArgs = append(cmdArgs[:1], append([]string{args[i]}, cmdArgs[1:]...)...)
515 }
516 }
517
Earl Lee2e463fb2025-04-17 11:22:22 -0700518 if out, err := combinedOutput(ctx, "docker", cmdArgs...); err != nil {
519 return fmt.Errorf("docker create: %s, %w", out, err)
520 }
521 return nil
522}
523
David Crawshawb5f6a002025-05-05 08:27:16 -0700524func buildLinuxSketchBin(ctx context.Context) (string, error) {
Pokey Rulea9a786b2025-05-12 10:52:34 +0100525 // Change to directory containing dockerimg.go for module detection
526 _, codeFile, _, _ := runtime.Caller(0)
527 codeDir := filepath.Dir(codeFile)
528 if currentDir, err := os.Getwd(); err != nil {
529 slog.WarnContext(ctx, "could not get current directory", "err", err)
530 } else {
531 if err := os.Chdir(codeDir); err != nil {
532 slog.WarnContext(ctx, "could not change to code directory for module check", "err", err)
533 } else {
534 defer func() {
535 _ = os.Chdir(currentDir)
536 }()
537 }
538 }
539
David Crawshaw8a617cb2025-04-18 01:28:43 -0700540 homeDir, err := os.UserHomeDir()
David Crawshaw69c67312025-04-17 13:42:00 -0700541 if err != nil {
542 return "", err
543 }
David Crawshaw8a617cb2025-04-18 01:28:43 -0700544 linuxGopath := filepath.Join(homeDir, ".cache", "sketch", "linuxgo")
545 if err := os.MkdirAll(linuxGopath, 0o777); err != nil {
546 return "", err
547 }
548
549 verToInstall := "@latest"
550 if out, err := exec.Command("go", "list", "-m").CombinedOutput(); err != nil {
551 return "", fmt.Errorf("failed to run go list -m: %s: %v", out, err)
552 } else {
553 if strings.TrimSpace(string(out)) == "sketch.dev" {
David Crawshaw094e4d22025-04-24 11:35:14 -0700554 slog.DebugContext(ctx, "built linux agent from currently checked out module")
David Crawshaw8a617cb2025-04-18 01:28:43 -0700555 verToInstall = ""
556 }
557 }
David Crawshaw69c67312025-04-17 13:42:00 -0700558
Earl Lee2e463fb2025-04-17 11:22:22 -0700559 start := time.Now()
David Crawshaw8a617cb2025-04-18 01:28:43 -0700560 cmd := exec.CommandContext(ctx, "go", "install", "sketch.dev/cmd/sketch"+verToInstall)
David Crawshawb9eaef52025-04-17 15:23:18 -0700561 cmd.Env = append(
562 os.Environ(),
563 "GOOS=linux",
564 "CGO_ENABLED=0",
565 "GOTOOLCHAIN=auto",
David Crawshaw8a617cb2025-04-18 01:28:43 -0700566 "GOPATH="+linuxGopath,
Josh Bleecher Snyderfae17572025-04-21 11:48:05 -0700567 "GOBIN=",
David Crawshawb9eaef52025-04-17 15:23:18 -0700568 )
Earl Lee2e463fb2025-04-17 11:22:22 -0700569
Earl Lee2e463fb2025-04-17 11:22:22 -0700570 out, err := cmd.CombinedOutput()
571 if err != nil {
David Crawshawc7e77962025-05-03 13:20:18 -0700572 slog.ErrorContext(ctx, "go", slog.Duration("elapsed", time.Since(start)), slog.String("err", err.Error()), slog.String("path", cmd.Path), slog.String("args", fmt.Sprintf("%v", skribe.Redact(cmd.Args))))
Earl Lee2e463fb2025-04-17 11:22:22 -0700573 return "", fmt.Errorf("failed to build linux sketch binary: %s: %w", out, err)
574 } else {
David Crawshawc7e77962025-05-03 13:20:18 -0700575 slog.DebugContext(ctx, "go", slog.Duration("elapsed", time.Since(start)), slog.String("path", cmd.Path), slog.String("args", fmt.Sprintf("%v", skribe.Redact(cmd.Args))))
Earl Lee2e463fb2025-04-17 11:22:22 -0700576 }
577
Philip Zeyliger5e227dd2025-04-21 15:55:29 -0700578 if runtime.GOOS != "linux" {
David Crawshawc7e77962025-05-03 13:20:18 -0700579 return filepath.Join(linuxGopath, "bin", "linux_"+runtime.GOARCH, "sketch"), nil
Philip Zeyliger5e227dd2025-04-21 15:55:29 -0700580 }
David Crawshawc7e77962025-05-03 13:20:18 -0700581 // If we are already on Linux, there's no extra platform name in the path
582 return filepath.Join(linuxGopath, "bin", "sketch"), nil
Earl Lee2e463fb2025-04-17 11:22:22 -0700583}
584
Sean McCulloughae3480f2025-04-23 15:28:20 -0700585func getContainerPort(ctx context.Context, cntrName, cntrPort string) (string, error) {
Earl Lee2e463fb2025-04-17 11:22:22 -0700586 localAddr := ""
Sean McCulloughae3480f2025-04-23 15:28:20 -0700587 if out, err := combinedOutput(ctx, "docker", "port", cntrName, cntrPort); err != nil {
Earl Lee2e463fb2025-04-17 11:22:22 -0700588 return "", fmt.Errorf("failed to find container port: %s: %v", out, err)
589 } else {
590 v4, _, found := strings.Cut(string(out), "\n")
591 if !found {
592 return "", fmt.Errorf("failed to find container port: %s: %v", out, err)
593 }
594 localAddr = v4
595 if strings.HasPrefix(localAddr, "0.0.0.0") {
596 localAddr = "127.0.0.1" + strings.TrimPrefix(localAddr, "0.0.0.0")
597 }
598 }
599 return localAddr, nil
600}
601
602// Contact the container and configure it.
Sean McCullough7013e9e2025-05-14 02:03:58 +0000603func postContainerInitConfig(ctx context.Context, localAddr, commit, gitPort, gitPass string, sshAvailable bool, sshError string, sshServerIdentity, sshAuthorizedKeys, sshContainerCAKey, sshHostCertificate []byte) error {
Earl Lee2e463fb2025-04-17 11:22:22 -0700604 localURL := "http://" + localAddr
Sean McCulloughbaa2b592025-04-23 10:40:08 -0700605
606 initMsg, err := json.Marshal(
607 server.InitRequest{
Sean McCullough7013e9e2025-05-14 02:03:58 +0000608 Commit: commit,
609 OutsideHTTP: fmt.Sprintf("http://sketch:%s@host.docker.internal:%s", gitPass, gitPort),
610 GitRemoteAddr: fmt.Sprintf("http://sketch:%s@host.docker.internal:%s/.git", gitPass, gitPort),
611 HostAddr: localAddr,
612 SSHAuthorizedKeys: sshAuthorizedKeys,
613 SSHServerIdentity: sshServerIdentity,
614 SSHContainerCAKey: sshContainerCAKey,
615 SSHHostCertificate: sshHostCertificate,
616 SSHAvailable: sshAvailable,
617 SSHError: sshError,
Sean McCulloughbaa2b592025-04-23 10:40:08 -0700618 })
Earl Lee2e463fb2025-04-17 11:22:22 -0700619 if err != nil {
620 return fmt.Errorf("init msg: %w", err)
621 }
622
Earl Lee2e463fb2025-04-17 11:22:22 -0700623 // Note: this /init POST is handled in loop/server/loophttp.go:
624 initMsgByteReader := bytes.NewReader(initMsg)
625 req, err := http.NewRequest("POST", localURL+"/init", initMsgByteReader)
626 if err != nil {
627 return err
628 }
629
630 var res *http.Response
631 for i := 0; ; i++ {
632 time.Sleep(100 * time.Millisecond)
633 // If you DON'T reset this byteReader, then subsequent retries may end up sending 0 bytes.
634 initMsgByteReader.Reset(initMsg)
635 res, err = http.DefaultClient.Do(req)
636 if err != nil {
David Crawshaw99231ba2025-05-03 10:48:26 -0700637 if i < 100 {
638 if i%10 == 0 {
639 slog.DebugContext(ctx, "postContainerInitConfig retrying", slog.Int("retry", i), slog.String("err", err.Error()))
640 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700641 continue
642 }
643 return fmt.Errorf("failed to %s/init sketch in container, NOT retrying: err: %v", localURL, err)
644 }
645 break
646 }
647 resBytes, _ := io.ReadAll(res.Body)
648 if res.StatusCode != http.StatusOK {
649 return fmt.Errorf("failed to initialize sketch in container, response status code %d: %s", res.StatusCode, resBytes)
650 }
651 return nil
652}
653
David Crawshaw5a7b3692025-05-05 16:49:15 -0700654func findOrBuildDockerImage(ctx context.Context, cwd, gitRoot, model, modelURL, modelAPIKey string, forceRebuild, verbose bool) (imgName string, err error) {
Earl Lee2e463fb2025-04-17 11:22:22 -0700655 h := sha256.Sum256([]byte(gitRoot))
656 imgName = "sketch-" + hex.EncodeToString(h[:6])
657
658 var curImgInitFilesHash string
659 if out, err := combinedOutput(ctx, "docker", "inspect", "--format", "{{json .Config.Labels}}", imgName); err != nil {
660 if strings.Contains(string(out), "No such object") {
661 // Image does not exist, continue and build it.
662 curImgInitFilesHash = ""
663 } else {
664 return "", fmt.Errorf("docker inspect failed: %s, %v", out, err)
665 }
666 } else {
667 m := map[string]string{}
668 if err := json.Unmarshal(bytes.TrimSpace(out), &m); err != nil {
669 return "", fmt.Errorf("docker inspect output unparsable: %s, %v", out, err)
670 }
671 curImgInitFilesHash = m["sketch_context"]
672 }
673
674 candidates, err := findRepoDockerfiles(cwd, gitRoot)
675 if err != nil {
676 return "", fmt.Errorf("find dockerfile: %w", err)
677 }
678
679 var initFiles map[string]string
680 var dockerfilePath string
David Crawshawff2df6a2025-05-12 14:45:29 -0700681 var generatedDockerfile string
Earl Lee2e463fb2025-04-17 11:22:22 -0700682
683 // TODO: prefer a "Dockerfile.sketch" so users can tailor any env to this tool.
684 if len(candidates) == 1 && strings.ToLower(filepath.Base(candidates[0])) == "dockerfile" {
685 dockerfilePath = candidates[0]
686 contents, err := os.ReadFile(dockerfilePath)
687 if err != nil {
688 return "", err
689 }
690 fmt.Printf("using %s as dev env\n", candidates[0])
691 if hashInitFiles(map[string]string{dockerfilePath: string(contents)}) == curImgInitFilesHash && !forceRebuild {
Earl Lee2e463fb2025-04-17 11:22:22 -0700692 return imgName, nil
693 }
694 } else {
695 initFiles, err = readInitFiles(os.DirFS(gitRoot))
696 if err != nil {
697 return "", err
698 }
699 subPathWorkingDir, err := filepath.Rel(gitRoot, cwd)
700 if err != nil {
701 return "", err
702 }
703 initFileHash := hashInitFiles(initFiles)
704 if curImgInitFilesHash == initFileHash && !forceRebuild {
Earl Lee2e463fb2025-04-17 11:22:22 -0700705 return imgName, nil
706 }
707
David Crawshaw5a7b3692025-05-05 16:49:15 -0700708 if model == "gemini" {
709 if strings.HasSuffix(modelURL, "/gemmsgs") {
710 // Horrible hack! Switch back to anthropic for container building.
David Crawshaw3659d872025-05-05 17:52:23 -0700711 // We can do this because we are talking to skaband and know the address.
David Crawshaw5a7b3692025-05-05 16:49:15 -0700712 modelURL = strings.Replace(modelURL, "/gemmsgs", "/antmsgs", 1)
713 } else {
714 return "", fmt.Errorf("building docker image with gemini model is not supported yet; start with -model=anthropic first then use gemini")
715 }
716 }
717
Earl Lee2e463fb2025-04-17 11:22:22 -0700718 start := time.Now()
Josh Bleecher Snyder4f84ab72025-04-22 16:40:54 -0700719 srv := &ant.Service{
David Crawshaw5a7b3692025-05-05 16:49:15 -0700720 URL: modelURL,
721 APIKey: modelAPIKey,
Josh Bleecher Snyder4f84ab72025-04-22 16:40:54 -0700722 HTTPC: http.DefaultClient,
723 }
Pokey Rulec31e2962025-05-13 10:53:33 +0000724 generatedDockerfile, err = createDockerfile(ctx, srv, initFiles, subPathWorkingDir, verbose)
Earl Lee2e463fb2025-04-17 11:22:22 -0700725 if err != nil {
726 return "", fmt.Errorf("create dockerfile: %w", err)
727 }
Josh Bleecher Snyder7c58b022025-05-14 17:30:39 +0000728 // Create a unique temporary directory for the Dockerfile
729 tmpDir, err := os.MkdirTemp("", "sketch-docker-*")
730 if err != nil {
731 return "", fmt.Errorf("failed to create temporary directory: %w", err)
732 }
733 dockerfilePath = filepath.Join(tmpDir, tmpSketchDockerfile)
David Crawshawff2df6a2025-05-12 14:45:29 -0700734 if err := os.WriteFile(dockerfilePath, []byte(generatedDockerfile), 0o666); err != nil {
Earl Lee2e463fb2025-04-17 11:22:22 -0700735 return "", err
736 }
Josh Bleecher Snyder7c58b022025-05-14 17:30:39 +0000737 // Remove the temporary directory and all contents when done
738 defer os.RemoveAll(tmpDir)
Earl Lee2e463fb2025-04-17 11:22:22 -0700739
David Crawshawb5f6a002025-05-05 08:27:16 -0700740 if verbose {
David Crawshawff2df6a2025-05-12 14:45:29 -0700741 fmt.Fprintf(os.Stderr, "generated Dockerfile in %s:\n\t%s\n\n", time.Since(start).Round(time.Millisecond), strings.Replace(generatedDockerfile, "\n", "\n\t", -1))
David Crawshawb5f6a002025-05-05 08:27:16 -0700742 }
Earl Lee2e463fb2025-04-17 11:22:22 -0700743 }
744
745 var gitUserEmail, gitUserName string
746 if out, err := combinedOutput(ctx, "git", "config", "--get", "user.email"); err != nil {
747 return "", fmt.Errorf("git config: %s: %v", out, err)
748 } else {
749 gitUserEmail = strings.TrimSpace(string(out))
750 }
751 if out, err := combinedOutput(ctx, "git", "config", "--get", "user.name"); err != nil {
752 return "", fmt.Errorf("git config: %s: %v", out, err)
753 } else {
754 gitUserName = strings.TrimSpace(string(out))
755 }
756
757 start := time.Now()
758 cmd := exec.CommandContext(ctx,
759 "docker", "build",
760 "-t", imgName,
761 "-f", dockerfilePath,
762 "--build-arg", "GIT_USER_EMAIL="+gitUserEmail,
763 "--build-arg", "GIT_USER_NAME="+gitUserName,
David Crawshaw31f15242025-05-06 16:03:49 -0700764 ".",
Earl Lee2e463fb2025-04-17 11:22:22 -0700765 )
David Crawshawb5f6a002025-05-05 08:27:16 -0700766 cmd.Dir = gitRoot
David Crawshaw31f15242025-05-06 16:03:49 -0700767 // We print the docker build output whether or not the user
768 // has selected --verbose. Building an image takes a while
769 // and this gives good context.
David Crawshawb5f6a002025-05-05 08:27:16 -0700770 cmd.Stdout = os.Stdout
771 cmd.Stderr = os.Stderr
772 fmt.Printf("🏗️ building docker image %s... (use -verbose to see build output)\n", imgName)
Earl Lee2e463fb2025-04-17 11:22:22 -0700773
774 err = run(ctx, "docker build", cmd)
775 if err != nil {
David Crawshawff2df6a2025-05-12 14:45:29 -0700776 var msg string
777 if generatedDockerfile != "" {
778 if !verbose {
779 fmt.Fprintf(os.Stderr, "Generated Dockerfile:\n\t%s\n\n", strings.Replace(generatedDockerfile, "\n", "\n\t", -1))
780 }
781 msg = fmt.Sprintf("\n\nThe generated Dockerfile failed to build.\nYou can override it by committing a Dockerfile to your project.")
782 }
783 return "", fmt.Errorf("docker build failed: %v%s", err, msg)
Earl Lee2e463fb2025-04-17 11:22:22 -0700784 }
785 fmt.Printf("built docker image %s in %s\n", imgName, time.Since(start).Round(time.Millisecond))
786 return imgName, nil
787}
788
789func findRepoDockerfiles(cwd, gitRoot string) ([]string, error) {
790 files, err := findDirDockerfiles(cwd)
791 if err != nil {
792 return nil, err
793 }
794 if len(files) > 0 {
795 return files, nil
796 }
797
798 path := cwd
799 for path != gitRoot {
800 path = filepath.Dir(path)
801 files, err := findDirDockerfiles(path)
802 if err != nil {
803 return nil, err
804 }
805 if len(files) > 0 {
806 return files, nil
807 }
808 }
809 return files, nil
810}
811
812// findDirDockerfiles finds all "Dockerfile*" files in a directory.
813func findDirDockerfiles(root string) (res []string, err error) {
814 err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
815 if err != nil {
816 return err
817 }
818 if info.IsDir() && root != path {
819 return filepath.SkipDir
820 }
821 name := strings.ToLower(info.Name())
822 if name == "dockerfile" || strings.HasPrefix(name, "dockerfile.") {
823 res = append(res, path)
824 }
825 return nil
826 })
827 if err != nil {
828 return nil, err
829 }
830 return res, nil
831}
832
833func findGitRoot(ctx context.Context, path string) (string, error) {
834 cmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-common-dir")
835 cmd.Dir = path
836 out, err := cmd.CombinedOutput()
837 if err != nil {
838 if strings.Contains(string(out), "not a git repository") {
839 return "", fmt.Errorf(`sketch needs to run from within a git repo, but %s is not part of a git repo.
840Consider one of the following options:
841 - cd to a different dir that is already part of a git repo first, or
842 - to create a new git repo from this directory (%s), run this command:
843
844 git init . && git commit --allow-empty -m "initial commit"
845
846and try running sketch again.
847`, path, path)
848 }
849 return "", fmt.Errorf("git rev-parse --git-common-dir: %s: %w", out, err)
850 }
851 gitDir := strings.TrimSpace(string(out)) // location of .git dir, often as a relative path
852 absGitDir := filepath.Join(path, gitDir)
853 return filepath.Dir(absGitDir), err
854}
855
Josh Bleecher Snyder2772f632025-05-01 21:42:35 +0000856// getEnvForwardingFromGitConfig retrieves environment variables to pass through to Docker
857// from git config using the sketch.envfwd multi-valued key.
858func getEnvForwardingFromGitConfig(ctx context.Context) []string {
859 outb, err := exec.CommandContext(ctx, "git", "config", "--get-all", "sketch.envfwd").CombinedOutput()
860 out := string(outb)
861 if err != nil {
862 if strings.Contains(out, "key does not exist") {
863 return nil
864 }
865 slog.ErrorContext(ctx, "failed to get sketch.envfwd from git config", "err", err, "output", out)
866 return nil
867 }
868
869 var envVars []string
870 for envVar := range strings.Lines(out) {
871 envVar = strings.TrimSpace(envVar)
872 if envVar == "" {
873 continue
874 }
875 envVars = append(envVars, envVar+"="+os.Getenv(envVar))
876 }
877 return envVars
878}
Philip Zeyliger1dc21372025-05-05 19:54:44 +0000879
880// parseDockerArgs parses a string containing space-separated Docker arguments into an array of strings.
881// It handles quoted arguments and escaped characters.
882//
883// Examples:
884//
885// --memory=2g --cpus=2 -> ["--memory=2g", "--cpus=2"]
886// --label="my label" --env=FOO=bar -> ["--label=my label", "--env=FOO=bar"]
887// --env="KEY=\"quoted value\"" -> ["--env=KEY=\"quoted value\""]
888func parseDockerArgs(args string) []string {
889 if args = strings.TrimSpace(args); args == "" {
890 return []string{}
891 }
892
893 var result []string
894 var current strings.Builder
895 inQuotes := false
896 escapeNext := false
897 quoteChar := rune(0)
898
899 for _, char := range args {
900 if escapeNext {
901 current.WriteRune(char)
902 escapeNext = false
903 continue
904 }
905
906 if char == '\\' {
907 escapeNext = true
908 continue
909 }
910
911 if char == '"' || char == '\'' {
912 if !inQuotes {
913 inQuotes = true
914 quoteChar = char
915 continue
916 } else if char == quoteChar {
917 inQuotes = false
918 quoteChar = rune(0)
919 continue
920 }
921 // Non-matching quote character inside quotes
922 current.WriteRune(char)
923 continue
924 }
925
926 // Space outside of quotes is an argument separator
927 if char == ' ' && !inQuotes {
928 if current.Len() > 0 {
929 result = append(result, current.String())
930 current.Reset()
931 }
932 continue
933 }
934
935 current.WriteRune(char)
936 }
937
938 // Add the last argument if there is one
939 if current.Len() > 0 {
940 result = append(result, current.String())
941 }
942
943 return result
944}