budget: remove num-iterations and wall clock time limits
Remove iteration count and time-based budget limits from the agent budget system
while preserving the budget concept for dollar-based limits only.
Problem Analysis:
The budget system previously supported three types of limits:
1. MaxResponses (iteration count limit)
2. MaxWallTime (wall clock time limit)
3. MaxDollars (cost limit)
This created complexity in both implementation and user experience, with
multiple overlapping budget mechanisms that could trigger independently.
The iteration and time limits added limited value compared to the more
practical dollar-based budget control.
Implementation Changes:
1. Budget Structure Simplification:
- Remove MaxResponses and MaxWallTime fields from Budget struct
- Keep only MaxDollars field for cost-based budget control
- Simplify Budget to single-field struct with clear purpose
2. CLI Flag Removal:
- Remove -max-iterations flag from command line interface
- Remove -max-wall-time flag from command line interface
- Keep -max-dollars flag with same functionality
- Remove unused time import from cmd/sketch/main.go
3. Budget Logic Streamlining:
- Simplify ResetBudget() to only adjust MaxDollars based on usage
- Simplify overBudget() to only check dollar limits
- Remove iteration and time checking logic throughout
4. Container Configuration Updates:
- Remove MaxIterations and MaxWallTime from ContainerConfig struct
- Remove corresponding docker command arguments
- Maintain MaxDollars configuration and passing
5. UI Cleanup:
- Remove budget display of max responses and max wall time from termui
- Keep dollar-based budget information display
6. Test Updates:
- Update test Budget initialization to use only MaxDollars
- Verify all existing tests continue to pass
Technical Details:
- Budget struct now contains only MaxDollars float64 field
- ResetBudget adjusts budget by adding current TotalCostUSD to MaxDollars
- overBudget checks only dollar spending against MaxDollars limit
- CLI help shows only -max-dollars option, no iteration/time options
- Docker container launch passes only max-dollars parameter
Benefits:
- Simplified budget system with single, clear cost control mechanism
- Reduced complexity in budget logic and error handling
- Cleaner CLI interface with fewer confusing options
- More predictable budget behavior focused on practical cost limits
- Easier to understand and configure for users
Testing:
- All existing tests pass with updated budget structure
- CLI help verification shows only max-dollars option
- Build verification confirms no compilation errors
- Budget functionality preserved for dollar-based limits
This change streamlines the budget system to focus on practical cost control
while maintaining all existing dollar-based budget functionality and removing
complexity around iteration and time-based limits.
Co-Authored-By: sketch <hello@sketch.dev>
Change-ID: sa7be127e12d43ee7k
diff --git a/cmd/sketch/main.go b/cmd/sketch/main.go
index 726e59a..8067af7 100644
--- a/cmd/sketch/main.go
+++ b/cmd/sketch/main.go
@@ -13,7 +13,6 @@
"runtime"
"runtime/debug"
"strings"
- "time"
"sketch.dev/experiment"
"sketch.dev/llm"
@@ -152,24 +151,22 @@
}
type CLIFlags struct {
- addr string
- skabandAddr string
- unsafe bool
- openBrowser bool
- httprrFile string
- maxIterations uint64
- maxWallTime time.Duration
- maxDollars float64
- oneShot bool
- prompt string
- modelName string
- llmAPIKey string
- listModels bool
- verbose bool
- version bool
- workingDir string
- sshPort int
- forceRebuild bool
+ addr string
+ skabandAddr string
+ unsafe bool
+ openBrowser bool
+ httprrFile string
+ maxDollars float64
+ oneShot bool
+ prompt string
+ modelName string
+ llmAPIKey string
+ listModels bool
+ verbose bool
+ version bool
+ workingDir string
+ sshPort int
+ forceRebuild bool
gitUsername string
gitEmail string
@@ -203,8 +200,6 @@
userFlags.StringVar(&flags.skabandAddr, "skaband-addr", "https://sketch.dev", "URL of the skaband server; set to empty to disable sketch.dev integration")
userFlags.BoolVar(&flags.unsafe, "unsafe", false, "run without a docker container")
userFlags.BoolVar(&flags.openBrowser, "open", true, "open sketch URL in system browser; on by default except if -one-shot is used or a ssh connection is detected")
- userFlags.Uint64Var(&flags.maxIterations, "max-iterations", 0, "maximum number of iterations the agent should perform per turn, 0 to disable limit")
- userFlags.DurationVar(&flags.maxWallTime, "max-wall-time", 0, "maximum time the agent should run per turn, 0 to disable limit")
userFlags.Float64Var(&flags.maxDollars, "max-dollars", 10.0, "maximum dollars the agent should spend per turn, 0 to disable limit")
userFlags.BoolVar(&flags.oneShot, "one-shot", false, "exit after the first turn without termui")
userFlags.StringVar(&flags.prompt, "prompt", "", "prompt to send to sketch")
@@ -365,8 +360,6 @@
ExperimentFlag: flags.experimentFlag.String(),
TermUI: flags.termUI,
MaxDollars: flags.maxDollars,
- MaxIterations: flags.maxIterations,
- MaxWallTime: flags.maxWallTime,
}
if err := dockerimg.LaunchContainer(ctx, config); err != nil {
@@ -459,9 +452,7 @@
return fmt.Errorf("failed to initialize LLM service: %w", err)
}
budget := conversation.Budget{
- MaxResponses: flags.maxIterations,
- MaxWallTime: flags.maxWallTime,
- MaxDollars: flags.maxDollars,
+ MaxDollars: flags.maxDollars,
}
agentConfig := loop.AgentConfig{
diff --git a/dockerimg/dockerimg.go b/dockerimg/dockerimg.go
index 79d4b28..0759b22 100644
--- a/dockerimg/dockerimg.go
+++ b/dockerimg/dockerimg.go
@@ -106,9 +106,7 @@
TermUI bool
// Budget configuration
- MaxDollars float64
- MaxIterations uint64
- MaxWallTime time.Duration
+ MaxDollars float64
GitRemoteUrl string
@@ -539,8 +537,6 @@
"-outside-os="+config.OutsideOS,
"-outside-working-dir="+config.OutsideWorkingDir,
fmt.Sprintf("-max-dollars=%f", config.MaxDollars),
- fmt.Sprintf("-max-iterations=%d", config.MaxIterations),
- fmt.Sprintf("-max-wall-time=%s", config.MaxWallTime.String()),
"-open=false",
"-termui="+fmt.Sprintf("%t", config.TermUI),
"-verbose="+fmt.Sprintf("%t", config.Verbose),
diff --git a/llm/conversation/convo.go b/llm/conversation/convo.go
index f4ed0bd..95e4fba 100644
--- a/llm/conversation/convo.go
+++ b/llm/conversation/convo.go
@@ -607,9 +607,7 @@
// A Budget represents the maximum amount of resources that may be spent on a conversation.
// Note that the default (zero) budget is unlimited.
type Budget struct {
- MaxResponses uint64 // if > 0, max number of iterations (=responses)
- MaxDollars float64 // if > 0, max dollars that may be spent
- MaxWallTime time.Duration // if > 0, max wall time that may be spent
+ MaxDollars float64 // if > 0, max dollars that may be spent
}
// OverBudget returns an error if the convo (or any of its parents) has exceeded its budget.
@@ -630,28 +628,15 @@
if c.Budget.MaxDollars > 0 {
c.Budget.MaxDollars += c.CumulativeUsage().TotalCostUSD
}
- if c.Budget.MaxResponses > 0 {
- c.Budget.MaxResponses += c.CumulativeUsage().Responses
- }
- if c.Budget.MaxWallTime > 0 {
- c.Budget.MaxWallTime += c.usage.WallTime()
- }
}
func (c *Convo) overBudget() error {
usage := c.CumulativeUsage()
// TODO: stop before we exceed the budget instead of after?
- // Top priority is money, then time, then response count.
var err error
cont := "Continuing to chat will reset the budget."
if c.Budget.MaxDollars > 0 && usage.TotalCostUSD >= c.Budget.MaxDollars {
err = errors.Join(err, fmt.Errorf("$%.2f spent, budget is $%.2f. %s", usage.TotalCostUSD, c.Budget.MaxDollars, cont))
}
- if c.Budget.MaxWallTime > 0 && usage.WallTime() >= c.Budget.MaxWallTime {
- err = errors.Join(err, fmt.Errorf("%v elapsed, budget is %v. %s", usage.WallTime().Truncate(time.Second), c.Budget.MaxWallTime.Truncate(time.Second), cont))
- }
- if c.Budget.MaxResponses > 0 && usage.Responses >= c.Budget.MaxResponses {
- err = errors.Join(err, fmt.Errorf("%d responses received, budget is %d. %s", usage.Responses, c.Budget.MaxResponses, cont))
- }
return err
}
diff --git a/loop/agent_test.go b/loop/agent_test.go
index 8e6a75d..30affbe 100644
--- a/loop/agent_test.go
+++ b/loop/agent_test.go
@@ -60,7 +60,7 @@
if err := os.Chdir("/"); err != nil {
t.Fatal(err)
}
- budget := conversation.Budget{MaxResponses: 100}
+ budget := conversation.Budget{MaxDollars: 10.0}
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
diff --git a/termui/termui.go b/termui/termui.go
index b14593d..1890231 100644
--- a/termui/termui.go
+++ b/termui/termui.go
@@ -247,12 +247,7 @@
case "budget":
originalBudget := ui.agent.OriginalBudget()
ui.AppendSystemMessage("💰 Budget summary:")
- if originalBudget.MaxResponses > 0 {
- ui.AppendSystemMessage("- Max responses: %d", originalBudget.MaxResponses)
- }
- if originalBudget.MaxWallTime > 0 {
- ui.AppendSystemMessage("- Max wall time: %v", originalBudget.MaxWallTime)
- }
+
ui.AppendSystemMessage("- Max total cost: %0.2f", originalBudget.MaxDollars)
case "browser", "open", "b":
if ui.httpURL != "" {