webui: fix IDLE/WORKING indicator to ignore system messages
Fix bug where IDLE/WORKING status indicator in top-right incorrectly used
system messages (like commit detection) to determine agent state instead
of only considering user and agent messages.
Co-Authored-By: sketch <hello@sketch.dev>
Change-ID: s7aecd377d88b46f8k
diff --git a/webui/src/web-components/sketch-app-shell.test.ts b/webui/src/web-components/sketch-app-shell.test.ts
index ead14ff..5d4ca0f 100644
--- a/webui/src/web-components/sketch-app-shell.test.ts
+++ b/webui/src/web-components/sketch-app-shell.test.ts
@@ -19,7 +19,7 @@
const component = await mount(SketchAppShell);
// Wait for initial data to load
- await page.waitForTimeout(500);
+ await page.waitForTimeout(1000);
// For now, skip the title verification since it requires more complex testing setup
// Test other core components instead
@@ -52,7 +52,7 @@
const component = await mount(SketchAppShell);
// Wait for initial data to load
- await page.waitForTimeout(500);
+ await page.waitForTimeout(1000);
// Ensure we're in chat view initially
await expect(component.locator(".chat-view.view-active")).toBeVisible();
@@ -109,7 +109,7 @@
const component = await mount(SketchAppShell);
// Wait for initial data to load
- await page.waitForTimeout(500);
+ await page.waitForTimeout(1000);
// For now, skip the title verification since it requires more complex testing setup
@@ -186,7 +186,7 @@
}, targetScrollPosition);
// Wait for scroll to take effect and verify it was set
- await page.waitForTimeout(200);
+ await page.waitForTimeout(500);
const actualScrollPosition = await scrollContainer.evaluate(
(el) => el.scrollTop,
@@ -223,3 +223,239 @@
10,
);
});
+
+test("correctly determines idle state ignoring system messages", async ({
+ page,
+ mount,
+}) => {
+ // Create test messages with various types including system messages
+ const testMessages = [
+ {
+ idx: 0,
+ type: "user" as const,
+ content: "Hello",
+ timestamp: "2023-05-15T12:00:00Z",
+ end_of_turn: true,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ {
+ idx: 1,
+ type: "agent" as const,
+ content: "Hi there",
+ timestamp: "2023-05-15T12:01:00Z",
+ end_of_turn: true,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ {
+ idx: 2,
+ type: "commit" as const,
+ content: "Commit detected: abc123",
+ timestamp: "2023-05-15T12:02:00Z",
+ end_of_turn: false,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ {
+ idx: 3,
+ type: "tool" as const,
+ content: "Running bash command",
+ timestamp: "2023-05-15T12:03:00Z",
+ end_of_turn: false,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ ];
+
+ // Mock the state API response
+ await page.route("**/state", async (route) => {
+ await route.fulfill({
+ json: {
+ ...initialState,
+ outstanding_llm_calls: 0,
+ outstanding_tool_calls: [],
+ },
+ });
+ });
+
+ // Mock the messages API response
+ await page.route("**/messages*", async (route) => {
+ await route.fulfill({ json: testMessages });
+ });
+
+ // Mock the SSE stream endpoint to prevent connection attempts
+ await page.route("**/stream*", async (route) => {
+ // Block the SSE connection request to prevent it from interfering
+ await route.abort();
+ });
+
+ // Mount the component
+ const component = await mount(SketchAppShell);
+
+ // Wait for initial data to load
+ await page.waitForTimeout(1000);
+
+ // Simulate connection established by setting the connection status property
+ await component.evaluate(async () => {
+ const appShell = document.querySelector('sketch-app-shell') as any;
+ if (appShell) {
+ appShell.connectionStatus = 'connected';
+ appShell.requestUpdate();
+ // Force an update cycle to complete
+ await appShell.updateComplete;
+ }
+ });
+
+ // Wait a bit more for the status to update and for any async operations
+ await page.waitForTimeout(1000);
+
+ // Check that the call status component shows IDLE
+ // The last user/agent message (agent with end_of_turn: true) should make it idle
+ // even though there are commit and tool messages after it
+ const callStatus = component.locator("sketch-call-status");
+ await expect(callStatus).toBeVisible();
+
+ // Check that the status banner shows IDLE
+ const statusBanner = callStatus.locator(".status-banner");
+ await expect(statusBanner).toBeVisible();
+ await expect(statusBanner).toHaveClass(/status-idle/);
+ await expect(statusBanner).toHaveText("IDLE");
+});
+
+test("correctly determines working state with non-end-of-turn agent message", async ({
+ page,
+ mount,
+}) => {
+ // Create test messages where the last agent message doesn't have end_of_turn
+ const testMessages = [
+ {
+ idx: 0,
+ type: "user" as const,
+ content: "Please help me",
+ timestamp: "2023-05-15T12:00:00Z",
+ end_of_turn: true,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ {
+ idx: 1,
+ type: "agent" as const,
+ content: "Working on it...",
+ timestamp: "2023-05-15T12:01:00Z",
+ end_of_turn: false, // Agent is still working
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ {
+ idx: 2,
+ type: "commit" as const,
+ content: "Commit detected: def456",
+ timestamp: "2023-05-15T12:02:00Z",
+ end_of_turn: false,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ ];
+
+ // Skip SSE mocking for this test - we'll set data directly
+ await page.route("**/stream*", async (route) => {
+ await route.abort();
+ });
+
+ // Mount the component
+ const component = await mount(SketchAppShell);
+
+ // Wait for initial data to load
+ await page.waitForTimeout(1000);
+
+ // Test the isIdle calculation logic directly
+ const isIdleResult = await component.evaluate(() => {
+ const appShell = document.querySelector('sketch-app-shell') as any;
+ if (!appShell) return { error: 'No app shell found' };
+
+ // Create test messages directly in the browser context
+ const testMessages = [
+ {
+ idx: 0,
+ type: "user",
+ content: "Please help me",
+ timestamp: "2023-05-15T12:00:00Z",
+ end_of_turn: true,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ {
+ idx: 1,
+ type: "agent",
+ content: "Working on it...",
+ timestamp: "2023-05-15T12:01:00Z",
+ end_of_turn: false, // Agent is still working
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ {
+ idx: 2,
+ type: "commit",
+ content: "Commit detected: def456",
+ timestamp: "2023-05-15T12:02:00Z",
+ end_of_turn: false,
+ conversation_id: "conv123",
+ parent_conversation_id: null,
+ },
+ ];
+
+ // Set the messages
+ appShell.messages = testMessages;
+
+ // Call the getLastUserOrAgentMessage method directly
+ const lastMessage = appShell.getLastUserOrAgentMessage();
+ const isIdle = lastMessage ?
+ lastMessage.end_of_turn && !lastMessage.parent_conversation_id :
+ true;
+
+ return {
+ messagesCount: testMessages.length,
+ lastMessage: lastMessage,
+ isIdle: isIdle,
+ expectedWorking: !isIdle
+ };
+ });
+
+ // The isIdle should be false because the last agent message has end_of_turn: false
+ expect(isIdleResult.isIdle).toBe(false);
+ expect(isIdleResult.expectedWorking).toBe(true);
+
+ // Now test the full component interaction
+ await component.evaluate(() => {
+ const appShell = document.querySelector('sketch-app-shell') as any;
+ if (appShell) {
+ // Set connection status to connected
+ appShell.connectionStatus = 'connected';
+
+ // Set container state with active LLM calls
+ appShell.containerState = {
+ outstanding_llm_calls: 1,
+ outstanding_tool_calls: [],
+ agent_state: null
+ };
+
+ // The messages are already set from the previous test
+ // Force a re-render
+ appShell.requestUpdate();
+ }
+ });
+
+ // Wait for the component to update
+ await page.waitForTimeout(500);
+
+ // Now check that the call status component shows WORKING
+ const callStatus = component.locator("sketch-call-status");
+ await expect(callStatus).toBeVisible();
+
+ // Check that the status banner shows WORKING
+ const statusBanner = callStatus.locator(".status-banner");
+ await expect(statusBanner).toBeVisible();
+ await expect(statusBanner).toHaveClass(/status-working/);
+ await expect(statusBanner).toHaveText("WORKING");
+});
diff --git a/webui/src/web-components/sketch-app-shell.ts b/webui/src/web-components/sketch-app-shell.ts
index 0f0177c..f545b82 100644
--- a/webui/src/web-components/sketch-app-shell.ts
+++ b/webui/src/web-components/sketch-app-shell.ts
@@ -998,6 +998,19 @@
this._windowFocused = false;
}
+ // Get the last user or agent message (ignore system messages like commit, error, etc.)
+ // For example, when Sketch notices a new commit, it'll send a message,
+ // but it's still idle!
+ private getLastUserOrAgentMessage(): AgentMessage | null {
+ for (let i = this.messages.length - 1; i >= 0; i--) {
+ const message = this.messages[i];
+ if (message.type === "user" || message.type === "agent") {
+ return message;
+ }
+ }
+ return null;
+ }
+
// Show notification for message with EndOfTurn=true
private async showEndOfTurnNotification(
message: AgentMessage,
@@ -1447,10 +1460,13 @@
.agentState=${this.containerState?.agent_state}
.llmCalls=${this.containerState?.outstanding_llm_calls || 0}
.toolCalls=${this.containerState?.outstanding_tool_calls || []}
- .isIdle=${this.messages.length > 0
- ? this.messages[this.messages.length - 1]?.end_of_turn &&
- !this.messages[this.messages.length - 1]?.parent_conversation_id
- : true}
+ .isIdle=${(() => {
+ const lastUserOrAgentMessage = this.getLastUserOrAgentMessage();
+ return lastUserOrAgentMessage
+ ? lastUserOrAgentMessage.end_of_turn &&
+ !lastUserOrAgentMessage.parent_conversation_id
+ : true;
+ })()}
.isDisconnected=${this.connectionStatus === "disconnected"}
></sketch-call-status>