compaction fail safe

Author: hugodutka

packages/scout-agent/lib/compaction.test.ts

@@ -8,11 +8,13 @@ import {
   COMPACT_CONVERSATION_TOOL_NAME,
   COMPACTION_MARKER_TOOL_NAME,
   countCompactionMarkers,
+  maxConsecutiveCompactionAttempts,
   createCompactionMarkerPart,
   createCompactionTool,
   findAPICallError,
   findCompactionSummary,
   isOutOfContextError,
+  MAX_CONSECUTIVE_COMPACTION_ATTEMPTS,
 } from "./compaction";
 import type { Message } from "./types";
 
@@ -302,6 +304,41 @@ describe("countCompactionMarkers", () => {
   });
 });
 
+describe("maxConsecutiveCompactionAttempts", () => {
+  test("counts consecutive assistant compaction attempts", () => {
+    const messages: Message[] = [
+      userMsg("1", "Hello"),
+      summaryMsg("summary-1", "Summary output 1"),
+      summaryMsg("summary-2", "Summary output 2"),
+    ];
+
+    expect(maxConsecutiveCompactionAttempts(messages)).toBe(2);
+  });
+
+  test("ignores summary acknowledgment when counting", () => {
+    const messages: Message[] = [
+      summaryMsg("summary-1", "First summary"),
+      {
+        id: "compaction-summary-response",
+        role: "assistant",
+        parts: [{ type: "text", text: "Acknowledged." }],
+      },
+      summaryMsg("summary-2", "Second summary"),
+    ];
+
+    expect(maxConsecutiveCompactionAttempts(messages)).toBe(1);
+  });
+
+  test("stops at non-compaction assistant message", () => {
+    const messages: Message[] = [
+      markerMsg("marker1"),
+      assistantMsg("assistant", "Normal reply"),
+    ];
+
+    expect(maxConsecutiveCompactionAttempts(messages)).toBe(0);
+  });
+});
+
 describe("buildCompactionRequestMessage", () => {
   test("creates user message with correct role", () => {
     const message = buildCompactionRequestMessage();
@@ -328,6 +365,20 @@ describe("applyCompactionToMessages", () => {
     expect(result).toEqual(messages);
   });
 
+  test("throws when consecutive compaction attempts hit the limit", () => {
+    const attempts = MAX_CONSECUTIVE_COMPACTION_ATTEMPTS + 1;
+    const messages: Message[] = [
+      userMsg("1", "Hello"),
+      ...Array.from({ length: attempts }, (_, idx) =>
+        summaryMsg(`summary-${idx}`, `Summary ${idx}`)
+      ),
+    ];
+
+    expect(() => applyCompactionToMessages(messages)).toThrow(
+      /Compaction loop detected/
+    );
+  });
+
   test("excludes correct number of messages based on marker count", () => {
     const messages: Message[] = [
       userMsg("1", "Message 1"),
@@ -599,6 +650,10 @@ describe("applyCompactionToMessages", () => {
       userMsg("3", "Third message"),
       userMsg("4", "Fourth message"),
       markerMsg("marker1"),
+      assistantMsg(
+        "assistant-buffer",
+        "Normal reply between compaction attempts"
+      ),
       markerMsg("marker2"),
       userMsg("interrupted", "User interrupted compaction with this message"),
       markerMsg("marker3"),

packages/scout-agent/lib/compaction.ts

@@ -12,6 +12,7 @@ import type { Message } from "./types";
 // Constants
 export const COMPACTION_MARKER_TOOL_NAME = "__compaction_marker";
 export const COMPACT_CONVERSATION_TOOL_NAME = "compact_conversation";
+export const MAX_CONSECUTIVE_COMPACTION_ATTEMPTS = 5;
 
 // Error patterns for out-of-context detection (regex)
 const OUT_OF_CONTEXT_PATTERNS = [
@@ -165,6 +166,15 @@ function isCompactionSummaryPart(part: Message["parts"][number]): boolean {
   );
 }
 
+function isCompactConversationPart(part: Message["parts"][number]): boolean {
+  return (
+    part.type === `tool-${COMPACT_CONVERSATION_TOOL_NAME}` ||
+    (part.type === "dynamic-tool" &&
+      "toolName" in part &&
+      part.toolName === COMPACT_CONVERSATION_TOOL_NAME)
+  );
+}
+
 export interface CompactionMarkerPart {
   type: "dynamic-tool";
   toolName: typeof COMPACTION_MARKER_TOOL_NAME;
@@ -259,6 +269,38 @@ export function countCompactionMarkers(
   return count;
 }
 
+/**
+ * Finds the maximum number of consecutive assistant messages that contain
+ * compaction tool calls. The streak resets when a non-assistant message
+ * is encountered.
+ *
+ * @param messages - The message history to analyze
+ * @returns The longest streak of consecutive compaction attempts
+ */
+export function maxConsecutiveCompactionAttempts(messages: Message[]): number {
+  let maxAttempts = 0;
+  let attempts = 0;
+
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const message = messages[i];
+    if (!message) {
+      continue;
+    }
+    if (message.role !== "assistant") {
+      attempts = 0;
+    }
+    const hasCompactionPart = message.parts.some((part) =>
+      isCompactConversationPart(part)
+    );
+    if (hasCompactionPart) {
+      attempts++;
+      maxAttempts = Math.max(maxAttempts, attempts);
+    }
+  }
+
+  return maxAttempts;
+}
+
 /**
  * Build the compaction request message that instructs the model to compact.
  */
@@ -438,6 +480,14 @@ function transformMessagesForCompaction(messages: Message[]): Message[] {
  * @throws {CompactionError} If compaction would leave no messages (too many retries)
  */
 export function applyCompactionToMessages(messages: Message[]): Message[] {
+  const compactionAttempts = maxConsecutiveCompactionAttempts(messages);
+  if (compactionAttempts >= MAX_CONSECUTIVE_COMPACTION_ATTEMPTS) {
+    throw new CompactionError(
+      `Compaction loop detected after ${compactionAttempts} attempts`,
+      compactionAttempts
+    );
+  }
+
   const currentConversation = applySummaryToMessages(messages);
   const transformedMessages =
     transformMessagesForCompaction(currentConversation);

packages/scout-agent/lib/core.test.ts

@@ -1128,6 +1128,24 @@ describe("compaction", () => {
     return textPart ? (textPart as { text: string }).text : undefined;
   };
 
+  const createCompactionSummaryMessage = (id: string): Message => ({
+    id,
+    role: "assistant",
+    parts: [
+      {
+        type: "dynamic-tool",
+        toolName: "compact_conversation",
+        toolCallId: `${id}-call`,
+        state: "output-available",
+        input: { summary: "Test summary" },
+        output: {
+          summary: "Test summary",
+          compacted_at: "2024-01-01T00:00:00Z",
+        },
+      } as Message["parts"][number],
+    ],
+  });
+
   test("buildStreamTextParams always includes compact_conversation tool by default", async () => {
     const agent = new blink.Agent<Message>();
     const scout = new Scout({
@@ -1163,11 +1181,46 @@ describe("compaction", () => {
     expect(params.tools.compact_conversation).toBeUndefined();
   });
 
-  test("buildStreamTextParams throws when exclusion would leave insufficient messages", async () => {
+  test("buildStreamTextParams disables compaction after repeated compaction attempts", async () => {
+    const warn = mock();
+    const logger = { ...noopLogger, warn };
     const agent = new blink.Agent<Message>();
     const scout = new Scout({
       agent,
-      logger: noopLogger,
+      logger,
+    });
+
+    const messages: Message[] = [
+      {
+        id: "user-1",
+        role: "user",
+        parts: [{ type: "text", text: "Hello" }],
+      },
+      createCompactionSummaryMessage("summary-1"),
+      createCompactionSummaryMessage("summary-2"),
+      createCompactionSummaryMessage("summary-3"),
+      createCompactionSummaryMessage("summary-4"),
+      createCompactionSummaryMessage("summary-5"),
+    ];
+
+    const params = await scout.buildStreamTextParams({
+      chatID: "test-chat-id" as blink.ID,
+      messages,
+      model: newMockModel({ textResponse: "test" }),
+    });
+
+    expect(params.tools.compact_conversation).toBeUndefined();
+    expect(params.experimental_transform).toBeUndefined();
+    expect(warn).toHaveBeenCalled();
+  });
+
+  test("buildStreamTextParams disables compaction when exclusion would leave insufficient messages", async () => {
+    const warn = mock();
+    const logger = { ...noopLogger, warn };
+    const agent = new blink.Agent<Message>();
+    const scout = new Scout({
+      agent,
+      logger,
     });
 
     // Create messages with insufficient content to summarize after exclusion
@@ -1196,13 +1249,15 @@ describe("compaction", () => {
       },
     ];
 
-    await expect(
-      scout.buildStreamTextParams({
-        chatID: "test-chat-id" as blink.ID,
-        messages,
-        model: newMockModel({ textResponse: "test" }),
-      })
-    ).rejects.toThrow(/Cannot compact/);
+    const params = await scout.buildStreamTextParams({
+      chatID: "test-chat-id" as blink.ID,
+      messages,
+      model: newMockModel({ textResponse: "test" }),
+    });
+
+    expect(params.tools.compact_conversation).toBeUndefined();
+    expect(params.experimental_transform).toBeUndefined();
+    expect(warn).toHaveBeenCalled();
   });
 
   test("e2e: complete compaction flow using scout methods directly", async () => {

packages/scout-agent/lib/core.ts

@@ -14,6 +14,7 @@ import {
 import type * as blink from "blink";
 import {
   applyCompactionToMessages,
+  CompactionError,
   createCompactionTransform,
   createCompactionTool,
 } from "./compaction";
@@ -469,6 +470,25 @@ export class Scout {
       }
     }
 
+    let compactionEnabled = compaction;
+    let messagesToConvert = messages;
+    if (compactionEnabled) {
+      try {
+        messagesToConvert = applyCompactionToMessages(messages);
+      } catch (error) {
+        if (error instanceof CompactionError) {
+          this.logger.warn(
+            "Disabling compaction due to repeated compaction failures",
+            error
+          );
+          compactionEnabled = false;
+          messagesToConvert = messages;
+        } else {
+          throw error;
+        }
+      }
+    }
+
     const tools = {
       ...(this.webSearch.config
         ? createWebSearchTools({ exaApiKey: this.webSearch.config.exaApiKey })
@@ -485,7 +505,7 @@ export class Scout {
         : undefined),
       ...computeTools,
       // Always include compaction tool when compaction is enabled (for caching purposes)
-      ...(compaction ? createCompactionTool() : {}),
+      ...(compactionEnabled ? createCompactionTool() : {}),
       ...providedTools,
     };
 
@@ -498,10 +518,6 @@ ${slack.formattingRules}
 </formatting-rules>`;
     }
 
-    const messagesToConvert = compaction
-      ? applyCompactionToMessages(messages)
-      : messages;
-
     const converted = convertToModelMessages(messagesToConvert, {
       ignoreIncompleteToolCalls: true,
       tools,
@@ -525,7 +541,7 @@ ${slack.formattingRules}
       maxOutputTokens: 64_000,
       providerOptions,
       tools: withModelIntent(tools),
-      experimental_transform: compaction
+      experimental_transform: compactionEnabled
         ? createCompactionTransform()
         : undefined,
     };