better out of context error detection

Author: hugodutka

packages/scout-agent/agent.ts

@@ -64,8 +64,7 @@ agent.on("chat", async ({ id, messages }) => {
       }),
     },
   });
-  const stream = streamText(params);
-  return scout.processStreamTextOutput(stream);
+  return streamText(params);
 });
 
 agent.serve();

packages/scout-agent/lib/compaction.test.ts

@@ -1,6 +1,7 @@
 /** biome-ignore-all lint/style/noNonNullAssertion: fine for tests */
 /** biome-ignore-all lint/suspicious/noExplicitAny: fine for tests */
 import { describe, expect, test } from "bun:test";
+import { APICallError } from "ai";
 import {
   applyCompactionToMessages,
   buildCompactionRequestMessage,
@@ -9,6 +10,7 @@ import {
   countCompactionMarkers,
   createCompactionMarkerPart,
   createCompactionTool,
+  findAPICallError,
   findCompactionSummary,
   isOutOfContextError,
 } from "./compaction";
@@ -55,62 +57,77 @@ function summaryMsg(
 }
 
 describe("isOutOfContextError", () => {
-  test("returns true for Anthropic context limit errors", () => {
-    expect(isOutOfContextError(new Error("max_tokens_exceeded"))).toBe(true);
+  const createApiError = (message: string) =>
+    new APICallError({
+      message,
+      url: "https://api.example.com",
+      requestBodyValues: {},
+      statusCode: 400,
+    });
+
+  test("returns true for APICallError with context limit message", () => {
     expect(
-      isOutOfContextError(new Error("The context window has been exceeded"))
+      isOutOfContextError(
+        createApiError("Input is too long for requested model")
+      )
     ).toBe(true);
-  });
-
-  test("returns true for OpenAI context_length_exceeded errors", () => {
-    expect(isOutOfContextError(new Error("context_length_exceeded"))).toBe(
+    expect(isOutOfContextError(createApiError("context_length_exceeded"))).toBe(
       true
     );
   });
 
-  test("returns true for generic token limit exceeded messages", () => {
-    expect(isOutOfContextError(new Error("token limit exceeded"))).toBe(true);
-    expect(
-      isOutOfContextError(new Error("Token limit has been exceeded"))
-    ).toBe(true);
-    expect(isOutOfContextError(new Error("maximum tokens reached"))).toBe(true);
+  test("returns true for APICallError in cause chain", () => {
+    const apiError = createApiError("max_tokens_exceeded");
+    const wrapper = new Error("Gateway error");
+    (wrapper as { cause?: unknown }).cause = apiError;
+    expect(isOutOfContextError(wrapper)).toBe(true);
   });
 
-  test("returns true for context window errors", () => {
-    expect(isOutOfContextError(new Error("context window exceeded"))).toBe(
-      true
-    );
-    expect(isOutOfContextError(new Error("context length exceeded"))).toBe(
-      true
+  test("returns false for APICallError with unrelated message", () => {
+    expect(isOutOfContextError(createApiError("authentication failed"))).toBe(
+      false
     );
   });
 
-  test("returns true for input too long errors", () => {
-    expect(isOutOfContextError(new Error("input is too long"))).toBe(true);
-    expect(isOutOfContextError(new Error("prompt is too long"))).toBe(true);
+  test("returns false for non-APICallError even if message matches pattern", () => {
+    expect(isOutOfContextError(new Error("context_length_exceeded"))).toBe(
+      false
+    );
+    expect(isOutOfContextError("input too long")).toBe(false);
   });
+});
+
+describe("findAPICallError", () => {
+  const createApiError = (message: string) =>
+    new APICallError({
+      message,
+      url: "https://api.example.com",
+      requestBodyValues: {},
+      statusCode: 400,
+    });
 
-  test("returns false for unrelated errors", () => {
-    expect(isOutOfContextError(new Error("network error"))).toBe(false);
-    expect(isOutOfContextError(new Error("authentication failed"))).toBe(false);
-    expect(isOutOfContextError(new Error("rate limit exceeded"))).toBe(false);
+  test("returns the APICallError when provided directly", () => {
+    const error = createApiError("test");
+    expect(findAPICallError(error)).toBe(error);
   });
 
-  test("handles string messages", () => {
-    expect(isOutOfContextError("token limit exceeded")).toBe(true);
-    expect(isOutOfContextError("some other error")).toBe(false);
+  test("returns APICallError from single-level cause", () => {
+    const apiError = createApiError("test");
+    const wrapper = new Error("wrapper");
+    (wrapper as { cause?: unknown }).cause = apiError;
+    expect(findAPICallError(wrapper)).toBe(apiError);
   });
 
-  test("handles objects with message property", () => {
-    expect(isOutOfContextError({ message: "token limit exceeded" })).toBe(true);
-    expect(isOutOfContextError({ message: "some other error" })).toBe(false);
+  test("returns APICallError from deep cause chain", () => {
+    const apiError = createApiError("test");
+    const wrapper = { cause: { cause: apiError } };
+    expect(findAPICallError(wrapper)).toBe(apiError);
   });
 
-  test("returns false for non-error values", () => {
-    expect(isOutOfContextError(null)).toBe(false);
-    expect(isOutOfContextError(undefined)).toBe(false);
-    expect(isOutOfContextError(123)).toBe(false);
-    expect(isOutOfContextError({})).toBe(false);
+  test("returns null when no APICallError present", () => {
+    expect(findAPICallError(new Error("other"))).toBeNull();
+    expect(findAPICallError("string")).toBeNull();
+    expect(findAPICallError(null)).toBeNull();
   });
 });
 

packages/scout-agent/lib/compaction.ts

@@ -1,4 +1,11 @@
-import { type Tool, tool } from "ai";
+import {
+  APICallError,
+  type StreamTextTransform,
+  type TextStreamPart,
+  type Tool,
+  type ToolSet,
+  tool,
+} from "ai";
 import { z } from "zod";
 import type { Message } from "./types";
 
@@ -20,28 +27,77 @@ const OUT_OF_CONTEXT_PATTERNS = [
   /maximum.*tokens/i,
 ];
 
+/**
+ * Recursively search for an APICallError in the error's cause chain.
+ */
+export function findAPICallError(error: unknown): APICallError | null {
+  if (APICallError.isInstance(error)) {
+    return error;
+  }
+  if (error && typeof error === "object" && "cause" in error) {
+    const cause = (error as { cause?: unknown }).cause;
+    return findAPICallError(cause);
+  }
+  return null;
+}
+
 /**
  * Check if an error is an out-of-context error based on known patterns.
  */
 export function isOutOfContextError(error: unknown): boolean {
-  let message: string;
-
-  if (error instanceof Error) {
-    message = error.message;
-  } else if (typeof error === "string") {
-    message = error;
-  } else if (
-    error !== null &&
-    typeof error === "object" &&
-    "message" in error &&
-    typeof error.message === "string"
-  ) {
-    message = error.message;
-  } else {
+  const apiError = findAPICallError(error);
+  if (!apiError) {
     return false;
   }
+  return OUT_OF_CONTEXT_PATTERNS.some((pattern) =>
+    pattern.test(apiError.message)
+  );
+}
 
-  return OUT_OF_CONTEXT_PATTERNS.some((pattern) => pattern.test(message));
+/**
+ * Creates a stream transform that detects out-of-context errors and emits a compaction marker.
+ */
+export function createCompactionTransform<T extends ToolSet>(
+  onCompactionTriggered?: () => void
+): StreamTextTransform<T> {
+  return ({ stopStream }) =>
+    new TransformStream<TextStreamPart<T>, TextStreamPart<T>>({
+      transform(chunk, controller) {
+        if (
+          chunk?.type === "error" &&
+          isOutOfContextError((chunk as { error?: unknown }).error)
+        ) {
+          onCompactionTriggered?.();
+          const markerPart = createCompactionMarkerPart();
+          controller.enqueue({
+            type: "tool-call",
+            toolCallType: "function",
+            toolCallId: markerPart.toolCallId,
+            toolName: markerPart.toolName,
+            input: markerPart.input,
+            dynamic: true,
+          } as TextStreamPart<T>);
+          controller.enqueue({
+            type: "tool-result",
+            toolCallId: markerPart.toolCallId,
+            toolName: markerPart.toolName,
+            input: markerPart.input,
+            output: markerPart.output,
+            providerExecuted: false,
+            dynamic: true,
+          } as TextStreamPart<T>);
+          controller.enqueue({
+            type: "finish",
+            finishReason: "tool-calls",
+            logprobs: undefined,
+            totalUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+          } as TextStreamPart<T>);
+          stopStream();
+          return;
+        }
+        controller.enqueue(chunk);
+      },
+    });
 }
 
 /**
@@ -371,7 +427,7 @@ function transformMessagesForCompaction(messages: Message[]): Message[] {
  * would be excluded, throws `CompactionError`.
  *
  * ## Flow example
- * 1. Model hits context limit → `processStreamTextOutput` emits compaction marker
+ * 1. Model hits context limit → compaction transform emits compaction marker
  * 2. Next iteration calls this function
  * 3. Messages are truncated + compaction request appended
  * 4. Model calls `compact_conversation` with summary