fix(agents): fallback on upstream provider errors

openclaw · altaywtf · Jun 29, 2026 · Jun 21, 2026 · Jun 29, 2026 · Jun 29, 2026
commit e67b78af640f5030a25345cf8d1993d970c0398e
diff --git a/src/agents/embedded-agent-helpers.formatassistanterrortext.test.ts b/src/agents/embedded-agent-helpers.formatassistanterrortext.test.ts
@@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest";
 import { MALFORMED_STREAMING_FRAGMENT_ERROR_MESSAGE } from "../shared/assistant-error-format.js";
 import {
   BILLING_ERROR_USER_MESSAGE,
+  classifyAssistantFailoverReason,
   formatBillingErrorMessage,
   formatAssistantErrorText,
   formatUserFacingAssistantErrorText,
@@ -116,6 +117,21 @@ describe("formatAssistantErrorText", () => {
     );
     expect(formatAssistantErrorText(msg)).toBe("LLM error server_error: Something exploded");
   });
+  it("classifies provider upstream_error payloads as server errors for fallback", () => {
+    const msg = makeAssistantMessageFixture({
+      errorMessage: "Upstream request failed",
+      errorType: "upstream_error",
+    });
+
+    expect(classifyAssistantFailoverReason(msg, { provider: "openai" })).toBe("server_error");
+    expect(
+      classifyAssistantFailoverReason(
+        makeAssistantError(
+          '{"error":{"message":"Upstream request failed","type":"upstream_error","param":"","code":null}}',
+        ),
+      ),
+    ).toBe("server_error");
+  });
   it("uses generic user-facing copy for escaped structured provider messages", () => {
     // The internal formatter keeps detail for logs, while user-facing text must
     // not expose arbitrary provider-controlled structured payload content.

diff --git a/src/agents/embedded-agent-helpers/errors.ts b/src/agents/embedded-agent-helpers/errors.ts
@@ -922,6 +922,19 @@ function classifyFailoverReasonFromCode(raw: string | undefined): FailoverReason
   }
 }
 
+function classifyFailoverReasonFromErrorType(raw: string | undefined): FailoverReason | null {
+  const normalized = normalizeOptionalLowercaseString(raw);
+  switch (normalized) {
+    case "server_error":
+    case "upstream_error":
+      return "server_error";
+    case "overloaded_error":
+      return "overloaded";
+    default:
+      return null;
+  }
+}
+
 function isProvider(provider: string | undefined, match: string): boolean {
   const normalized = normalizeOptionalLowercaseString(provider);
   return Boolean(normalized && normalized.includes(match));
@@ -1181,9 +1194,14 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
           errorType: signal.errorType,
         })
       : null;
+  const errorTypeReason = classifyFailoverReasonFromErrorType(signal.errorType);
+  const genericErrorTypeClassification = errorTypeReason
+    ? toReasonClassification(errorTypeReason)
+    : null;
   const effectiveMessageClassification = providerPluginReason
     ? toReasonClassification(providerPluginReason)
-    : mergeMessageAndDetailClassification(messageClassification, detailClassification);
+    : (mergeMessageAndDetailClassification(messageClassification, detailClassification) ??
+      genericErrorTypeClassification);
   const codeReason = classifyFailoverReasonFromCode(signal.code);
   if (codeReason === "auth_permanent") {
     return toReasonClassification(codeReason);
@@ -1658,8 +1676,17 @@ function isStructuredServerErrorMessage(raw: string): boolean {
   if (!raw) {
     return false;
   }
+  const parsedType = normalizeOptionalLowercaseString(parseApiErrorInfo(raw)?.type);
+  if (parsedType === "server_error" || parsedType === "upstream_error") {
+    return true;
+  }
   const value = normalizeLowercaseStringOrEmpty(raw);
-  return value.includes('"type":"server_error"') || value.includes('"code":"server_error"');
+  return (
+    value.includes('"type":"server_error"') ||
+    value.includes('"code":"server_error"') ||
+    value.includes('"type":"upstream_error"') ||
+    value.includes('"code":"upstream_error"')
+  );
 }
 
 export function parseImageDimensionError(raw: string): {

diff --git a/src/agents/embedded-agent-runner/result-fallback-classifier.test.ts b/src/agents/embedded-agent-runner/result-fallback-classifier.test.ts
@@ -50,6 +50,62 @@ describe("classifyEmbeddedAgentRunResultForModelFallback", () => {
     });
   });
 
+  it("classifies structured provider upstream_error payloads as fallback-worthy", () => {
+    const rawError =
+      '{"error":{"message":"Upstream request failed","type":"upstream_error","param":"","code":null}}';
+
+    const result = classifyEmbeddedAgentRunResultForModelFallback({
+      provider: "openai-compatible",
+      model: "primary-model",
+      result: {
+        payloads: [
+          {
+            isError: true,
+            text: rawError,
+          },
+        ],
+        meta: {
+          durationMs: 42,
+        },
+      },
+    });
+
+    expect(result).toEqual({
+      message: `openai-compatible/primary-model ended with a provider error: ${rawError}`,
+      reason: "server_error",
+      code: "embedded_error_payload",
+      rawError,
+    });
+  });
+
+  it("classifies structured provider overloaded_error payloads as fallback-worthy", () => {
+    const rawError =
+      '{"error":{"message":"Provider overloaded","type":"overloaded_error","param":"","code":null}}';
+
+    const result = classifyEmbeddedAgentRunResultForModelFallback({
+      provider: "openai-compatible",
+      model: "primary-model",
+      result: {
+        payloads: [
+          {
+            isError: true,
+            text: rawError,
+          },
+        ],
+        meta: {
+          durationMs: 42,
+        },
+      },
+    });
+
+    expect(result).toEqual({
+      message: `openai-compatible/primary-model ended with a provider error: ${rawError}`,
+      reason: "overloaded",
+      code: "embedded_error_payload",
+      rawError,
+    });
+  });
+
   it("classifies generic external runner failure text as fallback-worthy", () => {
     const result = classifyEmbeddedAgentRunResultForModelFallback({
       provider: "claude-cli",

diff --git a/src/agents/embedded-agent-runner/result-fallback-classifier.ts b/src/agents/embedded-agent-runner/result-fallback-classifier.ts
@@ -13,6 +13,11 @@ import {
 } from "./delivery-evidence.js";
 import type { EmbeddedAgentRunResult } from "./types.js";
 
+type ProviderErrorPayloadFailoverReason = Extract<
+  FailoverReason,
+  "auth" | "auth_permanent" | "billing" | "rate_limit" | "server_error" | "overloaded"
+>;
+
 /**
  * Classifies embedded-agent terminal results for model fallback decisions.
  *
@@ -146,11 +151,10 @@ function classifyHarnessResult(params: {
   }
 }
 
-/** Maps provider error payloads to fallback-safe business reasons. */
-function classifyBusinessDenialErrorPayloadReason(
+function classifyProviderErrorPayloadReason(
   errorText: string,
   provider: string,
-): Extract<FailoverReason, "auth" | "auth_permanent" | "billing" | "rate_limit"> | null {
+): ProviderErrorPayloadFailoverReason | null {
   if (!errorText.trim()) {
     return null;
   }
@@ -160,6 +164,8 @@ function classifyBusinessDenialErrorPayloadReason(
     case "auth_permanent":
     case "billing":
     case "rate_limit":
+    case "server_error":
+    case "overloaded":
       return failoverReason;
     default:
       return null;
@@ -252,7 +258,7 @@ export function classifyEmbeddedAgentRunResultForModelFallback(params: {
     .filter((payload) => payload?.isError === true)
     .map((payload) => (typeof payload.text === "string" ? payload.text : ""))
     .join("\n");
-  const failoverReason = classifyBusinessDenialErrorPayloadReason(errorText, params.provider);
+  const failoverReason = classifyProviderErrorPayloadReason(errorText, params.provider);
   if (failoverReason) {
     return {
       message: `${params.provider}/${params.model} ended with a provider error: ${errorText}`,