🐛 fix: resolve agent runtime service error serialization producing [object Object] (#13704)

✨ feat: add remote snapshot fetch for agent-tracing CLI and fix error serialization

packages/agent-tracing/src/cli/inspect.ts

@@ -1,6 +1,12 @@
 import type { Command } from 'commander';
 
 import { FileSnapshotStore } from '../store/file-store';
+import {
+  buildRemoteUrl,
+  isOperationId,
+  loadBaseUrl,
+  RemoteSnapshotStore,
+} from '../store/remote-store';
 import type { ExecutionSnapshot, StepSnapshot } from '../types';
 import {
   renderDiff,
@@ -112,6 +118,35 @@ export function registerInspectCommand(program: Command) {
 
         if (traceId && isUrl(traceId)) {
           snapshot = await fetchSnapshotFromUrl(traceId);
+        } else if (traceId && isOperationId(traceId)) {
+          // Try local store first, then fetch from remote
+          const fileStore = new FileSnapshotStore();
+          snapshot = await fileStore.get(traceId);
+          if (!snapshot) {
+            const remoteStore = new RemoteSnapshotStore();
+            const cached = await remoteStore.getCached(traceId);
+            if (cached) {
+              snapshot = cached;
+              console.error(`✓ Loaded from cache: _remote/${traceId}.json`);
+            } else {
+              const baseUrl = await loadBaseUrl();
+              if (!baseUrl) {
+                console.error(
+                  'Remote fetch requires TRACING_BASE_URL.\n' +
+                    'Set it via:\n' +
+                    '  1. Environment variable: export TRACING_BASE_URL=https://...\n' +
+                    '  2. File: .agent-tracing/.env with TRACING_BASE_URL=https://...',
+                );
+                process.exit(1);
+              }
+              const url = buildRemoteUrl(baseUrl, traceId);
+              if (!url) {
+                console.error(`Failed to parse operation ID: ${traceId}`);
+                process.exit(1);
+              }
+              snapshot = await remoteStore.fetch(url, traceId);
+            }
+          }
         } else {
           const store = new FileSnapshotStore();
           snapshot = traceId ? await store.get(traceId) : await store.getLatest();

packages/agent-tracing/src/store/remote-store.ts

@@ -0,0 +1,109 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import type { ExecutionSnapshot } from '../types';
+
+const REMOTE_DIR = '_remote';
+const ENV_FILE = '.env';
+const DEFAULT_DIR = '.agent-tracing';
+
+/**
+ * Parse an operation ID to extract agentId and topicId for URL construction.
+ *
+ * Format: op_{timestamp}_agt_{agentHash}_tpc_{topicHash}_{suffix}
+ * Example: op_1775743208456_agt_6OfrfD6sRP2x_tpc_lMs3V4bpXa5x_9fRnPApi
+ */
+export function parseOperationId(opId: string): {
+  agentId: string;
+  operationId: string;
+  topicId: string;
+} | null {
+  const agtMatch = opId.match(/(agt_[A-Za-z0-9]+)/);
+  const tpcMatch = opId.match(/(tpc_[A-Za-z0-9]+)/);
+  if (!agtMatch || !tpcMatch) return null;
+  return { agentId: agtMatch[1], operationId: opId, topicId: tpcMatch[1] };
+}
+
+export function isOperationId(input: string): boolean {
+  return input.startsWith('op_') && input.includes('_agt_') && input.includes('_tpc_');
+}
+
+export function buildRemoteUrl(baseUrl: string, opId: string): string | null {
+  const parsed = parseOperationId(opId);
+  if (!parsed) return null;
+  const base = baseUrl.replace(/\/$/, '');
+  return `${base}/${parsed.agentId}/${parsed.topicId}/${parsed.operationId}.json`;
+}
+
+/**
+ * Load TRACING_BASE_URL from environment variable or .agent-tracing/.env file.
+ */
+export async function loadBaseUrl(rootDir?: string): Promise<string | null> {
+  // 1. Check environment variable
+  if (process.env.TRACING_BASE_URL) return process.env.TRACING_BASE_URL;
+
+  // 2. Check .agent-tracing/.env
+  const dir = path.resolve(rootDir ?? process.cwd(), DEFAULT_DIR);
+  const envPath = path.join(dir, ENV_FILE);
+  try {
+    const content = await fs.readFile(envPath, 'utf8');
+    for (const line of content.split('\n')) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith('#') || !trimmed) continue;
+      if (!trimmed.startsWith('TRACING_BASE_URL')) continue;
+      const eqIdx = trimmed.indexOf('=');
+      if (eqIdx < 0) continue;
+      const value = trimmed
+        .slice(eqIdx + 1)
+        .trim()
+        .replaceAll(/^["']|["']$/g, '');
+      if (value) return value;
+    }
+  } catch {
+    // no .env file
+  }
+  return null;
+}
+
+export class RemoteSnapshotStore {
+  private cacheDir: string;
+
+  constructor(rootDir?: string) {
+    this.cacheDir = path.resolve(rootDir ?? process.cwd(), DEFAULT_DIR, REMOTE_DIR);
+  }
+
+  async getCached(operationId: string): Promise<ExecutionSnapshot | null> {
+    try {
+      const filePath = path.join(this.cacheDir, `${operationId}.json`);
+      const content = await fs.readFile(filePath, 'utf8');
+      return JSON.parse(content) as ExecutionSnapshot;
+    } catch {
+      return null;
+    }
+  }
+
+  async fetch(url: string, operationId: string): Promise<ExecutionSnapshot> {
+    // Check cache first
+    const cached = await this.getCached(operationId);
+    if (cached) {
+      console.error(`✓ Loaded from cache: _remote/${operationId}.json`);
+      return cached;
+    }
+
+    // Download
+    console.error(`↓ Downloading: ${url}`);
+    const res = await fetch(url);
+    if (!res.ok) {
+      throw new Error(`Failed to fetch snapshot: ${res.status} ${res.statusText}\n  URL: ${url}`);
+    }
+    const snapshot = (await res.json()) as ExecutionSnapshot;
+
+    // Cache locally
+    await fs.mkdir(this.cacheDir, { recursive: true });
+    const filePath = path.join(this.cacheDir, `${operationId}.json`);
+    await fs.writeFile(filePath, JSON.stringify(snapshot, null, 2), 'utf8');
+    console.error(`✓ Cached to: _remote/${operationId}.json`);
+
+    return snapshot;
+  }
+}

src/server/services/agentRuntime/AgentRuntimeService.ts

@@ -980,10 +980,14 @@ export class AgentRuntimeService {
                 completionReason: reason,
                 error: stepResult.newState.error
                   ? {
-                      message: String(
-                        stepResult.newState.error.message ?? stepResult.newState.error,
+                      message:
+                        this.extractErrorMessage(stepResult.newState.error) ??
+                        JSON.stringify(stepResult.newState.error),
+                      type: String(
+                        stepResult.newState.error.type ??
+                          stepResult.newState.error.errorType ??
+                          'unknown',
                       ),
-                      type: String(stepResult.newState.error.type ?? 'unknown'),
                     }
                   : undefined,
                 model: partial.model,

src/server/services/agentRuntime/__tests__/extractErrorMessage.test.ts

@@ -0,0 +1,135 @@
+// @vitest-environment node
+import { describe, expect, it, vi } from 'vitest';
+
+import { AgentRuntimeService } from '../AgentRuntimeService';
+
+// Mock all heavy dependencies to isolate extractErrorMessage logic
+vi.mock('@/envs/app', () => ({ appEnv: { APP_URL: 'http://localhost:3010' } }));
+vi.mock('@/database/models/message', () => ({
+  MessageModel: vi.fn().mockImplementation(() => ({})),
+}));
+vi.mock('@/server/modules/AgentRuntime', () => ({
+  AgentRuntimeCoordinator: vi.fn().mockImplementation(() => ({})),
+  createStreamEventManager: vi.fn(() => ({})),
+}));
+vi.mock('@/server/modules/AgentRuntime/RuntimeExecutors', () => ({
+  createRuntimeExecutors: vi.fn(() => ({})),
+}));
+vi.mock('@/server/services/mcp', () => ({ mcpService: {} }));
+vi.mock('@/server/services/queue', () => ({
+  QueueService: vi.fn().mockImplementation(() => ({
+    getImpl: vi.fn(() => ({})),
+    scheduleMessage: vi.fn(),
+  })),
+}));
+vi.mock('@/server/services/queue/impls', () => ({
+  LocalQueueServiceImpl: class {},
+}));
+vi.mock('@/server/services/toolExecution', () => ({
+  ToolExecutionService: vi.fn().mockImplementation(() => ({})),
+}));
+vi.mock('@/server/services/toolExecution/builtin', () => ({
+  BuiltinToolsExecutor: vi.fn().mockImplementation(() => ({})),
+}));
+vi.mock('@lobechat/builtin-tools/dynamicInterventionAudits', () => ({
+  dynamicInterventionAudits: [],
+}));
+
+describe('AgentRuntimeService.extractErrorMessage', () => {
+  const createService = () => {
+    return new AgentRuntimeService({} as any, 'user-1', { queueService: null });
+  };
+
+  it('should extract message from ChatCompletionErrorPayload (InsufficientBudgetForModel)', () => {
+    const service = createService();
+    const error = {
+      error: { message: 'Budget exceeded' },
+      errorType: 'InsufficientBudgetForModel',
+      provider: 'lobehub',
+      _responseBody: { provider: 'lobehub' },
+    };
+
+    const result = (service as any).extractErrorMessage(error);
+    expect(result).toBe('Budget exceeded');
+  });
+
+  it('should extract message from ChatCompletionErrorPayload (InvalidProviderAPIKey)', () => {
+    const service = createService();
+    const error = {
+      endpoint: 'https://cdn.example.com/v1',
+      error: {
+        code: '',
+        error: { code: '', message: '无效的令牌', type: 'new_api_error' },
+        message: '无效的令牌',
+        status: 401,
+        type: 'new_api_error',
+      },
+      errorType: 'InvalidProviderAPIKey',
+      provider: 'openai',
+    };
+
+    const result = (service as any).extractErrorMessage(error);
+    expect(result).toBe('无效的令牌');
+  });
+
+  it('should extract message from formatted ChatMessageError with body.error.message', () => {
+    const service = createService();
+    const error = {
+      body: { error: { message: 'Rate limit exceeded' } },
+      message: 'InvalidProviderAPIKey',
+      type: 'InvalidProviderAPIKey',
+    };
+
+    const result = (service as any).extractErrorMessage(error);
+    expect(result).toBe('Rate limit exceeded');
+  });
+
+  it('should extract message from ChatMessageError with body.message', () => {
+    const service = createService();
+    const error = {
+      body: { message: 'Something went wrong' },
+      message: 'error',
+      type: 'InternalServerError',
+    };
+
+    const result = (service as any).extractErrorMessage(error);
+    expect(result).toBe('Something went wrong');
+  });
+
+  it('should fallback to error.message when body is absent', () => {
+    const service = createService();
+    const error = { message: 'Connection timeout', type: 'NetworkError' };
+
+    const result = (service as any).extractErrorMessage(error);
+    expect(result).toBe('Connection timeout');
+  });
+
+  it('should fallback to errorType when message is "error"', () => {
+    const service = createService();
+    const error = { message: 'error', errorType: 'InsufficientBudgetForModel' };
+
+    const result = (service as any).extractErrorMessage(error);
+    expect(result).toBe('InsufficientBudgetForModel');
+  });
+
+  it('should return undefined for null/undefined', () => {
+    const service = createService();
+    expect((service as any).extractErrorMessage(null)).toBeUndefined();
+    expect((service as any).extractErrorMessage(undefined)).toBeUndefined();
+  });
+
+  it('should never return [object Object] for nested error objects', () => {
+    const service = createService();
+    const error = {
+      error: { message: 'Budget exceeded' },
+      errorType: 'InsufficientBudgetForModel',
+      provider: 'lobehub',
+      _responseBody: { provider: 'lobehub' },
+    };
+
+    const result = (service as any).extractErrorMessage(error);
+    expect(result).not.toBe('[object Object]');
+    expect(typeof result).toBe('string');
+    expect(result).toBe('Budget exceeded');
+  });
+});