From 432a1766afce98b2e596d4da271a2480cec1755c Mon Sep 17 00:00:00 2001 From: Andrew Pareles Date: Sun, 16 Feb 2025 20:50:07 -0800 Subject: [PATCH] anthropic tool use fix --- .../contrib/void/browser/chatThreadService.ts | 24 +++++++------ .../contrib/void/common/llmMessageTypes.ts | 17 +++++++--- .../contrib/void/common/toolsService.ts | 2 +- .../electron-main/llmMessage/anthropic.ts | 6 ++-- .../void/electron-main/llmMessage/openai.ts | 9 +++-- ...ssMessages.ts => preprocessLLMMessages.ts} | 34 +++++++++++++------ .../llmMessage/sendLLMMessage.ts | 8 ++--- .../void/electron-main/llmMessageChannel.ts | 2 +- 8 files changed, 65 insertions(+), 37 deletions(-) rename src/vs/workbench/contrib/void/electron-main/llmMessage/{processMessages.ts => preprocessLLMMessages.ts} (88%) diff --git a/src/vs/workbench/contrib/void/browser/chatThreadService.ts b/src/vs/workbench/contrib/void/browser/chatThreadService.ts index 81e4a434..03286d38 100644 --- a/src/vs/workbench/contrib/void/browser/chatThreadService.ts +++ b/src/vs/workbench/contrib/void/browser/chatThreadService.ts @@ -46,24 +46,25 @@ const defaultStaging: StagingInfo = { isBeingEdited: false, selections: [] } // WARNING: changing this format is a big deal!!!!!! need to migrate old format to new format on users' computers so people don't get errors. export type ChatMessage = | { + role: 'system'; + content: string; + displayContent?: undefined; + } | { role: 'user'; content: string | null; // content sent to the llm - allowed to be '', will be replaced with (empty) displayContent: string | null; // content displayed to user - allowed to be '', will be ignored selections: StagingSelectionItem[] | null; // the user's selection staging: StagingInfo | null - } - | { + } | { role: 'assistant'; - tool_calls?: { name: string, id: string, params: string }[]; + tool_calls?: { + name: string, + id: string, + params: string + }[]; content: string | null; // content received from LLM - allowed to be '', will be replaced with (empty) displayContent: string | null; // content displayed to user (this is the same as content for now) - allowed to be '', will be ignored - } - | { - role: 'system'; - content: string; - displayContent?: undefined; - } - | { + } | { role: 'tool'; name: string; // internal use params: string; // internal use @@ -325,7 +326,8 @@ class ChatThreadService extends Disposable implements IChatThreadService { onText: ({ fullText }) => { this._setStreamState(threadId, { messageSoFar: fullText }) }, - onFinalMessage: async ({ fullText, tools }) => { + onFinalMessage: async ({ fullText, toolCalls: tools }) => { + console.log('FINAL MESSAGE', fullText, tools) if ((tools?.length ?? 0) === 0) { this._finishStreamingTextMessage(threadId, fullText) diff --git a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts index e82da2cb..79960826 100644 --- a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts +++ b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts @@ -22,10 +22,6 @@ export const errorDetails = (fullError: Error | null): string | null => { return null } -export type OnText = (p: { newText: string, fullText: string }) => void -export type OnFinalMessage = (p: { fullText: string, tools?: { name: string, params: string, id: string, }[] }) => void // id is tool_use_id -export type OnError = (p: { message: string, fullError: Error | null }) => void -export type AbortRef = { current: (() => void) | null } export type LLMChatMessage = { role: 'system' | 'user'; @@ -41,6 +37,19 @@ export type LLMChatMessage = { id: string; } +export type LLMToolCallType = { + name: string; + params: string; + id: string; +} + + +export type OnText = (p: { newText: string, fullText: string }) => void +export type OnFinalMessage = (p: { fullText: string, toolCalls?: LLMToolCallType[] }) => void // id is tool_use_id +export type OnError = (p: { message: string, fullError: Error | null }) => void +export type AbortRef = { current: (() => void) | null } + + export const toLLMChatMessage = (c: ChatMessage): LLMChatMessage => { if (c.role === 'system' || c.role === 'user') { return { role: c.role, content: c.content ?? '(empty)' } diff --git a/src/vs/workbench/contrib/void/common/toolsService.ts b/src/vs/workbench/contrib/void/common/toolsService.ts index 32004ff2..0c7c7f82 100644 --- a/src/vs/workbench/contrib/void/common/toolsService.ts +++ b/src/vs/workbench/contrib/void/common/toolsService.ts @@ -142,7 +142,7 @@ export class ToolsService implements IToolsService { const queryBuilder = instantiationService.createInstance(QueryBuilder); - const parseObj = (s: string): { [s: string]: unknown } | null => { + const parseObj = (s: string): { [s: string]: unknown } | null => { try { const o = JSON.parse(s) return o diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts index 8d93b0f7..fec9dc07 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts @@ -7,7 +7,7 @@ import Anthropic from '@anthropic-ai/sdk'; import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; import { anthropicMaxPossibleTokens, developerInfoOfModelName, developerInfoOfProviderName } from '../../common/voidSettingsTypes.js'; import { InternalToolInfo } from '../../common/toolsService.js'; -import { addSystemMessageAndToolSupport } from './processMessages.js'; +import { addSystemMessageAndToolSupport } from './preprocessLLMMessages.js'; @@ -86,9 +86,9 @@ export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages: stream.on('finalMessage', (response) => { // stringify the response's content const content = response.content.map(c => c.type === 'text' ? c.text : '').join('\n\n') - // const tools = response.content.map(c => c.type === 'tool_use' ? { name: c.name, args: JSON.stringify(c.input), tool_use_id: c.id } : null).filter(c => !!c) + const tools = response.content.map(c => c.type === 'tool_use' ? { name: c.name, params: JSON.stringify(c.input), id: c.id } : null).filter(c => !!c) - onFinalMessage({ fullText: content, tools: [] }) + onFinalMessage({ fullText: content, toolCalls: tools }) }) stream.on('error', (error) => { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts index 86c41a9c..49dd0bfd 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts @@ -7,7 +7,7 @@ import OpenAI from 'openai'; import { _InternalModelListFnType, _InternalSendLLMFIMMessageFnType, _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; import { Model } from 'openai/resources/models.js'; import { InternalToolInfo } from '../../common/toolsService.js'; -import { addSystemMessageAndToolSupport } from './processMessages.js'; +import { addSystemMessageAndToolSupport } from './preprocessLLMMessages.js'; import { developerInfoOfModelName, developerInfoOfProviderName } from '../../common/voidSettingsTypes.js'; // import { parseMaxTokensStr } from './util.js'; @@ -192,7 +192,12 @@ export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages: me onText({ newText, fullText }); } - onFinalMessage({ fullText, tools: Object.keys(toolCallOfIndex).map(index => toolCallOfIndex[index]) }); + onFinalMessage({ + fullText, toolCalls: Object.keys(toolCallOfIndex).map(index => { + const tool = toolCallOfIndex[index] + return { name: tool.name, id: tool.id, params: tool.params } + }) + }); }) // when error/fail - this catches errors of both .create() and .then(for await) .catch(error => { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/processMessages.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts similarity index 88% rename from src/vs/workbench/contrib/void/electron-main/llmMessage/processMessages.ts rename to src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts index 2ae792fb..8bde8459 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/processMessages.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts @@ -5,7 +5,14 @@ import { developerInfoOfModelName, developerInfoOfProviderName, ProviderName } f import { deepClone } from '../../../../../base/common/objects.js'; - +export const parseObject = (args: unknown) => { + if (typeof args === 'object') + return args + if (typeof args === 'string') + try { return JSON.parse(args) } + catch (e) { return { args } } + return {} +} // no matter whether the model supports a system message or not (or what format it supports), add it in some way // also take into account tools if the model doesn't support tool use @@ -118,7 +125,7 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: } | { type: 'tool_use'; name: string; - input: string; + input: Record; id: string; })[] } | { @@ -127,7 +134,7 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: type: 'text'; text: string; } | { - type: 'tool_response'; + type: 'tool_result'; tool_use_id: string; content: string; })[] @@ -141,17 +148,22 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: if (currMsg.role !== 'tool') continue const prevMsg = 0 <= i - 1 && i - 1 <= newMessagesTools.length ? newMessagesTools[i - 1] : undefined - const nextMsg = 0 <= i + 1 && i + 1 <= newMessagesTools.length ? newMessagesTools[i + 1] : undefined if (prevMsg?.role === 'assistant') { - if (typeof prevMsg.content === 'string') prevMsg.content = [{ type: 'text', text: typeof prevMsg.content }] - prevMsg.content.push({ type: 'tool_use', name: currMsg.name, input: currMsg.params, id: currMsg.id }) + if (typeof prevMsg.content === 'string') prevMsg.content = [{ type: 'text', text: prevMsg.content }] + prevMsg.content.push({ type: 'tool_use', id: currMsg.id, name: currMsg.name, input: parseObject(currMsg.params) }) } - if (nextMsg?.role === 'user') { - if (typeof nextMsg.content === 'string') nextMsg.content = [{ type: 'text', text: typeof nextMsg.content }] - nextMsg.content.push({ type: 'tool_response', tool_use_id: currMsg.id, content: currMsg.content }) + + // turn each tool into a user message with tool results at the end + newMessagesTools[i] = { + role: 'user', + content: [ + ...[{ type: 'tool_result', tool_use_id: currMsg.id, content: currMsg.content }] as const, + ...currMsg.content ? [{ type: 'text', text: currMsg.content }] as const : [], + ] } } + finalMessages = newMessagesTools } @@ -212,7 +224,7 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: id: currMsg.id, function: { name: currMsg.name, - arguments: currMsg.params + arguments: JSON.stringify(currMsg.params) } }] } @@ -236,7 +248,7 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: console.log('SYSMG', separateSystemMessage) - console.log('FINAL MESSAGES', finalMessages) + console.log('FINAL MESSAGES', JSON.stringify(finalMessages, null, 2)) return { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts index e568d3b5..9d179fd8 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts @@ -62,10 +62,10 @@ export const sendLLMMessage = ({ _fullTextSoFar = fullText } - const onFinalMessage: OnFinalMessage = ({ fullText, tools }) => { + const onFinalMessage: OnFinalMessage = ({ fullText, toolCalls: tools }) => { if (_didAbort) return captureLLMEvent(`${loggingName} - Received Full Message`, { messageLength: fullText.length, duration: new Date().getMilliseconds() - submit_time.getMilliseconds() }) - onFinalMessage_({ fullText, tools }) + onFinalMessage_({ fullText, toolCalls: tools }) } const onError: OnError = ({ message: error, fullError }) => { @@ -103,11 +103,11 @@ export const sendLLMMessage = ({ case 'ollama': case 'groq': case 'gemini': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI FIM', tools: [] }) + if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI FIM', toolCalls: [] }) else /* */ sendOpenAIChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }); break; case 'anthropic': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM', tools: [] }) + if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM', toolCalls: [] }) else /* */ sendAnthropicChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }); break; default: diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts b/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts index 9db9a68f..333f3919 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts @@ -100,7 +100,7 @@ export class LLMMessageChannel implements IServerChannel { const mainThreadParams: SendLLMMessageParams = { ...params, onText: ({ newText, fullText }) => { this._onText_llm.fire({ requestId, newText, fullText }); }, - onFinalMessage: ({ fullText, tools }) => { this._onFinalMessage_llm.fire({ requestId, fullText, tools }); }, + onFinalMessage: ({ fullText, toolCalls: tools }) => { this._onFinalMessage_llm.fire({ requestId, fullText, toolCalls: tools }); }, onError: ({ message: error, fullError }) => { console.log('sendLLM: firing err'); this._onError_llm.fire({ requestId, message: error, fullError }); }, abortRef: this._abortRefOfRequestId_llm[requestId], }