From 131493b5e17581dc3f043fa875b3d2824bd8fe20 Mon Sep 17 00:00:00 2001 From: Andrew Pareles Date: Sun, 16 Feb 2025 00:43:46 -0800 Subject: [PATCH] tool structure --- .../contrib/void/browser/chatThreadService.ts | 122 +++++----- .../contrib/void/browser/editCodeService.ts | 13 +- .../react/src/void-settings-tsx/Settings.tsx | 39 +-- .../contrib/void/common/llmMessageTypes.ts | 27 ++- .../void/common/voidSettingsService.ts | 4 +- .../contrib/void/common/voidSettingsTypes.ts | 43 ++-- .../void/electron-main/llmMessage/_old.ts | 96 ++++++++ .../electron-main/llmMessage/addSupport.ts | 177 ++++++++++++++ .../electron-main/llmMessage/anthropic.ts | 17 +- .../void/electron-main/llmMessage/gemini.ts | 43 ---- .../void/electron-main/llmMessage/ollama.ts | 137 +++++------ .../void/electron-main/llmMessage/openai.ts | 54 +++-- .../llmMessage/sendLLMMessage.ts | 224 +----------------- 13 files changed, 530 insertions(+), 466 deletions(-) create mode 100644 src/vs/workbench/contrib/void/electron-main/llmMessage/_old.ts create mode 100644 src/vs/workbench/contrib/void/electron-main/llmMessage/addSupport.ts delete mode 100644 src/vs/workbench/contrib/void/electron-main/llmMessage/gemini.ts diff --git a/src/vs/workbench/contrib/void/browser/chatThreadService.ts b/src/vs/workbench/contrib/void/browser/chatThreadService.ts index 98165ce0..ec1719b8 100644 --- a/src/vs/workbench/contrib/void/browser/chatThreadService.ts +++ b/src/vs/workbench/contrib/void/browser/chatThreadService.ts @@ -15,6 +15,7 @@ import { ILLMMessageService } from '../common/llmMessageService.js'; import { IModelService } from '../../../../editor/common/services/model.js'; import { chat_userMessage, chat_systemMessage } from './prompt/prompts.js'; import { IToolsService, ToolName, voidTools } from '../common/toolsService.js'; +import { toLLMChatMessage } from '../common/llmMessageTypes.js'; // one of the square items that indicates a selection in a chat bubble (NOT a file, a Selection of text) export type CodeSelection = { @@ -53,6 +54,7 @@ export type ChatMessage = } | { role: 'assistant'; + tool_calls?: { name: string, id: string, params: string }[]; content: string | null; // content received from LLM - allowed to be '', will be replaced with (empty) displayContent: string | null; // content displayed to user (this is the same as content for now) - allowed to be '', will be ignored } @@ -65,7 +67,7 @@ export type ChatMessage = role: 'tool'; name: string; // internal use params: string | null; // internal use - tool_use_id: string; // apis require this + id: string; // apis require this tool use id content: string | null; // summary of the tool to the LLM displayContent: string | null; // text message of result } @@ -296,82 +298,64 @@ class ChatThreadService extends Disposable implements IChatThreadService { // agent loop - let shouldContinue = false - do { - shouldContinue = false + const agentLoop = async () => { - let res_: () => void - const awaitable = new Promise((res, rej) => { res_ = res }) + let shouldContinue = false + do { + shouldContinue = false - const llmCancelToken = this._llmMessageService.sendLLMMessage({ - messagesType: 'chatMessages', - useProviderFor: 'Ctrl+L', - logging: { loggingName: `Agent` }, - messages: [ - { role: 'system', content: chat_systemMessage }, - ...this.getCurrentThread().messages.map(m => ({ ...m, content: m.content || '(empty model output)' })), - ], - tools: [voidTools['read_file']], // TODO!!!!! make this change on agent | chat | search + let res_: () => void + const awaitable = new Promise((res, rej) => { res_ = res }) - onText: ({ fullText }) => { - this._setStreamState(threadId, { messageSoFar: fullText }) - }, - onFinalMessage: async ({ fullText, tools }) => { - if (tools.length === 0) { - this._finishStreamingTextMessage(threadId, fullText) - } - else { - for (const tool of tools) { - if (!(tool.name in this._toolsService.toolFns)) { - this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, tool_use_id: tool.tool_use_id, content: `Error: This tool was not recognized, so it was not called.`, displayContent: `Error: tool not recognized.`, }) - } - else { - const toolName = tool.name as ToolName - const toolResult = await this._toolsService.toolFns[toolName](tool.args) - const string = this._toolsService.toolResultToString[toolName](toolResult as any) // typescript is so bad it doesn't even couple the type of ToolResult with the type of the function being called here - this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, tool_use_id: tool.tool_use_id, content: string, displayContent: string, }) - shouldContinue = true + const llmCancelToken = this._llmMessageService.sendLLMMessage({ + messagesType: 'chatMessages', + useProviderFor: 'Ctrl+L', + logging: { loggingName: `Agent` }, + messages: [ + { role: 'system', content: chat_systemMessage }, + ...this.getCurrentThread().messages.map(m => (toLLMChatMessage(m))), + ], + + // TODO!!!!! make this change on 'agent' | 'chat' + tools: Object.keys(voidTools).map(toolName => voidTools[toolName as ToolName]), + + onText: ({ fullText }) => { + this._setStreamState(threadId, { messageSoFar: fullText }) + }, + onFinalMessage: async ({ fullText, tools }) => { + if (tools.length === 0) { + this._finishStreamingTextMessage(threadId, fullText) + } + else { + for (const tool of tools) { + if (!(tool.name in this._toolsService.toolFns)) { + this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, id: tool.id, content: `Error: This tool was not recognized, so it was not called.`, displayContent: `Error: tool not recognized.`, }) + } + else { + const toolName = tool.name as ToolName + const toolResult = await this._toolsService.toolFns[toolName](tool.args) + const string = this._toolsService.toolResultToString[toolName](toolResult as any) // typescript is so bad it doesn't even couple the type of ToolResult with the type of the function being called here + this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, id: tool.id, content: string, displayContent: string, }) + shouldContinue = true + } } } - } - res_() - }, - onError: (error) => { - this._finishStreamingTextMessage(threadId, this.streamState[threadId]?.messageSoFar ?? '', error) - res_() - }, - }) - if (llmCancelToken === null) return - this._setStreamState(threadId, { streamingToken: llmCancelToken }) + res_() + }, + onError: (error) => { + this._finishStreamingTextMessage(threadId, this.streamState[threadId]?.messageSoFar ?? '', error) + res_() + }, + }) + if (llmCancelToken === null) break + this._setStreamState(threadId, { streamingToken: llmCancelToken }) - await awaitable + await awaitable + } + while (shouldContinue); } - while (shouldContinue); - - - - // const llmCancelToken = this._llmMessageService.sendLLMMessage({ - // messagesType: 'chatMessages', - // logging: { loggingName: 'Chat' }, - // useProviderFor: 'Ctrl+L', - // messages: [ - // { role: 'system', content: chat_systemMessage }, - // ...this.getCurrentThread().messages.map(m => ({ role: m.role, content: m.content || '(empty model output)' })), - // ], - // onText: ({ newText, fullText }) => { - // this._setStreamState(threadId, { messageSoFar: fullText }) - // }, - // onFinalMessage: ({ fullText: content }) => { - // this._finishStreaming(threadId, content) - // }, - // onError: (error) => { - // this._finishStreaming(threadId, this.streamState[threadId]?.messageSoFar ?? '', error) - // }, - - // }) - // if (llmCancelToken === null) return - // this._setStreamState(threadId, { streamingToken: llmCancelToken }) + agentLoop() // DO NOT AWAIT THIS, this fn should resolve when ready to clear inputs } diff --git a/src/vs/workbench/contrib/void/browser/editCodeService.ts b/src/vs/workbench/contrib/void/browser/editCodeService.ts index 36173d68..c2828084 100644 --- a/src/vs/workbench/contrib/void/browser/editCodeService.ts +++ b/src/vs/workbench/contrib/void/browser/editCodeService.ts @@ -39,7 +39,7 @@ import { Emitter } from '../../../../base/common/event.js'; import { VOID_OPEN_SETTINGS_ACTION_ID } from './voidSettingsPane.js'; import { ICommandService } from '../../../../platform/commands/common/commands.js'; import { ILLMMessageService } from '../common/llmMessageService.js'; -import { LLMChatMessage, _InternalLLMChatMessage, errorDetails } from '../common/llmMessageTypes.js'; +import { LLMChatMessage, errorDetails } from '../common/llmMessageTypes.js'; import { IMetricsService } from '../common/metricsService.js'; import { VSReadFile } from './helpers/readFile.js'; @@ -1178,13 +1178,16 @@ class EditCodeService extends Disposable implements IEditCodeService { private async _initializeSearchAndReplaceStream({ applyStr }: { applyStr: string }) { + console.log('SEARCHREPLACE') const uri_ = this._getActiveEditorURI() if (!uri_) return const uri = uri_ + console.log('/* AAAA */') // generate search/replace block text const fileContents = await VSReadFile(this._modelService, uri) if (fileContents === null) return + console.log('/* BBB*/') @@ -1236,9 +1239,7 @@ class EditCodeService extends Disposable implements IEditCodeService { - - - // TODO turn this into a service and provide it + // TODO!!! turn this into a service and provide it streamRequestIdRef.current = this._llmMessageService.sendLLMMessage({ messagesType: 'chatMessages', useProviderFor: 'Apply', @@ -1304,6 +1305,8 @@ class EditCodeService extends Disposable implements IEditCodeService { this._refreshStylesAndDiffsInURI(uri) }, onFinalMessage: async ({ fullText }) => { + console.log('/* ONFIN */', fullText) + // 1. wait 500ms and fix lint errors - call lint error workflow // (update react state to say "Fixing errors") const blocks = extractSearchReplaceBlocks(fullText) @@ -1322,6 +1325,8 @@ class EditCodeService extends Disposable implements IEditCodeService { onDone(false) }, onError: (e) => { + console.log('/* ERRRRRR */') + console.log('ERROR', e); onDone(true) }, diff --git a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx index 62511563..b1079587 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx @@ -395,7 +395,16 @@ export const AIInstructionsBox = () => { export const FeaturesTab = () => { return <> -

Local Providers

+

Models

+ + + + + + + + +

Local Providers

{/*

{`Keep your data private by hosting AI locally on your computer.`}

*/} {/*

{`Instructions:`}

*/} {/*

{`Void can access any model that you host locally. We automatically detect your local models by default.`}

*/} @@ -420,13 +429,20 @@ export const FeaturesTab = () => { -

Models

+ + +

Feature Options

- - - - + {featureNames.map(featureName => +
+

{displayInfoOfFeatureName(featureName)}

+ +
+ )} + } @@ -588,17 +604,6 @@ const GeneralTab = () => { -
-

Model Selection

- {featureNames.map(featureName => -
-

{displayInfoOfFeatureName(featureName)}

- -
- )} -
} diff --git a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts index 27ce34c1..0af2e31e 100644 --- a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts +++ b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts @@ -3,6 +3,7 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ +import { ChatMessage } from '../browser/chatThreadService.js' import { InternalToolInfo } from './toolsService.js' import { FeatureName, ProviderName, SettingsOfProvider } from './voidSettingsTypes.js' @@ -22,7 +23,7 @@ export const errorDetails = (fullError: Error | null): string | null => { } export type OnText = (p: { newText: string, fullText: string }) => void -export type OnFinalMessage = (p: { fullText: string, tools: { name: string, args: string, tool_use_id: string, }[] }) => void +export type OnFinalMessage = (p: { fullText: string, tools: { name: string, args: string, id: string, }[] }) => void // id is tool_use_id export type OnError = (p: { message: string, fullError: Error | null }) => void export type AbortRef = { current: (() => void) | null } @@ -30,20 +31,32 @@ export type LLMChatMessage = { role: 'system' | 'user'; content: string; } | { - role: 'tool'; - tool_use_id: string; + role: 'assistant', + tool_calls?: { name: string, id: string, params: string }[]; content: string; } | { - role: 'assistant', - tool_calls?: { name: string, tool_use_id: string, params: string }[]; + role: 'tool'; + id: string; content: string; } +export const toLLMChatMessage = (c: ChatMessage): LLMChatMessage => { + if (c.role === 'system' || c.role === 'user') { + return { role: c.role, content: c.content ?? '(empty)' } + } + else if (c.role === 'assistant') + return { role: c.role, tool_calls: c.tool_calls, content: c.content ?? '(empty model output)' } + else if (c.role === 'tool') + return { role: c.role, id: c.id, content: c.content ?? '(empty output)' } + else { + throw 1 + } +} export type _InternalLLMChatMessage = { role: any; - tool_use_id?: any; + id?: any; content: string; } @@ -112,7 +125,7 @@ export type _InternalSendLLMChatMessageFnType = ( tools?: InternalToolInfo[], - messages: _InternalLLMChatMessage[]; + messages: LLMChatMessage[]; } ) => void diff --git a/src/vs/workbench/contrib/void/common/voidSettingsService.ts b/src/vs/workbench/contrib/void/common/voidSettingsService.ts index af68ea38..eac87692 100644 --- a/src/vs/workbench/contrib/void/common/voidSettingsService.ts +++ b/src/vs/workbench/contrib/void/common/voidSettingsService.ts @@ -11,7 +11,7 @@ import { registerSingleton, InstantiationType } from '../../../../platform/insta import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js'; import { IMetricsService } from './metricsService.js'; -import { defaultSettingsOfProvider, FeatureName, ProviderName, ModelSelectionOfFeature, SettingsOfProvider, SettingName, providerNames, ModelSelection, modelSelectionsEqual, featureNames, VoidModelInfo, GlobalSettings, GlobalSettingName, defaultGlobalSettings, displayInfoOfProviderName, defaultProviderSettings, developerInfoOfModelName, modelInfoOfAutodetectedModelNames } from './voidSettingsTypes.js'; +import { defaultSettingsOfProvider, FeatureName, ProviderName, ModelSelectionOfFeature, SettingsOfProvider, SettingName, providerNames, ModelSelection, modelSelectionsEqual, featureNames, VoidModelInfo, GlobalSettings, GlobalSettingName, defaultGlobalSettings, defaultProviderSettings, developerInfoOfModelName, modelInfoOfAutodetectedModelNames } from './voidSettingsTypes.js'; const STORAGE_KEY = 'void.settingsServiceStorage' @@ -89,7 +89,7 @@ const _updatedValidatedState = (state: Omit) // update model options let newModelOptions: ModelOption[] = [] for (const providerName of providerNames) { - const providerTitle = displayInfoOfProviderName(providerName).title.toLowerCase() // looks better lowercase, best practice to not use raw providerName + const providerTitle = providerName // displayInfoOfProviderName(providerName).title.toLowerCase() // looks better lowercase, best practice to not use raw providerName if (!newSettingsOfProvider[providerName]._didFillInProviderSettings) continue // if disabled, don't display model options for (const { modelName, isHidden } of newSettingsOfProvider[providerName].models) { if (isHidden) continue diff --git a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts index da5aa81b..9bc1638d 100644 --- a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts +++ b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts @@ -25,9 +25,7 @@ export type DeveloperInfoAtModel = { } export type DeveloperInfoAtProvider = { - separateSystemMessage?: boolean; - toolsGoInRole?: boolean; // whether to do {role:'tool'} or {role:'user' tool:...} - modelOverrides?: Partial; // any overrides for models that a provider might have (e.g. if a provider always supports tool use, even if we don't recognize the model we can set tools to true) + overrideSettingsForAllModels?: Partial; // any overrides for models that a provider might have (e.g. if a provider always supports tool use, even if we don't recognize the model we can set tools to true) } @@ -99,37 +97,34 @@ export function recognizedModelOfModelName(modelName: string): RecognizedModelNa const developerInfoAtProvider: { [providerName in ProviderName]: DeveloperInfoAtProvider } = { 'anthropic': { - separateSystemMessage: true, - toolsGoInRole: false, - modelOverrides: { + overrideSettingsForAllModels: { + supportsSystemMessage: 'system', supportsTools: true, + supportsAutocompleteFIM: false, + supportsStreaming: true, } }, 'deepseek': { - separateSystemMessage: true, - }, - 'openAI': { - separateSystemMessage: false, - toolsGoInRole: true, - }, - 'gemini': { - separateSystemMessage: true, - toolsGoInRole: false - }, - 'mistral': { - separateSystemMessage: true, - }, - 'groq': { - separateSystemMessage: true, + overrideSettingsForAllModels: { + supportsSystemMessage: false, + supportsTools: false, + supportsAutocompleteFIM: false, + supportsStreaming: true, + } }, 'ollama': { - separateSystemMessage: false, }, 'openRouter': { - separateSystemMessage: true, }, 'openAICompatible': { - separateSystemMessage: true, + }, + 'openAI': { + }, + 'gemini': { + }, + 'mistral': { + }, + 'groq': { }, } export const developerInfoOfProviderName = (providerName: ProviderName): Partial => { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/_old.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/_old.ts new file mode 100644 index 00000000..e1e90245 --- /dev/null +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/_old.ts @@ -0,0 +1,96 @@ +// /*-------------------------------------------------------------------------------------- +// * Copyright 2025 Glass Devtools, Inc. All rights reserved. +// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. +// *--------------------------------------------------------------------------------------*/ + +// import Groq from 'groq-sdk'; +// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; + +// // Groq +// export const sendGroqChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { +// let fullText = ''; + +// const thisConfig = settingsOfProvider.groq + +// const groq = new Groq({ +// apiKey: thisConfig.apiKey, +// dangerouslyAllowBrowser: true +// }); + +// await groq.chat.completions +// .create({ +// messages: messages, +// model: modelName, +// stream: true, +// }) +// .then(async response => { +// _setAborter(() => response.controller.abort()) +// // when receive text +// for await (const chunk of response) { +// const newText = chunk.choices[0]?.delta?.content || ''; +// fullText += newText; +// onText({ newText, fullText }); +// } + +// onFinalMessage({ fullText, tools: [] }); +// }) +// .catch(error => { +// onError({ message: error + '', fullError: error }); +// }) + + +// }; + + + +// /*-------------------------------------------------------------------------------------- +// * Copyright 2025 Glass Devtools, Inc. All rights reserved. +// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. +// *--------------------------------------------------------------------------------------*/ + +// import { Mistral } from '@mistralai/mistralai'; +// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; + +// // Mistral +// export const sendMistralChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { +// let fullText = ''; + +// const thisConfig = settingsOfProvider.mistral; + +// const mistral = new Mistral({ +// apiKey: thisConfig.apiKey, +// }) + +// await mistral.chat +// .stream({ +// messages: messages, +// model: modelName, +// stream: true, +// }) +// .then(async response => { +// // Mistral has a really nonstandard API - no interrupt and weird stream types +// _setAborter(() => { console.log('Mistral does not support interrupts! Further messages will just be ignored.') }); +// // when receive text +// for await (const chunk of response) { +// const c = chunk.data.choices[0].delta.content || '' +// const newText = ( +// typeof c === 'string' ? c +// : c?.map(c => c.type === 'text' ? c.text : c.type).join('\n') +// ) +// fullText += newText; +// onText({ newText, fullText }); +// } + +// onFinalMessage({ fullText, tools: [] }); +// }) +// .catch(error => { +// onError({ message: error + '', fullError: error }); +// }) +// } + + + + + + + diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/addSupport.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/addSupport.ts new file mode 100644 index 00000000..ef9dfdd5 --- /dev/null +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/addSupport.ts @@ -0,0 +1,177 @@ +import { _InternalLLMChatMessage, LLMChatMessage } from '../../common/llmMessageTypes.js'; +import { DeveloperInfoAtModel, developerInfoOfModelName, developerInfoOfProviderName, ProviderName } from '../../common/voidSettingsTypes.js'; + + +// no matter whether the model supports a system message or not (or what format it supports), add it in some way +// also take into account tools if the model doesn't support tool use +export const addSystemMessageAndToolSupport = (modelName: string, providerName: ProviderName, messages_: LLMChatMessage[], { separateSystemMessage }: { separateSystemMessage: boolean }): { separateSystemMessageStr?: string, messages: _InternalLLMChatMessage[], devInfo: DeveloperInfoAtModel } => { + + const messages: _InternalLLMChatMessage[] = messages_.map(m => ({ ...m, content: m.content.trim(), })) + + const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName) + const devInfo = developerInfoOfModelName(modelName, overrideSettingsForAllModels) + const { supportsSystemMessage } = devInfo + + // 1. SYSTEM MESSAGE + // find system messages and concatenate them + let systemMessageStr = messages + .filter(msg => msg.role === 'system') + .map(msg => msg.content) + .join('\n') || undefined; + + let separateSystemMessageStr: string | undefined = undefined + + // remove all system messages + const newMessages: _InternalLLMChatMessage[] = messages.filter(msg => msg.role !== 'system') + + + // if (!supportsTools) { + // if (!systemMessageStr) systemMessageStr = '' + // systemMessageStr += '' // TODO!!! add tool use system message here + // } + + + if (systemMessageStr) { + // if supports system message + if (supportsSystemMessage) { + if (separateSystemMessage) + separateSystemMessageStr = systemMessageStr + else { + newMessages.unshift({ role: supportsSystemMessage, content: systemMessageStr }) // add new first message + } + } + // if does not support system message + else { + if (supportsSystemMessage) { + if (newMessages.length === 0) + newMessages.push({ role: 'user', content: systemMessageStr }) + // add system mesasges to first message (should be a user message) + else { + const newFirstMessage = { + role: newMessages[0].role, + content: ('' + + '\n' + + systemMessageStr + + '\n' + + '\n' + + newMessages[0].content + ) + } + newMessages.splice(0, 1) // delete first message + newMessages.unshift(newFirstMessage) // add new first message + } + } + } + } + + + return { + separateSystemMessageStr, + messages: newMessages, + devInfo, + } +} + + + + + +// const { maxTokens, supportsTools, supportsAutocompleteFIM, supportsStreaming, } = developerInfoOfModelName(recognizedModel) + + + + + + +// let index = 0; +// while (index < newMessages.length) { + +// merge tool with the previous assistant and the following user message + +// take prev message and add +/* +openai assistant message will have: https://platform.openai.com/docs/guides/function-calling#function-calling-steps +"tool_calls":[ +{ +"id": "call_12345xyz", +"type": "function", +"function": { +"name": "get_weather", +"arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}" +} +}] + +openai user response will be: +{ +"role": "tool", +"tool_call_id": tool_call.id, +"content": str(result) +} + +anthropic assistant message will have: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples +"content": [ +{ +"type": "text", +"text": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." +}, +{ +"type": "tool_use", +"id": "toolu_01A09q90qw90lq917835lq9", +"name": "get_weather", +"input": {"location": "San Francisco, CA", "unit": "celsius"} +} +] + +anthropic user message response will be: +"content": [ +{ +"type": "tool_result", +"tool_use_id": "toolu_01A09q90qw90lq917835lq9", +"content": "15 degrees" +} +] + + +*/ + + + +/* + + +ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message) +gemini request: { +"role": "assistant", +"content": null, +"function_call": { +"name": "get_weather", +"arguments": { +"latitude": 48.8566, +"longitude": 2.3522 +} +} +} +gemini response: +{ +"role": "assistant", +"function_response": { +"name": "get_weather", +"response": { +"temperature": "15°C", +"condition": "Cloudy" +} +} +} + + ++ anthropic + ++ openai-compat (4) ++ gemini + +ollama + + +mistral: same as openai + +*/ diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts index b443cca1..77bbcb82 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts @@ -7,6 +7,7 @@ import Anthropic from '@anthropic-ai/sdk'; import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; import { anthropicMaxPossibleTokens } from '../../common/voidSettingsTypes.js'; import { InternalToolInfo } from '../../common/toolsService.js'; +import { addSystemMessageAndToolSupport } from './addSupport.js'; @@ -28,7 +29,7 @@ export const toAnthropicTool = (toolInfo: InternalToolInfo) => { -export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, tools }) => { +export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, tools: tools_ }) => { const thisConfig = settingsOfProvider.anthropic @@ -38,15 +39,19 @@ export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages, return } + const { messages, separateSystemMessageStr, devInfo } = addSystemMessageAndToolSupport(modelName, providerName, messages_, { separateSystemMessage: true }) + const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true }); + const tools = devInfo?.supportsTools && (tools_?.length ?? 0) !== 0 ? tools_?.map(tool => toAnthropicTool(tool)) : undefined + const stream = anthropic.messages.stream({ - // system: systemMessage, + system: separateSystemMessageStr, messages: messages, model: modelName, max_tokens: maxTokens, - tools: tools?.map(tool => toAnthropicTool(tool)), - tool_choice: { type: 'auto', disable_parallel_tool_use: true } // one tool use at a time + tools: tools, + tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time }) @@ -78,9 +83,9 @@ export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages, stream.on('finalMessage', (response) => { // stringify the response's content const content = response.content.map(c => c.type === 'text' ? c.text : '').join('\n\n') - const tools = response.content.map(c => c.type === 'tool_use' ? { name: c.name, args: JSON.stringify(c.input), tool_use_id: c.id } : null).filter(c => !!c) + // const tools = response.content.map(c => c.type === 'tool_use' ? { name: c.name, args: JSON.stringify(c.input), tool_use_id: c.id } : null).filter(c => !!c) - onFinalMessage({ fullText: content, tools }) + onFinalMessage({ fullText: content, tools: [] }) }) stream.on('error', (error) => { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/gemini.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/gemini.ts deleted file mode 100644 index 2732fbe4..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/gemini.ts +++ /dev/null @@ -1,43 +0,0 @@ -/*-------------------------------------------------------------------------------------- - * Copyright 2025 Glass Devtools, Inc. All rights reserved. - * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. - *--------------------------------------------------------------------------------------*/ - -import { Content, GoogleGenerativeAI } from '@google/generative-ai'; -import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; - -// Gemini -export const sendGeminiChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { - - let fullText = '' - - const thisConfig = settingsOfProvider.gemini - - const genAI = new GoogleGenerativeAI(thisConfig.apiKey); - const model = genAI.getGenerativeModel({ model: modelName }); - - // Convert messages to Gemini format - const geminiMessages: Content[] = messages - .map((msg, i) => ({ - parts: [{ text: msg.content }], - role: msg.role === 'assistant' ? 'model' : 'user' - })) - - model.generateContentStream({ - // systemInstruction: systemMessage, - contents: geminiMessages, - }) - .then(async response => { - _setAborter(() => response.stream.return(fullText)) - - for await (const chunk of response.stream) { - const newText = chunk.text(); - fullText += newText; - onText({ newText, fullText }); - } - onFinalMessage({ fullText, tools: [] }); - }) - .catch((error) => { - onError({ message: error + '', fullError: error }) - }) -} diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/ollama.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/ollama.ts index daff3a29..da6715c0 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/ollama.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/ollama.ts @@ -1,3 +1,4 @@ + /*-------------------------------------------------------------------------------------- * Copyright 2025 Glass Devtools, Inc. All rights reserved. * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. @@ -38,84 +39,86 @@ export const ollamaList: _InternalModelListFnType = async ( } -export const sendOllamaFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { - - const thisConfig = settingsOfProvider.ollama - // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in - if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`) - - let fullText = '' - - const ollama = new Ollama({ host: thisConfig.endpoint }) - - ollama.generate({ - model: modelName, - prompt: messages.prefix, - suffix: messages.suffix, - options: { - stop: messages.stopTokens, - num_predict: 300, // max tokens - // repeat_penalty: 1, - }, - raw: true, - stream: true, - }) - .then(async stream => { - _setAborter(() => stream.abort()) - // iterate through the stream - for await (const chunk of stream) { - const newText = chunk.response; - fullText += newText; - onText({ newText, fullText }); - } - onFinalMessage({ fullText, tools: [] }); - }) - // when error/fail - .catch((error) => { - onError({ message: error + '', fullError: error }) - }) -}; -// Ollama -export const sendOllamaChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { +// export const sendOllamaFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { - const thisConfig = settingsOfProvider.ollama - // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in - if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`) +// const thisConfig = settingsOfProvider.ollama +// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in +// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`) - let fullText = '' +// let fullText = '' - const ollama = new Ollama({ host: thisConfig.endpoint }) +// const ollama = new Ollama({ host: thisConfig.endpoint }) - ollama.chat({ - model: modelName, - messages: messages, - stream: true, - // options: { num_predict: parseMaxTokensStr(thisConfig.maxTokens) } // this is max_tokens - }) - .then(async stream => { - _setAborter(() => stream.abort()) - // iterate through the stream - for await (const chunk of stream) { - const newText = chunk.message.content; +// ollama.generate({ +// model: modelName, +// prompt: messages.prefix, +// suffix: messages.suffix, +// options: { +// stop: messages.stopTokens, +// num_predict: 300, // max tokens +// // repeat_penalty: 1, +// }, +// raw: true, +// stream: true, +// }) +// .then(async stream => { +// _setAborter(() => stream.abort()) +// // iterate through the stream +// for await (const chunk of stream) { +// const newText = chunk.response; +// fullText += newText; +// onText({ newText, fullText }); +// } +// onFinalMessage({ fullText, tools: [] }); +// }) +// // when error/fail +// .catch((error) => { +// onError({ message: error + '', fullError: error }) +// }) +// }; - // chunk.message.tool_calls[0].function.arguments - fullText += newText; - onText({ newText, fullText }); - } +// // Ollama +// export const sendOllamaChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { - onFinalMessage({ fullText, tools: [] }); +// const thisConfig = settingsOfProvider.ollama +// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in +// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`) - }) - // when error/fail - .catch((error) => { - onError({ message: error + '', fullError: error }) - }) +// let fullText = '' -}; +// const ollama = new Ollama({ host: thisConfig.endpoint }) + +// ollama.chat({ +// model: modelName, +// messages: messages, +// stream: true, +// // options: { num_predict: parseMaxTokensStr(thisConfig.maxTokens) } // this is max_tokens +// }) +// .then(async stream => { +// _setAborter(() => stream.abort()) +// // iterate through the stream +// for await (const chunk of stream) { +// const newText = chunk.message.content; + +// // chunk.message.tool_calls[0].function.arguments + +// fullText += newText; +// onText({ newText, fullText }); +// } + +// onFinalMessage({ fullText, tools: [] }); + +// }) +// // when error/fail +// .catch((error) => { +// onError({ message: error + '', fullError: error }) +// }) + +// }; -// ['codestral', 'qwen2.5-coder', 'qwen2.5-coder:0.5b', 'qwen2.5-coder:1.5b', 'qwen2.5-coder:3b', 'qwen2.5-coder:7b', 'qwen2.5-coder:14b', 'qwen2.5-coder:32b', 'codegemma', 'codegemma:2b', 'codegemma:7b', 'codellama', 'codellama:7b', 'codellama:13b', 'codellama:34b', 'codellama:70b', 'codellama:code', 'codellama:python', 'command-r', 'command-r:35b', 'command-r-plus', 'command-r-plus:104b', 'deepseek-coder-v2', 'deepseek-coder-v2:16b', 'deepseek-coder-v2:236b', 'falcon2', 'falcon2:11b', 'firefunction-v2', 'firefunction-v2:70b', 'gemma', 'gemma:2b', 'gemma:7b', 'gemma2', 'gemma2:2b', 'gemma2:9b', 'gemma2:27b', 'llama2', 'llama2:7b', 'llama2:13b', 'llama2:70b', 'llama3', 'llama3:8b', 'llama3:70b', 'llama3-chatqa', 'llama3-chatqa:8b', 'llama3-chatqa:70b', 'llama3-gradient', 'llama3-gradient:8b', 'llama3-gradient:70b', 'llama3.1', 'llama3.1:8b', 'llama3.1:70b', 'llama3.1:405b', 'llava', 'llava:7b', 'llava:13b', 'llava:34b', 'llava-llama3', 'llava-llama3:8b', 'llava-phi3', 'llava-phi3:3.8b', 'mistral', 'mistral:7b', 'mistral-large', 'mistral-large:123b', 'mistral-nemo', 'mistral-nemo:12b', 'mixtral', 'mixtral:8x7b', 'mixtral:8x22b', 'moondream', 'moondream:1.8b', 'openhermes', 'openhermes:v2.5', 'phi3', 'phi3:3.8b', 'phi3:14b', 'phi3.5', 'phi3.5:3.8b', 'qwen', 'qwen:7b', 'qwen:14b', 'qwen:32b', 'qwen:72b', 'qwen:110b', 'qwen2', 'qwen2:0.5b', 'qwen2:1.5b', 'qwen2:7b', 'qwen2:72b', 'smollm', 'smollm:135m', 'smollm:360m', 'smollm:1.7b',] +// // ['codestral', 'qwen2.5-coder', 'qwen2.5-coder:0.5b', 'qwen2.5-coder:1.5b', 'qwen2.5-coder:3b', 'qwen2.5-coder:7b', 'qwen2.5-coder:14b', 'qwen2.5-coder:32b', 'codegemma', 'codegemma:2b', 'codegemma:7b', 'codellama', 'codellama:7b', 'codellama:13b', 'codellama:34b', 'codellama:70b', 'codellama:code', 'codellama:python', 'command-r', 'command-r:35b', 'command-r-plus', 'command-r-plus:104b', 'deepseek-coder-v2', 'deepseek-coder-v2:16b', 'deepseek-coder-v2:236b', 'falcon2', 'falcon2:11b', 'firefunction-v2', 'firefunction-v2:70b', 'gemma', 'gemma:2b', 'gemma:7b', 'gemma2', 'gemma2:2b', 'gemma2:9b', 'gemma2:27b', 'llama2', 'llama2:7b', 'llama2:13b', 'llama2:70b', 'llama3', 'llama3:8b', 'llama3:70b', 'llama3-chatqa', 'llama3-chatqa:8b', 'llama3-chatqa:70b', 'llama3-gradient', 'llama3-gradient:8b', 'llama3-gradient:70b', 'llama3.1', 'llama3.1:8b', 'llama3.1:70b', 'llama3.1:405b', 'llava', 'llava:7b', 'llava:13b', 'llava:34b', 'llava-llama3', 'llava-llama3:8b', 'llava-phi3', 'llava-phi3:3.8b', 'mistral', 'mistral:7b', 'mistral-large', 'mistral-large:123b', 'mistral-nemo', 'mistral-nemo:12b', 'mixtral', 'mixtral:8x7b', 'mixtral:8x22b', 'moondream', 'moondream:1.8b', 'openhermes', 'openhermes:v2.5', 'phi3', 'phi3:3.8b', 'phi3:14b', 'phi3.5', 'phi3.5:3.8b', 'qwen', 'qwen:7b', 'qwen:14b', 'qwen:32b', 'qwen:72b', 'qwen:110b', 'qwen2', 'qwen2:0.5b', 'qwen2:1.5b', 'qwen2:7b', 'qwen2:72b', 'smollm', 'smollm:135m', 'smollm:360m', 'smollm:1.7b',] diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts index 370d411a..b7c81563 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts @@ -7,6 +7,7 @@ import OpenAI from 'openai'; import { _InternalModelListFnType, _InternalSendLLMFIMMessageFnType, _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; import { Model } from 'openai/resources/models.js'; import { InternalToolInfo } from '../../common/toolsService.js'; +import { addSystemMessageAndToolSupport } from './addSupport.js'; // import { parseMaxTokensStr } from './util.js'; @@ -38,11 +39,19 @@ type NewParams = Pick[0] & Paramet const newOpenAI = ({ settingsOfProvider, providerName }: NewParams) => { if (providerName === 'openAI') { - const thisConfig = settingsOfProvider.openAI - return new OpenAI({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true }); + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ + apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true + }) + } + else if (providerName === 'ollama') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ + baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true, + }) } else if (providerName === 'openRouter') { - const thisConfig = settingsOfProvider.openRouter + const thisConfig = settingsOfProvider[providerName] return new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, defaultHeaders: { @@ -51,33 +60,38 @@ const newOpenAI = ({ settingsOfProvider, providerName }: NewParams) => { }, }) } + else if (providerName === 'gemini') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ + baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, + }) + } else if (providerName === 'deepseek') { - const thisConfig = settingsOfProvider.deepseek + const thisConfig = settingsOfProvider[providerName] return new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, }) - } else if (providerName === 'openAICompatible') { - const thisConfig = settingsOfProvider.openAICompatible + const thisConfig = settingsOfProvider[providerName] return new OpenAI({ - baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true + baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, }) } else if (providerName === 'mistral') { - const thisConfig = settingsOfProvider.mistral + const thisConfig = settingsOfProvider[providerName] return new OpenAI({ baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, }) } else if (providerName === 'groq') { - const thisConfig = settingsOfProvider.groq + const thisConfig = settingsOfProvider[providerName] return new OpenAI({ - baseURL: '"https://api.groq.com/openai/v1"', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, + baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, }) } else { - console.error(`sendOpenAIMsg: invalid providerName: ${providerName}`) + console.error(`sendOpenAICompatibleMsg: invalid providerName: ${providerName}`) throw new Error(`providerName was invalid: ${providerName}`) } } @@ -130,10 +144,14 @@ export const sendOpenAIFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onTe // OpenAI, OpenRouter, OpenAICompatible -export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }) => { +export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools: tools_ }) => { let fullText = '' - const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {} + const toolCallOfIndex: { [index: string]: { name: string, args: string, id: string } } = {} + + const { messages, devInfo } = addSystemMessageAndToolSupport(modelName, providerName, messages_, { separateSystemMessage: false }) + + const tools = devInfo?.supportsTools && (tools_?.length ?? 0) !== 0 ? tools_?.map(tool => toOpenAITool(tool)) : undefined const openai: OpenAI = newOpenAI({ providerName, settingsOfProvider }) @@ -141,7 +159,9 @@ export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages, on model: modelName, messages: messages, stream: true, - tools: tools?.map(tool => toOpenAITool(tool)), + tools: tools, + tool_choice: tools ? 'auto' : undefined, + parallel_tool_calls: tools ? false : undefined, } openai.chat.completions @@ -155,9 +175,11 @@ export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages, on // tool call for (const tool of chunk.choices[0]?.delta?.tool_calls ?? []) { const index = tool.index - if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '' } + if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '', id: '' } toolCallOfIndex[index].name += tool.function?.name ?? '' - toolCallOfIndex[index].args += tool.function?.arguments ?? '' + toolCallOfIndex[index].args += tool.function?.arguments ?? ''; + toolCallOfIndex[index].id = tool.id ?? '' + } // message diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts index 22255292..dec84841 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts @@ -3,197 +3,12 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ -import { SendLLMMessageParams, OnText, OnFinalMessage, OnError, LLMChatMessage, _InternalLLMChatMessage } from '../../common/llmMessageTypes.js'; +import { SendLLMMessageParams, OnText, OnFinalMessage, OnError, _InternalLLMChatMessage } from '../../common/llmMessageTypes.js'; import { IMetricsService } from '../../common/metricsService.js'; +import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js'; import { sendAnthropicChat } from './anthropic.js'; -import { sendOllamaFIM, sendOllamaChat } from './ollama.js'; import { sendOpenAIChat } from './openai.js'; -import { sendGeminiChat } from './gemini.js'; -import { developerInfoOfModelName, developerInfoOfProviderName, displayInfoOfProviderName, ProviderName, recognizedModelOfModelName } from '../../common/voidSettingsTypes.js'; - - -const cleanChatMessages = (modelName: string, providerName: ProviderName, messages: LLMChatMessage[]): { separateSystemMessageStr?: string, messages: _InternalLLMChatMessage[] } => { - const recognizedModel = recognizedModelOfModelName(modelName) - const { separateSystemMessage, toolsGoInRole, modelOverrides } = developerInfoOfProviderName(providerName) - - const { supportsSystemMessage, maxTokens, /* supportsTools, supportsAutocompleteFIM, supportsStreaming */ } = developerInfoOfModelName(recognizedModel, modelOverrides) - - - // trim message content (Anthropic and other providers give an error if there is trailing whitespace) - messages = messages.map(m => ({ ...m, content: m.content.trim() })) - - - // 1. SYSTEM MESSAGE - // find system messages and concatenate them - const systemMessageStr = messages - .filter(msg => msg.role === 'system') - .map(msg => msg.content) - .join('\n') || undefined; - - let separateSystemMessageStr = undefined - - // remove all system messages - const noSystemMessages: _InternalLLMChatMessage[] = messages.filter(msg => msg.role !== 'system') - - if (systemMessageStr) { - // if supports system message - if (supportsSystemMessage) { - if (separateSystemMessage) - separateSystemMessageStr = systemMessageStr - else { - noSystemMessages.unshift({ role: supportsSystemMessage, content: systemMessageStr }) // add new first message - } - } - // if does not support system message - else { - if (supportsSystemMessage) { - if (noSystemMessages.length === 0) - noSystemMessages.push({ role: 'user', content: systemMessageStr }) - // add system mesasges to first message (should be a user message) - else { - const newFirstMessage = { - role: noSystemMessages[0].role, - content: ('' - + '\n' - + systemMessageStr - + '\n' - + '\n' - + noSystemMessages[0].content - ) - } - noSystemMessages.splice(0, 1) // delete first message - noSystemMessages.unshift(newFirstMessage) // add new first message - } - } - } - } - - // 2. TOOLS - - const newMessages = noSystemMessages; - - if (toolsGoInRole) { - let index = 0; - while (index < newMessages.length) { - - // merge tool with the previous assistant and the following user message - - // take prev message and add - /* -openai assistant message will have: https://platform.openai.com/docs/guides/function-calling#function-calling-steps -"tool_calls":[ -{ - "id": "call_12345xyz", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}" - } -}] - -openai user response will be: -{ - "role": "tool", - "tool_call_id": tool_call.id, - "content": str(result) -} - -anthropic assistant message will have: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples -"content": [ - { - "type": "text", - "text": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." - }, - { - "type": "tool_use", - "id": "toolu_01A09q90qw90lq917835lq9", - "name": "get_weather", - "input": {"location": "San Francisco, CA", "unit": "celsius"} - } - ] - -anthropic user message response will be: -"content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_01A09q90qw90lq917835lq9", - "content": "15 degrees" - } -] - - -ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message) -gemini request: { - "role": "assistant", - "content": null, - "function_call": { - "name": "get_weather", - "arguments": { - "latitude": 48.8566, - "longitude": 2.3522 - } - } -} -gemini response: -{ - "role": "assistant", - "function_response": { - "name": "get_weather", - "response": { - "temperature": "15°C", - "condition": "Cloudy" - } - } -} - - -+ anthropic - -+ openai-compat (4) - + gemini - -ollama - - -mistral: same as openai - - */ - - - if (newMessages[index].role === 'tool') { - const toolMessage = newMessages[index]; - const assistantMessage = newMessages[index - 1]; - const userMessage = newMessages[index + 1]; - - // while ((toolIndex = newMessages.findIndex((msg, idx) => idx > toolIndex && msg.role === 'tool')) !== -1) { - - // tool_use goes in assistant - if (assistantMessage?.role === 'assistant') { - assistantMessage.tool_use += `\n${toolMessage.content}`; - } - - // tool_result goes in user - if (userMessage?.role === 'user') { - - userMessage.content = `${toolMessage.content}\n${userMessage.content}`; - } - - // Remove the tool message after merging its content - newMessages.splice(index, 1); - } else { - index++; - } - } - } - - - - return { - separateSystemMessageStr, - messages: newMessages - } -} export const sendLLMMessage = ({ @@ -214,16 +29,6 @@ export const sendLLMMessage = ({ metricsService: IMetricsService ) => { - let messagesArr: _InternalLLMChatMessage[] = [] - - // TODO!!! move this to the actual providers - if (messagesType === 'chatMessages') { - const { messages: cleanedMessages, separateSystemMessageStr } = cleanChatMessages(modelName, providerName, [ - { role: 'system', content: aiInstructions }, - ...messages_ - ]) - messagesArr = cleanedMessages - } // only captures number of messages and message "shape", no actual code, instructions, prompts, etc const captureLLMEvent = (eventId: string, extras?: object) => { @@ -231,8 +36,8 @@ export const sendLLMMessage = ({ providerName, modelName, ...messagesType === 'chatMessages' ? { - numMessages: messagesArr?.length, - messagesShape: messagesArr?.map(msg => ({ role: msg.role, length: msg.content.length })), + numMessages: messages_?.length, + messagesShape: messages_?.map(msg => ({ role: msg.role, length: msg.content.length })), origNumMessages: messages_?.length, origMessagesShape: messages_?.map(msg => ({ role: msg.role, length: msg.content.length })), @@ -283,7 +88,10 @@ export const sendLLMMessage = ({ } abortRef_.current = onAbort - captureLLMEvent(`${loggingName} - Sending Message`, { messageLength: messagesArr[messagesArr.length - 1]?.content.length }) + if (messagesType === 'chatMessages') + captureLLMEvent(`${loggingName} - Sending Message`, { messageLength: messages_[messages_.length - 1]?.content.length }) + else if (messagesType === 'FIMMessage') + captureLLMEvent(`${loggingName} - Sending FIM`, {}) // TODO!!! add more metrics try { switch (providerName) { @@ -292,21 +100,15 @@ export const sendLLMMessage = ({ case 'deepseek': case 'openAICompatible': case 'mistral': - case 'groq': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI FIM', tools: [] }) - else /* */ sendOpenAIChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }); - break; case 'ollama': - if (messagesType === 'FIMMessage') sendOllamaFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName }); - else /* */ sendOllamaChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }); + case 'groq': + case 'gemini': + if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI FIM', tools: [] }) + else /* */ sendOpenAIChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }); break; case 'anthropic': if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM', tools: [] }) - else /* */ sendAnthropicChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }); - break; - case 'gemini': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Gemini FIM', tools: [] }) - else /* */ sendGeminiChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }); + else /* */ sendAnthropicChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }); break; default: onError({ message: `Error: Void provider was "${providerName}", which is not recognized.`, fullError: null })