diff --git a/src/vs/workbench/contrib/void/browser/autocompleteService.ts b/src/vs/workbench/contrib/void/browser/autocompleteService.ts index 5fc8ac76..f9cbec7b 100644 --- a/src/vs/workbench/contrib/void/browser/autocompleteService.ts +++ b/src/vs/workbench/contrib/void/browser/autocompleteService.ts @@ -795,26 +795,27 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ }, useProviderFor: 'Autocomplete', logging: { loggingName: 'Autocomplete' }, - onText: async ({ fullText, newText }) => { + onText: () => { }, // unused in FIMMessage + // onText: async ({ fullText, newText }) => { - newAutocompletion.insertText = fullText + // newAutocompletion.insertText = fullText - // count newlines in newText - const numNewlines = newText.match(/\n|\r\n/g)?.length || 0 - newAutocompletion._newlineCount += numNewlines + // // count newlines in newText + // const numNewlines = newText.match(/\n|\r\n/g)?.length || 0 + // newAutocompletion._newlineCount += numNewlines - // if too many newlines, resolve up to last newline - if (newAutocompletion._newlineCount > 10) { - const lastNewlinePos = fullText.lastIndexOf('\n') - newAutocompletion.insertText = fullText.substring(0, lastNewlinePos) - resolve(newAutocompletion.insertText) - return - } + // // if too many newlines, resolve up to last newline + // if (newAutocompletion._newlineCount > 10) { + // const lastNewlinePos = fullText.lastIndexOf('\n') + // newAutocompletion.insertText = fullText.substring(0, lastNewlinePos) + // resolve(newAutocompletion.insertText) + // return + // } - // if (!getAutocompletionMatchup({ prefix: this._lastPrefix, autocompletion: newAutocompletion })) { - // reject('LLM response did not match user\'s text.') - // } - }, + // // if (!getAutocompletionMatchup({ prefix: this._lastPrefix, autocompletion: newAutocompletion })) { + // // reject('LLM response did not match user\'s text.') + // // } + // }, onFinalMessage: ({ fullText }) => { // console.log('____res: ', JSON.stringify(newAutocompletion.insertText)) diff --git a/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts b/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts index cd3276ff..806676da 100644 --- a/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts +++ b/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts @@ -59,7 +59,7 @@ class SurroundingsRemover { // return offset === suffix.length // } - removeFromStartUntil = (until: string, alsoRemoveUntilStr: boolean) => { + removeFromStartUntilFullMatch = (until: string, alsoRemoveUntilStr: boolean) => { const index = this.originalS.indexOf(until, this.i) if (index === -1) { @@ -86,7 +86,7 @@ class SurroundingsRemover { const foundCodeBlock = pm.removePrefix('```') if (!foundCodeBlock) return false - pm.removeFromStartUntil('\n', true) // language + pm.removeFromStartUntilFullMatch('\n', true) // language const j = pm.j let foundCodeBlockEnd = pm.removeSuffix('```') @@ -159,27 +159,10 @@ export const extractCodeFromFIM = ({ text, recentlyAddedTextLen, midTag, }: { te const [delta, ignoredSuffix] = pm.deltaInfo(recentlyAddedTextLen) return [s, delta, ignoredSuffix] - - - // // const regex = /[\s\S]*?(?:`{1,3}\s*([a-zA-Z_]+[\w]*)?[\s\S]*?)?([\s\S]*?)(?:<\/MID>|`{1,3}|$)/; - // const regex = new RegExp( - // `[\\s\\S]*?(?:\`{1,3}\\s*([a-zA-Z_]+[\\w]*)?[\\s\\S]*?)?<${midTag}>([\\s\\S]*?)(?:|\`{1,3}|$)`, - // '' - // ); - // const match = text.match(regex); - // if (match) { - // const [_, languageName, codeBetweenMidTags] = match; - // return [languageName, codeBetweenMidTags] as const - - // } else { - // return [undefined, extractCodeFromRegular(text)] as const - // } - } - export type ExtractedSearchReplaceBlock = { state: 'writingOriginal' | 'writingFinal' | 'done', orig: string, diff --git a/src/vs/workbench/contrib/void/common/llmMessageService.ts b/src/vs/workbench/contrib/void/common/llmMessageService.ts index bb6cf09c..b2266213 100644 --- a/src/vs/workbench/contrib/void/common/llmMessageService.ts +++ b/src/vs/workbench/contrib/void/common/llmMessageService.ts @@ -3,7 +3,7 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ -import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from './llmMessageTypes.js'; +import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, VLLMModelResponse, } from './llmMessageTypes.js'; import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js'; @@ -24,27 +24,39 @@ export interface ILLMMessageService { sendLLMMessage: (params: ServiceSendLLMMessageParams) => string | null; abort: (requestId: string) => void; ollamaList: (params: ServiceModelListParams) => void; - openAICompatibleList: (params: ServiceModelListParams) => void; + vLLMList: (params: ServiceModelListParams) => void; } + +// open this file side by side with llmMessageChannel export class LLMMessageService extends Disposable implements ILLMMessageService { readonly _serviceBrand: undefined; private readonly channel: IChannel // LLMMessageChannel - // llmMessage - private readonly onTextHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) } = {} - private readonly onFinalMessageHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) } = {} - private readonly onErrorHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) } = {} + // sendLLMMessage + private readonly llmMessageHooks = { + onText: {} as { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) }, + onFinalMessage: {} as { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) }, + onError: {} as { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) }, + } - - // ollamaList - private readonly onSuccess_ollama: { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) } = {} - private readonly onError_ollama: { [eventId: string]: ((params: EventModelListOnErrorParams) => void) } = {} - - // openAICompatibleList - private readonly onSuccess_openAICompatible: { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) } = {} - private readonly onError_openAICompatible: { [eventId: string]: ((params: EventModelListOnErrorParams) => void) } = {} + // list hooks + private readonly listHooks = { + ollama: { + success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) }, + error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams) => void) }, + }, + vLLM: { + success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) }, + error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams) => void) }, + } + } satisfies { + [providerName: string]: { + success: { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) }, + error: { [eventId: string]: ((params: EventModelListOnErrorParams) => void) }, + } + } constructor( @IMainProcessService private readonly mainProcessService: IMainProcessService, // used as a renderer (only usable on client side) @@ -59,32 +71,14 @@ export class LLMMessageService extends Disposable implements ILLMMessageService // .listen sets up an IPC channel and takes a few ms, so we set up listeners immediately and add hooks to them instead // llm - this._register((this.channel.listen('onText_llm') satisfies Event)(e => { - this.onTextHooks_llm[e.requestId]?.(e) - })) - this._register((this.channel.listen('onFinalMessage_llm') satisfies Event)(e => { - this.onFinalMessageHooks_llm[e.requestId]?.(e) - this._onRequestIdDone(e.requestId) - })) - this._register((this.channel.listen('onError_llm') satisfies Event)(e => { - console.error('Error in LLMMessageService:', JSON.stringify(e)) - this.onErrorHooks_llm[e.requestId]?.(e) - this._onRequestIdDone(e.requestId) - })) + this._register((this.channel.listen('onText_sendLLMMessage') satisfies Event)(e => { this.llmMessageHooks.onText[e.requestId]?.(e) })) + this._register((this.channel.listen('onFinalMessage_sendLLMMessage') satisfies Event)(e => { this.llmMessageHooks.onFinalMessage[e.requestId]?.(e); this._onRequestIdDone(e.requestId) })) + this._register((this.channel.listen('onError_sendLLMMessage') satisfies Event)(e => { this.llmMessageHooks.onError[e.requestId]?.(e); this._onRequestIdDone(e.requestId); console.error('Error in LLMMessageService:', JSON.stringify(e)) })) // ollama .list() - this._register((this.channel.listen('onSuccess_ollama') satisfies Event>)(e => { - this.onSuccess_ollama[e.requestId]?.(e) - })) - this._register((this.channel.listen('onError_ollama') satisfies Event>)(e => { - this.onError_ollama[e.requestId]?.(e) - })) - // openaiCompatible .list() - this._register((this.channel.listen('onSuccess_openAICompatible') satisfies Event>)(e => { - this.onSuccess_openAICompatible[e.requestId]?.(e) - })) - this._register((this.channel.listen('onError_openAICompatible') satisfies Event>)(e => { - this.onError_openAICompatible[e.requestId]?.(e) - })) + this._register((this.channel.listen('onSuccess_list_ollama') satisfies Event>)(e => { this.listHooks.ollama.success[e.requestId]?.(e) })) + this._register((this.channel.listen('onError_list_ollama') satisfies Event>)(e => { this.listHooks.ollama.error[e.requestId]?.(e) })) + this._register((this.channel.listen('onSuccess_list_vLLM') satisfies Event>)(e => { this.listHooks.vLLM.success[e.requestId]?.(e) })) + this._register((this.channel.listen('onError_list_vLLM') satisfies Event>)(e => { this.listHooks.vLLM.error[e.requestId]?.(e) })) } @@ -117,9 +111,9 @@ export class LLMMessageService extends Disposable implements ILLMMessageService // add state for request id const requestId = generateUuid(); - this.onTextHooks_llm[requestId] = onText - this.onFinalMessageHooks_llm[requestId] = onFinalMessage - this.onErrorHooks_llm[requestId] = onError + this.llmMessageHooks.onText[requestId] = onText + this.llmMessageHooks.onFinalMessage[requestId] = onFinalMessage + this.llmMessageHooks.onError[requestId] = onError const { aiInstructions } = this.voidSettingsService.state.globalSettings const { settingsOfProvider } = this.voidSettingsService.state @@ -151,43 +145,46 @@ export class LLMMessageService extends Disposable implements ILLMMessageService // add state for request id const requestId_ = generateUuid(); - this.onSuccess_ollama[requestId_] = onSuccess - this.onError_ollama[requestId_] = onError + this.listHooks.ollama.success[requestId_] = onSuccess + this.listHooks.ollama.error[requestId_] = onError this.channel.call('ollamaList', { ...proxyParams, settingsOfProvider, + providerName: 'ollama', requestId: requestId_, } satisfies MainModelListParams) } - openAICompatibleList = (params: ServiceModelListParams) => { + vLLMList = (params: ServiceModelListParams) => { const { onSuccess, onError, ...proxyParams } = params const { settingsOfProvider } = this.voidSettingsService.state // add state for request id const requestId_ = generateUuid(); - this.onSuccess_openAICompatible[requestId_] = onSuccess - this.onError_openAICompatible[requestId_] = onError + this.listHooks.vLLM.success[requestId_] = onSuccess + this.listHooks.vLLM.error[requestId_] = onError - this.channel.call('openAICompatibleList', { + this.channel.call('vLLMList', { ...proxyParams, settingsOfProvider, + providerName: 'vLLM', requestId: requestId_, - } satisfies MainModelListParams) + } satisfies MainModelListParams) } - - _onRequestIdDone(requestId: string) { - delete this.onTextHooks_llm[requestId] - delete this.onFinalMessageHooks_llm[requestId] - delete this.onErrorHooks_llm[requestId] + delete this.llmMessageHooks.onText[requestId] + delete this.llmMessageHooks.onFinalMessage[requestId] + delete this.llmMessageHooks.onError[requestId] - delete this.onSuccess_ollama[requestId] - delete this.onError_ollama[requestId] + delete this.listHooks.ollama.success[requestId] + delete this.listHooks.ollama.error[requestId] + + delete this.listHooks.vLLM.success[requestId] + delete this.listHooks.vLLM.error[requestId] } } diff --git a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts index 0956b08b..abe88970 100644 --- a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts +++ b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts @@ -45,7 +45,7 @@ export type ToolCallType = { } -export type OnText = (p: { newText: string, fullText: string }) => void +export type OnText = (p: { newText: string, fullText: string; newReasoning: string; fullReasoning: string }) => void export type OnFinalMessage = (p: { fullText: string, toolCalls?: ToolCallType[] }) => void // id is tool_use_id export type OnError = (p: { message: string, fullError: Error | null }) => void export type AbortRef = { current: (() => void) | null } @@ -65,7 +65,7 @@ export const toLLMChatMessage = (c: ChatMessage): LLMChatMessage => { } -type _InternalSendFIMMessage = { +export type LLMFIMMessage = { prefix: string; suffix: string; stopTokens: string[]; @@ -77,7 +77,7 @@ type SendLLMType = { tools?: InternalToolInfo[]; } | { messagesType: 'FIMMessage'; - messages: _InternalSendFIMMessage; + messages: LLMFIMMessage; tools?: undefined; } @@ -118,38 +118,6 @@ export type EventLLMMessageOnFinalMessageParams = Parameters[0] export type EventLLMMessageOnErrorParams = Parameters[0] & { requestId: string } -export type _InternalSendLLMChatMessageFnType = ( - params: { - aiInstructions: string; - - onText: OnText; - onFinalMessage: OnFinalMessage; - onError: OnError; - providerName: ProviderName; - settingsOfProvider: SettingsOfProvider; - modelName: string; - _setAborter: (aborter: () => void) => void; - - tools?: InternalToolInfo[], - - messages: LLMChatMessage[]; - } -) => void - -export type _InternalSendLLMFIMMessageFnType = ( - params: { - onText: OnText; - onFinalMessage: OnFinalMessage; - onError: OnError; - providerName: ProviderName; - settingsOfProvider: SettingsOfProvider; - modelName: string; - _setAborter: (aborter: () => void) => void; - - messages: _InternalSendFIMMessage; - } -) => void - // service -> main -> internal -> event (back to main) // (browser) @@ -181,18 +149,22 @@ export type OllamaModelResponse = { size_vram: number; } -export type OpenaiCompatibleModelResponse = { +type OpenaiCompatibleModelResponse = { id: string; created: number; object: 'model'; owned_by: string; } +export type VLLMModelResponse = OpenaiCompatibleModelResponse + + // params to the true list fn -export type ModelListParams = { +export type ModelListParams = { + providerName: ProviderName; settingsOfProvider: SettingsOfProvider; - onSuccess: (param: { models: modelResponse[] }) => void; + onSuccess: (param: { models: ModelResponse[] }) => void; onError: (param: { error: string }) => void; } @@ -211,4 +183,3 @@ export type EventModelListOnErrorParams = Parameters = (params: ModelListParams) => void diff --git a/src/vs/workbench/contrib/void/common/refreshModelService.ts b/src/vs/workbench/contrib/void/common/refreshModelService.ts index 1c95a4ad..1d68b304 100644 --- a/src/vs/workbench/contrib/void/common/refreshModelService.ts +++ b/src/vs/workbench/contrib/void/common/refreshModelService.ts @@ -8,7 +8,7 @@ import { ILLMMessageService } from './llmMessageService.js'; import { Emitter, Event } from '../../../../base/common/event.js'; import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js'; import { RefreshableProviderName, refreshableProviderNames, SettingsOfProvider } from './voidSettingsTypes.js'; -import { OllamaModelResponse, OpenaiCompatibleModelResponse } from './llmMessageTypes.js'; +import { OllamaModelResponse, VLLMModelResponse } from './llmMessageTypes.js'; import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js'; import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; @@ -160,9 +160,8 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ } } const listFn = providerName === 'ollama' ? this.llmMessageService.ollamaList - : providerName === 'vLLM' ? this.llmMessageService.openAICompatibleList - : providerName === 'openAICompatible' ? this.llmMessageService.openAICompatibleList - : () => { } + : providerName === 'vLLM' ? this.llmMessageService.vLLMList + : () => { } listFn({ onSuccess: ({ models }) => { @@ -172,8 +171,7 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ providerName, models.map(model => { if (providerName === 'ollama') return (model as OllamaModelResponse).name; - else if (providerName === 'vLLM') return (model as OpenaiCompatibleModelResponse).id; - else if (providerName === 'openAICompatible') return (model as OpenaiCompatibleModelResponse).id; + else if (providerName === 'vLLM') return (model as VLLMModelResponse).id; else throw new Error('refreshMode fn: unknown provider', providerName); }), { enableProviderOnSuccess: options.enableProviderOnSuccess, hideRefresh: options.doNotFire } diff --git a/src/vs/workbench/contrib/void/common/toolsService.ts b/src/vs/workbench/contrib/void/common/toolsService.ts index 0b2b5a37..81198d02 100644 --- a/src/vs/workbench/contrib/void/common/toolsService.ts +++ b/src/vs/workbench/contrib/void/common/toolsService.ts @@ -89,6 +89,13 @@ export const voidTools = { export type ToolName = keyof typeof voidTools export const toolNames = Object.keys(voidTools) as ToolName[] +const toolNamesSet = new Set(toolNames) +export const isAToolName = (toolName: string): toolName is ToolName => { + const isAToolName = toolNamesSet.has(toolName) + return isAToolName +} + + export type ToolParamNames = keyof typeof voidTools[T]['params'] export type ToolParamsObj = { [paramName in ToolParamNames]: unknown } diff --git a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts index 529a872c..fb387bc1 100644 --- a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts +++ b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts @@ -4,367 +4,13 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ +import { defaultModelsOfProvider } from '../electron-main/llmMessage/MODELS.js'; import { VoidSettingsState } from './voidSettingsService.js' - -// developer info used in sendLLMMessage -export type DeveloperInfoAtModel = { - // USED: - supportsSystemMessage: 'developer' | boolean, // if null, we will just do a string of system message. this is independent from separateSystemMessage, which takes priority and is passed directly in each provider's implementation. - supportsTools: boolean, // we will just do a string of tool use if it doesn't support - - // UNUSED (coming soon): - // TODO!!! think tokens - deepseek - _recognizedModelName: RecognizedModelName, // used to show user if model was auto-recognized - _supportsStreaming: boolean, // we will just dump the final result if doesn't support it - _supportsAutocompleteFIM: boolean, // we will just do a description of FIM if it doens't support <|fim_hole|> - _maxTokens: number, // required -} - -export type DeveloperInfoAtProvider = { - overrideSettingsForAllModels?: Partial; // any overrides for models that a provider might have (e.g. if a provider always supports tool use, even if we don't recognize the model we can set tools to true) -} - - - - - -export type VoidModelInfo = { // <-- STATEFUL - modelName: string, - isDefault: boolean, // whether or not it's a default for its provider - isHidden: boolean, // whether or not the user is hiding it (switched off) - isAutodetected?: boolean, // whether the model was autodetected by polling -} & DeveloperInfoAtModel - - - - - -export const recognizedModels = [ - // chat - 'OpenAI 4o', - 'Anthropic Claude', - 'Llama 3.x', - 'Deepseek Chat', // deepseek coder v2 is now merged into chat (V3) https://api-docs.deepseek.com/updates#deepseek-coder--deepseek-chat-upgraded-to-deepseek-v25-model - 'xAI Grok', - // 'xAI Grok', - // 'Google Gemini, Gemma', - // 'Microsoft Phi4', - - - // coding (autocomplete) - 'Alibaba Qwen2.5 Coder Instruct', // we recommend this over Qwen2.5 - 'Mistral Codestral', - - // thinking - 'OpenAI o1', - 'Deepseek R1', - - // general - // 'Mixtral 8x7b' - // 'Qwen2.5', - -] as const - -type RecognizedModelName = (typeof recognizedModels)[number] | '' - - -export function recognizedModelOfModelName(modelName: string): RecognizedModelName { - const lower = modelName.toLowerCase(); - - if (lower.includes('gpt-4o')) - return 'OpenAI 4o'; - if (lower.includes('claude')) - return 'Anthropic Claude'; - if (lower.includes('llama')) - return 'Llama 3.x'; - if (lower.includes('qwen2.5-coder')) - return 'Alibaba Qwen2.5 Coder Instruct'; - if (lower.includes('mistral')) - return 'Mistral Codestral'; - if (/\bo1\b/.test(lower) || /\bo3\b/.test(lower)) // o1, o3 - return 'OpenAI o1'; - if (lower.includes('deepseek-r1') || lower.includes('deepseek-reasoner')) - return 'Deepseek R1'; - if (lower.includes('deepseek')) - return 'Deepseek Chat' - if (lower.includes('grok')) - return 'xAI Grok' - - return ''; -} - - -const developerInfoAtProvider: { [providerName in ProviderName]: DeveloperInfoAtProvider } = { - 'anthropic': { - overrideSettingsForAllModels: { - supportsSystemMessage: true, - supportsTools: true, - _supportsAutocompleteFIM: false, - _supportsStreaming: true, - } - }, - 'deepseek': { - overrideSettingsForAllModels: { - } - }, - 'ollama': { - }, - 'openRouter': { - }, - 'openAICompatible': { - }, - 'openAI': { - }, - 'gemini': { - }, - 'mistral': { - }, - 'groq': { - }, - 'xAI': { - }, - 'vLLM': { - }, -} -export const developerInfoOfProviderName = (providerName: ProviderName): Partial => { - return developerInfoAtProvider[providerName] ?? {} -} - - - - -// providerName is optional, but gives some extra fallbacks if provided -const developerInfoOfRecognizedModelName: { [recognizedModel in RecognizedModelName]: Omit } = { - 'OpenAI 4o': { - supportsSystemMessage: true, - supportsTools: true, - _supportsAutocompleteFIM: false, - _supportsStreaming: true, - _maxTokens: 4096, - }, - - 'Anthropic Claude': { - supportsSystemMessage: true, - supportsTools: false, - _supportsAutocompleteFIM: false, - _supportsStreaming: false, - _maxTokens: 4096, - }, - - 'Llama 3.x': { - supportsSystemMessage: true, - supportsTools: true, - _supportsAutocompleteFIM: false, - _supportsStreaming: false, - _maxTokens: 4096, - }, - - 'xAI Grok': { - supportsSystemMessage: true, - supportsTools: true, - _supportsAutocompleteFIM: false, - _supportsStreaming: true, - _maxTokens: 4096, - - }, - - 'Deepseek Chat': { - supportsSystemMessage: true, - supportsTools: false, - _supportsAutocompleteFIM: false, - _supportsStreaming: false, - _maxTokens: 4096, - }, - - 'Alibaba Qwen2.5 Coder Instruct': { - supportsSystemMessage: true, - supportsTools: true, - _supportsAutocompleteFIM: false, - _supportsStreaming: false, - _maxTokens: 4096, - }, - - 'Mistral Codestral': { - supportsSystemMessage: true, - supportsTools: true, - _supportsAutocompleteFIM: false, - _supportsStreaming: false, - _maxTokens: 4096, - }, - - 'OpenAI o1': { - supportsSystemMessage: 'developer', - supportsTools: false, - _supportsAutocompleteFIM: false, - _supportsStreaming: true, - _maxTokens: 4096, - }, - - 'Deepseek R1': { - supportsSystemMessage: false, - supportsTools: false, - _supportsAutocompleteFIM: false, - _supportsStreaming: false, - _maxTokens: 4096, - }, - - - '': { - supportsSystemMessage: false, - supportsTools: false, - _supportsAutocompleteFIM: false, - _supportsStreaming: false, - _maxTokens: 4096, - }, -} -export const developerInfoOfModelName = (modelName: string, overrides?: Partial): DeveloperInfoAtModel => { - const recognizedModelName = recognizedModelOfModelName(modelName) - return { - _recognizedModelName: recognizedModelName, - ...developerInfoOfRecognizedModelName[recognizedModelName], - ...overrides - } -} - - - - - - -// creates `modelInfo` from `modelNames` -export const modelInfoOfDefaultModelNames = (defaultModelNames: string[]): VoidModelInfo[] => { - return defaultModelNames.map((modelName, i) => ({ - modelName, - isDefault: true, - isAutodetected: false, - isHidden: defaultModelNames.length >= 10, // hide all models if there are a ton of them, and make user enable them individually - ...developerInfoOfModelName(modelName), - })) -} - -export const modelInfoOfAutodetectedModelNames = (defaultModelNames: string[], options: { existingModels: VoidModelInfo[] }) => { - const { existingModels } = options - - const existingModelsMap: Record = {} - for (const existingModel of existingModels) { - existingModelsMap[existingModel.modelName] = existingModel - } - - return defaultModelNames.map((modelName, i) => ({ - modelName, - isDefault: true, - isAutodetected: true, - isHidden: !!existingModelsMap[modelName]?.isHidden, - ...developerInfoOfModelName(modelName) - })) -} - - - - - -// https://docs.anthropic.com/en/docs/about-claude/models -export const defaultAnthropicModels = modelInfoOfDefaultModelNames([ - 'claude-3-5-sonnet-20241022', - 'claude-3-5-haiku-20241022', - 'claude-3-opus-20240229', - 'claude-3-sonnet-20240229', - // 'claude-3-haiku-20240307', -]) - - -// https://platform.openai.com/docs/models/gp -export const defaultOpenAIModels = modelInfoOfDefaultModelNames([ - 'o1', - 'o1-mini', - 'o3-mini', - 'gpt-4o', - 'gpt-4o-mini', - // 'gpt-4o-2024-05-13', - // 'gpt-4o-2024-08-06', - // 'gpt-4o-mini-2024-07-18', - // 'gpt-4-turbo', - // 'gpt-4-turbo-2024-04-09', - // 'gpt-4-turbo-preview', - // 'gpt-4-0125-preview', - // 'gpt-4-1106-preview', - // 'gpt-4', - // 'gpt-4-0613', - // 'gpt-3.5-turbo-0125', - // 'gpt-3.5-turbo', - // 'gpt-3.5-turbo-1106', -]) - -// https://platform.openai.com/docs/models/gp -export const defaultDeepseekModels = modelInfoOfDefaultModelNames([ - 'deepseek-chat', - 'deepseek-reasoner', -]) - - -// https://console.groq.com/docs/models -export const defaultGroqModels = modelInfoOfDefaultModelNames([ - "llama3-70b-8192", - "llama-3.3-70b-versatile", - "llama-3.1-8b-instant", - "gemma2-9b-it", - "mixtral-8x7b-32768" -]) - - -export const defaultGeminiModels = modelInfoOfDefaultModelNames([ - 'gemini-1.5-flash', - 'gemini-1.5-pro', - 'gemini-1.5-flash-8b', - 'gemini-2.0-flash-exp', - 'gemini-2.0-flash-thinking-exp-1219', - 'learnlm-1.5-pro-experimental' -]) - -export const defaultMistralModels = modelInfoOfDefaultModelNames([ - "codestral-latest", - "open-codestral-mamba", - "open-mistral-nemo", - "mistral-large-latest", - "pixtral-large-latest", - "ministral-3b-latest", - "ministral-8b-latest", - "mistral-small-latest", -]) - -export const defaultXAIModels = modelInfoOfDefaultModelNames([ - 'grok-2-latest', - 'grok-3-latest', -]) -// export const parseMaxTokensStr = (maxTokensStr: string) => { -// // parse the string but only if the full string is a valid number, eg parseInt('100abc') should return NaN -// const int = isNaN(Number(maxTokensStr)) ? undefined : parseInt(maxTokensStr) -// if (Number.isNaN(int)) -// return undefined -// return int -// } - - - - -export const anthropicMaxPossibleTokens = (modelName: string) => { - if (modelName === 'claude-3-5-sonnet-20241022' - || modelName === 'claude-3-5-haiku-20241022') - return 8192 - if (modelName === 'claude-3-opus-20240229' - || modelName === 'claude-3-sonnet-20240229' - || modelName === 'claude-3-haiku-20240307') - return 4096 - return 1024 // return a reasonably small number if they're using a different model -} - - type UnionOfKeys = T extends T ? keyof T : never; - export const defaultProviderSettings = { anthropic: { apiKey: '', @@ -418,6 +64,14 @@ export const customSettingNamesOfProvider = (providerName: ProviderName) => { +export type VoidModelInfo = { // <-- STATEFUL + modelName: string, + isDefault: boolean, // whether or not it's a default for its provider + isHidden: boolean, // whether or not the user is hiding it (switched off) + isAutodetected?: boolean, // whether the model was autodetected by polling +} // TODO!!! eventually we'd want to let the user change supportsFIM, etc on the model themselves + + type CommonProviderSettings = { _didFillInProviderSettings: boolean | undefined, // undefined initially, computed when user types in all fields @@ -434,10 +88,6 @@ export type SettingsOfProvider = { export type SettingName = keyof SettingsAtProvider - - - - type DisplayInfoForProviderName = { title: string, desc?: string, @@ -584,110 +234,83 @@ const defaultCustomSettings: Record = { } - -export const voidInitModelOptions = { - anthropic: { - models: defaultAnthropicModels, - }, - openAI: { - models: defaultOpenAIModels, - }, - deepseek: { - models: defaultDeepseekModels, - }, - ollama: { - models: [], - }, - vLLM: { - models: [], - }, - openRouter: { - models: [], // any string - }, - openAICompatible: { - models: [], - }, - gemini: { - models: defaultGeminiModels, - }, - groq: { - models: defaultGroqModels, - }, - mistral: { - models: defaultMistralModels, - }, - xAI: { - models: defaultXAIModels, +const modelInfoOfDefaultModelNames = (defaultModelNames: string[]): { models: VoidModelInfo[] } => { + return { + models: defaultModelNames.map((modelName, i) => ({ + modelName, + isDefault: true, + isAutodetected: false, + isHidden: defaultModelNames.length >= 10, // hide all models if there are a ton of them, and make user enable them individually + })) } -} satisfies Record - +} // used when waiting and for a type reference export const defaultSettingsOfProvider: SettingsOfProvider = { anthropic: { ...defaultCustomSettings, ...defaultProviderSettings.anthropic, - ...voidInitModelOptions.anthropic, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.anthropic), _didFillInProviderSettings: undefined, }, openAI: { ...defaultCustomSettings, ...defaultProviderSettings.openAI, - ...voidInitModelOptions.openAI, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openAI), _didFillInProviderSettings: undefined, }, deepseek: { ...defaultCustomSettings, ...defaultProviderSettings.deepseek, - ...voidInitModelOptions.deepseek, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.deepseek), _didFillInProviderSettings: undefined, }, gemini: { ...defaultCustomSettings, ...defaultProviderSettings.gemini, - ...voidInitModelOptions.gemini, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.gemini), _didFillInProviderSettings: undefined, }, mistral: { ...defaultCustomSettings, ...defaultProviderSettings.mistral, - ...voidInitModelOptions.mistral, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.mistral), _didFillInProviderSettings: undefined, }, xAI: { ...defaultCustomSettings, ...defaultProviderSettings.xAI, - ...voidInitModelOptions.xAI, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.xAI), _didFillInProviderSettings: undefined, }, groq: { // aggregator ...defaultCustomSettings, ...defaultProviderSettings.groq, - ...voidInitModelOptions.groq, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.groq), _didFillInProviderSettings: undefined, }, openRouter: { // aggregator ...defaultCustomSettings, ...defaultProviderSettings.openRouter, - ...voidInitModelOptions.openRouter, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openRouter), _didFillInProviderSettings: undefined, }, openAICompatible: { // aggregator ...defaultCustomSettings, ...defaultProviderSettings.openAICompatible, - ...voidInitModelOptions.openAICompatible, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openAICompatible), _didFillInProviderSettings: undefined, }, ollama: { // aggregator ...defaultCustomSettings, ...defaultProviderSettings.ollama, - ...voidInitModelOptions.ollama, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.ollama), _didFillInProviderSettings: undefined, }, vLLM: { // aggregator ...defaultCustomSettings, ...defaultProviderSettings.vLLM, - ...voidInitModelOptions.vLLM, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.vLLM), _didFillInProviderSettings: undefined, }, } diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts new file mode 100644 index 00000000..ce0d0537 --- /dev/null +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts @@ -0,0 +1,776 @@ +/*-------------------------------------------------------------------------------------- + * Copyright 2025 Glass Devtools, Inc. All rights reserved. + * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. + *--------------------------------------------------------------------------------------*/ + +import OpenAI, { ClientOptions } from 'openai'; +import { Model as OpenAIModel } from 'openai/resources/models.js'; +import { OllamaModelResponse, OnText, OnFinalMessage, OnError, LLMChatMessage, LLMFIMMessage, ModelListParams } from '../../common/llmMessageTypes.js'; +import { InternalToolInfo, isAToolName } from '../../common/toolsService.js'; +import { defaultProviderSettings, displayInfoOfProviderName, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js'; +import { prepareMessages } from './preprocessLLMMessages.js'; +import Anthropic from '@anthropic-ai/sdk'; +import { Ollama } from 'ollama'; + + + +export const defaultModelsOfProvider = { + anthropic: [ // https://docs.anthropic.com/en/docs/about-claude/models + 'claude-3-5-sonnet-latest', + 'claude-3-5-haiku-latest', + 'claude-3-opus-latest', + ], + openAI: [ // https://platform.openai.com/docs/models/gp + 'o1', + 'o1-mini', + 'o3-mini', + 'gpt-4o', + 'gpt-4o-mini', + ], + deepseek: [ // https://platform.openai.com/docs/models/gp + 'deepseek-chat', + 'deepseek-reasoner', + ], + ollama: [], + vLLM: [], + openRouter: [], + openAICompatible: [], + gemini: [ + 'gemini-1.5-flash', + 'gemini-1.5-pro', + 'gemini-1.5-flash-8b', + 'gemini-2.0-flash-exp', + 'gemini-2.0-flash-thinking-exp-1219', + 'learnlm-1.5-pro-experimental' + ], + groq: [ // https://console.groq.com/docs/models + "llama3-70b-8192", + "llama-3.3-70b-versatile", + "llama-3.1-8b-instant", + "gemma2-9b-it", + "mixtral-8x7b-32768" + ], + mistral: [ // https://docs.mistral.ai/getting-started/models/models_overview/ + "codestral-latest", + "open-codestral-mamba", + "open-mistral-nemo", + "mistral-large-latest", + "pixtral-large-latest", + "ministral-3b-latest", + "ministral-8b-latest", + "mistral-small-latest", + ], + xAI: [ // https://docs.x.ai/docs/models?cluster=us-east-1 + 'grok-3-latest', + 'grok-2-latest', + ], +} satisfies Record + + + +type ModelOptions = { + contextWindow: number; + cost: { + input: number; + output: number; + cache_read?: number; + cache_write?: number; + } + supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated'; + supportsTools: false | 'anthropic-style' | 'openai-style'; + supportsFIM: false | 'TODO_FIM_FORMAT'; + + supportsReasoning: boolean; // not whether it reasons, but whether it outputs reasoning tokens + manualMatchReasoningTokens?: [string, string]; // reasoning tokens if it's an OSS model +} + +type ProviderReasoningOptions = { + // include this in payload to get reasoning + input?: { includeInPayload?: { [key: string]: any }, }; + // nameOfFieldInDelta: reasoning output is in response.choices[0].delta[deltaReasoningField] + // needsManualParse: whether we must manually parse out the tags + output?: + | { nameOfFieldInDelta?: string, needsManualParse?: undefined, } + | { nameOfFieldInDelta?: undefined, needsManualParse?: true, }; +} + +type ProviderSettings = { + providerReasoningOptions?: ProviderReasoningOptions; + modelOptions: { [key: string]: ModelOptions }; + modelOptionsFallback: (modelName: string) => ModelOptions; // allowed to throw error if modeName is totally invalid +} + + +type ModelSettingsOfProvider = { + [providerName in ProviderName]: ProviderSettings +} + + + + + +const modelNotRecognizedErrorMessage = (modelName: string, providerName: ProviderName) => `Void could not find a model matching ${modelName} for ${displayInfoOfProviderName(providerName).title}.` + + + +// ---------------- OPENAI ---------------- +const openAIModelOptions = { + "o1": { + contextWindow: 128_000, + cost: { input: 15.00, cache_read: 7.50, output: 60.00, }, + supportsFIM: false, + supportsTools: false, + supportsSystemMessage: 'developer-role', + supportsReasoning: false, + }, + "o3-mini": { + contextWindow: 200_000, + cost: { input: 1.10, cache_read: 0.55, output: 4.40, }, + supportsFIM: false, + supportsTools: false, + supportsSystemMessage: 'developer-role', + supportsReasoning: false, + }, + "gpt-4o": { + contextWindow: 128_000, + cost: { input: 2.50, cache_read: 1.25, output: 10.00, }, + supportsFIM: false, + supportsTools: 'openai-style', + supportsSystemMessage: 'system-role', + supportsReasoning: false, + }, +} as const + +const openAISettings: ProviderSettings = { + modelOptions: openAIModelOptions, + modelOptionsFallback: (modelName) => { + if (modelName.includes('o1')) return openAIModelOptions['o1'] + if (modelName.includes('o3-mini')) return openAIModelOptions['o3-mini'] + if (modelName.includes('gpt-4o')) return openAIModelOptions['gpt-4o'] + throw new Error(modelNotRecognizedErrorMessage(modelName, 'openAI')) + } +} + +// ---------------- ANTHROPIC ---------------- +const anthropicModelOptions = { + "claude-3-5-sonnet-20241022": { + contextWindow: 200_000, + cost: { input: 3.00, cache_read: 0.30, cache_write: 3.75, output: 15.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + + }, + "claude-3-5-haiku-20241022": { + contextWindow: 200_000, + cost: { input: 0.80, cache_read: 0.08, cache_write: 1.00, output: 4.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + }, + "claude-3-opus-20240229": { + contextWindow: 200_000, + cost: { input: 15.00, cache_read: 1.50, cache_write: 18.75, output: 75.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + }, + "claude-3-sonnet-20240229": { + contextWindow: 200_000, cost: { input: 3.00, output: 15.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + } +} as const + +const anthropicSettings: ProviderSettings = { + modelOptions: anthropicModelOptions, + modelOptionsFallback: (modelName) => { + throw new Error(modelNotRecognizedErrorMessage(modelName, 'anthropic')) + } +} + + +// ---------------- XAI ---------------- +const XAIModelOptions = { + "grok-2-latest": { + contextWindow: 131_072, + cost: { input: 2.00, output: 10.00 }, + supportsFIM: false, + supportsSystemMessage: 'system-role', + supportsTools: 'openai-style', + supportsReasoning: false, + }, +} as const + +const XAISettings: ProviderSettings = { + modelOptions: XAIModelOptions, + modelOptionsFallback: (modelName) => { + throw new Error(modelNotRecognizedErrorMessage(modelName, 'xAI')) + } +} + + + +const modelSettingsOfProvider: ModelSettingsOfProvider = { + openAI: openAISettings, + anthropic: anthropicSettings, + xAI: XAISettings, + gemini: { + modelOptions: { + + } + }, + googleVertex: { + + }, + microsoftAzure: { + + }, + openRouter: { + providerReasoningOptions: { + // reasoning: OAICompat + response.choices[0].delta.reasoning : payload should have {include_reasoning: true} https://openrouter.ai/announcements/reasoning-tokens-for-thinking-models + input: { includeInPayload: { include_reasoning: true } }, + output: { nameOfFieldInDelta: 'reasoning' }, + } + }, + vLLM: { + providerReasoningOptions: { + // reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions + output: { nameOfFieldInDelta: 'reasoning_content' }, + } + }, + deepseek: { + providerReasoningOptions: { + // reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://api-docs.deepseek.com/guides/reasoning_model + output: { nameOfFieldInDelta: 'reasoning_content' }, + }, + }, + ollama: { + providerReasoningOptions: { + // reasoning: we need to filter out reasoning tags manually + output: { needsManualParse: true }, + }, + }, + + openAICompatible: { + }, + mistral: { + }, + groq: { + }, + + + +} as const satisfies ModelSettingsOfProvider + + +const modelOptionsOfProvider = (providerName: ProviderName, modelName: string) => { + const { modelOptions, modelOptionsFallback } = modelSettingsOfProvider[providerName] + if (modelName in modelOptions) return modelOptions[modelName] + return modelOptionsFallback(modelName) +} + + + +type InternalCommonMessageParams = { + aiInstructions: string; + onText: OnText; + onFinalMessage: OnFinalMessage; + onError: OnError; + providerName: ProviderName; + settingsOfProvider: SettingsOfProvider; + modelName: string; + _setAborter: (aborter: () => void) => void; +} + +type SendChatParams_Internal = InternalCommonMessageParams & { messages: LLMChatMessage[]; tools?: InternalToolInfo[] } +type SendFIMParams_Internal = InternalCommonMessageParams & { messages: LLMFIMMessage; } +export type ListParams_Internal = ModelListParams + + +// ------------ OPENAI-COMPATIBLE (HELPERS) ------------ +const toOpenAICompatibleTool = (toolInfo: InternalToolInfo) => { + const { name, description, params, required } = toolInfo + return { + type: 'function', + function: { + name: name, + description: description, + parameters: { + type: 'object', + properties: params, + required: required, + } + } + } satisfies OpenAI.Chat.Completions.ChatCompletionTool +} + +type ToolCallOfIndex = { [index: string]: { name: string, params: string, id: string } } + +const toolCallsFrom_OpenAICompat = (toolCallOfIndex: ToolCallOfIndex) => { + return Object.keys(toolCallOfIndex).map(index => { + const tool = toolCallOfIndex[index] + return isAToolName(tool.name) ? { name: tool.name, id: tool.id, params: tool.params } : null + }).filter(t => !!t) +} + + +const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => { + const commonPayloadOpts: ClientOptions = { + dangerouslyAllowBrowser: true, + ...includeInPayload, + } + if (providerName === 'openAI') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'ollama') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts }) + } + else if (providerName === 'vLLM') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts }) + } + else if (providerName === 'openRouter') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ + baseURL: 'https://openrouter.ai/api/v1', + apiKey: thisConfig.apiKey, + defaultHeaders: { + 'HTTP-Referer': 'https://voideditor.com', // Optional, for including your app on openrouter.ai rankings. + 'X-Title': 'Void', // Optional. Shows in rankings on openrouter.ai. + }, + ...commonPayloadOpts, + }) + } + else if (providerName === 'gemini') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'deepseek') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'openAICompatible') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'mistral') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'groq') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'xAI') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.x.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + + else throw new Error(`Void providerName was invalid: ${providerName}.`) +} + + + +const manualParseOnText = ( + providerName: ProviderName, + modelName: string, + onText_: OnText +): OnText => { + return onText_ +} + + +const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => { + const { + supportsReasoning: modelSupportsReasoning, + supportsSystemMessage, + supportsTools, + } = modelOptionsOfProvider(providerName, modelName) + + const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, }) + const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toOpenAICompatibleTool(tool)) : undefined + + const includeInPayload = modelSupportsReasoning ? {} : modelSettingsOfProvider[providerName].providerReasoningOptions?.input?.includeInPayload || {} + + const toolsObj = tools ? { tools: tools, tool_choice: 'auto', parallel_tool_calls: false, } as const : {} + const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload }) + const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelName, messages: messages, stream: true, ...toolsObj } + + const { nameOfFieldInDelta: nameOfReasoningFieldInDelta, needsManualParse: needsManualReasoningParse } = modelSettingsOfProvider[providerName].providerReasoningOptions?.output ?? {} + if (needsManualReasoningParse) onText = manualParseOnText(providerName, modelName, onText) + + let fullReasoning = '' + let fullText = '' + const toolCallOfIndex: ToolCallOfIndex = {} + openai.chat.completions + .create(options) + .then(async response => { + _setAborter(() => response.controller.abort()) + // when receive text + for await (const chunk of response) { + // tool call + for (const tool of chunk.choices[0]?.delta?.tool_calls ?? []) { + const index = tool.index + if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', params: '', id: '' } + toolCallOfIndex[index].name += tool.function?.name ?? '' + toolCallOfIndex[index].params += tool.function?.arguments ?? ''; + toolCallOfIndex[index].id = tool.id ?? '' + } + // message + const newText = chunk.choices[0]?.delta?.content ?? '' + fullText += newText + + // reasoning + let newReasoning = '' + if (nameOfReasoningFieldInDelta) { + // @ts-ignore + newReasoning = (chunk.choices[0]?.delta?.[nameOfFieldInDelta] || '') + '' + fullReasoning += newReasoning + } + + onText({ newText, fullText, newReasoning, fullReasoning }) + } + onFinalMessage({ fullText, toolCalls: toolCallsFrom_OpenAICompat(toolCallOfIndex) }); + }) + // when error/fail - this catches errors of both .create() and .then(for await) + .catch(error => { + if (error instanceof OpenAI.APIError && error.status === 401) { onError({ message: 'Invalid API key.', fullError: error }); } + else { onError({ message: error + '', fullError: error }); } + }) +} + + +const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider, providerName }: ListParams_Internal) => { + const onSuccess = ({ models }: { models: OpenAIModel[] }) => { + onSuccess_({ models }) + } + const onError = ({ error }: { error: string }) => { + onError_({ error }) + } + try { + const openai = newOpenAICompatibleSDK({ providerName, settingsOfProvider }) + openai.models.list() + .then(async (response) => { + const models: OpenAIModel[] = [] + models.push(...response.data) + while (response.hasNextPage()) { + models.push(...(await response.getNextPage()).data) + } + onSuccess({ models }) + }) + .catch((error) => { + onError({ error: error + '' }) + }) + } + catch (error) { + onError({ error: error + '' }) + } +} + + + +// ------------ OPENAI ------------ +const sendOpenAIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ ANTHROPIC ------------ +const toAnthropicTool = (toolInfo: InternalToolInfo) => { + const { name, description, params, required } = toolInfo + return { + name: name, + description: description, + input_schema: { + type: 'object', + properties: params, + required: required, + } + } satisfies Anthropic.Messages.Tool +} + +const toolCallsFromAnthropicContent = (content: Anthropic.Messages.ContentBlock[]) => { + return content.map(c => { + if (c.type !== 'tool_use') return null + if (!isAToolName(c.name)) return null + return c.type === 'tool_use' ? { name: c.name, params: JSON.stringify(c.input), id: c.id } : null + }).filter(t => !!t) +} + +const sendAnthropicChat = ({ messages: messages_, onText, providerName, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }: SendChatParams_Internal) => { + const { + // supportsReasoning: modelSupportsReasoning, + supportsSystemMessage, + supportsTools, + contextWindow, + } = modelOptionsOfProvider(providerName, modelName) + + const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, }) + + const thisConfig = settingsOfProvider.anthropic + const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true }); + const tools = ((tools_?.length ?? 0) !== 0) ? tools_?.map(tool => toAnthropicTool(tool)) : undefined + + const stream = anthropic.messages.stream({ + system: separateSystemMessageStr, + messages: messages, + model: modelName, + max_tokens: contextWindow, + tools: tools, + tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time + }) + // when receive text + stream.on('text', (newText, fullText) => { + onText({ newText, fullText, newReasoning: '', fullReasoning: '' }) + }) + // when we get the final message on this stream (or when error/fail) + stream.on('finalMessage', (response) => { + const content = response.content.map(c => c.type === 'text' ? c.text : '').join('\n\n') + const toolCalls = toolCallsFromAnthropicContent(response.content) + onFinalMessage({ fullText: content, toolCalls }) + }) + // on error + stream.on('error', (error) => { + if (error instanceof Anthropic.APIError && error.status === 401) { onError({ message: 'Invalid API key.', fullError: error }) } + else { onError({ message: error + '', fullError: error }) } + }) + _setAborter(() => stream.controller.abort()) +} + +// // in future, can do tool_use streaming in anthropic, but it's pretty fast even without streaming... +// const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {} +// stream.on('streamEvent', e => { +// if (e.type === 'content_block_start') { +// if (e.content_block.type !== 'tool_use') return +// const index = e.index +// if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '' } +// toolCallOfIndex[index].name += e.content_block.name ?? '' +// toolCallOfIndex[index].args += e.content_block.input ?? '' +// } +// else if (e.type === 'content_block_delta') { +// if (e.delta.type !== 'input_json_delta') return +// toolCallOfIndex[e.index].args += e.delta.partial_json +// } +// }) + + +// ------------ XAI ------------ +const sendXAIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ GEMINI ------------ +const sendGeminiAPIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ OLLAMA ------------ +const newOllamaSDK = ({ endpoint }: { endpoint: string }) => { + // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in + if (!endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} in Void if you want the default url).`) + const ollama = new Ollama({ host: endpoint }) + return ollama +} + +const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }: ListParams_Internal) => { + const onSuccess = ({ models }: { models: OllamaModelResponse[] }) => { + onSuccess_({ models }) + } + const onError = ({ error }: { error: string }) => { + onError_({ error }) + } + try { + const thisConfig = settingsOfProvider.ollama + const ollama = newOllamaSDK({ endpoint: thisConfig.endpoint }) + ollama.list() + .then((response) => { + const { models } = response + onSuccess({ models }) + }) + .catch((error) => { + onError({ error: error + '' }) + }) + } + catch (error) { + onError({ error: error + '' }) + } +} + +const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => { + const thisConfig = settingsOfProvider.ollama + const ollama = newOllamaSDK({ endpoint: thisConfig.endpoint }) + + let fullText = '' + ollama.generate({ + model: modelName, + prompt: messages.prefix, + suffix: messages.suffix, + options: { + stop: messages.stopTokens, + num_predict: 300, // max tokens + // repeat_penalty: 1, + }, + raw: true, + stream: true, // stream is not necessary but lets us expose the + }) + .then(async stream => { + _setAborter(() => stream.abort()) + for await (const chunk of stream) { + const newText = chunk.response + fullText += newText + } + onFinalMessage({ fullText }) + }) + // when error/fail + .catch((error) => { + onError({ message: error + '', fullError: error }) + }) +} + + +// ollama's implementation of openai-compatible SDK dumps all reasoning tokens out with message, and supports tools, so we can use it for chat! +const sendOllamaChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ OPENAI-COMPATIBLE ------------ +// TODO!!! FIM + +// using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration +const sendOpenAICompatibleChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ OPENROUTER ------------ +const sendOpenRouterChat = (params: SendChatParams_Internal) => { + _sendOpenAICompatibleChat(params) +} + +// ------------ VLLM ------------ +const vLLMList = async (params: ListParams_Internal) => { + return _openaiCompatibleList(params) +} +const sendVLLMFIM = (params: SendFIMParams_Internal) => { + // TODO!!! +} + +// using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration +const sendVLLMChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ DEEPSEEK API ------------ +const sendDeepSeekAPIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ MISTRAL ------------ +const sendMistralAPIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ GROQ ------------ +const sendGroqAPIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + + + + +/* +FIM: + +qwen2.5-coder https://ollama.com/library/qwen2.5-coder/blobs/e94a8ecb9327 +<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> + +codestral https://ollama.com/library/codestral/blobs/51707752a87c +[SUFFIX]{{ .Suffix }}[PREFIX] {{ .Prompt }} + +deepseek-coder-v2 https://ollama.com/library/deepseek-coder-v2/blobs/22091531faf0 +<｜fim▁begin｜>{{ .Prompt }}<｜fim▁hole｜>{{ .Suffix }}<｜fim▁end｜> + +starcoder2 https://ollama.com/library/starcoder2/blobs/3b190e68fefe + + +{{ .Prompt }}{{ .Suffix }} +<|end_of_text|> + +codegemma https://ollama.com/library/codegemma:2b/blobs/48d9a8140749 +<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> + +*/ + + + +type CallFnOfProvider = { + [providerName in ProviderName]: { + sendChat: (params: SendChatParams_Internal) => void; + sendFIM: ((params: SendFIMParams_Internal) => void) | null; + list: ((params: ListParams_Internal) => void) | null; + } +} +export const sendLLMMessageToProviderImplementation = { + openAI: { + sendChat: sendOpenAIChat, + sendFIM: null, + list: null, + }, + anthropic: { + sendChat: sendAnthropicChat, + sendFIM: null, + list: null, + }, + xAI: { + sendChat: sendXAIChat, + sendFIM: null, + list: null, + }, + gemini: { + sendChat: sendGeminiAPIChat, + sendFIM: null, + list: null, + }, + ollama: { + sendChat: sendOllamaChat, + sendFIM: sendOllamaFIM, + list: ollamaList, + }, + openAICompatible: { + sendChat: sendOpenAICompatibleChat, + sendFIM: null, + list: null, + }, + openRouter: { + sendChat: sendOpenRouterChat, + sendFIM: null, + list: null, + }, + vLLM: { + sendChat: sendVLLMChat, + sendFIM: sendVLLMFIM, + list: vLLMList, + }, + deepseek: { + sendChat: sendDeepSeekAPIChat, + sendFIM: null, + list: null, + }, + groq: { + sendChat: sendGroqAPIChat, + sendFIM: null, + list: null, + }, + mistral: { + sendChat: sendMistralAPIChat, + sendFIM: null, + list: null, + }, + +} satisfies CallFnOfProvider diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/_old.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/_old.ts deleted file mode 100644 index e1e90245..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/_old.ts +++ /dev/null @@ -1,96 +0,0 @@ -// /*-------------------------------------------------------------------------------------- -// * Copyright 2025 Glass Devtools, Inc. All rights reserved. -// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. -// *--------------------------------------------------------------------------------------*/ - -// import Groq from 'groq-sdk'; -// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; - -// // Groq -// export const sendGroqChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { -// let fullText = ''; - -// const thisConfig = settingsOfProvider.groq - -// const groq = new Groq({ -// apiKey: thisConfig.apiKey, -// dangerouslyAllowBrowser: true -// }); - -// await groq.chat.completions -// .create({ -// messages: messages, -// model: modelName, -// stream: true, -// }) -// .then(async response => { -// _setAborter(() => response.controller.abort()) -// // when receive text -// for await (const chunk of response) { -// const newText = chunk.choices[0]?.delta?.content || ''; -// fullText += newText; -// onText({ newText, fullText }); -// } - -// onFinalMessage({ fullText, tools: [] }); -// }) -// .catch(error => { -// onError({ message: error + '', fullError: error }); -// }) - - -// }; - - - -// /*-------------------------------------------------------------------------------------- -// * Copyright 2025 Glass Devtools, Inc. All rights reserved. -// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. -// *--------------------------------------------------------------------------------------*/ - -// import { Mistral } from '@mistralai/mistralai'; -// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; - -// // Mistral -// export const sendMistralChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { -// let fullText = ''; - -// const thisConfig = settingsOfProvider.mistral; - -// const mistral = new Mistral({ -// apiKey: thisConfig.apiKey, -// }) - -// await mistral.chat -// .stream({ -// messages: messages, -// model: modelName, -// stream: true, -// }) -// .then(async response => { -// // Mistral has a really nonstandard API - no interrupt and weird stream types -// _setAborter(() => { console.log('Mistral does not support interrupts! Further messages will just be ignored.') }); -// // when receive text -// for await (const chunk of response) { -// const c = chunk.data.choices[0].delta.content || '' -// const newText = ( -// typeof c === 'string' ? c -// : c?.map(c => c.type === 'text' ? c.text : c.type).join('\n') -// ) -// fullText += newText; -// onText({ newText, fullText }); -// } - -// onFinalMessage({ fullText, tools: [] }); -// }) -// .catch(error => { -// onError({ message: error + '', fullError: error }); -// }) -// } - - - - - - - diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts deleted file mode 100644 index c4338ebb..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/anthropic.ts +++ /dev/null @@ -1,114 +0,0 @@ -/*-------------------------------------------------------------------------------------- - * Copyright 2025 Glass Devtools, Inc. All rights reserved. - * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. - *--------------------------------------------------------------------------------------*/ - -import Anthropic from '@anthropic-ai/sdk'; -import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; -import { anthropicMaxPossibleTokens, developerInfoOfModelName, developerInfoOfProviderName } from '../../common/voidSettingsTypes.js'; -import { InternalToolInfo } from '../../common/toolsService.js'; -import { addSystemMessageAndToolSupport } from './preprocessLLMMessages.js'; -import { isAToolName } from './postprocessToolCalls.js'; - - - - -export const toAnthropicTool = (toolInfo: InternalToolInfo) => { - const { name, description, params, required } = toolInfo - return { - name: name, - description: description, - input_schema: { - type: 'object', - properties: params, - required: required, - } - } satisfies Anthropic.Messages.Tool -} - - - - - -export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }) => { - - const thisConfig = settingsOfProvider.anthropic - - const maxTokens = anthropicMaxPossibleTokens(modelName) - if (maxTokens === undefined) { - onError({ message: `Please set a value for Max Tokens.`, fullError: null }) - return - } - - const { messages, separateSystemMessageStr } = addSystemMessageAndToolSupport(modelName, providerName, messages_, aiInstructions, { separateSystemMessage: true }) - - const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName) - const { supportsTools } = developerInfoOfModelName(modelName, overrideSettingsForAllModels) - - const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true }); - - const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toAnthropicTool(tool)) : undefined - - const stream = anthropic.messages.stream({ - system: separateSystemMessageStr, - messages: messages, - model: modelName, - max_tokens: maxTokens, - tools: tools, - tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time - }) - - - // when receive text - stream.on('text', (newText, fullText) => { - onText({ newText, fullText }) - }) - - - // // can do tool use streaming - // const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {} - // stream.on('streamEvent', e => { - // if (e.type === 'content_block_start') { - // if (e.content_block.type !== 'tool_use') return - // const index = e.index - // if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '' } - // toolCallOfIndex[index].name += e.content_block.name ?? '' - // toolCallOfIndex[index].args += e.content_block.input ?? '' - // } - // else if (e.type === 'content_block_delta') { - // if (e.delta.type !== 'input_json_delta') return - // toolCallOfIndex[e.index].args += e.delta.partial_json - // } - // // TODO!!!!! - // // onText({}) - // }) - - // when we get the final message on this stream (or when error/fail) - stream.on('finalMessage', (response) => { - // stringify the response's content - const content = response.content.map(c => c.type === 'text' ? c.text : '').join('\n\n') - const toolCalls = response.content - .map(c => { - if (c.type !== 'tool_use') return null - if (!isAToolName(c.name)) return null - return c.type === 'tool_use' ? { name: c.name, params: JSON.stringify(c.input), id: c.id } : null - }) - .filter(t => !!t) - - onFinalMessage({ fullText: content, toolCalls }) - }) - - stream.on('error', (error) => { - // the most common error will be invalid API key (401), so we handle this with a nice message - if (error instanceof Anthropic.APIError && error.status === 401) { - onError({ message: 'Invalid API key.', fullError: error }) - } - else { - onError({ message: error + '', fullError: error }) // anthropic errors can be stringified nicely like this - } - }) - - // TODO need to test this to make sure it works, it might throw an error - _setAborter(() => stream.controller.abort()) - -}; diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/ollama.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/ollama.ts deleted file mode 100644 index da6715c0..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/ollama.ts +++ /dev/null @@ -1,124 +0,0 @@ - -/*-------------------------------------------------------------------------------------- - * Copyright 2025 Glass Devtools, Inc. All rights reserved. - * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. - *--------------------------------------------------------------------------------------*/ - -import { Ollama } from 'ollama'; -import { _InternalModelListFnType, _InternalSendLLMFIMMessageFnType, _InternalSendLLMChatMessageFnType, OllamaModelResponse } from '../../common/llmMessageTypes.js'; -import { defaultProviderSettings } from '../../common/voidSettingsTypes.js'; - -export const ollamaList: _InternalModelListFnType = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }) => { - - const onSuccess = ({ models }: { models: OllamaModelResponse[] }) => { - onSuccess_({ models }) - } - - const onError = ({ error }: { error: string }) => { - onError_({ error }) - } - - try { - const thisConfig = settingsOfProvider.ollama - // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in - if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} in Void if you want the default url).`) - - const ollama = new Ollama({ host: thisConfig.endpoint }) - ollama.list() - .then((response) => { - const { models } = response - onSuccess({ models }) - }) - .catch((error) => { - onError({ error: error + '' }) - }) - } - catch (error) { - onError({ error: error + '' }) - } -} - - - - -// export const sendOllamaFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { - -// const thisConfig = settingsOfProvider.ollama -// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in -// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`) - -// let fullText = '' - -// const ollama = new Ollama({ host: thisConfig.endpoint }) - -// ollama.generate({ -// model: modelName, -// prompt: messages.prefix, -// suffix: messages.suffix, -// options: { -// stop: messages.stopTokens, -// num_predict: 300, // max tokens -// // repeat_penalty: 1, -// }, -// raw: true, -// stream: true, -// }) -// .then(async stream => { -// _setAborter(() => stream.abort()) -// // iterate through the stream -// for await (const chunk of stream) { -// const newText = chunk.response; -// fullText += newText; -// onText({ newText, fullText }); -// } -// onFinalMessage({ fullText, tools: [] }); -// }) -// // when error/fail -// .catch((error) => { -// onError({ message: error + '', fullError: error }) -// }) -// }; - - -// // Ollama -// export const sendOllamaChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => { - -// const thisConfig = settingsOfProvider.ollama -// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in -// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`) - -// let fullText = '' - -// const ollama = new Ollama({ host: thisConfig.endpoint }) - -// ollama.chat({ -// model: modelName, -// messages: messages, -// stream: true, -// // options: { num_predict: parseMaxTokensStr(thisConfig.maxTokens) } // this is max_tokens -// }) -// .then(async stream => { -// _setAborter(() => stream.abort()) -// // iterate through the stream -// for await (const chunk of stream) { -// const newText = chunk.message.content; - -// // chunk.message.tool_calls[0].function.arguments - -// fullText += newText; -// onText({ newText, fullText }); -// } - -// onFinalMessage({ fullText, tools: [] }); - -// }) -// // when error/fail -// .catch((error) => { -// onError({ message: error + '', fullError: error }) -// }) - -// }; - - - -// // ['codestral', 'qwen2.5-coder', 'qwen2.5-coder:0.5b', 'qwen2.5-coder:1.5b', 'qwen2.5-coder:3b', 'qwen2.5-coder:7b', 'qwen2.5-coder:14b', 'qwen2.5-coder:32b', 'codegemma', 'codegemma:2b', 'codegemma:7b', 'codellama', 'codellama:7b', 'codellama:13b', 'codellama:34b', 'codellama:70b', 'codellama:code', 'codellama:python', 'command-r', 'command-r:35b', 'command-r-plus', 'command-r-plus:104b', 'deepseek-coder-v2', 'deepseek-coder-v2:16b', 'deepseek-coder-v2:236b', 'falcon2', 'falcon2:11b', 'firefunction-v2', 'firefunction-v2:70b', 'gemma', 'gemma:2b', 'gemma:7b', 'gemma2', 'gemma2:2b', 'gemma2:9b', 'gemma2:27b', 'llama2', 'llama2:7b', 'llama2:13b', 'llama2:70b', 'llama3', 'llama3:8b', 'llama3:70b', 'llama3-chatqa', 'llama3-chatqa:8b', 'llama3-chatqa:70b', 'llama3-gradient', 'llama3-gradient:8b', 'llama3-gradient:70b', 'llama3.1', 'llama3.1:8b', 'llama3.1:70b', 'llama3.1:405b', 'llava', 'llava:7b', 'llava:13b', 'llava:34b', 'llava-llama3', 'llava-llama3:8b', 'llava-phi3', 'llava-phi3:3.8b', 'mistral', 'mistral:7b', 'mistral-large', 'mistral-large:123b', 'mistral-nemo', 'mistral-nemo:12b', 'mixtral', 'mixtral:8x7b', 'mixtral:8x22b', 'moondream', 'moondream:1.8b', 'openhermes', 'openhermes:v2.5', 'phi3', 'phi3:3.8b', 'phi3:14b', 'phi3.5', 'phi3.5:3.8b', 'qwen', 'qwen:7b', 'qwen:14b', 'qwen:32b', 'qwen:72b', 'qwen:110b', 'qwen2', 'qwen2:0.5b', 'qwen2:1.5b', 'qwen2:7b', 'qwen2:72b', 'smollm', 'smollm:135m', 'smollm:360m', 'smollm:1.7b',] diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts deleted file mode 100644 index 7769a983..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/openai.ts +++ /dev/null @@ -1,231 +0,0 @@ -/*-------------------------------------------------------------------------------------- - * Copyright 2025 Glass Devtools, Inc. All rights reserved. - * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. - *--------------------------------------------------------------------------------------*/ - -import OpenAI from 'openai'; -import { _InternalModelListFnType, _InternalSendLLMFIMMessageFnType, _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js'; -import { Model } from 'openai/resources/models.js'; -import { InternalToolInfo } from '../../common/toolsService.js'; -import { addSystemMessageAndToolSupport } from './preprocessLLMMessages.js'; -import { developerInfoOfModelName, developerInfoOfProviderName } from '../../common/voidSettingsTypes.js'; -import { isAToolName } from './postprocessToolCalls.js'; - - -// developer command - https://cdn.openai.com/spec/model-spec-2024-05-08.html#follow-the-chain-of-command -// prompting - https://platform.openai.com/docs/guides/reasoning#advice-on-prompting - -// npm i @openrouter/ai-sdk-provider ai ollama-ai-provider - -export const toOpenAITool = (toolInfo: InternalToolInfo) => { - const { name, description, params, required } = toolInfo - return { - type: 'function', - function: { - name: name, - description: description, - parameters: { - type: 'object', - properties: params, - required: required, - } - } - } satisfies OpenAI.Chat.Completions.ChatCompletionTool -} - - - - - -type NewParams = Pick[0] & Parameters<_InternalSendLLMFIMMessageFnType>[0], 'settingsOfProvider' | 'providerName'> -const newOpenAI = ({ settingsOfProvider, providerName }: NewParams) => { - - if (providerName === 'openAI') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true - }) - } - else if (providerName === 'ollama') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true, - }) - } - else if (providerName === 'vLLM') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true, - }) - } - else if (providerName === 'openRouter') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: 'https://openrouter.ai/api/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, - defaultHeaders: { - 'HTTP-Referer': 'https://voideditor.com', // Optional, for including your app on openrouter.ai rankings. - 'X-Title': 'Void Editor', // Optional. Shows in rankings on openrouter.ai. - }, - }) - } - else if (providerName === 'gemini') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, - }) - } - else if (providerName === 'deepseek') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, - }) - } - else if (providerName === 'openAICompatible') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, - }) - } - else if (providerName === 'mistral') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, - }) - } - else if (providerName === 'groq') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, - }) - } - else if (providerName === 'xAI') { - const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ - baseURL: 'https://api.x.ai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, - }) - } - else { - console.error(`sendOpenAICompatibleMsg: invalid providerName: ${providerName}`) - throw new Error(`Void providerName was invalid: ${providerName}`) - } -} - - - -// might not currently be used in the code -export const openaiCompatibleList: _InternalModelListFnType = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }) => { - const onSuccess = ({ models }: { models: Model[] }) => { - onSuccess_({ models }) - } - - const onError = ({ error }: { error: string }) => { - onError_({ error }) - } - - try { - const openai = newOpenAI({ providerName: 'openAICompatible', settingsOfProvider }) - - openai.models.list() - .then(async (response) => { - const models: Model[] = [] - models.push(...response.data) - while (response.hasNextPage()) { - models.push(...(await response.getNextPage()).data) - } - onSuccess({ models }) - }) - .catch((error) => { - onError({ error: error + '' }) - }) - } - catch (error) { - onError({ error: error + '' }) - } -} - - - - -export const sendOpenAIFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName }) => { - - - // openai.completions has a FIM parameter called `suffix`, but it's deprecated and only works for ~GPT 3 era models - - - -} - - - -// OpenAI, OpenRouter, OpenAICompatible -export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }) => { - - let fullText = '' - const toolCallOfIndex: { [index: string]: { name: string, params: string, id: string } } = {} - - const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName) - const { supportsTools } = developerInfoOfModelName(modelName, overrideSettingsForAllModels) - - const { messages } = addSystemMessageAndToolSupport(modelName, providerName, messages_, aiInstructions, { separateSystemMessage: false }) - - const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toOpenAITool(tool)) : undefined - - const openai: OpenAI = newOpenAI({ providerName, settingsOfProvider }) - const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { - model: modelName, - messages: messages, - stream: true, - tools: tools, - tool_choice: tools ? 'auto' : undefined, - parallel_tool_calls: tools ? false : undefined, - } - - openai.chat.completions - .create(options) - .then(async response => { - _setAborter(() => response.controller.abort()) - - // when receive text - for await (const chunk of response) { - - // tool call - for (const tool of chunk.choices[0]?.delta?.tool_calls ?? []) { - const index = tool.index - if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', params: '', id: '' } - toolCallOfIndex[index].name += tool.function?.name ?? '' - toolCallOfIndex[index].params += tool.function?.arguments ?? ''; - toolCallOfIndex[index].id = tool.id ?? '' - - } - - // message - let newText = '' - newText += chunk.choices[0]?.delta?.content ?? '' - console.log('!!!!', JSON.stringify(chunk, null, 2)) - fullText += newText; - - onText({ newText, fullText }); - } - onFinalMessage({ - fullText, - toolCalls: Object.keys(toolCallOfIndex) - .map(index => { - const tool = toolCallOfIndex[index] - if (isAToolName(tool.name)) - return { name: tool.name, id: tool.id, params: tool.params } - return null - }) - .filter(t => !!t) - }); - }) - // when error/fail - this catches errors of both .create() and .then(for await) - .catch(error => { - if (error instanceof OpenAI.APIError && error.status === 401) { - onError({ message: 'Invalid API key.', fullError: error }); - } - else { - onError({ message: error + '', fullError: error }); - } - }) - -} diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/postprocessToolCalls.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/postprocessToolCalls.ts deleted file mode 100644 index aee52dcb..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/postprocessToolCalls.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { ToolName, toolNames } from '../../common/toolsService.js'; - -const toolNamesSet = new Set(toolNames) - -export const isAToolName = (toolName: string): toolName is ToolName => { - const isAToolName = toolNamesSet.has(toolName) - return isAToolName -} diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts index 689e44de..1d388338 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts @@ -1,7 +1,6 @@ import { LLMChatMessage } from '../../common/llmMessageTypes.js'; -import { developerInfoOfModelName, developerInfoOfProviderName, ProviderName } from '../../common/voidSettingsTypes.js'; import { deepClone } from '../../../../../base/common/objects.js'; @@ -14,16 +13,24 @@ export const parseObject = (args: unknown) => { return {} } -// no matter whether the model supports a system message or not (or what format it supports), add it in some way -// also take into account tools if the model doesn't support tool use -export const addSystemMessageAndToolSupport = (modelName: string, providerName: ProviderName, messages_: LLMChatMessage[], aiInstructions: string, { separateSystemMessage }: { separateSystemMessage: boolean }): { separateSystemMessageStr?: string, messages: any[] } => { +const prepareMessages_cloneAndTrim = ({ messages: messages_ }: { messages: LLMChatMessage[] }) => { const messages = deepClone(messages_).map(m => ({ ...m, content: m.content.trim(), })) + return { messages } +} - const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName) - const { supportsSystemMessage, supportsTools } = developerInfoOfModelName(modelName, overrideSettingsForAllModels) +// no matter whether the model supports a system message or not (or what format it supports), add it in some way +const prepareMessages_systemMessage = ({ + messages, + aiInstructions, + supportsSystemMessage, +}: { + messages: LLMChatMessage[], + aiInstructions: string, + supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated', +}) + : { separateSystemMessageStr?: string, messages: any[] } => { - // 1. SYSTEM MESSAGE // find system messages and concatenate them let systemMessageStr = messages .filter(msg => msg.role === 'system') @@ -33,7 +40,6 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: if (aiInstructions) systemMessageStr = `${(systemMessageStr ? `${systemMessageStr}\n\n` : '')}GUIDELINES\n${aiInstructions}` - let separateSystemMessageStr: string | undefined = undefined // remove all system messages @@ -49,11 +55,12 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: if (systemMessageStr) { // if supports system message if (supportsSystemMessage) { - if (separateSystemMessage) + if (supportsSystemMessage === 'separated') separateSystemMessageStr = systemMessageStr - else { - newMessages.unshift({ role: supportsSystemMessage === 'developer' ? 'developer' : 'system', content: systemMessageStr }) // add new first message - } + else if (supportsSystemMessage === 'system-role') + newMessages.unshift({ role: 'system', content: systemMessageStr }) // add new first message + else if (supportsSystemMessage === 'developer-role') + newMessages.unshift({ role: 'developer', content: systemMessageStr }) // add new first message } // if does not support system message else { @@ -79,225 +86,239 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName: } } - - - - - - // 2. MAKE TOOLS FORMAT CORRECT in messages - let finalMessages: any[] - if (!supportsTools) { - // do nothing - finalMessages = newMessages - } - - // anthropic assistant message will have: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples - // "content": [ - // { - // "type": "text", - // "text": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." - // }, - // { - // "type": "tool_use", - // "id": "toolu_01A09q90qw90lq917835lq9", - // "name": "get_weather", - // "input": { "location": "San Francisco, CA", "unit": "celsius" } - // } - // ] - - // anthropic user message response will be: - // "content": [ - // { - // "type": "tool_result", - // "tool_use_id": "toolu_01A09q90qw90lq917835lq9", - // "content": "15 degrees" - // } - // ] - - - else if (providerName === 'anthropic') { // convert role:'tool' to anthropic's type - const newMessagesTools: ( - Exclude | { - role: 'assistant', - content: string | ({ - type: 'text'; - text: string; - } | { - type: 'tool_use'; - name: string; - input: Record; - id: string; - })[] - } | { - role: 'user', - content: string | ({ - type: 'text'; - text: string; - } | { - type: 'tool_result'; - tool_use_id: string; - content: string; - })[] - } - )[] = newMessages; - - - for (let i = 0; i < newMessagesTools.length; i += 1) { - const currMsg = newMessagesTools[i] - - if (currMsg.role !== 'tool') continue - - const prevMsg = 0 <= i - 1 && i - 1 <= newMessagesTools.length ? newMessagesTools[i - 1] : undefined - - if (prevMsg?.role === 'assistant') { - if (typeof prevMsg.content === 'string') prevMsg.content = [{ type: 'text', text: prevMsg.content }] - prevMsg.content.push({ type: 'tool_use', id: currMsg.id, name: currMsg.name, input: parseObject(currMsg.params) }) - } - - // turn each tool into a user message with tool results at the end - newMessagesTools[i] = { - role: 'user', - content: [ - ...[{ type: 'tool_result', tool_use_id: currMsg.id, content: currMsg.content }] as const, - ...currMsg.content ? [{ type: 'text', text: currMsg.content }] as const : [], - ] - } - } - - finalMessages = newMessagesTools - } - - // openai assistant message will have: https://platform.openai.com/docs/guides/function-calling#function-calling-steps - // "tool_calls":[ - // { - // "type": "function", - // "id": "call_12345xyz", - // "function": { - // "name": "get_weather", - // "arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}" - // } - // }] - - // openai user response will be: - // { - // "role": "tool", - // "tool_call_id": tool_call.id, - // "content": str(result) - // } - - // treat all other providers like openai tool message for now - else { - - const newMessagesTools: ( - Exclude | { - role: 'assistant', - content: string; - tool_calls?: { - type: 'function'; - id: string; - function: { - name: string; - arguments: string; - } - }[] - } | { - role: 'tool', - id: string; // old val - tool_call_id: string; // new val - content: string; - } - )[] = []; - - for (let i = 0; i < newMessages.length; i += 1) { - const currMsg = newMessages[i] - - if (currMsg.role !== 'tool') { - newMessagesTools.push(currMsg) - continue - } - - // edit previous assistant message to have called the tool - const prevMsg = 0 <= i - 1 && i - 1 <= newMessagesTools.length ? newMessagesTools[i - 1] : undefined - if (prevMsg?.role === 'assistant') { - prevMsg.tool_calls = [{ - type: 'function', - id: currMsg.id, - function: { - name: currMsg.name, - arguments: JSON.stringify(currMsg.params) - } - }] - } - - // add the tool - newMessagesTools.push({ - role: 'tool', - id: currMsg.id, - content: currMsg.content, - tool_call_id: currMsg.id, - }) - } - finalMessages = newMessagesTools - } - - - - - // 3. CROP MESSAGES SO EVERYTHING FITS IN CONTEXT - // TODO!!! - - - console.log('SYSMG', separateSystemMessage) - console.log('FINAL MESSAGES', JSON.stringify(finalMessages, null, 2)) - - - return { - separateSystemMessageStr, - messages: finalMessages, - } + return { messages: newMessages, separateSystemMessageStr } } +// convert messages as if about to send to openai +/* +reference - https://platform.openai.com/docs/guides/function-calling#function-calling-steps +openai MESSAGE (role=assistant): +"tool_calls":[{ + "type": "function", + "id": "call_12345xyz", + "function": { + "name": "get_weather", + "arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}" +}] + +openai RESPONSE (role=user): +{ "role": "tool", + "tool_call_id": tool_call.id, + "content": str(result) } + +also see +openai on prompting - https://platform.openai.com/docs/guides/reasoning#advice-on-prompting +openai on developer system message - https://cdn.openai.com/spec/model-spec-2024-05-08.html#follow-the-chain-of-command +*/ + +const prepareMessages_tools_openai = ({ messages }: { messages: LLMChatMessage[], }) => { + + const newMessages: ( + Exclude | { + role: 'assistant', + content: string; + tool_calls?: { + type: 'function'; + id: string; + function: { + name: string; + arguments: string; + } + }[] + } | { + role: 'tool', + id: string; // old val + tool_call_id: string; // new val + content: string; + } + )[] = []; + + for (let i = 0; i < messages.length; i += 1) { + const currMsg = messages[i] + + if (currMsg.role !== 'tool') { + newMessages.push(currMsg) + continue + } + + // edit previous assistant message to have called the tool + const prevMsg = 0 <= i - 1 && i - 1 <= newMessages.length ? newMessages[i - 1] : undefined + if (prevMsg?.role === 'assistant') { + prevMsg.tool_calls = [{ + type: 'function', + id: currMsg.id, + function: { + name: currMsg.name, + arguments: JSON.stringify(currMsg.params) + } + }] + } + + // add the tool + newMessages.push({ + role: 'tool', + id: currMsg.id, + content: currMsg.content, + tool_call_id: currMsg.id, + }) + } + return { messages: newMessages } + +} + + +// convert messages as if about to send to anthropic +/* +https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples +anthropic MESSAGE (role=assistant): +"content": [{ + "type": "text", + "text": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." +}, { + "type": "tool_use", + "id": "toolu_01A09q90qw90lq917835lq9", + "name": "get_weather", + "input": { "location": "San Francisco, CA", "unit": "celsius" } +}] +anthropic RESPONSE (role=user): +"content": [{ + "type": "tool_result", + "tool_use_id": "toolu_01A09q90qw90lq917835lq9", + "content": "15 degrees" +}] +*/ + +const prepareMessages_tools_anthropic = ({ messages }: { messages: LLMChatMessage[], }) => { + const newMessages: ( + Exclude | { + role: 'assistant', + content: string | ({ + type: 'text'; + text: string; + } | { + type: 'tool_use'; + name: string; + input: Record; + id: string; + })[] + } | { + role: 'user', + content: string | ({ + type: 'text'; + text: string; + } | { + type: 'tool_result'; + tool_use_id: string; + content: string; + })[] + } + )[] = messages; + + + for (let i = 0; i < newMessages.length; i += 1) { + const currMsg = newMessages[i] + + if (currMsg.role !== 'tool') continue + + const prevMsg = 0 <= i - 1 && i - 1 <= newMessages.length ? newMessages[i - 1] : undefined + + if (prevMsg?.role === 'assistant') { + if (typeof prevMsg.content === 'string') prevMsg.content = [{ type: 'text', text: prevMsg.content }] + prevMsg.content.push({ type: 'tool_use', id: currMsg.id, name: currMsg.name, input: parseObject(currMsg.params) }) + } + + // turn each tool into a user message with tool results at the end + newMessages[i] = { + role: 'user', + content: [ + ...[{ type: 'tool_result', tool_use_id: currMsg.id, content: currMsg.content }] as const, + ...currMsg.content ? [{ type: 'text', text: currMsg.content }] as const : [], + ] + } + } + return { messages: newMessages } +} + + + + + +const prepareMessages_tools = ({ messages, supportsTools }: { messages: LLMChatMessage[], supportsTools: false | 'anthropic-style' | 'openai-style' }) => { + if (!supportsTools) { + return { messages: messages } + } + else if (supportsTools === 'anthropic-style') { + return prepareMessages_tools_anthropic({ messages }) + } + else if (supportsTools === 'openai-style') { + return prepareMessages_tools_openai({ messages }) + } + else { + throw 1 + } +} + + + + /* +Gemini has this, but they're openai-compat so we don't need to implement this +gemini request: +{ "role": "assistant", + "content": null, + "function_call": { + "name": "get_weather", + "arguments": { + "latitude": 48.8566, + "longitude": 2.3522 + } + } +} - -ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message) -gemini request: { -"role": "assistant", -"content": null, -"function_call": { -"name": "get_weather", -"arguments": { -"latitude": 48.8566, -"longitude": 2.3522 -} -} -} gemini response: -{ -"role": "assistant", -"function_response": { -"name": "get_weather", -"response": { -"temperature": "15°C", -"condition": "Cloudy" +{ "role": "assistant", + "function_response": { + "name": "get_weather", + "response": { + "temperature": "15°C", + "condition": "Cloudy" + } + } } -} -} - - -+ anthropic - -+ openai-compat (4) -+ gemini - -ollama - - -mistral: same as openai - */ + + + + + + + + + + + +export const prepareMessages = ({ + messages, + aiInstructions, + supportsSystemMessage, + supportsTools, +}: { + messages: LLMChatMessage[], + aiInstructions: string, + supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated', + supportsTools: false | 'anthropic-style' | 'openai-style', +}) => { + const { messages: messages1 } = prepareMessages_cloneAndTrim({ messages }) + const { messages: messages2, separateSystemMessageStr } = prepareMessages_systemMessage({ messages: messages1, aiInstructions, supportsSystemMessage }) + const { messages: messages3 } = prepareMessages_tools({ messages: messages2, supportsTools }) + return { + messages: messages3 as any, + separateSystemMessageStr + } as const +} + diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts index 7bb4e141..90deffe2 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts @@ -6,9 +6,7 @@ import { SendLLMMessageParams, OnText, OnFinalMessage, OnError } from '../../common/llmMessageTypes.js'; import { IMetricsService } from '../../common/metricsService.js'; import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js'; - -import { sendAnthropicChat } from './anthropic.js'; -import { sendOpenAIChat } from './openai.js'; +import { sendLLMMessageToProviderImplementation } from './MODELS.js'; export const sendLLMMessage = ({ @@ -58,9 +56,10 @@ export const sendLLMMessage = ({ let _setAborter = (fn: () => void) => { _aborter = fn } let _didAbort = false - const onText: OnText = ({ newText, fullText }) => { + const onText: OnText = (params) => { + const { fullText } = params if (_didAbort) return - onText_({ newText, fullText }) + onText_(params) _fullTextSoFar = fullText } @@ -95,29 +94,27 @@ export const sendLLMMessage = ({ else if (messagesType === 'FIMMessage') captureLLMEvent(`${loggingName} - Sending FIM`, {}) // TODO!!! add more metrics + try { - switch (providerName) { - case 'openAI': - case 'openRouter': - case 'deepseek': - case 'openAICompatible': - case 'mistral': - case 'ollama': - case 'vLLM': - case 'groq': - case 'gemini': - case 'xAI': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI API FIM', toolCalls: [] }) - else /* */ sendOpenAIChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }); - break; - case 'anthropic': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM', toolCalls: [] }) - else /* */ sendAnthropicChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }); - break; - default: - onError({ message: `Error: Void provider was "${providerName}", which is not recognized.`, fullError: null }) - break; + const implementation = sendLLMMessageToProviderImplementation[providerName] + if (!implementation) { + onError({ message: `Error: Provider "${providerName}" not recognized.`, fullError: null }) + return } + const { sendFIM, sendChat } = implementation + if (messagesType === 'chatMessages') { + sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }) + return + } + if (messagesType === 'FIMMessage') { + if (sendFIM) { + sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions }) + return + } + onError({ message: `Error: This provider does not support Autocomplete yet.`, fullError: null }) + return + } + onError({ message: `Error: Message type "${messagesType}" not recognized.`, fullError: null }) } catch (error) { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts b/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts index b00ade9c..d2bceb4c 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts @@ -8,30 +8,42 @@ import { IServerChannel } from '../../../../base/parts/ipc/common/ipc.js'; import { Emitter, Event } from '../../../../base/common/event.js'; -import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, MainModelListParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from '../common/llmMessageTypes.js'; +import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, VLLMModelResponse, MainModelListParams, } from '../common/llmMessageTypes.js'; import { sendLLMMessage } from './llmMessage/sendLLMMessage.js' import { IMetricsService } from '../common/metricsService.js'; -import { ollamaList } from './llmMessage/ollama.js'; -import { openaiCompatibleList } from './llmMessage/openai.js'; +import { sendLLMMessageToProviderImplementation } from './llmMessage/MODELS.js'; // NODE IMPLEMENTATION - calls actual sendLLMMessage() and returns listeners to it export class LLMMessageChannel implements IServerChannel { + // sendLLMMessage - private readonly _onText_llm = new Emitter(); - private readonly _onFinalMessage_llm = new Emitter(); - private readonly _onError_llm = new Emitter(); + private readonly llmMessageEmitters = { + onText: new Emitter(), + onFinalMessage: new Emitter(), + onError: new Emitter(), + } - // abort - private readonly _abortRefOfRequestId_llm: Record = {} + // aborters for above + private readonly abortRefOfRequestId: Record = {} - // ollamaList - private readonly _onSuccess_ollama = new Emitter>(); - private readonly _onError_ollama = new Emitter>(); - // openaiCompatibleList - private readonly _onSuccess_openAICompatible = new Emitter>(); - private readonly _onError_openAICompatible = new Emitter>(); + // list + private readonly listEmitters = { + ollama: { + success: new Emitter>(), + error: new Emitter>(), + }, + vLLM: { + success: new Emitter>(), + error: new Emitter>(), + } + } satisfies { + [providerName: string]: { + success: Emitter>, + error: Emitter>, + } + } // stupidly, channels can't take in @IService constructor( @@ -40,30 +52,17 @@ export class LLMMessageChannel implements IServerChannel { // browser uses this to listen for changes listen(_: unknown, event: string): Event { - if (event === 'onText_llm') { - return this._onText_llm.event; - } - else if (event === 'onFinalMessage_llm') { - return this._onFinalMessage_llm.event; - } - else if (event === 'onError_llm') { - return this._onError_llm.event; - } - else if (event === 'onSuccess_ollama') { - return this._onSuccess_ollama.event; - } - else if (event === 'onError_ollama') { - return this._onError_ollama.event; - } - else if (event === 'onSuccess_openAICompatible') { - return this._onSuccess_openAICompatible.event; - } - else if (event === 'onError_openAICompatible') { - return this._onError_openAICompatible.event; - } - else { - throw new Error(`Event not found: ${event}`); - } + // text + if (event === 'onText_sendLLMMessage') return this.llmMessageEmitters.onText.event; + else if (event === 'onFinalMessage_sendLLMMessage') return this.llmMessageEmitters.onFinalMessage.event; + else if (event === 'onError_sendLLMMessage') return this.llmMessageEmitters.onError.event; + // list + else if (event === 'onSuccess_list_ollama') return this.listEmitters.ollama.success.event; + else if (event === 'onError_list_ollama') return this.listEmitters.ollama.error.event; + else if (event === 'onSuccess_list_vLLM') return this.listEmitters.vLLM.success.event; + else if (event === 'onError_list_vLLM') return this.listEmitters.vLLM.error.event; + + else throw new Error(`Event not found: ${event}`); } // browser uses this to call (see this.channel.call() in llmMessageService.ts for all usages) @@ -78,8 +77,8 @@ export class LLMMessageChannel implements IServerChannel { else if (command === 'ollamaList') { this._callOllamaList(params) } - else if (command === 'openAICompatibleList') { - this._callOpenAICompatibleList(params) + else if (command === 'vLLMList') { + this._callVLLMList(params) } else { throw new Error(`Void sendLLM: command "${command}" not recognized.`) @@ -94,47 +93,50 @@ export class LLMMessageChannel implements IServerChannel { private async _callSendLLMMessage(params: MainSendLLMMessageParams) { const { requestId } = params; - if (!(requestId in this._abortRefOfRequestId_llm)) - this._abortRefOfRequestId_llm[requestId] = { current: null } + if (!(requestId in this.abortRefOfRequestId)) + this.abortRefOfRequestId[requestId] = { current: null } const mainThreadParams: SendLLMMessageParams = { ...params, - onText: ({ newText, fullText }) => { this._onText_llm.fire({ requestId, newText, fullText }); }, - onFinalMessage: ({ fullText, toolCalls }) => { this._onFinalMessage_llm.fire({ requestId, fullText, toolCalls }); }, - onError: ({ message: error, fullError }) => { console.log('sendLLM: firing err'); this._onError_llm.fire({ requestId, message: error, fullError }); }, - abortRef: this._abortRefOfRequestId_llm[requestId], + onText: (p) => { this.llmMessageEmitters.onText.fire({ requestId, ...p }); }, + onFinalMessage: (p) => { this.llmMessageEmitters.onFinalMessage.fire({ requestId, ...p }); }, + onError: (p) => { console.log('sendLLM: firing err'); this.llmMessageEmitters.onError.fire({ requestId, ...p }); }, + abortRef: this.abortRefOfRequestId[requestId], } sendLLMMessage(mainThreadParams, this.metricsService); } - private _callAbort(params: MainLLMMessageAbortParams) { - const { requestId } = params; - if (!(requestId in this._abortRefOfRequestId_llm)) return - this._abortRefOfRequestId_llm[requestId].current?.() - delete this._abortRefOfRequestId_llm[requestId] - } - - private _callOllamaList(params: MainModelListParams) { - const { requestId } = params; - + _callOllamaList = (params: MainModelListParams) => { + const { requestId } = params + const emitters = this.listEmitters.ollama const mainThreadParams: ModelListParams = { ...params, - onSuccess: ({ models }) => { this._onSuccess_ollama.fire({ requestId, models }); }, - onError: ({ error }) => { this._onError_ollama.fire({ requestId, error }); }, + onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); }, + onError: (p) => { emitters.error.fire({ requestId, ...p }); }, } - ollamaList(mainThreadParams) + sendLLMMessageToProviderImplementation.ollama.list(mainThreadParams) } - private _callOpenAICompatibleList(params: MainModelListParams) { - const { requestId } = params; - - const mainThreadParams: ModelListParams = { + _callVLLMList = (params: MainModelListParams) => { + const { requestId } = params + const emitters = this.listEmitters.vLLM + const mainThreadParams: ModelListParams = { ...params, - onSuccess: ({ models }) => { this._onSuccess_openAICompatible.fire({ requestId, models }); }, - onError: ({ error }) => { this._onError_openAICompatible.fire({ requestId, error }); }, + onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); }, + onError: (p) => { emitters.error.fire({ requestId, ...p }); }, } - openaiCompatibleList(mainThreadParams) + sendLLMMessageToProviderImplementation.vLLM.list(mainThreadParams) } + + + + private _callAbort(params: MainLLMMessageAbortParams) { + const { requestId } = params; + if (!(requestId in this.abortRefOfRequestId)) return + this.abortRefOfRequestId[requestId].current?.() + delete this.abortRefOfRequestId[requestId] + } + }