From 9f20476eea682cdab5e9757b7844604b8eb5252b Mon Sep 17 00:00:00 2001 From: Andrew Pareles Date: Sun, 23 Feb 2025 21:37:34 -0800 Subject: [PATCH] provider support progress Co-authored-by: Mathew Pareles --- .../browser/helpers/extractCodeFromResult.ts | 21 +- .../contrib/void/common/llmMessageService.ts | 109 ++-- .../contrib/void/common/llmMessageTypes.ts | 7 +- .../void/common/refreshModelService.ts | 10 +- .../void/electron-main/llmMessage/MODELS.ts | 588 +++++++++++++----- .../llmMessage/preprocessLLMMessages.ts | 51 +- .../llmMessage/sendLLMMessage.ts | 47 +- .../void/electron-main/llmMessageChannel.ts | 133 ++-- 8 files changed, 602 insertions(+), 364 deletions(-) diff --git a/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts b/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts index cd3276ff..806676da 100644 --- a/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts +++ b/src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts @@ -59,7 +59,7 @@ class SurroundingsRemover { // return offset === suffix.length // } - removeFromStartUntil = (until: string, alsoRemoveUntilStr: boolean) => { + removeFromStartUntilFullMatch = (until: string, alsoRemoveUntilStr: boolean) => { const index = this.originalS.indexOf(until, this.i) if (index === -1) { @@ -86,7 +86,7 @@ class SurroundingsRemover { const foundCodeBlock = pm.removePrefix('```') if (!foundCodeBlock) return false - pm.removeFromStartUntil('\n', true) // language + pm.removeFromStartUntilFullMatch('\n', true) // language const j = pm.j let foundCodeBlockEnd = pm.removeSuffix('```') @@ -159,27 +159,10 @@ export const extractCodeFromFIM = ({ text, recentlyAddedTextLen, midTag, }: { te const [delta, ignoredSuffix] = pm.deltaInfo(recentlyAddedTextLen) return [s, delta, ignoredSuffix] - - - // // const regex = /[\s\S]*?(?:`{1,3}\s*([a-zA-Z_]+[\w]*)?[\s\S]*?)?([\s\S]*?)(?:<\/MID>|`{1,3}|$)/; - // const regex = new RegExp( - // `[\\s\\S]*?(?:\`{1,3}\\s*([a-zA-Z_]+[\\w]*)?[\\s\\S]*?)?<${midTag}>([\\s\\S]*?)(?:|\`{1,3}|$)`, - // '' - // ); - // const match = text.match(regex); - // if (match) { - // const [_, languageName, codeBetweenMidTags] = match; - // return [languageName, codeBetweenMidTags] as const - - // } else { - // return [undefined, extractCodeFromRegular(text)] as const - // } - } - export type ExtractedSearchReplaceBlock = { state: 'writingOriginal' | 'writingFinal' | 'done', orig: string, diff --git a/src/vs/workbench/contrib/void/common/llmMessageService.ts b/src/vs/workbench/contrib/void/common/llmMessageService.ts index c8f9ea2c..b2266213 100644 --- a/src/vs/workbench/contrib/void/common/llmMessageService.ts +++ b/src/vs/workbench/contrib/void/common/llmMessageService.ts @@ -3,7 +3,7 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ -import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from './llmMessageTypes.js'; +import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, VLLMModelResponse, } from './llmMessageTypes.js'; import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js'; @@ -24,27 +24,39 @@ export interface ILLMMessageService { sendLLMMessage: (params: ServiceSendLLMMessageParams) => string | null; abort: (requestId: string) => void; ollamaList: (params: ServiceModelListParams) => void; - openAICompatibleList: (params: ServiceModelListParams) => void; + vLLMList: (params: ServiceModelListParams) => void; } + +// open this file side by side with llmMessageChannel export class LLMMessageService extends Disposable implements ILLMMessageService { readonly _serviceBrand: undefined; private readonly channel: IChannel // LLMMessageChannel - // llmMessage - private readonly onTextHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) } = {} - private readonly onFinalMessageHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) } = {} - private readonly onErrorHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) } = {} + // sendLLMMessage + private readonly llmMessageHooks = { + onText: {} as { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) }, + onFinalMessage: {} as { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) }, + onError: {} as { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) }, + } - - // ollamaList - private readonly onSuccess_ollama: { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) } = {} - private readonly onError_ollama: { [eventId: string]: ((params: EventModelListOnErrorParams) => void) } = {} - - // openAICompatibleList - private readonly onSuccess_openAICompatible: { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) } = {} - private readonly onError_openAICompatible: { [eventId: string]: ((params: EventModelListOnErrorParams) => void) } = {} + // list hooks + private readonly listHooks = { + ollama: { + success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) }, + error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams) => void) }, + }, + vLLM: { + success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) }, + error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams) => void) }, + } + } satisfies { + [providerName: string]: { + success: { [eventId: string]: ((params: EventModelListOnSuccessParams) => void) }, + error: { [eventId: string]: ((params: EventModelListOnErrorParams) => void) }, + } + } constructor( @IMainProcessService private readonly mainProcessService: IMainProcessService, // used as a renderer (only usable on client side) @@ -59,32 +71,14 @@ export class LLMMessageService extends Disposable implements ILLMMessageService // .listen sets up an IPC channel and takes a few ms, so we set up listeners immediately and add hooks to them instead // llm - this._register((this.channel.listen('onText_llm') satisfies Event)(e => { - this.onTextHooks_llm[e.requestId]?.(e) - })) - this._register((this.channel.listen('onFinalMessage_llm') satisfies Event)(e => { - this.onFinalMessageHooks_llm[e.requestId]?.(e) - this._onRequestIdDone(e.requestId) - })) - this._register((this.channel.listen('onError_llm') satisfies Event)(e => { - console.error('Error in LLMMessageService:', JSON.stringify(e)) - this.onErrorHooks_llm[e.requestId]?.(e) - this._onRequestIdDone(e.requestId) - })) + this._register((this.channel.listen('onText_sendLLMMessage') satisfies Event)(e => { this.llmMessageHooks.onText[e.requestId]?.(e) })) + this._register((this.channel.listen('onFinalMessage_sendLLMMessage') satisfies Event)(e => { this.llmMessageHooks.onFinalMessage[e.requestId]?.(e); this._onRequestIdDone(e.requestId) })) + this._register((this.channel.listen('onError_sendLLMMessage') satisfies Event)(e => { this.llmMessageHooks.onError[e.requestId]?.(e); this._onRequestIdDone(e.requestId); console.error('Error in LLMMessageService:', JSON.stringify(e)) })) // ollama .list() - this._register((this.channel.listen('onSuccess_ollama') satisfies Event>)(e => { - this.onSuccess_ollama[e.requestId]?.(e) - })) - this._register((this.channel.listen('onError_ollama') satisfies Event>)(e => { - this.onError_ollama[e.requestId]?.(e) - })) - // openaiCompatible .list() - this._register((this.channel.listen('onSuccess_openAICompatible') satisfies Event>)(e => { - this.onSuccess_openAICompatible[e.requestId]?.(e) - })) - this._register((this.channel.listen('onError_openAICompatible') satisfies Event>)(e => { - this.onError_openAICompatible[e.requestId]?.(e) - })) + this._register((this.channel.listen('onSuccess_list_ollama') satisfies Event>)(e => { this.listHooks.ollama.success[e.requestId]?.(e) })) + this._register((this.channel.listen('onError_list_ollama') satisfies Event>)(e => { this.listHooks.ollama.error[e.requestId]?.(e) })) + this._register((this.channel.listen('onSuccess_list_vLLM') satisfies Event>)(e => { this.listHooks.vLLM.success[e.requestId]?.(e) })) + this._register((this.channel.listen('onError_list_vLLM') satisfies Event>)(e => { this.listHooks.vLLM.error[e.requestId]?.(e) })) } @@ -117,9 +111,9 @@ export class LLMMessageService extends Disposable implements ILLMMessageService // add state for request id const requestId = generateUuid(); - this.onTextHooks_llm[requestId] = onText - this.onFinalMessageHooks_llm[requestId] = onFinalMessage - this.onErrorHooks_llm[requestId] = onError + this.llmMessageHooks.onText[requestId] = onText + this.llmMessageHooks.onFinalMessage[requestId] = onFinalMessage + this.llmMessageHooks.onError[requestId] = onError const { aiInstructions } = this.voidSettingsService.state.globalSettings const { settingsOfProvider } = this.voidSettingsService.state @@ -151,8 +145,8 @@ export class LLMMessageService extends Disposable implements ILLMMessageService // add state for request id const requestId_ = generateUuid(); - this.onSuccess_ollama[requestId_] = onSuccess - this.onError_ollama[requestId_] = onError + this.listHooks.ollama.success[requestId_] = onSuccess + this.listHooks.ollama.error[requestId_] = onError this.channel.call('ollamaList', { ...proxyParams, @@ -163,33 +157,34 @@ export class LLMMessageService extends Disposable implements ILLMMessageService } - openAICompatibleList = (params: ServiceModelListParams) => { + vLLMList = (params: ServiceModelListParams) => { const { onSuccess, onError, ...proxyParams } = params const { settingsOfProvider } = this.voidSettingsService.state // add state for request id const requestId_ = generateUuid(); - this.onSuccess_openAICompatible[requestId_] = onSuccess - this.onError_openAICompatible[requestId_] = onError + this.listHooks.vLLM.success[requestId_] = onSuccess + this.listHooks.vLLM.error[requestId_] = onError - this.channel.call('openAICompatibleList', { + this.channel.call('vLLMList', { ...proxyParams, settingsOfProvider, - providerName: 'openAICompatible', + providerName: 'vLLM', requestId: requestId_, - } satisfies MainModelListParams) + } satisfies MainModelListParams) } - - _onRequestIdDone(requestId: string) { - delete this.onTextHooks_llm[requestId] - delete this.onFinalMessageHooks_llm[requestId] - delete this.onErrorHooks_llm[requestId] + delete this.llmMessageHooks.onText[requestId] + delete this.llmMessageHooks.onFinalMessage[requestId] + delete this.llmMessageHooks.onError[requestId] - delete this.onSuccess_ollama[requestId] - delete this.onError_ollama[requestId] + delete this.listHooks.ollama.success[requestId] + delete this.listHooks.ollama.error[requestId] + + delete this.listHooks.vLLM.success[requestId] + delete this.listHooks.vLLM.error[requestId] } } diff --git a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts index 01e03ad4..abe88970 100644 --- a/src/vs/workbench/contrib/void/common/llmMessageTypes.ts +++ b/src/vs/workbench/contrib/void/common/llmMessageTypes.ts @@ -45,7 +45,7 @@ export type ToolCallType = { } -export type OnText = (p: { newText: string, fullText: string }) => void +export type OnText = (p: { newText: string, fullText: string; newReasoning: string; fullReasoning: string }) => void export type OnFinalMessage = (p: { fullText: string, toolCalls?: ToolCallType[] }) => void // id is tool_use_id export type OnError = (p: { message: string, fullError: Error | null }) => void export type AbortRef = { current: (() => void) | null } @@ -149,13 +149,16 @@ export type OllamaModelResponse = { size_vram: number; } -export type OpenaiCompatibleModelResponse = { +type OpenaiCompatibleModelResponse = { id: string; created: number; object: 'model'; owned_by: string; } +export type VLLMModelResponse = OpenaiCompatibleModelResponse + + // params to the true list fn export type ModelListParams = { diff --git a/src/vs/workbench/contrib/void/common/refreshModelService.ts b/src/vs/workbench/contrib/void/common/refreshModelService.ts index 1c95a4ad..1d68b304 100644 --- a/src/vs/workbench/contrib/void/common/refreshModelService.ts +++ b/src/vs/workbench/contrib/void/common/refreshModelService.ts @@ -8,7 +8,7 @@ import { ILLMMessageService } from './llmMessageService.js'; import { Emitter, Event } from '../../../../base/common/event.js'; import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js'; import { RefreshableProviderName, refreshableProviderNames, SettingsOfProvider } from './voidSettingsTypes.js'; -import { OllamaModelResponse, OpenaiCompatibleModelResponse } from './llmMessageTypes.js'; +import { OllamaModelResponse, VLLMModelResponse } from './llmMessageTypes.js'; import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js'; import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; @@ -160,9 +160,8 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ } } const listFn = providerName === 'ollama' ? this.llmMessageService.ollamaList - : providerName === 'vLLM' ? this.llmMessageService.openAICompatibleList - : providerName === 'openAICompatible' ? this.llmMessageService.openAICompatibleList - : () => { } + : providerName === 'vLLM' ? this.llmMessageService.vLLMList + : () => { } listFn({ onSuccess: ({ models }) => { @@ -172,8 +171,7 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ providerName, models.map(model => { if (providerName === 'ollama') return (model as OllamaModelResponse).name; - else if (providerName === 'vLLM') return (model as OpenaiCompatibleModelResponse).id; - else if (providerName === 'openAICompatible') return (model as OpenaiCompatibleModelResponse).id; + else if (providerName === 'vLLM') return (model as VLLMModelResponse).id; else throw new Error('refreshMode fn: unknown provider', providerName); }), { enableProviderOnSuccess: options.enableProviderOnSuccess, hideRefresh: options.doNotFire } diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts index e6e035fe..ce0d0537 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts @@ -3,11 +3,11 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ -import OpenAI from 'openai'; +import OpenAI, { ClientOptions } from 'openai'; import { Model as OpenAIModel } from 'openai/resources/models.js'; import { OllamaModelResponse, OnText, OnFinalMessage, OnError, LLMChatMessage, LLMFIMMessage, ModelListParams } from '../../common/llmMessageTypes.js'; import { InternalToolInfo, isAToolName } from '../../common/toolsService.js'; -import { defaultProviderSettings, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js'; +import { defaultProviderSettings, displayInfoOfProviderName, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js'; import { prepareMessages } from './preprocessLLMMessages.js'; import Anthropic from '@anthropic-ai/sdk'; import { Ollama } from 'ollama'; @@ -68,129 +68,215 @@ export const defaultModelsOfProvider = { +type ModelOptions = { + contextWindow: number; + cost: { + input: number; + output: number; + cache_read?: number; + cache_write?: number; + } + supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated'; + supportsTools: false | 'anthropic-style' | 'openai-style'; + supportsFIM: false | 'TODO_FIM_FORMAT'; + + supportsReasoning: boolean; // not whether it reasons, but whether it outputs reasoning tokens + manualMatchReasoningTokens?: [string, string]; // reasoning tokens if it's an OSS model +} + +type ProviderReasoningOptions = { + // include this in payload to get reasoning + input?: { includeInPayload?: { [key: string]: any }, }; + // nameOfFieldInDelta: reasoning output is in response.choices[0].delta[deltaReasoningField] + // needsManualParse: whether we must manually parse out the tags + output?: + | { nameOfFieldInDelta?: string, needsManualParse?: undefined, } + | { nameOfFieldInDelta?: undefined, needsManualParse?: true, }; +} + type ProviderSettings = { - thinkingFormat: string; - toolsFormat: string; - FIMFormat: string; - modelOptions: { - [key: string]: { - contextWindow: number; - cost: { - input: number; - output: number; - cache_read?: number; - cache_write?: number; - } - supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated'; - supportsTools: false | 'anthropic-style' | 'openai-style'; - supportsFIM: false | 'TODO_FIM_FORMAT' + providerReasoningOptions?: ProviderReasoningOptions; + modelOptions: { [key: string]: ModelOptions }; + modelOptionsFallback: (modelName: string) => ModelOptions; // allowed to throw error if modeName is totally invalid +} + + +type ModelSettingsOfProvider = { + [providerName in ProviderName]: ProviderSettings +} + + + + + +const modelNotRecognizedErrorMessage = (modelName: string, providerName: ProviderName) => `Void could not find a model matching ${modelName} for ${displayInfoOfProviderName(providerName).title}.` + + + +// ---------------- OPENAI ---------------- +const openAIModelOptions = { + "o1": { + contextWindow: 128_000, + cost: { input: 15.00, cache_read: 7.50, output: 60.00, }, + supportsFIM: false, + supportsTools: false, + supportsSystemMessage: 'developer-role', + supportsReasoning: false, + }, + "o3-mini": { + contextWindow: 200_000, + cost: { input: 1.10, cache_read: 0.55, output: 4.40, }, + supportsFIM: false, + supportsTools: false, + supportsSystemMessage: 'developer-role', + supportsReasoning: false, + }, + "gpt-4o": { + contextWindow: 128_000, + cost: { input: 2.50, cache_read: 1.25, output: 10.00, }, + supportsFIM: false, + supportsTools: 'openai-style', + supportsSystemMessage: 'system-role', + supportsReasoning: false, + }, +} as const + +const openAISettings: ProviderSettings = { + modelOptions: openAIModelOptions, + modelOptionsFallback: (modelName) => { + if (modelName.includes('o1')) return openAIModelOptions['o1'] + if (modelName.includes('o3-mini')) return openAIModelOptions['o3-mini'] + if (modelName.includes('gpt-4o')) return openAIModelOptions['gpt-4o'] + throw new Error(modelNotRecognizedErrorMessage(modelName, 'openAI')) + } +} + +// ---------------- ANTHROPIC ---------------- +const anthropicModelOptions = { + "claude-3-5-sonnet-20241022": { + contextWindow: 200_000, + cost: { input: 3.00, cache_read: 0.30, cache_write: 3.75, output: 15.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + + }, + "claude-3-5-haiku-20241022": { + contextWindow: 200_000, + cost: { input: 0.80, cache_read: 0.08, cache_write: 1.00, output: 4.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + }, + "claude-3-opus-20240229": { + contextWindow: 200_000, + cost: { input: 15.00, cache_read: 1.50, cache_write: 18.75, output: 75.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + }, + "claude-3-sonnet-20240229": { + contextWindow: 200_000, cost: { input: 3.00, output: 15.00 }, + supportsFIM: false, + supportsSystemMessage: 'separated', + supportsTools: 'anthropic-style', + supportsReasoning: false, + } +} as const + +const anthropicSettings: ProviderSettings = { + modelOptions: anthropicModelOptions, + modelOptionsFallback: (modelName) => { + throw new Error(modelNotRecognizedErrorMessage(modelName, 'anthropic')) + } +} + + +// ---------------- XAI ---------------- +const XAIModelOptions = { + "grok-2-latest": { + contextWindow: 131_072, + cost: { input: 2.00, output: 10.00 }, + supportsFIM: false, + supportsSystemMessage: 'system-role', + supportsTools: 'openai-style', + supportsReasoning: false, + }, +} as const + +const XAISettings: ProviderSettings = { + modelOptions: XAIModelOptions, + modelOptionsFallback: (modelName) => { + throw new Error(modelNotRecognizedErrorMessage(modelName, 'xAI')) + } +} + + + +const modelSettingsOfProvider: ModelSettingsOfProvider = { + openAI: openAISettings, + anthropic: anthropicSettings, + xAI: XAISettings, + gemini: { + modelOptions: { + } - } -} + }, + googleVertex: { + }, + microsoftAzure: { -const openAIProviderSettings: ProviderSettings = { - - thinkingFormat: '', - - toolsFormat: '', - - FIMFormat: '', - - modelOptions: { - 'o1': { - contextWindow: 128_000, - cost: { input: 15.00, cache_read: 7.50, output: 60.00, }, - supportsFIM: false, - supportsTools: false, - supportsSystemMessage: 'developer-role', - }, - 'o3-mini': { - contextWindow: 200_000, - cost: { input: 1.10, cache_read: 0.55, output: 4.40, }, - supportsFIM: false, - supportsTools: false, - supportsSystemMessage: 'developer-role', - }, - 'gpt-4o': { - contextWindow: 128_000, - cost: { input: 2.50, cache_read: 1.25, output: 10.00, }, - supportsFIM: false, - supportsTools: 'openai-style', - supportsSystemMessage: 'system-role', - }, - } - -} - - - - - -const anthropicProviderSettings: ProviderSettings = { - thinkingFormat: '', - - toolsFormat: '', - - FIMFormat: '', - - modelOptions: { - "claude-3-5-sonnet-20241022": { - contextWindow: 200_000, - cost: { input: 3.00, cache_read: 0.30, cache_write: 3.75, output: 15.00 }, - supportsFIM: false, - supportsSystemMessage: 'system-role', - supportsTools: 'anthropic-style', - - }, - "claude-3-5-haiku-20241022": { - contextWindow: 200_000, - cost: { input: 0.80, cache_read: 0.08, cache_write: 1.00, output: 4.00 }, - supportsFIM: false, - supportsSystemMessage: 'system-role', - supportsTools: 'anthropic-style', - }, - "claude-3-opus-20240229": { - contextWindow: 200_000, - cost: { input: 15.00, cache_read: 1.50, cache_write: 18.75, output: 75.00 }, - supportsFIM: false, - supportsSystemMessage: 'system-role', - supportsTools: 'anthropic-style', - }, - "claude-3-sonnet-20240229": { - contextWindow: 200_000, cost: { input: 3.00, output: 15.00 }, - supportsFIM: false, - supportsSystemMessage: 'system-role', - supportsTools: 'anthropic-style', + }, + openRouter: { + providerReasoningOptions: { + // reasoning: OAICompat + response.choices[0].delta.reasoning : payload should have {include_reasoning: true} https://openrouter.ai/announcements/reasoning-tokens-for-thinking-models + input: { includeInPayload: { include_reasoning: true } }, + output: { nameOfFieldInDelta: 'reasoning' }, } - } -} - - - -const grokProviderSettings: ProviderSettings = { - thinkingFormat: '', - - toolsFormat: '', - - FIMFormat: '', - - modelOptions: { - "grok-2-latest": { - contextWindow: 131_072, - cost: { input: 2.00, output: 10.00 }, - supportsFIM: false, - supportsSystemMessage: 'system-role', - supportsTools: 'openai-style', + }, + vLLM: { + providerReasoningOptions: { + // reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions + output: { nameOfFieldInDelta: 'reasoning_content' }, + } + }, + deepseek: { + providerReasoningOptions: { + // reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://api-docs.deepseek.com/guides/reasoning_model + output: { nameOfFieldInDelta: 'reasoning_content' }, }, - } + }, + ollama: { + providerReasoningOptions: { + // reasoning: we need to filter out reasoning tags manually + output: { needsManualParse: true }, + }, + }, + openAICompatible: { + }, + mistral: { + }, + groq: { + }, + + + +} as const satisfies ModelSettingsOfProvider + + +const modelOptionsOfProvider = (providerName: ProviderName, modelName: string) => { + const { modelOptions, modelOptionsFallback } = modelSettingsOfProvider[providerName] + if (modelName in modelOptions) return modelOptions[modelName] + return modelOptionsFallback(modelName) } - - type InternalCommonMessageParams = { aiInstructions: string; onText: OnText; @@ -234,30 +320,94 @@ const toolCallsFrom_OpenAICompat = (toolCallOfIndex: ToolCallOfIndex) => { } -const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName }) => { +const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => { + const commonPayloadOpts: ClientOptions = { + dangerouslyAllowBrowser: true, + ...includeInPayload, + } if (providerName === 'openAI') { const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, }) + return new OpenAI({ apiKey: thisConfig.apiKey, ...commonPayloadOpts }) } else if (providerName === 'ollama') { const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true, }) + return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts }) } else if (providerName === 'vLLM') { const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true, }) + return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts }) } - else throw new Error(`Invalid providerName ${providerName}`) + else if (providerName === 'openRouter') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ + baseURL: 'https://openrouter.ai/api/v1', + apiKey: thisConfig.apiKey, + defaultHeaders: { + 'HTTP-Referer': 'https://voideditor.com', // Optional, for including your app on openrouter.ai rankings. + 'X-Title': 'Void', // Optional. Shows in rankings on openrouter.ai. + }, + ...commonPayloadOpts, + }) + } + else if (providerName === 'gemini') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'deepseek') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'openAICompatible') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'mistral') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'groq') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'xAI') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: 'https://api.x.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + + else throw new Error(`Void providerName was invalid: ${providerName}.`) } -export const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => { - const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage: '', supportsTools: '', }) + + +const manualParseOnText = ( + providerName: ProviderName, + modelName: string, + onText_: OnText +): OnText => { + return onText_ +} + + +const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => { + const { + supportsReasoning: modelSupportsReasoning, + supportsSystemMessage, + supportsTools, + } = modelOptionsOfProvider(providerName, modelName) + + const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, }) const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toOpenAICompatibleTool(tool)) : undefined + const includeInPayload = modelSupportsReasoning ? {} : modelSettingsOfProvider[providerName].providerReasoningOptions?.input?.includeInPayload || {} + const toolsObj = tools ? { tools: tools, tool_choice: 'auto', parallel_tool_calls: false, } as const : {} - const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider }) + const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload }) const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelName, messages: messages, stream: true, ...toolsObj } + const { nameOfFieldInDelta: nameOfReasoningFieldInDelta, needsManualParse: needsManualReasoningParse } = modelSettingsOfProvider[providerName].providerReasoningOptions?.output ?? {} + if (needsManualReasoningParse) onText = manualParseOnText(providerName, modelName, onText) + + let fullReasoning = '' let fullText = '' const toolCallOfIndex: ToolCallOfIndex = {} openai.chat.completions @@ -275,10 +425,18 @@ export const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinal toolCallOfIndex[index].id = tool.id ?? '' } // message - let newText = '' - newText += chunk.choices[0]?.delta?.content ?? '' + const newText = chunk.choices[0]?.delta?.content ?? '' fullText += newText - onText({ newText, fullText }) + + // reasoning + let newReasoning = '' + if (nameOfReasoningFieldInDelta) { + // @ts-ignore + newReasoning = (chunk.choices[0]?.delta?.[nameOfFieldInDelta] || '') + '' + fullReasoning += newReasoning + } + + onText({ newText, fullText, newReasoning, fullReasoning }) } onFinalMessage({ fullText, toolCalls: toolCallsFrom_OpenAICompat(toolCallOfIndex) }); }) @@ -290,7 +448,7 @@ export const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinal } -export const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider, providerName }: ListParams_Internal) => { +const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider, providerName }: ListParams_Internal) => { const onSuccess = ({ models }: { models: OpenAIModel[] }) => { onSuccess_({ models }) } @@ -318,8 +476,9 @@ export const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: on } + // ------------ OPENAI ------------ -export const sendOpenAIChat = (params: SendChatParams_Internal) => { +const sendOpenAIChat = (params: SendChatParams_Internal) => { return _sendOpenAICompatibleChat(params) } @@ -345,25 +504,31 @@ const toolCallsFromAnthropicContent = (content: Anthropic.Messages.ContentBlock[ }).filter(t => !!t) } -export const sendAnthropicChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }: SendChatParams_Internal) => { - const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage: 'separated', supportsTools: 'anthropic-style', }) +const sendAnthropicChat = ({ messages: messages_, onText, providerName, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }: SendChatParams_Internal) => { + const { + // supportsReasoning: modelSupportsReasoning, + supportsSystemMessage, + supportsTools, + contextWindow, + } = modelOptionsOfProvider(providerName, modelName) + + const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, }) const thisConfig = settingsOfProvider.anthropic const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true }); const tools = ((tools_?.length ?? 0) !== 0) ? tools_?.map(tool => toAnthropicTool(tool)) : undefined - const maxTokens = ; const stream = anthropic.messages.stream({ system: separateSystemMessageStr, messages: messages, model: modelName, - max_tokens: maxTokens, + max_tokens: contextWindow, tools: tools, tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time }) // when receive text stream.on('text', (newText, fullText) => { - onText({ newText, fullText }) + onText({ newText, fullText, newReasoning: '', fullReasoning: '' }) }) // when we get the final message on this stream (or when error/fail) stream.on('finalMessage', (response) => { @@ -377,7 +542,7 @@ export const sendAnthropicChat = ({ messages: messages_, onText, onFinalMessage, else { onError({ message: error + '', fullError: error }) } }) _setAborter(() => stream.controller.abort()) -}; +} // // in future, can do tool_use streaming in anthropic, but it's pretty fast even without streaming... // const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {} @@ -396,6 +561,16 @@ export const sendAnthropicChat = ({ messages: messages_, onText, onFinalMessage, // }) +// ------------ XAI ------------ +const sendXAIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ GEMINI ------------ +const sendGeminiAPIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + // ------------ OLLAMA ------------ const newOllamaSDK = ({ endpoint }: { endpoint: string }) => { // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in @@ -404,7 +579,7 @@ const newOllamaSDK = ({ endpoint }: { endpoint: string }) => { return ollama } -export const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }: ListParams_Internal) => { +const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }: ListParams_Internal) => { const onSuccess = ({ models }: { models: OllamaModelResponse[] }) => { onSuccess_({ models }) } @@ -428,7 +603,7 @@ export const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, set } } -export const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => { +const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => { const thisConfig = settingsOfProvider.ollama const ollama = newOllamaSDK({ endpoint: thisConfig.endpoint }) @@ -461,58 +636,141 @@ export const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfPro // ollama's implementation of openai-compatible SDK dumps all reasoning tokens out with message, and supports tools, so we can use it for chat! -export const sendOllamaChat = (params: SendChatParams_Internal) => { +const sendOllamaChat = (params: SendChatParams_Internal) => { return _sendOpenAICompatibleChat(params) - // TODO!!! filter out reasoning tags... -} - - - -// ------------ OPENROUTER ------------ -export const sendOpenRouterFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => { - // TODO!!! -} - -export const sendOpenRouterChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => { - // reasoning: response.choices[0].delta.reasoning : payload should have {include_reasoning: true} https://openrouter.ai/announcements/reasoning-tokens-for-thinking-models - // } // ------------ OPENAI-COMPATIBLE ------------ -export const openAICompatibleList = async (params: ListParams_Internal) => { - return _openaiCompatibleList(params) -} - // TODO!!! FIM // using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration -export const sendOpenAICompatibleChat = (params: SendChatParams_Internal) => { +const sendOpenAICompatibleChat = (params: SendChatParams_Internal) => { return _sendOpenAICompatibleChat(params) } +// ------------ OPENROUTER ------------ +const sendOpenRouterChat = (params: SendChatParams_Internal) => { + _sendOpenAICompatibleChat(params) +} + // ------------ VLLM ------------ - -// TODO!!! FIM +const vLLMList = async (params: ListParams_Internal) => { + return _openaiCompatibleList(params) +} +const sendVLLMFIM = (params: SendFIMParams_Internal) => { + // TODO!!! +} // using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration -export const sendVLLMChat = (params: SendChatParams_Internal) => { +const sendVLLMChat = (params: SendChatParams_Internal) => { return _sendOpenAICompatibleChat(params) - // reasoning: response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions } - // ------------ DEEPSEEK API ------------ -export const sendDeepSeekAPIChat = (params: SendChatParams_Internal) => { +const sendDeepSeekAPIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ MISTRAL ------------ +const sendMistralAPIChat = (params: SendChatParams_Internal) => { + return _sendOpenAICompatibleChat(params) +} + +// ------------ GROQ ------------ +const sendGroqAPIChat = (params: SendChatParams_Internal) => { return _sendOpenAICompatibleChat(params) - // reasoning: response.choices[0].delta.reasoning_content // https://api-docs.deepseek.com/guides/reasoning_model } -// ------------ GEMINI ------------ -// ------------ MISTRAL ------------ -// ------------ GROQ ------------ -// ------------ GROK ------------ + + +/* +FIM: + +qwen2.5-coder https://ollama.com/library/qwen2.5-coder/blobs/e94a8ecb9327 +<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> + +codestral https://ollama.com/library/codestral/blobs/51707752a87c +[SUFFIX]{{ .Suffix }}[PREFIX] {{ .Prompt }} + +deepseek-coder-v2 https://ollama.com/library/deepseek-coder-v2/blobs/22091531faf0 +<｜fim▁begin｜>{{ .Prompt }}<｜fim▁hole｜>{{ .Suffix }}<｜fim▁end｜> + +starcoder2 https://ollama.com/library/starcoder2/blobs/3b190e68fefe + + +{{ .Prompt }}{{ .Suffix }} +<|end_of_text|> + +codegemma https://ollama.com/library/codegemma:2b/blobs/48d9a8140749 +<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> + +*/ +type CallFnOfProvider = { + [providerName in ProviderName]: { + sendChat: (params: SendChatParams_Internal) => void; + sendFIM: ((params: SendFIMParams_Internal) => void) | null; + list: ((params: ListParams_Internal) => void) | null; + } +} +export const sendLLMMessageToProviderImplementation = { + openAI: { + sendChat: sendOpenAIChat, + sendFIM: null, + list: null, + }, + anthropic: { + sendChat: sendAnthropicChat, + sendFIM: null, + list: null, + }, + xAI: { + sendChat: sendXAIChat, + sendFIM: null, + list: null, + }, + gemini: { + sendChat: sendGeminiAPIChat, + sendFIM: null, + list: null, + }, + ollama: { + sendChat: sendOllamaChat, + sendFIM: sendOllamaFIM, + list: ollamaList, + }, + openAICompatible: { + sendChat: sendOpenAICompatibleChat, + sendFIM: null, + list: null, + }, + openRouter: { + sendChat: sendOpenRouterChat, + sendFIM: null, + list: null, + }, + vLLM: { + sendChat: sendVLLMChat, + sendFIM: sendVLLMFIM, + list: vLLMList, + }, + deepseek: { + sendChat: sendDeepSeekAPIChat, + sendFIM: null, + list: null, + }, + groq: { + sendChat: sendGroqAPIChat, + sendFIM: null, + list: null, + }, + mistral: { + sendChat: sendMistralAPIChat, + sendFIM: null, + list: null, + }, +} satisfies CallFnOfProvider diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts index 3cef5327..1d388338 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts @@ -266,6 +266,31 @@ const prepareMessages_tools = ({ messages, supportsTools }: { messages: LLMChatM +/* +Gemini has this, but they're openai-compat so we don't need to implement this +gemini request: +{ "role": "assistant", + "content": null, + "function_call": { + "name": "get_weather", + "arguments": { + "latitude": 48.8566, + "longitude": 2.3522 + } + } +} + +gemini response: +{ "role": "assistant", + "function_response": { + "name": "get_weather", + "response": { + "temperature": "15°C", + "condition": "Cloudy" + } + } +} +*/ @@ -297,29 +322,3 @@ export const prepareMessages = ({ } as const } - -/* -ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message) -gemini request: -{ "role": "assistant", - "content": null, - "function_call": { - "name": "get_weather", - "arguments": { - "latitude": 48.8566, - "longitude": 2.3522 - } - } -} - -gemini response: -{ "role": "assistant", - "function_response": { - "name": "get_weather", - "response": { - "temperature": "15°C", - "condition": "Cloudy" - } - } -} -*/ diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts index 1c9ac21d..90deffe2 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts @@ -6,7 +6,7 @@ import { SendLLMMessageParams, OnText, OnFinalMessage, OnError } from '../../common/llmMessageTypes.js'; import { IMetricsService } from '../../common/metricsService.js'; import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js'; -import { sendAnthropicChat, sendOpenAIChat } from './MODELS.js'; +import { sendLLMMessageToProviderImplementation } from './MODELS.js'; export const sendLLMMessage = ({ @@ -56,9 +56,10 @@ export const sendLLMMessage = ({ let _setAborter = (fn: () => void) => { _aborter = fn } let _didAbort = false - const onText: OnText = ({ newText, fullText }) => { + const onText: OnText = (params) => { + const { fullText } = params if (_didAbort) return - onText_({ newText, fullText }) + onText_(params) _fullTextSoFar = fullText } @@ -93,29 +94,27 @@ export const sendLLMMessage = ({ else if (messagesType === 'FIMMessage') captureLLMEvent(`${loggingName} - Sending FIM`, {}) // TODO!!! add more metrics + try { - switch (providerName) { - case 'anthropic': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM' }) - else /* */ sendAnthropicChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }); - break; - case 'openAI': - case 'openRouter': - case 'deepseek': - case 'openAICompatible': - case 'mistral': - case 'ollama': - case 'vLLM': - case 'groq': - case 'gemini': - case 'xAI': - if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI API FIM' }) - else /* */ sendOpenAIChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }); - break; - default: - onError({ message: `Error: Void provider was "${providerName}", which is not recognized.`, fullError: null }) - break; + const implementation = sendLLMMessageToProviderImplementation[providerName] + if (!implementation) { + onError({ message: `Error: Provider "${providerName}" not recognized.`, fullError: null }) + return } + const { sendFIM, sendChat } = implementation + if (messagesType === 'chatMessages') { + sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools }) + return + } + if (messagesType === 'FIMMessage') { + if (sendFIM) { + sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions }) + return + } + onError({ message: `Error: This provider does not support Autocomplete yet.`, fullError: null }) + return + } + onError({ message: `Error: Message type "${messagesType}" not recognized.`, fullError: null }) } catch (error) { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts b/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts index 929c85e4..d2bceb4c 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessageChannel.ts @@ -8,29 +8,42 @@ import { IServerChannel } from '../../../../base/parts/ipc/common/ipc.js'; import { Emitter, Event } from '../../../../base/common/event.js'; -import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, MainModelListParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from '../common/llmMessageTypes.js'; +import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, VLLMModelResponse, MainModelListParams, } from '../common/llmMessageTypes.js'; import { sendLLMMessage } from './llmMessage/sendLLMMessage.js' import { IMetricsService } from '../common/metricsService.js'; -import { ollamaList } from './llmMessage/MODELS.js'; +import { sendLLMMessageToProviderImplementation } from './llmMessage/MODELS.js'; // NODE IMPLEMENTATION - calls actual sendLLMMessage() and returns listeners to it export class LLMMessageChannel implements IServerChannel { + // sendLLMMessage - private readonly _onText_llm = new Emitter(); - private readonly _onFinalMessage_llm = new Emitter(); - private readonly _onError_llm = new Emitter(); + private readonly llmMessageEmitters = { + onText: new Emitter(), + onFinalMessage: new Emitter(), + onError: new Emitter(), + } - // abort - private readonly _abortRefOfRequestId_llm: Record = {} + // aborters for above + private readonly abortRefOfRequestId: Record = {} - // ollamaList - private readonly _onSuccess_ollama = new Emitter>(); - private readonly _onError_ollama = new Emitter>(); - // openaiCompatibleList - private readonly _onSuccess_openAICompatible = new Emitter>(); - private readonly _onError_openAICompatible = new Emitter>(); + // list + private readonly listEmitters = { + ollama: { + success: new Emitter>(), + error: new Emitter>(), + }, + vLLM: { + success: new Emitter>(), + error: new Emitter>(), + } + } satisfies { + [providerName: string]: { + success: Emitter>, + error: Emitter>, + } + } // stupidly, channels can't take in @IService constructor( @@ -39,30 +52,17 @@ export class LLMMessageChannel implements IServerChannel { // browser uses this to listen for changes listen(_: unknown, event: string): Event { - if (event === 'onText_llm') { - return this._onText_llm.event; - } - else if (event === 'onFinalMessage_llm') { - return this._onFinalMessage_llm.event; - } - else if (event === 'onError_llm') { - return this._onError_llm.event; - } - else if (event === 'onSuccess_ollama') { - return this._onSuccess_ollama.event; - } - else if (event === 'onError_ollama') { - return this._onError_ollama.event; - } - else if (event === 'onSuccess_openAICompatible') { - return this._onSuccess_openAICompatible.event; - } - else if (event === 'onError_openAICompatible') { - return this._onError_openAICompatible.event; - } - else { - throw new Error(`Event not found: ${event}`); - } + // text + if (event === 'onText_sendLLMMessage') return this.llmMessageEmitters.onText.event; + else if (event === 'onFinalMessage_sendLLMMessage') return this.llmMessageEmitters.onFinalMessage.event; + else if (event === 'onError_sendLLMMessage') return this.llmMessageEmitters.onError.event; + // list + else if (event === 'onSuccess_list_ollama') return this.listEmitters.ollama.success.event; + else if (event === 'onError_list_ollama') return this.listEmitters.ollama.error.event; + else if (event === 'onSuccess_list_vLLM') return this.listEmitters.vLLM.success.event; + else if (event === 'onError_list_vLLM') return this.listEmitters.vLLM.error.event; + + else throw new Error(`Event not found: ${event}`); } // browser uses this to call (see this.channel.call() in llmMessageService.ts for all usages) @@ -77,8 +77,8 @@ export class LLMMessageChannel implements IServerChannel { else if (command === 'ollamaList') { this._callOllamaList(params) } - else if (command === 'openAICompatibleList') { - this._callOpenAICompatibleList(params) + else if (command === 'vLLMList') { + this._callVLLMList(params) } else { throw new Error(`Void sendLLM: command "${command}" not recognized.`) @@ -93,47 +93,50 @@ export class LLMMessageChannel implements IServerChannel { private async _callSendLLMMessage(params: MainSendLLMMessageParams) { const { requestId } = params; - if (!(requestId in this._abortRefOfRequestId_llm)) - this._abortRefOfRequestId_llm[requestId] = { current: null } + if (!(requestId in this.abortRefOfRequestId)) + this.abortRefOfRequestId[requestId] = { current: null } const mainThreadParams: SendLLMMessageParams = { ...params, - onText: ({ newText, fullText }) => { this._onText_llm.fire({ requestId, newText, fullText }); }, - onFinalMessage: ({ fullText, toolCalls }) => { this._onFinalMessage_llm.fire({ requestId, fullText, toolCalls }); }, - onError: ({ message: error, fullError }) => { console.log('sendLLM: firing err'); this._onError_llm.fire({ requestId, message: error, fullError }); }, - abortRef: this._abortRefOfRequestId_llm[requestId], + onText: (p) => { this.llmMessageEmitters.onText.fire({ requestId, ...p }); }, + onFinalMessage: (p) => { this.llmMessageEmitters.onFinalMessage.fire({ requestId, ...p }); }, + onError: (p) => { console.log('sendLLM: firing err'); this.llmMessageEmitters.onError.fire({ requestId, ...p }); }, + abortRef: this.abortRefOfRequestId[requestId], } sendLLMMessage(mainThreadParams, this.metricsService); } - private _callAbort(params: MainLLMMessageAbortParams) { - const { requestId } = params; - if (!(requestId in this._abortRefOfRequestId_llm)) return - this._abortRefOfRequestId_llm[requestId].current?.() - delete this._abortRefOfRequestId_llm[requestId] - } - - private _callOllamaList(params: MainModelListParams) { - const { requestId } = params; - + _callOllamaList = (params: MainModelListParams) => { + const { requestId } = params + const emitters = this.listEmitters.ollama const mainThreadParams: ModelListParams = { ...params, - onSuccess: ({ models }) => { this._onSuccess_ollama.fire({ requestId, models }); }, - onError: ({ error }) => { this._onError_ollama.fire({ requestId, error }); }, + onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); }, + onError: (p) => { emitters.error.fire({ requestId, ...p }); }, } - ollamaList(mainThreadParams) + sendLLMMessageToProviderImplementation.ollama.list(mainThreadParams) } - private _callOpenAICompatibleList(params: MainModelListParams) { - const { requestId } = params; - - const mainThreadParams: ModelListParams = { + _callVLLMList = (params: MainModelListParams) => { + const { requestId } = params + const emitters = this.listEmitters.vLLM + const mainThreadParams: ModelListParams = { ...params, - onSuccess: ({ models }) => { this._onSuccess_openAICompatible.fire({ requestId, models }); }, - onError: ({ error }) => { this._onError_openAICompatible.fire({ requestId, error }); }, + onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); }, + onError: (p) => { emitters.error.fire({ requestId, ...p }); }, } - openaiCompatibleList(mainThreadParams) + sendLLMMessageToProviderImplementation.vLLM.list(mainThreadParams) } + + + + private _callAbort(params: MainLLMMessageAbortParams) { + const { requestId } = params; + if (!(requestId in this.abortRefOfRequestId)) return + this.abortRefOfRequestId[requestId].current?.() + delete this.abortRefOfRequestId[requestId] + } + }