mirror of
https://github.com/voideditor/void
synced 2026-05-24 01:48:25 +00:00
provider support progress
Co-authored-by: Mathew Pareles <mathewpareles@users.noreply.github.com>
This commit is contained in:
Andrew Pareles
parent
fd5e523434
commit
9f20476eea
8 changed files with 602 additions and 364 deletions
|
|
@ -59,7 +59,7 @@ class SurroundingsRemover {
|
|||
// return offset === suffix.length
|
||||
// }
|
||||
|
||||
removeFromStartUntil = (until: string, alsoRemoveUntilStr: boolean) => {
|
||||
removeFromStartUntilFullMatch = (until: string, alsoRemoveUntilStr: boolean) => {
|
||||
const index = this.originalS.indexOf(until, this.i)
|
||||
|
||||
if (index === -1) {
|
||||
|
|
@ -86,7 +86,7 @@ class SurroundingsRemover {
|
|||
const foundCodeBlock = pm.removePrefix('```')
|
||||
if (!foundCodeBlock) return false
|
||||
|
||||
pm.removeFromStartUntil('\n', true) // language
|
||||
pm.removeFromStartUntilFullMatch('\n', true) // language
|
||||
|
||||
const j = pm.j
|
||||
let foundCodeBlockEnd = pm.removeSuffix('```')
|
||||
|
|
@ -159,27 +159,10 @@ export const extractCodeFromFIM = ({ text, recentlyAddedTextLen, midTag, }: { te
|
|||
const [delta, ignoredSuffix] = pm.deltaInfo(recentlyAddedTextLen)
|
||||
|
||||
return [s, delta, ignoredSuffix]
|
||||
|
||||
|
||||
// // const regex = /[\s\S]*?(?:`{1,3}\s*([a-zA-Z_]+[\w]*)?[\s\S]*?)?<MID>([\s\S]*?)(?:<\/MID>|`{1,3}|$)/;
|
||||
// const regex = new RegExp(
|
||||
// `[\\s\\S]*?(?:\`{1,3}\\s*([a-zA-Z_]+[\\w]*)?[\\s\\S]*?)?<${midTag}>([\\s\\S]*?)(?:</${midTag}>|\`{1,3}|$)`,
|
||||
// ''
|
||||
// );
|
||||
// const match = text.match(regex);
|
||||
// if (match) {
|
||||
// const [_, languageName, codeBetweenMidTags] = match;
|
||||
// return [languageName, codeBetweenMidTags] as const
|
||||
|
||||
// } else {
|
||||
// return [undefined, extractCodeFromRegular(text)] as const
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
export type ExtractedSearchReplaceBlock = {
|
||||
state: 'writingOriginal' | 'writingFinal' | 'done',
|
||||
orig: string,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from './llmMessageTypes.js';
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, VLLMModelResponse, } from './llmMessageTypes.js';
|
||||
|
||||
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
||||
import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js';
|
||||
|
|
@ -24,27 +24,39 @@ export interface ILLMMessageService {
|
|||
sendLLMMessage: (params: ServiceSendLLMMessageParams) => string | null;
|
||||
abort: (requestId: string) => void;
|
||||
ollamaList: (params: ServiceModelListParams<OllamaModelResponse>) => void;
|
||||
openAICompatibleList: (params: ServiceModelListParams<OpenaiCompatibleModelResponse>) => void;
|
||||
vLLMList: (params: ServiceModelListParams<VLLMModelResponse>) => void;
|
||||
}
|
||||
|
||||
|
||||
// open this file side by side with llmMessageChannel
|
||||
export class LLMMessageService extends Disposable implements ILLMMessageService {
|
||||
|
||||
readonly _serviceBrand: undefined;
|
||||
private readonly channel: IChannel // LLMMessageChannel
|
||||
|
||||
// llmMessage
|
||||
private readonly onTextHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) } = {}
|
||||
private readonly onFinalMessageHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) } = {}
|
||||
private readonly onErrorHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) } = {}
|
||||
// sendLLMMessage
|
||||
private readonly llmMessageHooks = {
|
||||
onText: {} as { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) },
|
||||
onFinalMessage: {} as { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) },
|
||||
onError: {} as { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) },
|
||||
}
|
||||
|
||||
|
||||
// ollamaList
|
||||
private readonly onSuccess_ollama: { [eventId: string]: ((params: EventModelListOnSuccessParams<OllamaModelResponse>) => void) } = {}
|
||||
private readonly onError_ollama: { [eventId: string]: ((params: EventModelListOnErrorParams<OllamaModelResponse>) => void) } = {}
|
||||
|
||||
// openAICompatibleList
|
||||
private readonly onSuccess_openAICompatible: { [eventId: string]: ((params: EventModelListOnSuccessParams<OpenaiCompatibleModelResponse>) => void) } = {}
|
||||
private readonly onError_openAICompatible: { [eventId: string]: ((params: EventModelListOnErrorParams<OpenaiCompatibleModelResponse>) => void) } = {}
|
||||
// list hooks
|
||||
private readonly listHooks = {
|
||||
ollama: {
|
||||
success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams<OllamaModelResponse>) => void) },
|
||||
error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams<OllamaModelResponse>) => void) },
|
||||
},
|
||||
vLLM: {
|
||||
success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams<VLLMModelResponse>) => void) },
|
||||
error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams<VLLMModelResponse>) => void) },
|
||||
}
|
||||
} satisfies {
|
||||
[providerName: string]: {
|
||||
success: { [eventId: string]: ((params: EventModelListOnSuccessParams<any>) => void) },
|
||||
error: { [eventId: string]: ((params: EventModelListOnErrorParams<any>) => void) },
|
||||
}
|
||||
}
|
||||
|
||||
constructor(
|
||||
@IMainProcessService private readonly mainProcessService: IMainProcessService, // used as a renderer (only usable on client side)
|
||||
|
|
@ -59,32 +71,14 @@ export class LLMMessageService extends Disposable implements ILLMMessageService
|
|||
|
||||
// .listen sets up an IPC channel and takes a few ms, so we set up listeners immediately and add hooks to them instead
|
||||
// llm
|
||||
this._register((this.channel.listen('onText_llm') satisfies Event<EventLLMMessageOnTextParams>)(e => {
|
||||
this.onTextHooks_llm[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onFinalMessage_llm') satisfies Event<EventLLMMessageOnFinalMessageParams>)(e => {
|
||||
this.onFinalMessageHooks_llm[e.requestId]?.(e)
|
||||
this._onRequestIdDone(e.requestId)
|
||||
}))
|
||||
this._register((this.channel.listen('onError_llm') satisfies Event<EventLLMMessageOnErrorParams>)(e => {
|
||||
console.error('Error in LLMMessageService:', JSON.stringify(e))
|
||||
this.onErrorHooks_llm[e.requestId]?.(e)
|
||||
this._onRequestIdDone(e.requestId)
|
||||
}))
|
||||
this._register((this.channel.listen('onText_sendLLMMessage') satisfies Event<EventLLMMessageOnTextParams>)(e => { this.llmMessageHooks.onText[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onFinalMessage_sendLLMMessage') satisfies Event<EventLLMMessageOnFinalMessageParams>)(e => { this.llmMessageHooks.onFinalMessage[e.requestId]?.(e); this._onRequestIdDone(e.requestId) }))
|
||||
this._register((this.channel.listen('onError_sendLLMMessage') satisfies Event<EventLLMMessageOnErrorParams>)(e => { this.llmMessageHooks.onError[e.requestId]?.(e); this._onRequestIdDone(e.requestId); console.error('Error in LLMMessageService:', JSON.stringify(e)) }))
|
||||
// ollama .list()
|
||||
this._register((this.channel.listen('onSuccess_ollama') satisfies Event<EventModelListOnSuccessParams<OllamaModelResponse>>)(e => {
|
||||
this.onSuccess_ollama[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onError_ollama') satisfies Event<EventModelListOnErrorParams<OllamaModelResponse>>)(e => {
|
||||
this.onError_ollama[e.requestId]?.(e)
|
||||
}))
|
||||
// openaiCompatible .list()
|
||||
this._register((this.channel.listen('onSuccess_openAICompatible') satisfies Event<EventModelListOnSuccessParams<OpenaiCompatibleModelResponse>>)(e => {
|
||||
this.onSuccess_openAICompatible[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onError_openAICompatible') satisfies Event<EventModelListOnErrorParams<OpenaiCompatibleModelResponse>>)(e => {
|
||||
this.onError_openAICompatible[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onSuccess_list_ollama') satisfies Event<EventModelListOnSuccessParams<OllamaModelResponse>>)(e => { this.listHooks.ollama.success[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onError_list_ollama') satisfies Event<EventModelListOnErrorParams<OllamaModelResponse>>)(e => { this.listHooks.ollama.error[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onSuccess_list_vLLM') satisfies Event<EventModelListOnSuccessParams<VLLMModelResponse>>)(e => { this.listHooks.vLLM.success[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onError_list_vLLM') satisfies Event<EventModelListOnErrorParams<VLLMModelResponse>>)(e => { this.listHooks.vLLM.error[e.requestId]?.(e) }))
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -117,9 +111,9 @@ export class LLMMessageService extends Disposable implements ILLMMessageService
|
|||
|
||||
// add state for request id
|
||||
const requestId = generateUuid();
|
||||
this.onTextHooks_llm[requestId] = onText
|
||||
this.onFinalMessageHooks_llm[requestId] = onFinalMessage
|
||||
this.onErrorHooks_llm[requestId] = onError
|
||||
this.llmMessageHooks.onText[requestId] = onText
|
||||
this.llmMessageHooks.onFinalMessage[requestId] = onFinalMessage
|
||||
this.llmMessageHooks.onError[requestId] = onError
|
||||
|
||||
const { aiInstructions } = this.voidSettingsService.state.globalSettings
|
||||
const { settingsOfProvider } = this.voidSettingsService.state
|
||||
|
|
@ -151,8 +145,8 @@ export class LLMMessageService extends Disposable implements ILLMMessageService
|
|||
|
||||
// add state for request id
|
||||
const requestId_ = generateUuid();
|
||||
this.onSuccess_ollama[requestId_] = onSuccess
|
||||
this.onError_ollama[requestId_] = onError
|
||||
this.listHooks.ollama.success[requestId_] = onSuccess
|
||||
this.listHooks.ollama.error[requestId_] = onError
|
||||
|
||||
this.channel.call('ollamaList', {
|
||||
...proxyParams,
|
||||
|
|
@ -163,33 +157,34 @@ export class LLMMessageService extends Disposable implements ILLMMessageService
|
|||
}
|
||||
|
||||
|
||||
openAICompatibleList = (params: ServiceModelListParams<OpenaiCompatibleModelResponse>) => {
|
||||
vLLMList = (params: ServiceModelListParams<VLLMModelResponse>) => {
|
||||
const { onSuccess, onError, ...proxyParams } = params
|
||||
|
||||
const { settingsOfProvider } = this.voidSettingsService.state
|
||||
|
||||
// add state for request id
|
||||
const requestId_ = generateUuid();
|
||||
this.onSuccess_openAICompatible[requestId_] = onSuccess
|
||||
this.onError_openAICompatible[requestId_] = onError
|
||||
this.listHooks.vLLM.success[requestId_] = onSuccess
|
||||
this.listHooks.vLLM.error[requestId_] = onError
|
||||
|
||||
this.channel.call('openAICompatibleList', {
|
||||
this.channel.call('vLLMList', {
|
||||
...proxyParams,
|
||||
settingsOfProvider,
|
||||
providerName: 'openAICompatible',
|
||||
providerName: 'vLLM',
|
||||
requestId: requestId_,
|
||||
} satisfies MainModelListParams<OpenaiCompatibleModelResponse>)
|
||||
} satisfies MainModelListParams<VLLMModelResponse>)
|
||||
}
|
||||
|
||||
|
||||
|
||||
_onRequestIdDone(requestId: string) {
|
||||
delete this.onTextHooks_llm[requestId]
|
||||
delete this.onFinalMessageHooks_llm[requestId]
|
||||
delete this.onErrorHooks_llm[requestId]
|
||||
delete this.llmMessageHooks.onText[requestId]
|
||||
delete this.llmMessageHooks.onFinalMessage[requestId]
|
||||
delete this.llmMessageHooks.onError[requestId]
|
||||
|
||||
delete this.onSuccess_ollama[requestId]
|
||||
delete this.onError_ollama[requestId]
|
||||
delete this.listHooks.ollama.success[requestId]
|
||||
delete this.listHooks.ollama.error[requestId]
|
||||
|
||||
delete this.listHooks.vLLM.success[requestId]
|
||||
delete this.listHooks.vLLM.error[requestId]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ export type ToolCallType = {
|
|||
}
|
||||
|
||||
|
||||
export type OnText = (p: { newText: string, fullText: string }) => void
|
||||
export type OnText = (p: { newText: string, fullText: string; newReasoning: string; fullReasoning: string }) => void
|
||||
export type OnFinalMessage = (p: { fullText: string, toolCalls?: ToolCallType[] }) => void // id is tool_use_id
|
||||
export type OnError = (p: { message: string, fullError: Error | null }) => void
|
||||
export type AbortRef = { current: (() => void) | null }
|
||||
|
|
@ -149,13 +149,16 @@ export type OllamaModelResponse = {
|
|||
size_vram: number;
|
||||
}
|
||||
|
||||
export type OpenaiCompatibleModelResponse = {
|
||||
type OpenaiCompatibleModelResponse = {
|
||||
id: string;
|
||||
created: number;
|
||||
object: 'model';
|
||||
owned_by: string;
|
||||
}
|
||||
|
||||
export type VLLMModelResponse = OpenaiCompatibleModelResponse
|
||||
|
||||
|
||||
|
||||
// params to the true list fn
|
||||
export type ModelListParams<ModelResponse> = {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import { ILLMMessageService } from './llmMessageService.js';
|
|||
import { Emitter, Event } from '../../../../base/common/event.js';
|
||||
import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js';
|
||||
import { RefreshableProviderName, refreshableProviderNames, SettingsOfProvider } from './voidSettingsTypes.js';
|
||||
import { OllamaModelResponse, OpenaiCompatibleModelResponse } from './llmMessageTypes.js';
|
||||
import { OllamaModelResponse, VLLMModelResponse } from './llmMessageTypes.js';
|
||||
import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js';
|
||||
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
||||
|
||||
|
|
@ -160,9 +160,8 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ
|
|||
}
|
||||
}
|
||||
const listFn = providerName === 'ollama' ? this.llmMessageService.ollamaList
|
||||
: providerName === 'vLLM' ? this.llmMessageService.openAICompatibleList
|
||||
: providerName === 'openAICompatible' ? this.llmMessageService.openAICompatibleList
|
||||
: () => { }
|
||||
: providerName === 'vLLM' ? this.llmMessageService.vLLMList
|
||||
: () => { }
|
||||
|
||||
listFn({
|
||||
onSuccess: ({ models }) => {
|
||||
|
|
@ -172,8 +171,7 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ
|
|||
providerName,
|
||||
models.map(model => {
|
||||
if (providerName === 'ollama') return (model as OllamaModelResponse).name;
|
||||
else if (providerName === 'vLLM') return (model as OpenaiCompatibleModelResponse).id;
|
||||
else if (providerName === 'openAICompatible') return (model as OpenaiCompatibleModelResponse).id;
|
||||
else if (providerName === 'vLLM') return (model as VLLMModelResponse).id;
|
||||
else throw new Error('refreshMode fn: unknown provider', providerName);
|
||||
}),
|
||||
{ enableProviderOnSuccess: options.enableProviderOnSuccess, hideRefresh: options.doNotFire }
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import OpenAI, { ClientOptions } from 'openai';
|
||||
import { Model as OpenAIModel } from 'openai/resources/models.js';
|
||||
import { OllamaModelResponse, OnText, OnFinalMessage, OnError, LLMChatMessage, LLMFIMMessage, ModelListParams } from '../../common/llmMessageTypes.js';
|
||||
import { InternalToolInfo, isAToolName } from '../../common/toolsService.js';
|
||||
import { defaultProviderSettings, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js';
|
||||
import { defaultProviderSettings, displayInfoOfProviderName, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js';
|
||||
import { prepareMessages } from './preprocessLLMMessages.js';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { Ollama } from 'ollama';
|
||||
|
|
@ -68,129 +68,215 @@ export const defaultModelsOfProvider = {
|
|||
|
||||
|
||||
|
||||
type ModelOptions = {
|
||||
contextWindow: number;
|
||||
cost: {
|
||||
input: number;
|
||||
output: number;
|
||||
cache_read?: number;
|
||||
cache_write?: number;
|
||||
}
|
||||
supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated';
|
||||
supportsTools: false | 'anthropic-style' | 'openai-style';
|
||||
supportsFIM: false | 'TODO_FIM_FORMAT';
|
||||
|
||||
supportsReasoning: boolean; // not whether it reasons, but whether it outputs reasoning tokens
|
||||
manualMatchReasoningTokens?: [string, string]; // reasoning tokens if it's an OSS model
|
||||
}
|
||||
|
||||
type ProviderReasoningOptions = {
|
||||
// include this in payload to get reasoning
|
||||
input?: { includeInPayload?: { [key: string]: any }, };
|
||||
// nameOfFieldInDelta: reasoning output is in response.choices[0].delta[deltaReasoningField]
|
||||
// needsManualParse: whether we must manually parse out the <think> tags
|
||||
output?:
|
||||
| { nameOfFieldInDelta?: string, needsManualParse?: undefined, }
|
||||
| { nameOfFieldInDelta?: undefined, needsManualParse?: true, };
|
||||
}
|
||||
|
||||
type ProviderSettings = {
|
||||
thinkingFormat: string;
|
||||
toolsFormat: string;
|
||||
FIMFormat: string;
|
||||
modelOptions: {
|
||||
[key: string]: {
|
||||
contextWindow: number;
|
||||
cost: {
|
||||
input: number;
|
||||
output: number;
|
||||
cache_read?: number;
|
||||
cache_write?: number;
|
||||
}
|
||||
supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated';
|
||||
supportsTools: false | 'anthropic-style' | 'openai-style';
|
||||
supportsFIM: false | 'TODO_FIM_FORMAT'
|
||||
providerReasoningOptions?: ProviderReasoningOptions;
|
||||
modelOptions: { [key: string]: ModelOptions };
|
||||
modelOptionsFallback: (modelName: string) => ModelOptions; // allowed to throw error if modeName is totally invalid
|
||||
}
|
||||
|
||||
|
||||
type ModelSettingsOfProvider = {
|
||||
[providerName in ProviderName]: ProviderSettings
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
const modelNotRecognizedErrorMessage = (modelName: string, providerName: ProviderName) => `Void could not find a model matching ${modelName} for ${displayInfoOfProviderName(providerName).title}.`
|
||||
|
||||
|
||||
|
||||
// ---------------- OPENAI ----------------
|
||||
const openAIModelOptions = {
|
||||
"o1": {
|
||||
contextWindow: 128_000,
|
||||
cost: { input: 15.00, cache_read: 7.50, output: 60.00, },
|
||||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"o3-mini": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 1.10, cache_read: 0.55, output: 4.40, },
|
||||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"gpt-4o": {
|
||||
contextWindow: 128_000,
|
||||
cost: { input: 2.50, cache_read: 1.25, output: 10.00, },
|
||||
supportsFIM: false,
|
||||
supportsTools: 'openai-style',
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
} as const
|
||||
|
||||
const openAISettings: ProviderSettings = {
|
||||
modelOptions: openAIModelOptions,
|
||||
modelOptionsFallback: (modelName) => {
|
||||
if (modelName.includes('o1')) return openAIModelOptions['o1']
|
||||
if (modelName.includes('o3-mini')) return openAIModelOptions['o3-mini']
|
||||
if (modelName.includes('gpt-4o')) return openAIModelOptions['gpt-4o']
|
||||
throw new Error(modelNotRecognizedErrorMessage(modelName, 'openAI'))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------- ANTHROPIC ----------------
|
||||
const anthropicModelOptions = {
|
||||
"claude-3-5-sonnet-20241022": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 3.00, cache_read: 0.30, cache_write: 3.75, output: 15.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
|
||||
},
|
||||
"claude-3-5-haiku-20241022": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 0.80, cache_read: 0.08, cache_write: 1.00, output: 4.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"claude-3-opus-20240229": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 15.00, cache_read: 1.50, cache_write: 18.75, output: 75.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"claude-3-sonnet-20240229": {
|
||||
contextWindow: 200_000, cost: { input: 3.00, output: 15.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
}
|
||||
} as const
|
||||
|
||||
const anthropicSettings: ProviderSettings = {
|
||||
modelOptions: anthropicModelOptions,
|
||||
modelOptionsFallback: (modelName) => {
|
||||
throw new Error(modelNotRecognizedErrorMessage(modelName, 'anthropic'))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ---------------- XAI ----------------
|
||||
const XAIModelOptions = {
|
||||
"grok-2-latest": {
|
||||
contextWindow: 131_072,
|
||||
cost: { input: 2.00, output: 10.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
} as const
|
||||
|
||||
const XAISettings: ProviderSettings = {
|
||||
modelOptions: XAIModelOptions,
|
||||
modelOptionsFallback: (modelName) => {
|
||||
throw new Error(modelNotRecognizedErrorMessage(modelName, 'xAI'))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
const modelSettingsOfProvider: ModelSettingsOfProvider = {
|
||||
openAI: openAISettings,
|
||||
anthropic: anthropicSettings,
|
||||
xAI: XAISettings,
|
||||
gemini: {
|
||||
modelOptions: {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
googleVertex: {
|
||||
|
||||
},
|
||||
microsoftAzure: {
|
||||
|
||||
const openAIProviderSettings: ProviderSettings = {
|
||||
|
||||
thinkingFormat: '',
|
||||
|
||||
toolsFormat: '',
|
||||
|
||||
FIMFormat: '',
|
||||
|
||||
modelOptions: {
|
||||
'o1': {
|
||||
contextWindow: 128_000,
|
||||
cost: { input: 15.00, cache_read: 7.50, output: 60.00, },
|
||||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
},
|
||||
'o3-mini': {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 1.10, cache_read: 0.55, output: 4.40, },
|
||||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
},
|
||||
'gpt-4o': {
|
||||
contextWindow: 128_000,
|
||||
cost: { input: 2.50, cache_read: 1.25, output: 10.00, },
|
||||
supportsFIM: false,
|
||||
supportsTools: 'openai-style',
|
||||
supportsSystemMessage: 'system-role',
|
||||
},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
const anthropicProviderSettings: ProviderSettings = {
|
||||
thinkingFormat: '',
|
||||
|
||||
toolsFormat: '',
|
||||
|
||||
FIMFormat: '',
|
||||
|
||||
modelOptions: {
|
||||
"claude-3-5-sonnet-20241022": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 3.00, cache_read: 0.30, cache_write: 3.75, output: 15.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'anthropic-style',
|
||||
|
||||
},
|
||||
"claude-3-5-haiku-20241022": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 0.80, cache_read: 0.08, cache_write: 1.00, output: 4.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'anthropic-style',
|
||||
},
|
||||
"claude-3-opus-20240229": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 15.00, cache_read: 1.50, cache_write: 18.75, output: 75.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'anthropic-style',
|
||||
},
|
||||
"claude-3-sonnet-20240229": {
|
||||
contextWindow: 200_000, cost: { input: 3.00, output: 15.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'anthropic-style',
|
||||
},
|
||||
openRouter: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning : payload should have {include_reasoning: true} https://openrouter.ai/announcements/reasoning-tokens-for-thinking-models
|
||||
input: { includeInPayload: { include_reasoning: true } },
|
||||
output: { nameOfFieldInDelta: 'reasoning' },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
const grokProviderSettings: ProviderSettings = {
|
||||
thinkingFormat: '',
|
||||
|
||||
toolsFormat: '',
|
||||
|
||||
FIMFormat: '',
|
||||
|
||||
modelOptions: {
|
||||
"grok-2-latest": {
|
||||
contextWindow: 131_072,
|
||||
cost: { input: 2.00, output: 10.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
},
|
||||
vLLM: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions
|
||||
output: { nameOfFieldInDelta: 'reasoning_content' },
|
||||
}
|
||||
},
|
||||
deepseek: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://api-docs.deepseek.com/guides/reasoning_model
|
||||
output: { nameOfFieldInDelta: 'reasoning_content' },
|
||||
},
|
||||
}
|
||||
},
|
||||
ollama: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: we need to filter out reasoning <think> tags manually
|
||||
output: { needsManualParse: true },
|
||||
},
|
||||
},
|
||||
|
||||
openAICompatible: {
|
||||
},
|
||||
mistral: {
|
||||
},
|
||||
groq: {
|
||||
},
|
||||
|
||||
|
||||
|
||||
} as const satisfies ModelSettingsOfProvider
|
||||
|
||||
|
||||
const modelOptionsOfProvider = (providerName: ProviderName, modelName: string) => {
|
||||
const { modelOptions, modelOptionsFallback } = modelSettingsOfProvider[providerName]
|
||||
if (modelName in modelOptions) return modelOptions[modelName]
|
||||
return modelOptionsFallback(modelName)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
type InternalCommonMessageParams = {
|
||||
aiInstructions: string;
|
||||
onText: OnText;
|
||||
|
|
@ -234,30 +320,94 @@ const toolCallsFrom_OpenAICompat = (toolCallOfIndex: ToolCallOfIndex) => {
|
|||
}
|
||||
|
||||
|
||||
const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName }) => {
|
||||
const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => {
|
||||
const commonPayloadOpts: ClientOptions = {
|
||||
dangerouslyAllowBrowser: true,
|
||||
...includeInPayload,
|
||||
}
|
||||
if (providerName === 'openAI') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true, })
|
||||
return new OpenAI({ apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'ollama') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true, })
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'vLLM') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true, })
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts })
|
||||
}
|
||||
else throw new Error(`Invalid providerName ${providerName}`)
|
||||
else if (providerName === 'openRouter') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://openrouter.ai/api/v1',
|
||||
apiKey: thisConfig.apiKey,
|
||||
defaultHeaders: {
|
||||
'HTTP-Referer': 'https://voideditor.com', // Optional, for including your app on openrouter.ai rankings.
|
||||
'X-Title': 'Void', // Optional. Shows in rankings on openrouter.ai.
|
||||
},
|
||||
...commonPayloadOpts,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'gemini') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'deepseek') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'openAICompatible') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'mistral') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'groq') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'xAI') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.x.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
|
||||
else throw new Error(`Void providerName was invalid: ${providerName}.`)
|
||||
}
|
||||
|
||||
export const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage: '', supportsTools: '', })
|
||||
|
||||
|
||||
const manualParseOnText = (
|
||||
providerName: ProviderName,
|
||||
modelName: string,
|
||||
onText_: OnText
|
||||
): OnText => {
|
||||
return onText_
|
||||
}
|
||||
|
||||
|
||||
const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const {
|
||||
supportsReasoning: modelSupportsReasoning,
|
||||
supportsSystemMessage,
|
||||
supportsTools,
|
||||
} = modelOptionsOfProvider(providerName, modelName)
|
||||
|
||||
const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, })
|
||||
const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toOpenAICompatibleTool(tool)) : undefined
|
||||
|
||||
const includeInPayload = modelSupportsReasoning ? {} : modelSettingsOfProvider[providerName].providerReasoningOptions?.input?.includeInPayload || {}
|
||||
|
||||
const toolsObj = tools ? { tools: tools, tool_choice: 'auto', parallel_tool_calls: false, } as const : {}
|
||||
const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider })
|
||||
const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload })
|
||||
const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelName, messages: messages, stream: true, ...toolsObj }
|
||||
|
||||
const { nameOfFieldInDelta: nameOfReasoningFieldInDelta, needsManualParse: needsManualReasoningParse } = modelSettingsOfProvider[providerName].providerReasoningOptions?.output ?? {}
|
||||
if (needsManualReasoningParse) onText = manualParseOnText(providerName, modelName, onText)
|
||||
|
||||
let fullReasoning = ''
|
||||
let fullText = ''
|
||||
const toolCallOfIndex: ToolCallOfIndex = {}
|
||||
openai.chat.completions
|
||||
|
|
@ -275,10 +425,18 @@ export const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinal
|
|||
toolCallOfIndex[index].id = tool.id ?? ''
|
||||
}
|
||||
// message
|
||||
let newText = ''
|
||||
newText += chunk.choices[0]?.delta?.content ?? ''
|
||||
const newText = chunk.choices[0]?.delta?.content ?? ''
|
||||
fullText += newText
|
||||
onText({ newText, fullText })
|
||||
|
||||
// reasoning
|
||||
let newReasoning = ''
|
||||
if (nameOfReasoningFieldInDelta) {
|
||||
// @ts-ignore
|
||||
newReasoning = (chunk.choices[0]?.delta?.[nameOfFieldInDelta] || '') + ''
|
||||
fullReasoning += newReasoning
|
||||
}
|
||||
|
||||
onText({ newText, fullText, newReasoning, fullReasoning })
|
||||
}
|
||||
onFinalMessage({ fullText, toolCalls: toolCallsFrom_OpenAICompat(toolCallOfIndex) });
|
||||
})
|
||||
|
|
@ -290,7 +448,7 @@ export const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinal
|
|||
}
|
||||
|
||||
|
||||
export const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider, providerName }: ListParams_Internal<OpenAIModel>) => {
|
||||
const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider, providerName }: ListParams_Internal<OpenAIModel>) => {
|
||||
const onSuccess = ({ models }: { models: OpenAIModel[] }) => {
|
||||
onSuccess_({ models })
|
||||
}
|
||||
|
|
@ -318,8 +476,9 @@ export const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: on
|
|||
}
|
||||
|
||||
|
||||
|
||||
// ------------ OPENAI ------------
|
||||
export const sendOpenAIChat = (params: SendChatParams_Internal) => {
|
||||
const sendOpenAIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
|
|
@ -345,25 +504,31 @@ const toolCallsFromAnthropicContent = (content: Anthropic.Messages.ContentBlock[
|
|||
}).filter(t => !!t)
|
||||
}
|
||||
|
||||
export const sendAnthropicChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage: 'separated', supportsTools: 'anthropic-style', })
|
||||
const sendAnthropicChat = ({ messages: messages_, onText, providerName, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const {
|
||||
// supportsReasoning: modelSupportsReasoning,
|
||||
supportsSystemMessage,
|
||||
supportsTools,
|
||||
contextWindow,
|
||||
} = modelOptionsOfProvider(providerName, modelName)
|
||||
|
||||
const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, })
|
||||
|
||||
const thisConfig = settingsOfProvider.anthropic
|
||||
const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true });
|
||||
const tools = ((tools_?.length ?? 0) !== 0) ? tools_?.map(tool => toAnthropicTool(tool)) : undefined
|
||||
|
||||
const maxTokens = ;
|
||||
const stream = anthropic.messages.stream({
|
||||
system: separateSystemMessageStr,
|
||||
messages: messages,
|
||||
model: modelName,
|
||||
max_tokens: maxTokens,
|
||||
max_tokens: contextWindow,
|
||||
tools: tools,
|
||||
tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time
|
||||
})
|
||||
// when receive text
|
||||
stream.on('text', (newText, fullText) => {
|
||||
onText({ newText, fullText })
|
||||
onText({ newText, fullText, newReasoning: '', fullReasoning: '' })
|
||||
})
|
||||
// when we get the final message on this stream (or when error/fail)
|
||||
stream.on('finalMessage', (response) => {
|
||||
|
|
@ -377,7 +542,7 @@ export const sendAnthropicChat = ({ messages: messages_, onText, onFinalMessage,
|
|||
else { onError({ message: error + '', fullError: error }) }
|
||||
})
|
||||
_setAborter(() => stream.controller.abort())
|
||||
};
|
||||
}
|
||||
|
||||
// // in future, can do tool_use streaming in anthropic, but it's pretty fast even without streaming...
|
||||
// const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {}
|
||||
|
|
@ -396,6 +561,16 @@ export const sendAnthropicChat = ({ messages: messages_, onText, onFinalMessage,
|
|||
// })
|
||||
|
||||
|
||||
// ------------ XAI ------------
|
||||
const sendXAIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ GEMINI ------------
|
||||
const sendGeminiAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ OLLAMA ------------
|
||||
const newOllamaSDK = ({ endpoint }: { endpoint: string }) => {
|
||||
// if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
|
|
@ -404,7 +579,7 @@ const newOllamaSDK = ({ endpoint }: { endpoint: string }) => {
|
|||
return ollama
|
||||
}
|
||||
|
||||
export const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }: ListParams_Internal<OllamaModelResponse>) => {
|
||||
const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }: ListParams_Internal<OllamaModelResponse>) => {
|
||||
const onSuccess = ({ models }: { models: OllamaModelResponse[] }) => {
|
||||
onSuccess_({ models })
|
||||
}
|
||||
|
|
@ -428,7 +603,7 @@ export const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, set
|
|||
}
|
||||
}
|
||||
|
||||
export const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => {
|
||||
const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => {
|
||||
const thisConfig = settingsOfProvider.ollama
|
||||
const ollama = newOllamaSDK({ endpoint: thisConfig.endpoint })
|
||||
|
||||
|
|
@ -461,58 +636,141 @@ export const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfPro
|
|||
|
||||
|
||||
// ollama's implementation of openai-compatible SDK dumps all reasoning tokens out with message, and supports tools, so we can use it for chat!
|
||||
export const sendOllamaChat = (params: SendChatParams_Internal) => {
|
||||
const sendOllamaChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
// TODO!!! filter out reasoning <think> tags...
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ------------ OPENROUTER ------------
|
||||
export const sendOpenRouterFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => {
|
||||
// TODO!!!
|
||||
}
|
||||
|
||||
export const sendOpenRouterChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
// reasoning: response.choices[0].delta.reasoning : payload should have {include_reasoning: true} https://openrouter.ai/announcements/reasoning-tokens-for-thinking-models
|
||||
//
|
||||
}
|
||||
|
||||
// ------------ OPENAI-COMPATIBLE ------------
|
||||
export const openAICompatibleList = async (params: ListParams_Internal<OpenAIModel>) => {
|
||||
return _openaiCompatibleList(params)
|
||||
}
|
||||
|
||||
// TODO!!! FIM
|
||||
|
||||
// using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration
|
||||
export const sendOpenAICompatibleChat = (params: SendChatParams_Internal) => {
|
||||
const sendOpenAICompatibleChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ OPENROUTER ------------
|
||||
const sendOpenRouterChat = (params: SendChatParams_Internal) => {
|
||||
_sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ VLLM ------------
|
||||
|
||||
// TODO!!! FIM
|
||||
const vLLMList = async (params: ListParams_Internal<OpenAIModel>) => {
|
||||
return _openaiCompatibleList(params)
|
||||
}
|
||||
const sendVLLMFIM = (params: SendFIMParams_Internal) => {
|
||||
// TODO!!!
|
||||
}
|
||||
|
||||
// using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration
|
||||
export const sendVLLMChat = (params: SendChatParams_Internal) => {
|
||||
const sendVLLMChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
// reasoning: response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions
|
||||
}
|
||||
|
||||
|
||||
// ------------ DEEPSEEK API ------------
|
||||
export const sendDeepSeekAPIChat = (params: SendChatParams_Internal) => {
|
||||
const sendDeepSeekAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ MISTRAL ------------
|
||||
const sendMistralAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ GROQ ------------
|
||||
const sendGroqAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
// reasoning: response.choices[0].delta.reasoning_content // https://api-docs.deepseek.com/guides/reasoning_model
|
||||
}
|
||||
|
||||
|
||||
// ------------ GEMINI ------------
|
||||
// ------------ MISTRAL ------------
|
||||
// ------------ GROQ ------------
|
||||
// ------------ GROK ------------
|
||||
|
||||
|
||||
/*
|
||||
FIM:
|
||||
|
||||
qwen2.5-coder https://ollama.com/library/qwen2.5-coder/blobs/e94a8ecb9327
|
||||
<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|>
|
||||
|
||||
codestral https://ollama.com/library/codestral/blobs/51707752a87c
|
||||
[SUFFIX]{{ .Suffix }}[PREFIX] {{ .Prompt }}
|
||||
|
||||
deepseek-coder-v2 https://ollama.com/library/deepseek-coder-v2/blobs/22091531faf0
|
||||
<｜fim▁begin｜>{{ .Prompt }}<｜fim▁hole｜>{{ .Suffix }}<｜fim▁end｜>
|
||||
|
||||
starcoder2 https://ollama.com/library/starcoder2/blobs/3b190e68fefe
|
||||
<file_sep>
|
||||
<fim_prefix>
|
||||
{{ .Prompt }}<fim_suffix>{{ .Suffix }}<fim_middle>
|
||||
<|end_of_text|>
|
||||
|
||||
codegemma https://ollama.com/library/codegemma:2b/blobs/48d9a8140749
|
||||
<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|>
|
||||
|
||||
*/
|
||||
|
||||
|
||||
|
||||
type CallFnOfProvider = {
|
||||
[providerName in ProviderName]: {
|
||||
sendChat: (params: SendChatParams_Internal) => void;
|
||||
sendFIM: ((params: SendFIMParams_Internal) => void) | null;
|
||||
list: ((params: ListParams_Internal<any>) => void) | null;
|
||||
}
|
||||
}
|
||||
export const sendLLMMessageToProviderImplementation = {
|
||||
openAI: {
|
||||
sendChat: sendOpenAIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
anthropic: {
|
||||
sendChat: sendAnthropicChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
xAI: {
|
||||
sendChat: sendXAIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
gemini: {
|
||||
sendChat: sendGeminiAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
ollama: {
|
||||
sendChat: sendOllamaChat,
|
||||
sendFIM: sendOllamaFIM,
|
||||
list: ollamaList,
|
||||
},
|
||||
openAICompatible: {
|
||||
sendChat: sendOpenAICompatibleChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
openRouter: {
|
||||
sendChat: sendOpenRouterChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
vLLM: {
|
||||
sendChat: sendVLLMChat,
|
||||
sendFIM: sendVLLMFIM,
|
||||
list: vLLMList,
|
||||
},
|
||||
deepseek: {
|
||||
sendChat: sendDeepSeekAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
groq: {
|
||||
sendChat: sendGroqAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
mistral: {
|
||||
sendChat: sendMistralAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
|
||||
} satisfies CallFnOfProvider
|
||||
|
|
|
|||
|
|
@ -266,6 +266,31 @@ const prepareMessages_tools = ({ messages, supportsTools }: { messages: LLMChatM
|
|||
|
||||
|
||||
|
||||
/*
|
||||
Gemini has this, but they're openai-compat so we don't need to implement this
|
||||
gemini request:
|
||||
{ "role": "assistant",
|
||||
"content": null,
|
||||
"function_call": {
|
||||
"name": "get_weather",
|
||||
"arguments": {
|
||||
"latitude": 48.8566,
|
||||
"longitude": 2.3522
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gemini response:
|
||||
{ "role": "assistant",
|
||||
"function_response": {
|
||||
"name": "get_weather",
|
||||
"response": {
|
||||
"temperature": "15°C",
|
||||
"condition": "Cloudy"
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
|
||||
|
||||
|
|
@ -297,29 +322,3 @@ export const prepareMessages = ({
|
|||
} as const
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message)
|
||||
gemini request:
|
||||
{ "role": "assistant",
|
||||
"content": null,
|
||||
"function_call": {
|
||||
"name": "get_weather",
|
||||
"arguments": {
|
||||
"latitude": 48.8566,
|
||||
"longitude": 2.3522
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gemini response:
|
||||
{ "role": "assistant",
|
||||
"function_response": {
|
||||
"name": "get_weather",
|
||||
"response": {
|
||||
"temperature": "15°C",
|
||||
"condition": "Cloudy"
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
import { SendLLMMessageParams, OnText, OnFinalMessage, OnError } from '../../common/llmMessageTypes.js';
|
||||
import { IMetricsService } from '../../common/metricsService.js';
|
||||
import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js';
|
||||
import { sendAnthropicChat, sendOpenAIChat } from './MODELS.js';
|
||||
import { sendLLMMessageToProviderImplementation } from './MODELS.js';
|
||||
|
||||
|
||||
export const sendLLMMessage = ({
|
||||
|
|
@ -56,9 +56,10 @@ export const sendLLMMessage = ({
|
|||
let _setAborter = (fn: () => void) => { _aborter = fn }
|
||||
let _didAbort = false
|
||||
|
||||
const onText: OnText = ({ newText, fullText }) => {
|
||||
const onText: OnText = (params) => {
|
||||
const { fullText } = params
|
||||
if (_didAbort) return
|
||||
onText_({ newText, fullText })
|
||||
onText_(params)
|
||||
_fullTextSoFar = fullText
|
||||
}
|
||||
|
||||
|
|
@ -93,29 +94,27 @@ export const sendLLMMessage = ({
|
|||
else if (messagesType === 'FIMMessage')
|
||||
captureLLMEvent(`${loggingName} - Sending FIM`, {}) // TODO!!! add more metrics
|
||||
|
||||
|
||||
try {
|
||||
switch (providerName) {
|
||||
case 'anthropic':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM' })
|
||||
else /* */ sendAnthropicChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools });
|
||||
break;
|
||||
case 'openAI':
|
||||
case 'openRouter':
|
||||
case 'deepseek':
|
||||
case 'openAICompatible':
|
||||
case 'mistral':
|
||||
case 'ollama':
|
||||
case 'vLLM':
|
||||
case 'groq':
|
||||
case 'gemini':
|
||||
case 'xAI':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI API FIM' })
|
||||
else /* */ sendOpenAIChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools });
|
||||
break;
|
||||
default:
|
||||
onError({ message: `Error: Void provider was "${providerName}", which is not recognized.`, fullError: null })
|
||||
break;
|
||||
const implementation = sendLLMMessageToProviderImplementation[providerName]
|
||||
if (!implementation) {
|
||||
onError({ message: `Error: Provider "${providerName}" not recognized.`, fullError: null })
|
||||
return
|
||||
}
|
||||
const { sendFIM, sendChat } = implementation
|
||||
if (messagesType === 'chatMessages') {
|
||||
sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools })
|
||||
return
|
||||
}
|
||||
if (messagesType === 'FIMMessage') {
|
||||
if (sendFIM) {
|
||||
sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions })
|
||||
return
|
||||
}
|
||||
onError({ message: `Error: This provider does not support Autocomplete yet.`, fullError: null })
|
||||
return
|
||||
}
|
||||
onError({ message: `Error: Message type "${messagesType}" not recognized.`, fullError: null })
|
||||
}
|
||||
|
||||
catch (error) {
|
||||
|
|
|
|||
|
|
@ -8,29 +8,42 @@
|
|||
|
||||
import { IServerChannel } from '../../../../base/parts/ipc/common/ipc.js';
|
||||
import { Emitter, Event } from '../../../../base/common/event.js';
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, MainModelListParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from '../common/llmMessageTypes.js';
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, VLLMModelResponse, MainModelListParams, } from '../common/llmMessageTypes.js';
|
||||
import { sendLLMMessage } from './llmMessage/sendLLMMessage.js'
|
||||
import { IMetricsService } from '../common/metricsService.js';
|
||||
import { ollamaList } from './llmMessage/MODELS.js';
|
||||
import { sendLLMMessageToProviderImplementation } from './llmMessage/MODELS.js';
|
||||
|
||||
// NODE IMPLEMENTATION - calls actual sendLLMMessage() and returns listeners to it
|
||||
|
||||
export class LLMMessageChannel implements IServerChannel {
|
||||
|
||||
// sendLLMMessage
|
||||
private readonly _onText_llm = new Emitter<EventLLMMessageOnTextParams>();
|
||||
private readonly _onFinalMessage_llm = new Emitter<EventLLMMessageOnFinalMessageParams>();
|
||||
private readonly _onError_llm = new Emitter<EventLLMMessageOnErrorParams>();
|
||||
private readonly llmMessageEmitters = {
|
||||
onText: new Emitter<EventLLMMessageOnTextParams>(),
|
||||
onFinalMessage: new Emitter<EventLLMMessageOnFinalMessageParams>(),
|
||||
onError: new Emitter<EventLLMMessageOnErrorParams>(),
|
||||
}
|
||||
|
||||
// abort
|
||||
private readonly _abortRefOfRequestId_llm: Record<string, AbortRef> = {}
|
||||
// aborters for above
|
||||
private readonly abortRefOfRequestId: Record<string, AbortRef> = {}
|
||||
|
||||
// ollamaList
|
||||
private readonly _onSuccess_ollama = new Emitter<EventModelListOnSuccessParams<OllamaModelResponse>>();
|
||||
private readonly _onError_ollama = new Emitter<EventModelListOnErrorParams<OllamaModelResponse>>();
|
||||
|
||||
// openaiCompatibleList
|
||||
private readonly _onSuccess_openAICompatible = new Emitter<EventModelListOnSuccessParams<OpenaiCompatibleModelResponse>>();
|
||||
private readonly _onError_openAICompatible = new Emitter<EventModelListOnErrorParams<OpenaiCompatibleModelResponse>>();
|
||||
// list
|
||||
private readonly listEmitters = {
|
||||
ollama: {
|
||||
success: new Emitter<EventModelListOnSuccessParams<OllamaModelResponse>>(),
|
||||
error: new Emitter<EventModelListOnErrorParams<OllamaModelResponse>>(),
|
||||
},
|
||||
vLLM: {
|
||||
success: new Emitter<EventModelListOnSuccessParams<VLLMModelResponse>>(),
|
||||
error: new Emitter<EventModelListOnErrorParams<VLLMModelResponse>>(),
|
||||
}
|
||||
} satisfies {
|
||||
[providerName: string]: {
|
||||
success: Emitter<EventModelListOnSuccessParams<any>>,
|
||||
error: Emitter<EventModelListOnErrorParams<any>>,
|
||||
}
|
||||
}
|
||||
|
||||
// stupidly, channels can't take in @IService
|
||||
constructor(
|
||||
|
|
@ -39,30 +52,17 @@ export class LLMMessageChannel implements IServerChannel {
|
|||
|
||||
// browser uses this to listen for changes
|
||||
listen(_: unknown, event: string): Event<any> {
|
||||
if (event === 'onText_llm') {
|
||||
return this._onText_llm.event;
|
||||
}
|
||||
else if (event === 'onFinalMessage_llm') {
|
||||
return this._onFinalMessage_llm.event;
|
||||
}
|
||||
else if (event === 'onError_llm') {
|
||||
return this._onError_llm.event;
|
||||
}
|
||||
else if (event === 'onSuccess_ollama') {
|
||||
return this._onSuccess_ollama.event;
|
||||
}
|
||||
else if (event === 'onError_ollama') {
|
||||
return this._onError_ollama.event;
|
||||
}
|
||||
else if (event === 'onSuccess_openAICompatible') {
|
||||
return this._onSuccess_openAICompatible.event;
|
||||
}
|
||||
else if (event === 'onError_openAICompatible') {
|
||||
return this._onError_openAICompatible.event;
|
||||
}
|
||||
else {
|
||||
throw new Error(`Event not found: ${event}`);
|
||||
}
|
||||
// text
|
||||
if (event === 'onText_sendLLMMessage') return this.llmMessageEmitters.onText.event;
|
||||
else if (event === 'onFinalMessage_sendLLMMessage') return this.llmMessageEmitters.onFinalMessage.event;
|
||||
else if (event === 'onError_sendLLMMessage') return this.llmMessageEmitters.onError.event;
|
||||
// list
|
||||
else if (event === 'onSuccess_list_ollama') return this.listEmitters.ollama.success.event;
|
||||
else if (event === 'onError_list_ollama') return this.listEmitters.ollama.error.event;
|
||||
else if (event === 'onSuccess_list_vLLM') return this.listEmitters.vLLM.success.event;
|
||||
else if (event === 'onError_list_vLLM') return this.listEmitters.vLLM.error.event;
|
||||
|
||||
else throw new Error(`Event not found: ${event}`);
|
||||
}
|
||||
|
||||
// browser uses this to call (see this.channel.call() in llmMessageService.ts for all usages)
|
||||
|
|
@ -77,8 +77,8 @@ export class LLMMessageChannel implements IServerChannel {
|
|||
else if (command === 'ollamaList') {
|
||||
this._callOllamaList(params)
|
||||
}
|
||||
else if (command === 'openAICompatibleList') {
|
||||
this._callOpenAICompatibleList(params)
|
||||
else if (command === 'vLLMList') {
|
||||
this._callVLLMList(params)
|
||||
}
|
||||
else {
|
||||
throw new Error(`Void sendLLM: command "${command}" not recognized.`)
|
||||
|
|
@ -93,47 +93,50 @@ export class LLMMessageChannel implements IServerChannel {
|
|||
private async _callSendLLMMessage(params: MainSendLLMMessageParams) {
|
||||
const { requestId } = params;
|
||||
|
||||
if (!(requestId in this._abortRefOfRequestId_llm))
|
||||
this._abortRefOfRequestId_llm[requestId] = { current: null }
|
||||
if (!(requestId in this.abortRefOfRequestId))
|
||||
this.abortRefOfRequestId[requestId] = { current: null }
|
||||
|
||||
const mainThreadParams: SendLLMMessageParams = {
|
||||
...params,
|
||||
onText: ({ newText, fullText }) => { this._onText_llm.fire({ requestId, newText, fullText }); },
|
||||
onFinalMessage: ({ fullText, toolCalls }) => { this._onFinalMessage_llm.fire({ requestId, fullText, toolCalls }); },
|
||||
onError: ({ message: error, fullError }) => { console.log('sendLLM: firing err'); this._onError_llm.fire({ requestId, message: error, fullError }); },
|
||||
abortRef: this._abortRefOfRequestId_llm[requestId],
|
||||
onText: (p) => { this.llmMessageEmitters.onText.fire({ requestId, ...p }); },
|
||||
onFinalMessage: (p) => { this.llmMessageEmitters.onFinalMessage.fire({ requestId, ...p }); },
|
||||
onError: (p) => { console.log('sendLLM: firing err'); this.llmMessageEmitters.onError.fire({ requestId, ...p }); },
|
||||
abortRef: this.abortRefOfRequestId[requestId],
|
||||
}
|
||||
sendLLMMessage(mainThreadParams, this.metricsService);
|
||||
}
|
||||
|
||||
private _callAbort(params: MainLLMMessageAbortParams) {
|
||||
const { requestId } = params;
|
||||
if (!(requestId in this._abortRefOfRequestId_llm)) return
|
||||
this._abortRefOfRequestId_llm[requestId].current?.()
|
||||
delete this._abortRefOfRequestId_llm[requestId]
|
||||
}
|
||||
|
||||
private _callOllamaList(params: MainModelListParams<OllamaModelResponse>) {
|
||||
const { requestId } = params;
|
||||
|
||||
_callOllamaList = (params: MainModelListParams<OllamaModelResponse>) => {
|
||||
const { requestId } = params
|
||||
const emitters = this.listEmitters.ollama
|
||||
const mainThreadParams: ModelListParams<OllamaModelResponse> = {
|
||||
...params,
|
||||
onSuccess: ({ models }) => { this._onSuccess_ollama.fire({ requestId, models }); },
|
||||
onError: ({ error }) => { this._onError_ollama.fire({ requestId, error }); },
|
||||
onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); },
|
||||
onError: (p) => { emitters.error.fire({ requestId, ...p }); },
|
||||
}
|
||||
ollamaList(mainThreadParams)
|
||||
sendLLMMessageToProviderImplementation.ollama.list(mainThreadParams)
|
||||
}
|
||||
|
||||
private _callOpenAICompatibleList(params: MainModelListParams<OpenaiCompatibleModelResponse>) {
|
||||
const { requestId } = params;
|
||||
|
||||
const mainThreadParams: ModelListParams<OpenaiCompatibleModelResponse> = {
|
||||
_callVLLMList = (params: MainModelListParams<VLLMModelResponse>) => {
|
||||
const { requestId } = params
|
||||
const emitters = this.listEmitters.vLLM
|
||||
const mainThreadParams: ModelListParams<VLLMModelResponse> = {
|
||||
...params,
|
||||
onSuccess: ({ models }) => { this._onSuccess_openAICompatible.fire({ requestId, models }); },
|
||||
onError: ({ error }) => { this._onError_openAICompatible.fire({ requestId, error }); },
|
||||
onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); },
|
||||
onError: (p) => { emitters.error.fire({ requestId, ...p }); },
|
||||
}
|
||||
openaiCompatibleList(mainThreadParams)
|
||||
sendLLMMessageToProviderImplementation.vLLM.list(mainThreadParams)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private _callAbort(params: MainLLMMessageAbortParams) {
|
||||
const { requestId } = params;
|
||||
if (!(requestId in this.abortRefOfRequestId)) return
|
||||
this.abortRefOfRequestId[requestId].current?.()
|
||||
delete this.abortRefOfRequestId[requestId]
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue